From 9d6b5ea2b6b14d398f6fcb95685463e102d354e3 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:31:09 +0000 Subject: [PATCH 001/149] feat(dev): add an update lab for multi-environment update testing Provider latest-version checks honour npm_config_registry, and t3 update reads the release index from T3CODE_RELEASE_BASE_URL when set, so the lab (and npm or release mirrors) can serve both. --- apps/server/src/cli/update.ts | 8 +- .../src/server/maintenanceResolver.ts | 13 +- packages/shared/src/cliRelease.ts | 11 +- scripts/update-lab.ts | 1198 +++++++++++++++++ 4 files changed, 1224 insertions(+), 6 deletions(-) create mode 100644 scripts/update-lab.ts diff --git a/apps/server/src/cli/update.ts b/apps/server/src/cli/update.ts index 576158e316bc2..fa167e3bdeeb7 100644 --- a/apps/server/src/cli/update.ts +++ b/apps/server/src/cli/update.ts @@ -64,12 +64,13 @@ const RELEASE_INDEX_MAX_PAGES = 10; /** Asks GitHub for the newest published version on a channel, page by page. */ const resolveNewestVersion = Effect.fn("cli.update.resolve_newest")(function* ( channel: CliReleaseChannel, + releaseBaseUrl: string | undefined, ) { const httpClient = yield* HttpClient.HttpClient; for (let page = 1; page <= RELEASE_INDEX_MAX_PAGES; page += 1) { const body = yield* httpClient .execute( - HttpClientRequest.get(cliReleaseIndexPageUrl(page)).pipe( + HttpClientRequest.get(cliReleaseIndexPageUrl(page, releaseBaseUrl)).pipe( HttpClientRequest.setHeader("Accept", "application/vnd.github+json"), ), ) @@ -347,6 +348,7 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: { const service = yield* BootService.BootService; const currentVersion = packageJson.version; + const releaseBaseUrl = environment[CLI_RELEASE_BASE_URL_ENV]?.trim() || undefined; const channel = input.channel ?? cliReleaseChannelOf(currentVersion); if (input.requestedVersion !== undefined && !isExactServiceVersion(input.requestedVersion)) { return yield* new CliUpdateError({ @@ -357,7 +359,7 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: { progress.status("Checking for updates..."); const targetVersion = yield* ( input.requestedVersion === undefined - ? resolveNewestVersion(channel) + ? resolveNewestVersion(channel, releaseBaseUrl) : Effect.succeed(input.requestedVersion) ).pipe(Effect.ensuring(Effect.sync(progress.finish))); const targetChannel = cliReleaseChannelOf(targetVersion); @@ -491,7 +493,7 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: { httpClient, platform, arch, - releaseBaseUrl: environment[CLI_RELEASE_BASE_URL_ENV]?.trim() || undefined, + releaseBaseUrl, validate: (paths) => runner .run({ diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index 65667281ac6a9..ae6352b2abeaf 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -750,10 +750,21 @@ export function createProviderVersionAdvisory(input: { }; } +const DEFAULT_NPM_REGISTRY_URL = "https://registry.npmjs.org"; + +/** npm's own registry override, so mirrors (and local fake registries) answer "latest". */ +const NpmRegistryUrlConfig = Config.String("npm_config_registry").pipe( + Config.withDefault(DEFAULT_NPM_REGISTRY_URL), + Config.map((url) => url.trim().replace(/\/+$/, "") || DEFAULT_NPM_REGISTRY_URL), +); + const fetchNpmLatestVersion = Effect.fn("fetchNpmLatestVersion")(function* (packageName: string) { const client = yield* HttpClient.HttpClient; + const registryUrl = yield* NpmRegistryUrlConfig.pipe( + Effect.orElseSucceed(() => DEFAULT_NPM_REGISTRY_URL), + ); const request = HttpClientRequest.get( - `https://registry.npmjs.org/${encodeURIComponent(packageName)}/latest`, + `${registryUrl}/${encodeURIComponent(packageName)}/latest`, ).pipe(HttpClientRequest.setHeader("accept", "application/json")); const response = yield* client.execute(request).pipe( Effect.timeoutOption(LATEST_VERSION_TIMEOUT_MS), diff --git a/packages/shared/src/cliRelease.ts b/packages/shared/src/cliRelease.ts index 28f0d530bb297..69d0daddb4fc0 100644 --- a/packages/shared/src/cliRelease.ts +++ b/packages/shared/src/cliRelease.ts @@ -96,9 +96,16 @@ export function cliReleaseChannelOf(version: string): CliReleaseChannel { * One page of GitHub's list-releases endpoint, newest first. Callers walk pages * until a channel match turns up; a busy nightly train can push the newest * preview or stable release past any single page. + * + * A mirror set through `T3CODE_RELEASE_BASE_URL` serves the same JSON (an + * array of `{ tag_name, draft? }`, newest first, empty past the last page) at + * `/index.json?per_page=100&page=N`. */ -export function cliReleaseIndexPageUrl(page: number): string { - return `https://api.github.com/repos/${CLI_RELEASE_REPOSITORY}/releases?per_page=100&page=${page}`; +export function cliReleaseIndexPageUrl(page: number, baseUrl?: string): string { + const mirror = baseUrl?.trim().replace(/\/+$/, ""); + return mirror + ? `${mirror}/index.json?per_page=100&page=${page}` + : `https://api.github.com/repos/${CLI_RELEASE_REPOSITORY}/releases?per_page=100&page=${page}`; } /** diff --git a/scripts/update-lab.ts b/scripts/update-lab.ts new file mode 100644 index 0000000000000..5f70c4f84f9e7 --- /dev/null +++ b/scripts/update-lab.ts @@ -0,0 +1,1198 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off globalFetch:off globalConsole:off preferSchemaOverJson:off - Host-side dev lab that spawns isolated servers and fake CLIs with Node APIs directly. +/** + * Update lab: several isolated T3 servers ("machines") with fake provider + * CLIs, a fake npm registry and a fake t3 release feed, for exercising + * provider, server and launcher update flows end to end. State lives in the + * gitignored `/.t3/update-lab`. Fake CLIs and package managers re-enter + * this file as `__tool`, and the registry/feed server as `__feed`, so the lab + * stays one file. Run `node scripts/update-lab.ts --help`. + */ +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeHttp from "node:http"; +import * as NodeNet from "node:net"; +import * as NodePath from "node:path"; +import * as NodeReadline from "node:readline"; + +const REPO = NodePath.resolve(import.meta.dirname, ".."); +const LAB = NodePath.join(REPO, ".t3", "update-lab"); +const STATE_PATH = NodePath.join(LAB, "state.json"); +const RELEASES = NodePath.join(LAB, "releases"); +const SERVER_BIN = NodePath.join(REPO, "apps/server/src/bin.ts"); +const MOCK_ACP_AGENT = NodePath.join(REPO, "apps/server/scripts/acp-mock-agent.ts"); +const NODE = process.execPath; +const SELF = import.meta.filename; +const REPO_VERSION: string = JSON.parse( + NodeFS.readFileSync(NodePath.join(REPO, "apps/server/package.json"), "utf8"), +).version; +// oxlint-disable-next-line t3code/no-global-process-runtime -- Plain Node dev script; the lab only targets this host. +const PLATFORM_KEY = `${process.platform}-${process.arch}`; +const ORCHESTRATION_PROTOCOL = "2"; + +const HELP = `Usage: node scripts/update-lab.ts + +Isolated T3 servers with fake provider CLIs, a fake npm registry and a fake +t3 release feed, all under .t3/update-lab (state survives down/up). + + up [--envs N] [--host IP] [--launcher N] [--from-version V] [--busy-seconds S] + Start the feed and N servers (default 3; host 127.0.0.1). The last N + envs (default 1) run under the real service launcher on runtime V + (default: repo version minus one patch) so self-update can be tried. + Prints each env's origin and one-time pairing URL. Rerun to resume. + down Stop only the processes the lab recorded. + status Envs, ports, liveness, installed vs latest. + pair Mint a fresh pairing URL for env (e.g. env1). + publish Set the registry's latest for a provider. + fail on|off Make that env's installer/updater exit 1. + delay Installer/updater duration (default 3000). + providers Print the provider list clients see for env. + update [v] Run server.updateProvider like the UI does. + busy Start a mock-acp turn that runs --busy-seconds. + self-update Run server.updateServer (launcher envs only). + build-runtime [--archive file] + Publish a t3 runtime to the release feed: a source runtime (copy of + apps/server stamped with , runs on this checkout's deps) or + a real CLI archive built by the release pipeline. + reset Delete the lab (only after down). + +Providers: codex claude opencode pi grok. Install methods rotate across envs: +npm prefix, pnpm, bun, yarn, volta, vite-plus, Homebrew formula and cask, +native installers (claude update, codex standalone, opencode upgrade, +grok update, pi update --self) and mise (manual-only). Each env also has a +"mock-acp" provider (apps/server/scripts/acp-mock-agent.ts) for busy turns. +Servers cache npm "latest" for an hour: down && up after publish. +Not covered: cursor, antigravity, muse (no npm advisory or network-only).`; + +type ProviderKey = "codex" | "claude" | "opencode" | "pi" | "grok"; +type Method = + | "npm" + | "pnpm" + | "bun" + | "yarn" + | "volta" + | "vite-plus" + | "homebrew" + | "native" + | "mise"; + +const PROVIDERS: Record< + ProviderKey, + { + readonly command: string; + readonly pkg: string; + readonly driver: string; + readonly installed: string; + readonly latest: string; + readonly brew: { readonly kind: "formula" | "cask"; readonly name: string } | null; + } +> = { + codex: { + command: "codex", + pkg: "@openai/codex", + driver: "codex", + installed: "0.158.0", + latest: "0.160.0", + brew: { kind: "formula", name: "codex" }, + }, + claude: { + command: "claude", + pkg: "@anthropic-ai/claude-code", + driver: "claudeAgent", + installed: "2.1.285", + latest: "2.1.290", + brew: { kind: "cask", name: "claude-code" }, + }, + opencode: { + command: "opencode", + pkg: "opencode-ai", + driver: "opencode", + installed: "1.14.19", + latest: "1.14.25", + brew: { kind: "formula", name: "opencode" }, + }, + pi: { + command: "pi", + pkg: "@earendil-works/pi-coding-agent", + driver: "pi", + installed: "0.80.6", + latest: "0.81.0", + brew: null, + }, + grok: { + command: "grok", + pkg: "@xai-official/grok", + driver: "grok", + installed: "1.0.13", + latest: "1.0.20", + brew: null, + }, +}; +const PROVIDER_KEYS = Object.keys(PROVIDERS) as ProviderKey[]; + +// Env N uses row (N-1) % rows; three envs already reach every resolver branch. +const INSTALL_MATRIX: ReadonlyArray> = [ + { codex: "npm", claude: "native", opencode: "homebrew", pi: "pnpm", grok: "native" }, + { codex: "native", claude: "homebrew", opencode: "bun", pi: "native", grok: "npm" }, + { codex: "volta", claude: "yarn", opencode: "native", pi: "vite-plus", grok: "bun" }, + { codex: "mise", claude: "pnpm", opencode: "npm", pi: "yarn", grok: "native" }, + { codex: "vite-plus", claude: "bun", opencode: "volta", pi: "npm", grok: "pnpm" }, +]; + +// The package manager each fake tool stands in for. +const TOOL_METHOD: Record = { + npm: "npm", + pnpm: "pnpm", + bun: "bun", + yarn: "yarn", + volta: "volta", + vp: "vite-plus", + brew: "homebrew", +}; + +interface Install { + method: Method; + version: string; + failing: boolean; +} + +interface LabEnv { + id: string; + port: number; + pid: number | null; + launcher: boolean; + projectId: string | null; + installs: Record; +} + +interface LabState { + host: string; + feedPort: number; + feedPid: number | null; + delayMs: number; + busySeconds: number; + fromVersion: string; + latest: Record; + envs: LabEnv[]; +} + +// --------------------------------------------------------------------------- +// State + +function readState(): LabState | null { + if (!NodeFS.existsSync(STATE_PATH)) return null; + return JSON.parse(NodeFS.readFileSync(STATE_PATH, "utf8")); +} + +function requireState(): LabState { + const state = readState(); + if (!state) fail("No lab yet. Run `node scripts/update-lab.ts up` first."); + return state; +} + +/** Fake installers and the CLI write concurrently, so writes are atomic renames. */ +function writeState(state: LabState): void { + const temp = `${STATE_PATH}.${process.pid}.tmp`; + NodeFS.writeFileSync(temp, `${JSON.stringify(state, null, 2)}\n`); + NodeFS.renameSync(temp, STATE_PATH); +} + +function updateState(mutate: (state: LabState) => void): LabState { + const state = requireState(); + mutate(state); + writeState(state); + return state; +} + +function requireEnv(state: LabState, id: string | undefined): LabEnv { + const env = state.envs.find((candidate) => candidate.id === id); + if (!env) fail(`Unknown env ${id ?? "(none)"}. Envs: ${state.envs.map((e) => e.id).join(", ")}`); + return env; +} + +function requireProvider(name: string | undefined): ProviderKey { + if (name && name in PROVIDERS) return name as ProviderKey; + fail(`Unknown provider ${name ?? "(none)"}. Providers: ${PROVIDER_KEYS.join(", ")}`); +} + +function fail(message: string): never { + process.stderr.write(`${message}\n`); + process.exit(1); +} + +const envDir = (id: string) => NodePath.join(LAB, "envs", id); +const homeDir = (id: string) => NodePath.join(envDir(id), "home"); +const t3Home = (id: string) => NodePath.join(envDir(id), "t3"); +const projectDir = (id: string) => NodePath.join(envDir(id), "project"); +const logPath = (id: string) => NodePath.join(envDir(id), "server.log"); +const originOf = (state: LabState, env: LabEnv) => `http://${state.host}:${env.port}`; + +// --------------------------------------------------------------------------- +// Fake installs + +function writeExecutable(path: string, content: string): void { + NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true }); + NodeFS.writeFileSync(path, content, { mode: 0o755 }); +} + +function link(target: string, path: string): void { + NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true }); + NodeFS.rmSync(path, { force: true }); + NodeFS.symlinkSync(target, path); +} + +/** A shell shim that re-enters this file as a fake tool; `$0` names a shared volta shim. */ +function toolShim(envId: string, tool: string | null): string { + const name = tool === null ? `"$(basename "$0")"` : `'${tool}'`; + return `#!/bin/sh\nexec '${NODE}' '${SELF}' __tool '${envId}' ${name} "$@"\n`; +} + +/** Search order shared by the server's PATH and the fake login shell profile. */ +function envPath(envId: string): string { + const home = homeDir(envId); + return [ + ".local/bin", + ".npm-global/bin", + ".local/share/pnpm", + ".bun/bin", + ".yarn/bin", + ".volta/bin", + ".vite-plus/bin", + "homebrew/bin", + ".opencode/bin", + ".grok/bin", + ".pi/bin", + ".local/share/mise/shims", + ] + .map((dir) => NodePath.join(home, dir)) + .concat([NodePath.join(envDir(envId), "tools"), "/usr/local/bin", "/usr/bin", "/bin"]) + .join(":"); +} + +/** Lay a provider out the way its installer would, so the server's resolver sees real paths. */ +function layOutInstall(envId: string, provider: ProviderKey, method: Method): void { + const home = homeDir(envId); + const { command, pkg, brew } = PROVIDERS[provider]; + const shim = toolShim(envId, command); + const at = (...parts: string[]) => NodePath.join(home, ...parts); + switch (method) { + case "npm": { + const real = at(".npm-global/lib/node_modules", pkg, "bin", `${command}.js`); + writeExecutable(real, shim); + link(real, at(".npm-global/bin", command)); + return; + } + case "yarn": { + const real = at(".config/yarn/global/node_modules", pkg, "bin", `${command}.js`); + writeExecutable(real, shim); + link(real, at(".yarn/bin", command)); + return; + } + case "pnpm": + return writeExecutable(at(".local/share/pnpm", command), shim); + case "bun": + return writeExecutable(at(".bun/bin", command), shim); + case "vite-plus": + return writeExecutable(at(".vite-plus/bin", command), shim); + case "volta": + writeExecutable(at(".volta/bin/volta-shim"), toolShim(envId, null)); + writeExecutable(at(".volta/bin/volta"), toolShim(envId, "volta")); + NodeFS.mkdirSync(at(".volta/tools/image/packages", ...pkg.split("/")), { recursive: true }); + link(at(".volta/bin/volta-shim"), at(".volta/bin", command)); + return; + case "homebrew": { + if (!brew) fail(`${provider} has no Homebrew package.`); + const keg = + brew.kind === "formula" + ? at("homebrew/Cellar", brew.name, PROVIDERS[provider].installed, "bin", command) + : at("homebrew/Caskroom", brew.name, PROVIDERS[provider].installed, command); + writeExecutable(keg, shim); + link(keg, at("homebrew/bin", command)); + writeExecutable(at("homebrew/bin/brew"), toolShim(envId, "brew")); + return; + } + case "mise": { + const real = at(".local/share/mise/installs", command, "latest", "bin", command); + writeExecutable(real, shim); + link(real, at(".local/share/mise/shims", command)); + return; + } + case "native": + switch (provider) { + case "claude": + return writeExecutable(at(".local/bin/claude"), shim); + case "codex": { + const real = at(".codex/packages/standalone/current/bin/codex"); + writeExecutable(real, shim); + link(real, at(".local/bin/codex")); + return; + } + case "opencode": + return writeExecutable(at(".opencode/bin/opencode"), shim); + case "grok": + return writeExecutable(at(".grok/bin/grok"), shim); + case "pi": + return writeExecutable(at(".pi/bin/pi"), shim); + } + } +} + +function seedEnvFiles(state: LabState, env: LabEnv): void { + const home = homeDir(env.id); + NodeFS.mkdirSync(home, { recursive: true }); + // The server reads PATH from a login shell first; these dotfiles keep it on the lab PATH. + const profile = `export PATH='${envPath(env.id)}'\n`; + for (const file of [".bash_profile", ".bashrc", ".profile", ".zprofile", ".zshrc"]) { + NodeFS.writeFileSync(NodePath.join(home, file), profile); + } + const tools = NodePath.join(envDir(env.id), "tools"); + for (const tool of ["npm", "pnpm", "bun", "yarn", "vp"]) { + writeExecutable(NodePath.join(tools, tool), toolShim(env.id, tool)); + } + link(NODE, NodePath.join(tools, "node")); + for (const provider of PROVIDER_KEYS) { + layOutInstall(env.id, provider, env.installs[provider].method); + } + + const project = projectDir(env.id); + if (!NodeFS.existsSync(NodePath.join(project, ".git"))) { + NodeFS.mkdirSync(project, { recursive: true }); + NodeFS.writeFileSync(NodePath.join(project, "README.md"), `# ${env.id} lab project\n`); + const git = (...args: string[]) => + NodeChildProcess.execFileSync("git", ["-C", project, ...args], { stdio: "ignore" }); + git("init", "-q"); + git("add", "."); + git("-c", "user.name=Update Lab", "-c", "user.email=lab@localhost", "commit", "-qm", "init"); + } + + const settingsPath = NodePath.join(t3Home(env.id), "userdata", "settings.json"); + NodeFS.mkdirSync(NodePath.dirname(settingsPath), { recursive: true }); + const settings = NodeFS.existsSync(settingsPath) + ? JSON.parse(NodeFS.readFileSync(settingsPath, "utf8")) + : {}; + settings.providerInstances = { + ...settings.providerInstances, + opencode: { driver: "opencode", enabled: true }, + pi: { driver: "pi", enabled: true }, + grok: { driver: "grok", enabled: true }, + "mock-acp": { + driver: "acpRegistry", + displayName: "Mock ACP", + enabled: true, + environment: [ + { name: "T3_ACP_PROMPT_DELAY_MS", value: String(state.busySeconds * 1000) }, + { name: "T3_ACP_PROMPT_RESPONSE_TEXT", value: `${env.id} finished its busy turn.` }, + ], + config: { source: "local", commandPath: NODE, commandArgs: [MOCK_ACP_AGENT] }, + }, + }; + settings.defaultModelSelection ??= { instanceId: "mock-acp", model: "default" }; + NodeFS.writeFileSync(settingsPath, `${JSON.stringify(settings, null, 2)}\n`); +} + +// --------------------------------------------------------------------------- +// Fake CLIs and package managers (`__tool ...args`) + +async function runTool(envId: string, tool: string, args: string[]): Promise { + const state = requireState(); + const env = requireEnv(state, envId); + const provider = PROVIDER_KEYS.find((key) => PROVIDERS[key].command === tool); + if (provider) return runProviderCli(state, env, provider, args); + const method = TOOL_METHOD[tool]; + if (!method) fail(`update-lab: unknown fake tool ${tool}`); + if (args[0] === "--version" || args[0] === "-v") return void console.log("0.0.0-update-lab"); + + if (tool === "brew") { + const prefix = NodePath.join(homeDir(env.id), "homebrew"); + if (args[0] === "--prefix") return void console.log(prefix); + const name = args.findLast((arg) => !arg.startsWith("-")); + const owner = PROVIDER_KEYS.find((key) => PROVIDERS[key].brew?.name === name); + if (!owner) fail(`Error: No available formula or cask with the name "${name}".`); + if (args[0] === "info") { + const latest = state.latest[owner]; + return void console.log( + JSON.stringify( + PROVIDERS[owner].brew!.kind === "formula" + ? { formulae: [{ versions: { stable: latest } }], casks: [] } + : { formulae: [], casks: [{ version: `${latest},lab` }] }, + ), + ); + } + if (args[0] === "upgrade") return install(env.id, owner, tool, undefined); + fail(`update-lab brew: unsupported ${args.join(" ")}`); + } + + // npm/pnpm/bun/yarn/volta/vp install commands name ``, `@latest` or `@x.y.z`. + for (const arg of args) { + for (const key of PROVIDER_KEYS) { + const { pkg } = PROVIDERS[key]; + if (arg !== pkg && !arg.startsWith(`${pkg}@`)) continue; + const requested = arg.slice(pkg.length + 1); + if (method === "npm") { + const prefix = args[args.indexOf("--prefix") + 1]; + if ( + env.installs[key].method !== "npm" || + prefix !== NodePath.join(homeDir(env.id), ".npm-global") + ) { + fail(`npm error ${pkg} is not installed under --prefix ${prefix ?? "(default)"}`); + } + } + return install( + env.id, + key, + tool, + requested && requested !== "latest" ? requested : undefined, + ); + } + } + fail(`update-lab ${tool}: unsupported ${args.join(" ")}`); +} + +async function runProviderCli( + state: LabState, + env: LabEnv, + provider: ProviderKey, + args: string[], +): Promise { + const version = env.installs[provider].version; + const [first, second] = args; + if (first === "--version" || first === "-v") { + const banner = { + codex: `codex-cli ${version}`, + claude: `${version} (Claude Code)`, + opencode: version, + pi: version, + grok: `grok ${version}`, + }[provider]; + return void console.log(banner); + } + const native = + (provider === "claude" && first === "update") || + (provider === "codex" && first === "update") || + (provider === "opencode" && first === "upgrade") || + (provider === "grok" && first === "update") || + (provider === "pi" && first === "update" && second === "--self"); + if (native) return install(env.id, provider, PROVIDERS[provider].command, undefined); + if (provider === "grok" && first === "models") return void console.log("You are logged in"); + if (provider === "codex" && first === "app-server") return codexAppServer(env, version); + // Session probes (claude stream-json, opencode serve, pi --mode rpc) are out of + // scope: a quiet failure keeps installed + version, so advisories still show. + process.stderr.write(`update-lab fake ${provider}: ${args.join(" ")} is not simulated\n`); + process.exit(1); +} + +/** Just enough of `codex app-server` for the status probe to read the version. */ +async function codexAppServer(env: LabEnv, version: string): Promise { + const codexHome = process.env.CODEX_HOME || NodePath.join(homeDir(env.id), ".codex"); + const results: Record = { + initialize: { + codexHome, + platformFamily: "unix", + platformOs: "linux", + userAgent: `codex_cli_rs/${version}`, + }, + "account/read": { account: null, requiresOpenaiAuth: false }, + "skills/list": { data: [] }, + "model/list": { data: [] }, + }; + const lines = NodeReadline.createInterface({ input: process.stdin }); + for await (const line of lines) { + let message: { id?: unknown; method?: string }; + try { + message = JSON.parse(line); + } catch { + continue; + } + if (message.id === undefined || message.method === undefined) continue; + const result = results[message.method]; + process.stdout.write( + `${JSON.stringify( + result === undefined + ? { id: message.id, error: { code: -32601, message: `${message.method} not simulated` } } + : { id: message.id, result }, + )}\n`, + ); + } +} + +/** Shared install path: progress output for `delayMs`, injected failure, then the new version. */ +async function install( + envId: string, + provider: ProviderKey, + tool: string, + requested: string | undefined, +): Promise { + const state = requireState(); + const env = requireEnv(state, envId); + const current = env.installs[provider]; + const method = TOOL_METHOD[tool]; + if (method && method !== current.method) { + fail(`${tool}: ${PROVIDERS[provider].pkg} is not managed by ${tool} here (${current.method}).`); + } + const target = requested ?? state.latest[provider]; + const steps = 5; + for (let step = 1; step <= steps; step++) { + console.log(`${tool}: fetching ${PROVIDERS[provider].pkg}@${target} [${step}/${steps}]`); + await new Promise((resolve) => setTimeout(resolve, state.delayMs / steps)); + } + if (requireState().envs.find((e) => e.id === envId)!.installs[provider].failing) { + process.stderr.write(`${tool}: injected failure installing ${PROVIDERS[provider].pkg}\n`); + process.exit(1); + } + updateState((next) => { + requireEnv(next, envId).installs[provider].version = target; + }); + console.log(`${tool}: installed ${PROVIDERS[provider].pkg}@${target} (was ${current.version})`); +} + +// --------------------------------------------------------------------------- +// Fake npm registry + t3 release feed (`__feed `) + +function runFeed(port: number): void { + const server = NodeHttp.createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://lab"); + const send = (status: number, body: unknown) => { + response.writeHead(status, { "content-type": "application/json" }); + response.end(JSON.stringify(body)); + }; + const state = readState(); + if (!state) return send(503, { error: "no lab state" }); + + // GET /npm//latest and /npm/; the server URI-encodes scoped names. + const npm = /^\/npm\/(.+?)(\/latest)?$/.exec(url.pathname); + if (npm) { + const name = decodeURIComponent(npm[1]!); + const key = PROVIDER_KEYS.find((candidate) => PROVIDERS[candidate].pkg === name); + if (!key) return send(404, { error: "Not found" }); + const version = state.latest[key]; + return npm[2] + ? send(200, { name, version }) + : send(200, { + name, + "dist-tags": { latest: version }, + versions: { [version]: { name, version } }, + }); + } + + // GET /releases/index.json?page=N: GitHub's list-releases shape, newest first. + if (url.pathname === "/releases/index.json") { + const page = Number(url.searchParams.get("page") ?? "1"); + return send( + 200, + page === 1 ? publishedRuntimes().map((v) => ({ tag_name: `v${v}`, draft: false })) : [], + ); + } + + // GET /releases/v/: SHA256SUMS and archives. + const asset = /^\/releases\/(v[^/]+)\/([^/]+)$/.exec(url.pathname); + if (asset) { + const file = NodePath.join(RELEASES, asset[1]!, asset[2]!); + if (!NodeFS.existsSync(file)) return send(404, { error: "Not found" }); + response.writeHead(200, { "content-length": NodeFS.statSync(file).size }); + return void NodeFS.createReadStream(file).pipe(response); + } + send(404, { error: "Not found" }); + }); + server.listen(port, "127.0.0.1"); +} + +function publishedRuntimes(): string[] { + if (!NodeFS.existsSync(RELEASES)) return []; + return NodeFS.readdirSync(RELEASES) + .filter((dir) => NodeFS.existsSync(NodePath.join(RELEASES, dir, "SHA256SUMS"))) + .map((dir) => dir.slice(1)) + .sort((a, b) => compareVersions(b, a)); +} + +function compareVersions(a: string, b: string): number { + const parse = (v: string) => v.split(/[.-]/).map((part) => Number(part) || 0); + const [left, right] = [parse(a), parse(b)]; + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const diff = (left[i] ?? 0) - (right[i] ?? 0); + if (diff !== 0) return diff; + } + return 0; +} + +// --------------------------------------------------------------------------- +// Runtimes for the release feed and the service launcher + +/** + * Publish `t3--.tar.gz` + SHA256SUMS. Without `--archive` + * this is a source runtime: a copy of apps/server whose package.json carries + * `version` (every version check reads it) and whose node_modules links back + * to this checkout, started by a `t3` wrapper. + */ +function buildRuntime(version: string, archive: string | undefined): string { + if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) + fail(`Not an exact version: ${version}`); + const releaseDir = NodePath.join(RELEASES, `v${version}`); + const fileName = `t3-${version}-${PLATFORM_KEY}.tar.gz`; + const archivePath = NodePath.join(releaseDir, fileName); + NodeFS.mkdirSync(releaseDir, { recursive: true }); + if (archive) { + NodeFS.copyFileSync(NodePath.resolve(archive), archivePath); + } else { + const stage = NodePath.join(LAB, "build", `t3-${version}-${PLATFORM_KEY}`); + NodeFS.rmSync(stage, { recursive: true, force: true }); + const serverSource = NodePath.join(REPO, "apps/server"); + NodeFS.cpSync(serverSource, NodePath.join(stage, "server"), { + recursive: true, + filter: (source) => + !["node_modules", "dist", "dist-exe"].includes(NodePath.relative(serverSource, source)), + }); + const packageJsonPath = NodePath.join(stage, "server/package.json"); + const packageJson = JSON.parse(NodeFS.readFileSync(packageJsonPath, "utf8")); + packageJson.version = version; + NodeFS.writeFileSync(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`); + NodeFS.symlinkSync( + NodePath.join(serverSource, "node_modules"), + NodePath.join(stage, "server/node_modules"), + ); + writeExecutable( + NodePath.join(stage, "t3"), + `#!/bin/sh\nexec '${NODE}' "$(dirname "$0")/server/src/bin.ts" "$@"\n`, + ); + NodeChildProcess.execFileSync("tar", [ + "-czf", + archivePath, + "-C", + NodePath.dirname(stage), + NodePath.basename(stage), + ]); + NodeFS.rmSync(stage, { recursive: true, force: true }); + } + const digest = NodeCrypto.createHash("sha256") + .update(NodeFS.readFileSync(archivePath)) + .digest("hex"); + NodeFS.writeFileSync(NodePath.join(releaseDir, "SHA256SUMS"), `${digest} ${fileName}\n`); + return archivePath; +} + +/** Install a published runtime the way pinnedRuntime does, and make it the launcher's active version. */ +function installLauncherRuntime(envId: string, version: string): void { + const runtimeDir = NodePath.join(t3Home(envId), "runtime"); + const versionDir = NodePath.join(runtimeDir, "versions", version); + if (!NodeFS.existsSync(NodePath.join(versionDir, ".install-complete"))) { + const archive = NodePath.join(RELEASES, `v${version}`, `t3-${version}-${PLATFORM_KEY}.tar.gz`); + if (!NodeFS.existsSync(archive)) buildRuntime(version, undefined); + NodeFS.mkdirSync(versionDir, { recursive: true }); + NodeChildProcess.execFileSync("tar", [ + "-xzf", + archive, + "-C", + versionDir, + "--strip-components=1", + ]); + NodeFS.writeFileSync(NodePath.join(versionDir, ".install-complete"), `${version}\n`); + } + const statePath = NodePath.join(runtimeDir, "service-state.json"); + if (!NodeFS.existsSync(statePath)) { + NodeFS.writeFileSync(statePath, `${JSON.stringify({ protocol: 3, activeVersion: version })}\n`); + } +} + +// --------------------------------------------------------------------------- +// Processes + +function listeningPorts(): Set { + const output = NodeChildProcess.execFileSync("ss", ["-H", "-ltn"], { encoding: "utf8" }); + return new Set( + output + .split("\n") + .map((line) => Number(line.trim().split(/\s+/)[3]?.split(":").pop())) + .filter((port) => Number.isInteger(port) && port > 0), + ); +} + +/** An OS-assigned port that `ss` does not already list and nobody else in the lab holds. */ +async function freePort(host: string, taken: Set): Promise { + for (let attempt = 0; attempt < 20; attempt++) { + const port = await new Promise((resolve, reject) => { + const probe = NodeNet.createServer(); + probe.once("error", reject); + probe.listen(0, host, () => { + const address = probe.address(); + probe.close(() => resolve(typeof address === "object" && address ? address.port : 0)); + }); + }); + if (port > 0 && !taken.has(port) && !listeningPorts().has(port)) { + taken.add(port); + return port; + } + } + fail("Could not find a free port."); +} + +/** True only for a live process whose command line is still one the lab started. */ +function isLabProcess(pid: number | null, marker: string): pid is number { + if (!pid) return false; + try { + return NodeFS.readFileSync(`/proc/${pid}/cmdline`, "utf8").includes(marker); + } catch { + return false; + } +} + +const serverMarker = (env: LabEnv) => (env.launcher ? "__service-launcher" : t3Home(env.id)); + +/** The runtime the service launcher runs now, which moves after a self-update. */ +function launcherLabel(env: LabEnv): string { + if (!env.launcher) return ""; + const statePath = NodePath.join(t3Home(env.id), "runtime", "service-state.json"); + const active = NodeFS.existsSync(statePath) + ? JSON.parse(NodeFS.readFileSync(statePath, "utf8")).activeVersion + : "?"; + return ` launcher@${active}`; +} + +/** The minimal environment a lab server sees; nothing from this shell leaks in. */ +function serverEnvironment(state: LabState, env: LabEnv): NodeJS.ProcessEnv { + const home = homeDir(env.id); + const keep = ["LANG", "LC_ALL", "TZ", "USER", "LOGNAME", "TMPDIR"]; + return { + ...Object.fromEntries( + keep.flatMap((key) => (process.env[key] ? [[key, process.env[key]]] : [])), + ), + HOME: home, + PATH: envPath(env.id), + SHELL: "/bin/bash", + XDG_CONFIG_HOME: NodePath.join(home, ".config"), + XDG_DATA_HOME: NodePath.join(home, ".local/share"), + XDG_CACHE_HOME: NodePath.join(home, ".cache"), + XDG_STATE_HOME: NodePath.join(home, ".local/state"), + npm_config_registry: `http://127.0.0.1:${state.feedPort}/npm`, + T3CODE_RELEASE_BASE_URL: `http://127.0.0.1:${state.feedPort}/releases`, + T3CODE_UPDATE_LAB: "1", + T3CODE_NO_BROWSER: "1", + }; +} + +function spawnDetached( + command: string, + args: string[], + options: { env: NodeJS.ProcessEnv; cwd: string; log: string }, +): number { + const fd = NodeFS.openSync(options.log, "a"); + const child = NodeChildProcess.spawn(command, args, { + cwd: options.cwd, + env: options.env, + detached: true, + stdio: ["ignore", fd, fd], + }); + NodeFS.closeSync(fd); + child.unref(); + if (!child.pid) fail(`Failed to start ${command} ${args.join(" ")}`); + return child.pid; +} + +function startServer(state: LabState, env: LabEnv): number { + NodeFS.writeFileSync(logPath(env.id), ""); + const environment = serverEnvironment(state, env); + if (env.launcher) { + installLauncherRuntime(env.id, state.fromVersion); + // The launcher runs `/t3 serve` and passes its environment through. + return spawnDetached(NODE, [SERVER_BIN, "__service-launcher"], { + env: { + ...environment, + T3CODE_HOME: t3Home(env.id), + T3CODE_PORT: String(env.port), + T3CODE_HOST: state.host, + }, + cwd: projectDir(env.id), + log: logPath(env.id), + }); + } + return spawnDetached( + NODE, + [ + SERVER_BIN, + "serve", + "--base-dir", + t3Home(env.id), + "--port", + String(env.port), + "--host", + state.host, + projectDir(env.id), + ], + { env: environment, cwd: projectDir(env.id), log: logPath(env.id) }, + ); +} + +async function waitForPairingUrl(env: LabEnv): Promise { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const log = NodeFS.readFileSync(logPath(env.id), "utf8"); + const url = /Pairing URL: (\S+)/.exec(log)?.[1]; + if (url) return url; + if (!isLabProcess(env.pid, serverMarker(env))) { + fail(`${env.id} exited during startup; see ${logPath(env.id)}\n${log.slice(-2000)}`); + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + fail(`${env.id} did not print a pairing URL within 120s; see ${logPath(env.id)}`); +} + +/** SIGTERM the process group we created, then SIGKILL it if it lingers. */ +async function stopGroup(pid: number, marker: string): Promise { + if (!isLabProcess(pid, marker)) return; + process.kill(-pid, "SIGTERM"); + const deadline = Date.now() + 15_000; + while (Date.now() < deadline && isLabProcess(pid, marker)) { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + try { + process.kill(-pid, "SIGKILL"); + } catch { + // The group is already gone. + } +} + +// --------------------------------------------------------------------------- +// Talking to a lab server like a client does + +async function sessionToken(env: LabEnv): Promise { + const tokenPath = NodePath.join(envDir(env.id), "session-token"); + if (NodeFS.existsSync(tokenPath)) return NodeFS.readFileSync(tokenPath, "utf8").trim(); + const token = NodeChildProcess.execFileSync( + NODE, + [ + SERVER_BIN, + "auth", + "session", + "issue", + "--base-dir", + t3Home(env.id), + "--label", + "update-lab", + "--token-only", + ], + { cwd: REPO, encoding: "utf8", env: { PATH: process.env.PATH, HOME: homeDir(env.id) } }, + ).trim(); + NodeFS.writeFileSync(tokenPath, token, { mode: 0o600 }); + return token; +} + +async function authHeaders(env: LabEnv): Promise> { + return { + Authorization: `Bearer ${await sessionToken(env)}`, + "Content-Type": "application/json", + "x-t3-orchestration-protocol": ORCHESTRATION_PROTOCOL, + }; +} + +/** One unary call over the server's Effect RPC WebSocket protocol (JSON frames). */ +async function rpc(state: LabState, env: LabEnv, tag: string, payload: unknown): Promise { + const origin = originOf(state, env); + const ticketResponse = await fetch(`${origin}/api/auth/websocket-ticket`, { + method: "POST", + headers: await authHeaders(env), + }); + if (!ticketResponse.ok) fail(`${env.id}: websocket ticket failed (${ticketResponse.status})`); + const { ticket } = (await ticketResponse.json()) as { ticket: string }; + const socket = new WebSocket( + `${origin.replace("http", "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL}&wsTicket=${encodeURIComponent(ticket)}`, + ); + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${tag} timed out`)), 10 * 60_000); + const finish = (settle: () => void) => { + clearTimeout(timer); + socket.close(); + settle(); + }; + socket.addEventListener("open", () => + socket.send(JSON.stringify({ _tag: "Request", id: "1", tag, payload, headers: [] })), + ); + socket.addEventListener("error", () => finish(() => reject(new Error(`${tag}: socket error`)))); + socket.addEventListener("message", (event) => { + const decoded = JSON.parse(String(event.data)); + for (const message of Array.isArray(decoded) ? decoded : [decoded]) { + if (message._tag !== "Exit" || message.requestId !== "1") continue; + const exit = message.exit; + finish(() => + exit._tag === "Success" + ? resolve(exit.value) + : reject(new Error(`${tag} failed: ${JSON.stringify(exit.cause)}`)), + ); + } + }); + }); +} + +function printProviders(providers: ReadonlyArray): void { + for (const provider of providers) { + const advisory = provider.versionAdvisory ?? {}; + const update = provider.updateState ?? null; + console.log( + [ + ` ${provider.instanceId.padEnd(10)} ${String(provider.status).padEnd(8)}`, + `installed=${provider.version ?? "-"} latest=${advisory.latestVersion ?? "-"} advisory=${advisory.status ?? "-"}`, + `update=${advisory.canUpdate ? JSON.stringify(advisory.updateCommand) : "manual"}`, + update ? `last=${update.status}: ${update.message ?? ""}` : "", + ].join(" "), + ); + } +} + +// --------------------------------------------------------------------------- +// Commands + +function option(args: string[], name: string): string | undefined { + const index = args.indexOf(`--${name}`); + return index >= 0 ? args[index + 1] : undefined; +} + +async function up(args: string[]): Promise { + const previous = readState(); + const host = option(args, "host") ?? previous?.host ?? "127.0.0.1"; + const envCount = Number(option(args, "envs") ?? previous?.envs.length ?? 3); + const launcherCount = Number(option(args, "launcher") ?? 1); + const [major, minor, patch] = REPO_VERSION.split(/[.-]/).map(Number); + const state: LabState = previous ?? { + host, + feedPort: 0, + feedPid: null, + delayMs: 3000, + busySeconds: 120, + fromVersion: `${major}.${minor}.${Math.max(0, patch! - 1)}`, + latest: Object.fromEntries( + PROVIDER_KEYS.map((key) => [key, PROVIDERS[key].latest]), + ) as LabState["latest"], + envs: [], + }; + state.host = host; + state.busySeconds = Number(option(args, "busy-seconds") ?? state.busySeconds); + state.fromVersion = option(args, "from-version") ?? state.fromVersion; + NodeFS.mkdirSync(LAB, { recursive: true }); + const taken = new Set(state.envs.map((env) => env.port).concat(state.feedPort)); + + if (!isLabProcess(state.feedPid, "__feed")) { + state.feedPort = await freePort("127.0.0.1", taken); + state.feedPid = spawnDetached(NODE, [SELF, "__feed", String(state.feedPort)], { + env: { PATH: process.env.PATH }, + cwd: REPO, + log: NodePath.join(LAB, "feed.log"), + }); + } + // The feed always offers this checkout's version, the update a launcher env is behind on. + if (!publishedRuntimes().includes(REPO_VERSION)) buildRuntime(REPO_VERSION, undefined); + + for (let index = state.envs.length; index < envCount; index++) { + const row = INSTALL_MATRIX[index % INSTALL_MATRIX.length]!; + state.envs.push({ + id: `env${index + 1}`, + port: 0, + pid: null, + launcher: index >= envCount - launcherCount, + projectId: null, + installs: Object.fromEntries( + PROVIDER_KEYS.map((key) => [ + key, + { method: row[key], version: PROVIDERS[key].installed, failing: false }, + ]), + ) as LabEnv["installs"], + }); + } + writeState(state); + + const started: LabEnv[] = []; + for (const env of state.envs) { + if (isLabProcess(env.pid, serverMarker(env))) continue; + seedEnvFiles(state, env); + if (!env.port || listeningPorts().has(env.port)) env.port = await freePort(host, taken); + env.pid = startServer(state, env); + started.push(env); + writeState(state); + } + console.log( + `feed http://127.0.0.1:${state.feedPort} (npm registry /npm, releases /releases)`, + ); + const urls = await Promise.all(started.map(waitForPairingUrl)); + for (const env of state.envs) { + const index = started.indexOf(env); + console.log(`${env.id.padEnd(9)} ${originOf(state, env)} pid ${env.pid}${launcherLabel(env)}`); + console.log(` ${index >= 0 ? urls[index] : "(already running; `pair` mints a URL)"}`); + } +} + +async function down(): Promise { + const state = readState(); + if (!state) return; + for (const env of state.envs) { + if (env.pid) await stopGroup(env.pid, serverMarker(env)); + env.pid = null; + } + if (state.feedPid) await stopGroup(state.feedPid, "__feed"); + state.feedPid = null; + writeState(state); + console.log("Lab stopped."); +} + +function status(): void { + const state = requireState(); + const feed = isLabProcess(state.feedPid, "__feed") ? "up" : "down"; + console.log( + `feed http://127.0.0.1:${state.feedPort} ${feed}; delay ${state.delayMs}ms; runtimes ${publishedRuntimes().join(", ") || "-"}`, + ); + for (const env of state.envs) { + const alive = isLabProcess(env.pid, serverMarker(env)); + console.log( + `${env.id} ${originOf(state, env)} pid ${env.pid ?? "-"} ${alive ? "up" : "down"}${launcherLabel(env)}`, + ); + for (const key of PROVIDER_KEYS) { + const install = env.installs[key]; + const behind = compareVersions(install.version, state.latest[key]) < 0 ? "behind" : "current"; + console.log( + ` ${key.padEnd(9)} ${install.method.padEnd(9)} installed ${install.version.padEnd(8)} latest ${state.latest[key].padEnd(8)} ${behind}${install.failing ? " FAILING" : ""}`, + ); + } + } +} + +async function busy(state: LabState, env: LabEnv): Promise { + const headers = await authHeaders(env); + if (!env.projectId) { + const projectId = NodeCrypto.randomUUID(); + const created = await fetch(`${originOf(state, env)}/api/projects/mutate`, { + method: "POST", + headers, + body: JSON.stringify({ + type: "project.create", + commandId: NodeCrypto.randomUUID(), + projectId, + title: `${env.id} lab project`, + workspaceRoot: projectDir(env.id), + }), + }); + if (!created.ok) fail(`Project creation failed: ${created.status} ${await created.text()}`); + updateState((next) => { + requireEnv(next, env.id).projectId = projectId; + }); + env.projectId = projectId; + } + const threadId = NodeCrypto.randomUUID(); + await rpc(state, env, "orchestration.launchThread", { + commandId: NodeCrypto.randomUUID(), + threadId, + projectId: env.projectId, + title: `Busy for ${state.busySeconds}s`, + modelSelection: { instanceId: "mock-acp", model: "default" }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { text: "Stay busy so updates have to wait.", attachments: [] }, + }); + console.log(`${env.id}: thread ${threadId} runs for ~${state.busySeconds}s on mock-acp.`); +} + +async function main(argv: string[]): Promise { + const [command, ...args] = argv; + switch (command) { + case "__tool": + return runTool(args[0]!, args[1]!, args.slice(2)); + case "__feed": + return runFeed(Number(args[0])); + case "up": + return up(args); + case "down": + return down(); + case "status": + return status(); + case "pair": { + const state = requireState(); + const env = requireEnv(state, args[0]); + const output = NodeChildProcess.execFileSync( + NODE, + [ + SERVER_BIN, + "auth", + "pairing", + "create", + "--base-dir", + t3Home(env.id), + "--base-url", + originOf(state, env), + ], + { cwd: REPO, encoding: "utf8", env: { PATH: process.env.PATH, HOME: homeDir(env.id) } }, + ); + return void console.log(/(http\S+\/pair#token=\S+)/.exec(output)?.[1] ?? output); + } + case "publish": { + const provider = requireProvider(args[0]); + if (!args[1]) fail("Usage: publish "); + updateState((state) => { + state.latest[provider] = args[1]!; + }); + return void console.log(`${PROVIDERS[provider].pkg} latest is now ${args[1]}.`); + } + case "fail": { + const provider = requireProvider(args[1]); + if (args[2] !== "on" && args[2] !== "off") fail("Usage: fail on|off"); + updateState((state) => { + requireEnv(state, args[0]).installs[provider].failing = args[2] === "on"; + }); + return void console.log( + `${args[0]} ${provider} installs will ${args[2] === "on" ? "fail" : "succeed"}.`, + ); + } + case "delay": { + const ms = Number(args[0]); + if (!Number.isInteger(ms) || ms < 0) fail("Usage: delay "); + updateState((state) => { + state.delayMs = ms; + }); + return void console.log(`Installers now take ${ms}ms.`); + } + case "providers": { + const state = requireState(); + const env = requireEnv(state, args[0]); + const result = await rpc(state, env, "server.getConfig", {}); + console.log(`${env.id} ${originOf(state, env)}`); + return printProviders(result.providers); + } + case "update": { + const state = requireState(); + const env = requireEnv(state, args[0]); + const instanceId = args[1] ?? fail("Usage: update [version]"); + const driver = + PROVIDER_KEYS.map((key) => PROVIDERS[key].driver).find((d) => d === instanceId) ?? + instanceId; + const result = await rpc(state, env, "server.updateProvider", { + provider: driver, + instanceId, + ...(args[2] ? { targetVersion: args[2] } : {}), + }); + return printProviders(result.providers.filter((p: any) => p.instanceId === instanceId)); + } + case "busy": { + const state = requireState(); + return busy(state, requireEnv(state, args[0])); + } + case "self-update": { + const state = requireState(); + const env = requireEnv(state, args[0]); + if (!args[1]) fail("Usage: self-update "); + const result = await rpc(state, env, "server.updateServer", { targetVersion: args[1] }); + return void console.log(JSON.stringify(result)); + } + case "build-runtime": { + if (!args[0]) fail("Usage: build-runtime [--archive file]"); + return void console.log(`Published ${buildRuntime(args[0], option(args, "archive"))}`); + } + case "reset": { + const state = readState(); + if (state && [state.feedPid, ...state.envs.map((e) => e.pid)].some((pid) => pid !== null)) { + fail("Run `down` first."); + } + NodeFS.rmSync(LAB, { recursive: true, force: true }); + return void console.log(`Removed ${LAB}.`); + } + default: + console.log(HELP); + if (command && command !== "--help" && command !== "help") process.exit(1); + } +} + +await main(process.argv.slice(2)); From 25b6b08e4b4c9b126551f336df879f27c27625b4 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:06:27 +0000 Subject: [PATCH 002/149] feat(server): update provider CLIs in the background when nothing is running Adds an update window that stays closed while threads run or wait for input, for 15 minutes after any client interaction, and while a scheduled task or usage-limit resume is due within 10 minutes. A new Update automatically setting (on by default) installs provider CLI updates inside that window. The provider maintenance runner is now one server-lifetime instance, so RPC and background updates share the per-installer lock. It was previously built per WebSocket connection, so two clients' updates never queued. --- .../SettingsServerControlsRouteScreen.tsx | 19 ++ .../server/src/background/BackgroundPolicy.ts | 8 + .../src/environment/ServerEnvironment.ts | 1 + .../src/mcp/toolkits/environment/handlers.ts | 2 + .../src/mcp/toolkits/environment/tools.ts | 2 + .../src/orchestration-v2/ProjectionStore.ts | 36 +++ apps/server/src/server.ts | 11 + .../server/src/updates/ProviderAutoUpdater.ts | 100 ++++++++ apps/server/src/updates/UpdateWindow.ts | 232 ++++++++++++++++++ apps/server/src/ws.ts | 9 +- .../components/settings/SettingsPanels.tsx | 42 ++++ .../src/components/settings/settingsSearch.ts | 7 + docs/user/updating.md | 6 + packages/contracts/src/environment.ts | 2 + packages/contracts/src/settings.ts | 3 + 15 files changed, 479 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/updates/ProviderAutoUpdater.ts create mode 100644 apps/server/src/updates/UpdateWindow.ts diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index ae4f43b5fb376..964f2a3f8c888 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -230,6 +230,9 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { (target) => target.environment.serverConfig.environment.capabilities.threadRestartContinuation === true, ); + const supportsAutomaticUpdates = targets.every( + (target) => target.environment.serverConfig.environment.capabilities.automaticUpdates === true, + ); const disabledFor = (key: string) => disabled || (projectSelected && @@ -462,6 +465,22 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { disabled={disabledFor("enableProviderUpdateChecks")} onValueChange={(value) => write({ enableProviderUpdateChecks: value })} /> + + write({ automaticUpdates: value })} + /> + Effect.Effect; readonly reportHostPowerState: (snapshot: HostPowerSnapshot) => Effect.Effect; readonly snapshot: Effect.Effect; + /** When any client last reported recent interaction, so updates can wait for quiet. */ + readonly lastClientInteractionAt: Effect.Effect>; readonly streamChanges: Stream.Stream; readonly subscribe: Effect.Effect< { @@ -213,6 +216,7 @@ export const make = Effect.fn("background.policy.make")(function* () { const hostPowerMonitor = yield* HostPowerMonitor.HostPowerMonitor; const serverSettings = yield* ServerSettings.ServerSettingsService; const leasesRef = yield* Ref.make(new Map()); + const lastClientInteractionAtRef = yield* Ref.make(Option.none()); const changes = yield* PubSub.sliding(1); const publishMutex = yield* Semaphore.make(1); @@ -266,6 +270,9 @@ export const make = Effect.fn("background.policy.make")(function* () { expiresAt, }; yield* Ref.update(leasesRef, (leases) => upsertClientActivityLease(leases, lease, now)); + if (input.recentlyInteracted) { + yield* Ref.set(lastClientInteractionAtRef, Option.some(now)); + } yield* publishSnapshotUnlocked; }), ); @@ -339,6 +346,7 @@ export const make = Effect.fn("background.policy.make")(function* () { removeRpcClient, reportHostPowerState: hostPowerMonitor.report, snapshot, + lastClientInteractionAt: Ref.get(lastClientInteractionAtRef), streamChanges: Stream.fromPubSub(changes), subscribe: subscribeBeforeSnapshot(changes, snapshot, publishMutex), hasDemand, diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 6381547468ae3..aa1373c0da73d 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -231,6 +231,7 @@ export const make = Effect.gen(function* () { projectWorktreeCleanup: true, worktreesDirectory: true, threadRestartContinuation: true, + automaticUpdates: true, projectSettingsOverrides: true, threadSnooze: true, environmentThemes: true, diff --git a/apps/server/src/mcp/toolkits/environment/handlers.ts b/apps/server/src/mcp/toolkits/environment/handlers.ts index d7b84ca0599c4..dc51226bcf227 100644 --- a/apps/server/src/mcp/toolkits/environment/handlers.ts +++ b/apps/server/src/mcp/toolkits/environment/handlers.ts @@ -13,6 +13,7 @@ export function preferences(settings: ServerSettings) { defaultThreadEnvMode, newWorktreesStartFromOrigin, enableProviderUpdateChecks, + automaticUpdates, backgroundActivity, sourceControlWritingStyle, } = settings; @@ -21,6 +22,7 @@ export function preferences(settings: ServerSettings) { defaultThreadEnvMode, newWorktreesStartFromOrigin, enableProviderUpdateChecks, + automaticUpdates, backgroundActivity: { profile: backgroundActivity.profile }, sourceControlWritingStyle: { ...sourceControlWritingStyle, diff --git a/apps/server/src/mcp/toolkits/environment/tools.ts b/apps/server/src/mcp/toolkits/environment/tools.ts index e441174900ce1..88e103ce0debe 100644 --- a/apps/server/src/mcp/toolkits/environment/tools.ts +++ b/apps/server/src/mcp/toolkits/environment/tools.ts @@ -18,6 +18,7 @@ const PreferenceFields = { defaultThreadEnvMode: ServerSettings.fields.defaultThreadEnvMode, newWorktreesStartFromOrigin: ServerSettings.fields.newWorktreesStartFromOrigin, enableProviderUpdateChecks: ServerSettings.fields.enableProviderUpdateChecks, + automaticUpdates: ServerSettings.fields.automaticUpdates, backgroundActivity: Schema.Struct({ profile: BackgroundActivityProfileSelection }), sourceControlWritingStyle: Schema.Struct({ mode: Schema.String, @@ -59,6 +60,7 @@ const EnvironmentPreferencesTool = Tool.make("t3_environment_preferences_update" defaultThreadEnvMode: ServerSettingsPatch.fields.defaultThreadEnvMode, newWorktreesStartFromOrigin: ServerSettingsPatch.fields.newWorktreesStartFromOrigin, enableProviderUpdateChecks: ServerSettingsPatch.fields.enableProviderUpdateChecks, + automaticUpdates: ServerSettingsPatch.fields.automaticUpdates, backgroundActivity: Schema.optionalKey(Schema.Struct({ profile: BackgroundActivityProfile })), sourceControlWritingStyle: ServerSettingsPatch.fields.sourceControlWritingStyle, }), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index a6a092ff84b7a..8119dba91fc95 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -147,6 +147,9 @@ export type ProjectionStoreV2Error = typeof ProjectionStoreV2Error.Type; export type ProjectionRecoveryKind = | "queued-runs" | "runtime" + // Work a restart or CLI replacement would cut off. Idle provider sessions + // do not count: they resume on the next turn. + | "active-work" | "subagent-results" | "delegated-completions"; @@ -564,6 +567,19 @@ function needsRecovery( ) ); } + case "active-work": + return ( + projection.runs.some( + (run) => + ["preparing", "starting", "running", "waiting"].includes(run.status) || + (run.status === "queued" && run.queueHeld !== true), + ) || + projection.runtimeRequests.some((request) => request.status === "pending") || + projection.providerThreads.some( + (thread) => + thread.status === "active" || (thread.pendingBackgroundTasks?.length ?? 0) > 0, + ) + ); case "runtime": return ( projection.runs.some( @@ -3592,6 +3608,26 @@ export const layer: Layer.Layer = ) ELSE 0 END `; + case "active-work": + return sql` + SELECT thread_id FROM orchestration_v2_projection_runs + WHERE status IN ('preparing', 'starting', 'running', 'waiting') + UNION + SELECT thread_id FROM orchestration_v2_projection_runs + WHERE status = 'queued' + AND CASE WHEN json_valid(payload_json) + THEN json_extract(payload_json, '$.queueHeld') IS NOT 1 + ELSE 1 END + UNION + SELECT thread_id FROM orchestration_v2_projection_runtime_requests + WHERE status = 'pending' + UNION + SELECT thread_id FROM orchestration_v2_projection_provider_threads + WHERE status = 'active' + OR CASE WHEN json_valid(payload_json) + THEN json_array_length(payload_json, '$.pendingBackgroundTasks') > 0 + ELSE 0 END + `; case "runtime": return sql` WITH pending_provider_threads AS MATERIALIZED ( diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 6a698c2e46ab1..c4be403306d84 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -157,6 +157,9 @@ import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts"; import * as CursorUsageReader from "./usage/cursorUsageReader.ts"; import * as UsageService from "./usage/UsageService.ts"; import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts"; +import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; +import * as ProviderAutoUpdater from "./updates/ProviderAutoUpdater.ts"; +import * as UpdateWindow from "./updates/UpdateWindow.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as ResourceCleanupService from "./orchestration-v2/ResourceCleanupService.ts"; @@ -528,7 +531,15 @@ const layerProviderInstallationRefresh = Layer.effectDiscard( }), ); +// One maintenance runner for RPCs and background updates, so both queue +// behind the same per-installer lock. +const layerProviderUpdates = ProviderAutoUpdater.layer.pipe( + Layer.provide(UpdateWindow.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), + Layer.provideMerge(ProviderMaintenanceRunner.layer), +); + const layerRuntimeCoreDependenciesBase = Layer.mergeAll( + layerProviderUpdates, AgentAwarenessRelay.layer, // Asks T3 Connect to deliver webhooks it held while this environment was offline. HeldHooksWaker.layer, diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts new file mode 100644 index 0000000000000..2c0c38a54b670 --- /dev/null +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -0,0 +1,100 @@ +import type { ServerProvider } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schedule from "effect/Schedule"; + +import * as ServerConfig from "../config.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ProviderMaintenanceRunner from "../provider/providerMaintenanceRunner.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as UpdateWindow from "./UpdateWindow.ts"; + +/** A failed install is retried for the same version only after this long. */ +const RETRY_AFTER = Duration.hours(6); + +function isAutoUpdatable(provider: ServerProvider): boolean { + const advisory = provider.versionAdvisory; + const updateStatus = provider.updateState?.status; + return ( + provider.enabled && + provider.installed && + advisory?.status === "behind_latest" && + advisory.canUpdate && + advisory.latestVersion !== null && + updateStatus !== "queued" && + updateStatus !== "running" + ); +} + +/** + * Installs provider CLI updates in the background when `automaticUpdates` is + * on, one instance at a time, only while the update window is open. + */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + // A dev server shares the host's CLIs with the real install; leave them to it. + if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return; + const providers = yield* ProviderRegistry.ProviderRegistry; + const runner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; + const updateWindow = yield* UpdateWindow.UpdateWindow; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const timings = yield* UpdateWindow.updateTimings; + // Keyed by instance and target version, so a newer release retries at once. + const attempts = new Map(); + + const pass = Effect.gen(function* () { + const settings = yield* serverSettings.getSettings; + if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; + const nowMs = yield* Clock.currentTimeMillis; + const candidates = (yield* providers.getProviders).filter((provider) => { + if (!isAutoUpdatable(provider)) return false; + const attemptedAt = attempts.get( + `${provider.instanceId}@${provider.versionAdvisory?.latestVersion}`, + ); + return attemptedAt === undefined || nowMs - attemptedAt >= Duration.toMillis(RETRY_AFTER); + }); + for (const provider of candidates) { + const targetVersion = provider.versionAdvisory?.latestVersion; + const install = Effect.gen(function* () { + attempts.set(`${provider.instanceId}@${targetVersion}`, yield* Clock.currentTimeMillis); + yield* Effect.logInfo("Updating provider in the background", { + instanceId: provider.instanceId, + fromVersion: provider.version, + targetVersion, + }); + yield* runner + .updateProvider({ provider: provider.driver, instanceId: provider.instanceId }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("Background provider update failed", { + instanceId: provider.instanceId, + cause, + }), + ), + ); + }); + // Rechecked per install: the previous one may have run for minutes. + const installed = yield* updateWindow.runIfOpen({ closesWindows: false }, install); + if (Option.isNone(installed)) { + yield* Effect.logDebug("Provider auto-update waiting for the update window"); + return; + } + } + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Provider auto-update pass failed", { cause }), + ), + Effect.withSpan("updates.ProviderAutoUpdater.pass"), + ); + + yield* forkParked( + Effect.sleep(timings.firstPassDelay).pipe( + Effect.andThen(pass.pipe(Effect.repeat(Schedule.spaced(timings.passInterval)))), + ), + ); + }), +); diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts new file mode 100644 index 0000000000000..be9e91055e840 --- /dev/null +++ b/apps/server/src/updates/UpdateWindow.ts @@ -0,0 +1,232 @@ +import type { ScheduledTaskId, ThreadId } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Semaphore from "effect/Semaphore"; + +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; +import * as ServerSettings from "../serverSettings.ts"; + +/** Work due this soon keeps the window closed, so it does not start late. */ +const UPDATE_LOOKAHEAD = Duration.minutes(5); +/** + * Tasks repeating at least this often always have a run due soon. Missed runs + * catch up after a restart, so they only block while running. + */ +const FREQUENT_TASK_INTERVAL = Duration.minutes(15); + +/** + * Background update pacing. `T3CODE_UPDATE_LAB=1` (scripts/update-lab.ts) + * shortens the waits so update flows can be exercised in seconds. + */ +export const updateTimings = Config.Boolean("T3CODE_UPDATE_LAB").pipe( + Config.withDefault(false), + Config.map((lab) => ({ + /** + * How long every client must be untouched before an update may interrupt + * it. Clients report activity about every 25s, so this must stay well above that. + */ + quietPeriod: lab ? Duration.seconds(60) : Duration.minutes(15), + /** How often a waiting update rechecks the window or looks for new versions. */ + recheck: lab ? Duration.seconds(5) : Duration.seconds(30), + firstPassDelay: lab ? Duration.seconds(10) : Duration.minutes(2), + passInterval: lab ? Duration.seconds(20) : Duration.minutes(10), + })), +); + +export type UpdateWindowBlocker = + | { readonly type: "active-threads"; readonly threadIds: ReadonlyArray } + | { readonly type: "recent-interaction"; readonly quietAt: DateTime.Utc } + | { readonly type: "focused-window" } + | { readonly type: "provider-update" } + | { + readonly type: "scheduled-task"; + readonly taskId: ScheduledTaskId; + readonly title: string; + /** Null while the task is running. */ + readonly runsAt: DateTime.Utc | null; + } + | { + readonly type: "usage-limit-resume"; + readonly threadId: ThreadId; + readonly resumesAt: DateTime.Utc; + }; + +export interface UpdateWindowOptions { + /** The update closes the app windows (a desktop install), not just a reconnect. */ + readonly closesWindows: boolean; +} + +export interface UpdateWindowStatus { + readonly open: boolean; + readonly blockers: ReadonlyArray; +} + +/** + * Decides when a background update may restart the server, replace a provider + * CLI, or relaunch the desktop app without cutting off someone's work. + */ +export class UpdateWindow extends Context.Service< + UpdateWindow, + { + readonly check: (options: UpdateWindowOptions) => Effect.Effect; + /** + * Runs `install` if the window is open, holding a lock every background + * install shares, so a server restart never cuts off a CLI install. + */ + readonly runIfOpen: ( + options: UpdateWindowOptions, + install: Effect.Effect, + ) => Effect.Effect, E, R>; + /** Waits for the window, then runs `install` like `runIfOpen`. */ + readonly runWhenOpen: ( + options: UpdateWindowOptions, + install: Effect.Effect, + ) => Effect.Effect; + } +>()("t3/updates/UpdateWindow") {} + +export const make = Effect.fn("updates.UpdateWindow.make")(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; + const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const providers = yield* ProviderRegistry.ProviderRegistry; + const timings = yield* updateTimings; + + const check: UpdateWindow["Service"]["check"] = Effect.fn("updates.UpdateWindow.check")( + function* (options) { + const now = yield* DateTime.now; + const horizon = DateTime.addDuration(now, UPDATE_LOOKAHEAD); + const blockers: Array = []; + + // A failed read keeps the window closed: guessing "idle" could cut off a turn. + const activeThreadIds = yield* projections.getRecoveryThreadIds("active-work").pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read threads", { cause }), + ), + Effect.option, + ); + if (Option.isNone(activeThreadIds)) { + return { open: false, blockers: [{ type: "active-threads", threadIds: [] }] }; + } + if (activeThreadIds.value.length > 0) { + blockers.push({ type: "active-threads", threadIds: activeThreadIds.value }); + } + + // Includes installs someone started by hand. + const providerUpdating = (yield* providers.getProviders).some( + (provider) => + provider.updateState?.status === "queued" || provider.updateState?.status === "running", + ); + if (providerUpdating) { + blockers.push({ type: "provider-update" }); + } + + const lastInteraction = yield* backgroundPolicy.lastClientInteractionAt; + if (Option.isSome(lastInteraction)) { + const quietAt = DateTime.addDuration(lastInteraction.value, timings.quietPeriod); + if (DateTime.isGreaterThan(quietAt, now)) { + blockers.push({ type: "recent-interaction", quietAt }); + } + } + + if (options.closesWindows) { + const policy = yield* backgroundPolicy.snapshot; + const hostIdleSeconds = policy.hostPower.stale ? null : policy.hostPower.idleSeconds; + const hostAway = + hostIdleSeconds !== null && hostIdleSeconds >= Duration.toSeconds(timings.quietPeriod); + const windowFocused = policy.leases.some( + (lease) => lease.clientKind === "desktop-renderer" && lease.focused, + ); + if (windowFocused && !hostAway) { + blockers.push({ type: "focused-window" }); + } + } + + const tasks = yield* scheduledTasks.list().pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read scheduled tasks", { cause }), + ), + Effect.option, + ); + if (Option.isNone(tasks)) { + return { open: false, blockers }; + } + for (const task of tasks.value.tasks) { + if (task.lastRunStatus === "running") { + blockers.push({ + type: "scheduled-task", + taskId: task.id, + title: task.title, + runsAt: null, + }); + continue; + } + if (!task.enabled || task.nextRunAt === null) continue; + if ( + task.schedule.type === "interval" && + task.schedule.everyMs < Duration.toMillis(FREQUENT_TASK_INTERVAL) + ) { + continue; + } + const runsAt = DateTime.make(task.nextRunAt); + if (Option.isSome(runsAt) && !DateTime.isGreaterThan(runsAt.value, horizon)) { + blockers.push({ + type: "scheduled-task", + taskId: task.id, + title: task.title, + runsAt: runsAt.value, + }); + } + } + + const settings = yield* serverSettings.getSettings.pipe(Effect.option); + if (Option.isSome(settings) && settings.value.autoResumeLimitedThreads) { + const limited = yield* projections + .getLimitRecoveryCandidates({ now: horizon, autoResume: true, snooze: false }) + .pipe(Effect.orElseSucceed(() => [])); + for (const thread of limited) { + if (thread.limitRecovery?.autoResume !== true) continue; + const resumesAt = DateTime.make(thread.limitRecovery.resetAt); + if (Option.isSome(resumesAt)) { + blockers.push({ + type: "usage-limit-resume", + threadId: thread.id, + resumesAt: resumesAt.value, + }); + } + } + } + + return { open: blockers.length === 0, blockers }; + }, + ); + + const installLock = yield* Semaphore.make(1); + const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (options, install) => + installLock.withPermits(1)( + check(options).pipe( + Effect.flatMap((status) => (status.open ? Effect.asSome(install) : Effect.succeedNone)), + ), + ); + const runWhenOpen: UpdateWindow["Service"]["runWhenOpen"] = (options, install) => + Effect.gen(function* () { + while (true) { + const result = yield* runIfOpen(options, install); + if (Option.isSome(result)) return result.value; + yield* Effect.sleep(timings.recheck); + } + }); + + return UpdateWindow.of({ check, runIfOpen, runWhenOpen }); +}); + +export const layer = Layer.effect(UpdateWindow, make()); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 1b045f7e963f1..a4cc87f2e1c1f 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -3131,6 +3131,8 @@ export const layer = Layer.unwrap( const previewAutomationBroker = yield* PreviewAutomationBroker.PreviewAutomationBroker; const serverBrowser = yield* ServerBrowser.ServerBrowser; const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; + // Shared with background updates so every caller queues behind one lock per installer. + const providerMaintenanceRunner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; const pullRequests = yield* PullRequestService.PullRequestService; const sql = yield* SqlClient.SqlClient; return HttpRouter.add( @@ -3193,7 +3195,12 @@ export const layer = Layer.unwrap( Layer.provide(DefectReporter.layer), Layer.provide(Layer.succeed(SqlClient.SqlClient, sql)), Layer.provide(AgentSessionScanner.layer), - Layer.provide(ProviderMaintenanceRunner.layer), + Layer.provide( + Layer.succeed( + ProviderMaintenanceRunner.ProviderMaintenanceRunner, + providerMaintenanceRunner, + ), + ), Layer.provide(Layer.succeed(ServerSelfUpdate.ServerSelfUpdate, serverSelfUpdate)), // One server-lifetime service means clients share the same PR caches, and a WS // mutation invalidates the HTTP diff cache that every client reads from. diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index bee57d21b2531..f5b07cfc8beee 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -629,6 +629,9 @@ export function useSettingsRestore(onRestored?: () => void) { DEFAULT_UNIFIED_SETTINGS.enableProviderUpdateChecks ? ["Provider update checks"] : []), + ...(settings.automaticUpdates !== DEFAULT_UNIFIED_SETTINGS.automaticUpdates + ? ["Update automatically"] + : []), ...(settings.continueThreadsAfterServerUpdate !== DEFAULT_UNIFIED_SETTINGS.continueThreadsAfterServerUpdate ? ["Continue threads after restarts"] @@ -705,6 +708,7 @@ export function useSettingsRestore(onRestored?: () => void) { settings.responseStreamingMode, settings.persistComposerContextStrip, settings.enableProviderUpdateChecks, + settings.automaticUpdates, settings.continueThreadsAfterServerUpdate, settings.sidebarAutoSettleAfterDays, settings.sidebarAutoSettleOnMerge, @@ -819,6 +823,7 @@ export function useSettingsRestore(onRestored?: () => void) { snoozeLimitedThreads: DEFAULT_UNIFIED_SETTINGS.snoozeLimitedThreads, responseStreamingMode: DEFAULT_UNIFIED_SETTINGS.responseStreamingMode, enableProviderUpdateChecks: DEFAULT_UNIFIED_SETTINGS.enableProviderUpdateChecks, + automaticUpdates: DEFAULT_UNIFIED_SETTINGS.automaticUpdates, continueThreadsAfterServerUpdate: DEFAULT_UNIFIED_SETTINGS.continueThreadsAfterServerUpdate, backgroundActivity: DEFAULT_UNIFIED_SETTINGS.backgroundActivity, backgroundActivityProfile: DEFAULT_UNIFIED_SETTINGS.backgroundActivityProfile, @@ -2201,6 +2206,11 @@ export function GeneralSettingsPanel() { connectedEnvironments.every( (target) => target.serverConfig?.environment.capabilities.threadAutoSettlement === true, ); + const supportsAutomaticUpdates = + connectedEnvironments.length > 0 && + connectedEnvironments.every( + (target) => target.serverConfig?.environment.capabilities.automaticUpdates === true, + ); const supportsRestartContinuation = connectedEnvironments.length > 0 && connectedEnvironments.every( @@ -2910,6 +2920,38 @@ export function GeneralSettingsPanel() { } /> + + updateSettings({ automaticUpdates: DEFAULT_UNIFIED_SETTINGS.automaticUpdates }) + } + /> + ) : null + } + control={ + updateSettings({ automaticUpdates: Boolean(checked) })} + aria-label="Update automatically" + /> + } + /> + Date: Fri, 9 Oct 2026 18:10:07 +0000 Subject: [PATCH 003/149] feat(server): environments update themselves when nothing is running Background service installs stage the newest release on their channel, then hand off to the launcher once the update window opens. Desktop-hosted servers relaunch the app into an update it already downloaded once no window is in use. Both follow the Update automatically setting. A rolled-back target is remembered by the launcher and not retried, and a release that changes the orchestration protocol is left for a manual update so connected clients are not locked out. --- .../SettingsServerControlsRouteScreen.tsx | 2 +- apps/server/src/cli/update.ts | 2 +- apps/server/src/cloud/selfUpdate.test.ts | 9 + apps/server/src/cloud/selfUpdate.ts | 210 +++++++++++------- .../server/src/cloud/serviceLauncherClient.ts | 4 + .../server/src/cloud/servicePreflight.test.ts | 2 + apps/server/src/cloud/servicePreflight.ts | 15 +- apps/server/src/server.ts | 6 + .../src/serverRuntimeStartup.autoPull.test.ts | 1 + apps/server/src/updates/ServerAutoUpdater.ts | 158 +++++++++++++ apps/server/src/updates/UpdateWindow.ts | 2 + .../components/settings/SettingsPanels.tsx | 2 +- docs/user/updating.md | 22 +- 13 files changed, 340 insertions(+), 95 deletions(-) create mode 100644 apps/server/src/updates/ServerAutoUpdater.ts diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 964f2a3f8c888..df70d72ad7be7 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -473,7 +473,7 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { projectSelected ? "Environment-wide setting. Select All projects to change it." : supportsAutomaticUpdates - ? "Install provider CLI updates while nothing is running." + ? "Install T3 Code and provider updates while nothing is running." : "Update older servers to control automatic updates." } value={uniform("automaticUpdates")} diff --git a/apps/server/src/cli/update.ts b/apps/server/src/cli/update.ts index fa167e3bdeeb7..f2b8f28632082 100644 --- a/apps/server/src/cli/update.ts +++ b/apps/server/src/cli/update.ts @@ -62,7 +62,7 @@ const RELEASE_INDEX_TIMEOUT = Duration.seconds(30); const RELEASE_INDEX_MAX_PAGES = 10; /** Asks GitHub for the newest published version on a channel, page by page. */ -const resolveNewestVersion = Effect.fn("cli.update.resolve_newest")(function* ( +export const resolveNewestVersion = Effect.fn("cli.update.resolve_newest")(function* ( channel: CliReleaseChannel, releaseBaseUrl: string | undefined, ) { diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 111b62c8a481e..577ad660c2a93 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -105,6 +105,7 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( return "launcher-id"; })), prepareTrial: Effect.undefined, + lastOutcome: undefined, }); const config = yield* ServerConfig.ServerConfig.pipe( Effect.provide(ServerConfig.layerTest(process.cwd(), baseDir)), @@ -127,6 +128,8 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( return { selfUpdate, order }; }); +const withoutStaging = { capability: null, stage: () => Effect.die("unused") } as const; + it.layer(NodeServices.layer)("server self update", (it) => { it.effect("marks running threads at the boot-service handoff", () => Effect.gen(function* () { @@ -134,6 +137,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({ mode: "web", selfUpdate: { + ...withoutStaging, update: (_input, reportProgress = () => Effect.void) => reportProgress("downloading").pipe( Effect.andThen(reportProgress("installing")), @@ -168,6 +172,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({ mode: "desktop", selfUpdate: { + ...withoutStaging, update: (_input, reportProgress = () => Effect.void) => reportProgress("installing").pipe( Effect.as({ @@ -216,6 +221,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({ mode: "web", selfUpdate: { + ...withoutStaging, update: (_input, reportProgress = () => Effect.void) => reportProgress("installing").pipe(Effect.andThen(Effect.fail(updateError))), commitDesktopUpdate: () => Effect.never, @@ -238,6 +244,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({ mode: "web", selfUpdate: { + ...withoutStaging, update: ( _input, reportProgress = () => Effect.void, @@ -271,6 +278,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({ mode: "desktop", selfUpdate: { + ...withoutStaging, update: () => Effect.succeed({ targetVersion: "1.2.0", @@ -307,6 +315,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { const selfUpdate = yield* ServerSelfUpdate.withRunningThreadContinuation({ mode: "desktop", selfUpdate: { + ...withoutStaging, update: () => Effect.succeed({ targetVersion: "1.2.0", diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 47c4949de7392..a6d8c70740cc2 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -15,6 +15,7 @@ import * as Effect from "effect/Effect"; import * as HashSet from "effect/HashSet"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; +import * as Semaphore from "effect/Semaphore"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -59,6 +60,15 @@ export class ServerSelfUpdate extends Context.Service< requestId: string, onHandoffAccepted?: () => Effect.Effect, ) => Effect.Effect; + /** How this server can update itself, or null when it cannot. */ + readonly capability: ServerSelfUpdateCapability | null; + /** Downloads and preflights a boot-service runtime without restarting. */ + readonly stage: ( + targetVersion: string, + ) => Effect.Effect< + { readonly orchestrationProtocol: number | undefined }, + ServerSelfUpdateError + >; } >()("t3/cloud/selfUpdate/ServerSelfUpdate") {} @@ -137,6 +147,7 @@ export const withRunningThreadContinuation = Effect.fn( }; return ServerSelfUpdate.of({ + ...input.selfUpdate, update, commitDesktopUpdate: (requestId) => Effect.gen(function* () { @@ -185,6 +196,8 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), ); const inFlight = yield* Ref.make(false); + // A background stage and a manual update may target the same version. + const stageLock = yield* Semaphore.make(1); const capability: ServerSelfUpdateCapability | null = serverConfig.mode === "desktop" ? "desktop-managed" : launcher.managed ? "boot-service" : null; @@ -193,6 +206,109 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { ? new ServerSelfUpdateError({ reason }) : new ServerSelfUpdateError({ reason, cause }); + /** Downloads and preflights a runtime without handing off, so a background + update can stage first and restart later. */ + const stageRuntime = Effect.fn("cloud.server_self_update.stage")(function* ( + targetVersion: string, + ) { + return yield* stageLock.withPermits(1)(stageRuntimeUnlocked(targetVersion)); + }); + const stageRuntimeUnlocked = Effect.fn("cloud.server_self_update.stageUnlocked")(function* ( + targetVersion: string, + ) { + let orchestrationProtocol: number | undefined; + const paths = yield* ensurePinnedRuntimeInstalled({ + baseDir: serverConfig.baseDir, + version: targetVersion, + fs, + path, + runner, + httpClient, + platform, + arch, + releaseBaseUrl, + validate: (runtime) => + runner + .run({ + command: pinnedRuntimeCommand(runtime).command, + args: [ + ...pinnedRuntimeCommand(runtime).args, + "__service-preflight", + "--database-path", + serverConfig.dbPath, + "--launcher-protocol", + String(SERVICE_LAUNCHER_PROTOCOL), + ], + timeout: PREFLIGHT_TIMEOUT, + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "running the staged service preflight", + cause, + }), + ), + Effect.flatMap( + ( + result, + ): Effect.Effect< + void, + PinnedRuntimeInstallError | PinnedRuntimePreflightBlockedError + > => { + if (result.code !== 0) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "running the staged service preflight", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout.trim()); + } catch (cause) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "decoding the staged service preflight", + cause, + }), + ); + } + const preflight = decodeServicePreflightResult(parsed); + if (preflight === undefined || preflight.version !== targetVersion) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "verifying the staged service preflight", + }), + ); + } + if (preflight.status === "ready") { + orchestrationProtocol = preflight.orchestrationProtocol; + } + return preflight.status === "ready" + ? Effect.void + : Effect.fail( + new PinnedRuntimePreflightBlockedError({ + version: targetVersion, + reason: preflight.reason, + }), + ); + }, + ), + ), + }).pipe( + Effect.mapError((error) => + error._tag === "PinnedRuntimePreflightBlockedError" + ? failWith(error.reason, error) + : failWith(`Could not prepare t3@${targetVersion}.`, error), + ), + ); + return { paths, orchestrationProtocol }; + }); + const update: ServerSelfUpdate["Service"]["update"] = Effect.fn( "cloud.server_self_update.update", )(function* (input, reportProgress = () => Effect.void, onHandoffAccepted = () => Effect.void) { @@ -223,92 +339,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { return yield* Effect.gen(function* () { yield* reportProgress("downloading"); - const paths = yield* ensurePinnedRuntimeInstalled({ - baseDir: serverConfig.baseDir, - version: targetVersion, - fs, - path, - runner, - httpClient, - platform, - arch, - releaseBaseUrl, - validate: (runtime) => - runner - .run({ - command: pinnedRuntimeCommand(runtime).command, - args: [ - ...pinnedRuntimeCommand(runtime).args, - "__service-preflight", - "--database-path", - serverConfig.dbPath, - "--launcher-protocol", - String(SERVICE_LAUNCHER_PROTOCOL), - ], - timeout: PREFLIGHT_TIMEOUT, - }) - .pipe( - Effect.mapError( - (cause) => - new PinnedRuntimeInstallError({ - step: "running the staged service preflight", - cause, - }), - ), - Effect.flatMap( - ( - result, - ): Effect.Effect< - void, - PinnedRuntimeInstallError | PinnedRuntimePreflightBlockedError - > => { - if (result.code !== 0) { - return Effect.fail( - new PinnedRuntimeInstallError({ - step: "running the staged service preflight", - exitCode: Number(result.code), - stdoutLength: result.stdout.length, - stderrLength: result.stderr.length, - }), - ); - } - let parsed: unknown; - try { - parsed = JSON.parse(result.stdout.trim()); - } catch (cause) { - return Effect.fail( - new PinnedRuntimeInstallError({ - step: "decoding the staged service preflight", - cause, - }), - ); - } - const preflight = decodeServicePreflightResult(parsed); - if (preflight === undefined || preflight.version !== targetVersion) { - return Effect.fail( - new PinnedRuntimeInstallError({ - step: "verifying the staged service preflight", - }), - ); - } - return preflight.status === "ready" - ? Effect.void - : Effect.fail( - new PinnedRuntimePreflightBlockedError({ - version: targetVersion, - reason: preflight.reason, - }), - ); - }, - ), - ), - }).pipe( - Effect.mapError((error) => - error._tag === "PinnedRuntimePreflightBlockedError" - ? failWith(error.reason, error) - : failWith(`Could not prepare t3@${targetVersion}.`, error), - ), - ); + const { paths } = yield* stageRuntime(targetVersion); yield* reportProgress("installing"); const updateId = yield* Effect.uninterruptible( @@ -338,6 +369,13 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { update, commitDesktopUpdate: (requestId, onHandoffAccepted) => desktopAppUpdate.commit(requestId, onHandoffAccepted), + capability, + stage: (targetVersion) => + capability === "boot-service" + ? stageRuntime(targetVersion).pipe( + Effect.map(({ orchestrationProtocol }) => ({ orchestrationProtocol })), + ) + : Effect.fail(failWith("Only background service installs can stage an update.")), }); }); diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.ts index 2d644157211fd..dad88d986beaa 100644 --- a/apps/server/src/cloud/serviceLauncherClient.ts +++ b/apps/server/src/cloud/serviceLauncherClient.ts @@ -108,6 +108,9 @@ export class ServiceLauncherClient extends Context.Service< ServerSelfUpdateOutcome | undefined, ServiceLauncherClientError >; + /** The launcher's last finished update, kept until the next one, so a + rolled-back target is not retried on every boot. */ + readonly lastOutcome: ServerSelfUpdateOutcome | undefined; } >()("t3/cloud/serviceLauncherClient") {} @@ -247,6 +250,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o managed, requestUpdate, prepareTrial, + lastOutcome: outcome, }); }); diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts index 1b9cf8e46c41e..f8b6055ce5c36 100644 --- a/apps/server/src/cloud/servicePreflight.test.ts +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -1,4 +1,5 @@ import { expect, it } from "@effect/vitest"; +import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; import { runServicePreflight } from "./servicePreflight.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; @@ -29,5 +30,6 @@ it("accepts the current launcher protocol", () => { status: "ready", version: "1.2.3", launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION, }); }); diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.ts index ee0f972baa3fc..000863be47d18 100644 --- a/apps/server/src/cloud/servicePreflight.ts +++ b/apps/server/src/cloud/servicePreflight.ts @@ -1,3 +1,5 @@ +import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; + import packageJson from "../../package.json" with { type: "json" }; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; @@ -6,6 +8,9 @@ export type ServicePreflightResult = readonly status: "ready"; readonly version: string; readonly launcherProtocol: typeof SERVICE_LAUNCHER_PROTOCOL; + /** Absent from runtimes that predate it. A background update skips a + target whose clients would need a newer protocol. */ + readonly orchestrationProtocol?: number; } | { readonly status: "blocked"; @@ -29,7 +34,12 @@ export function runServicePreflight(input: { }; } - return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; + return { + status: "ready", + version, + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION, + }; } export function decodeServicePreflightResult(value: unknown): ServicePreflightResult | undefined { @@ -46,6 +56,9 @@ export function decodeServicePreflightResult(value: unknown): ServicePreflightRe status: "ready", version: record.version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + ...(typeof record.orchestrationProtocol === "number" + ? { orchestrationProtocol: record.orchestrationProtocol } + : {}), }; } if ( diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c4be403306d84..103ed184693c4 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -159,6 +159,7 @@ import * as UsageService from "./usage/UsageService.ts"; import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts"; import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; import * as ProviderAutoUpdater from "./updates/ProviderAutoUpdater.ts"; +import * as ServerAutoUpdater from "./updates/ServerAutoUpdater.ts"; import * as UpdateWindow from "./updates/UpdateWindow.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; @@ -708,6 +709,11 @@ const layerMakeRoutes = Layer.mergeAll( Layer.provide(ProviderAdapterRegistry.layerFromProviderInstanceRegistry), Layer.provide(McpOAuth.layerMcpClientAuthenticator), ), + // Shares the routes' ServerSelfUpdate, so a manual and a background update never overlap. + ServerAutoUpdater.layer.pipe( + Layer.provide(UpdateWindow.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), + Layer.provide(layerDesktopTelemetryReceiver), + ), ).pipe( // Both transports consume the same service instance, so caches single-flight across clients // and mutations observed on WebSocket invalidate patches subsequently read over HTTP. diff --git a/apps/server/src/serverRuntimeStartup.autoPull.test.ts b/apps/server/src/serverRuntimeStartup.autoPull.test.ts index 8313d33dc220d..8b80251ceb95f 100644 --- a/apps/server/src/serverRuntimeStartup.autoPull.test.ts +++ b/apps/server/src/serverRuntimeStartup.autoPull.test.ts @@ -150,6 +150,7 @@ it.effect("parks automatic pull until activation without delaying command readin Layer.mock(ThreadLaunch.ThreadLaunchService)({}), Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ managed: true, + lastOutcome: undefined, prepareTrial: Deferred.succeed(prepared, undefined).pipe( Effect.andThen(Deferred.await(commitTrial)), Effect.as(undefined), diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts new file mode 100644 index 0000000000000..e5945feb463b6 --- /dev/null +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -0,0 +1,158 @@ +import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; +import { CLI_RELEASE_BASE_URL_ENV, cliReleaseChannelOf } from "@t3tools/shared/cliRelease"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/http"; + +import packageJson from "../../package.json" with { type: "json" }; +import { resolveNewestVersion } from "../cli/update.ts"; +import * as ServerSelfUpdate from "../cloud/selfUpdate.ts"; +import * as ServerConfig from "../config.ts"; +import * as ServiceLauncherClient from "../cloud/serviceLauncherClient.ts"; +import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts"; +import * as DesktopTelemetryReceiver from "../resourceTelemetry/DesktopTelemetryReceiver.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as UpdateWindow from "./UpdateWindow.ts"; + +/** + * Updates this environment in the background when `automaticUpdates` is on. + * + * - Background service installs stage the newest release on their channel, + * then hand off to the launcher once the update window opens. + * - Desktop-hosted servers wait for the app to finish its background + * download, then relaunch it through the existing prepare/commit handoff + * once no window is in use. + * - Foreground and `npx` servers cannot replace themselves and are skipped. + */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + // A dev server must not relaunch the desktop app or service it runs beside. + if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return; + const selfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; + const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; + const receiver = yield* DesktopTelemetryReceiver.DesktopTelemetryReceiver; + const updateWindow = yield* UpdateWindow.UpdateWindow; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const timings = yield* UpdateWindow.updateTimings; + const httpClient = yield* HttpClient.HttpClient; + // Same mirror the staged download uses, so a mirror lists what it serves. + const releaseBaseUrl = Option.getOrUndefined( + yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), + ); + const currentVersion = packageJson.version; + // Targets this process already gave up on; the launcher remembers rollbacks across restarts. + const skippedTargets = new Set(); + if (launcher.lastOutcome !== undefined && launcher.lastOutcome.status !== "committed") { + skippedTargets.add(launcher.lastOutcome.targetVersion); + } + + const automaticUpdatesEnabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.automaticUpdates), + Effect.orElseSucceed(() => false), + ); + + const updateBootService = Effect.gen(function* () { + if (!(yield* automaticUpdatesEnabled)) return; + const targetVersion = yield* resolveNewestVersion( + cliReleaseChannelOf(currentVersion), + releaseBaseUrl, + ).pipe(Effect.provideService(HttpClient.HttpClient, httpClient)); + if ( + compareExactServiceVersions(targetVersion, currentVersion) <= 0 || + skippedTargets.has(targetVersion) + ) { + return; + } + yield* Effect.logInfo("Staging a background server update", { targetVersion }); + const staged = yield* selfUpdate.stage(targetVersion); + if ( + staged.orchestrationProtocol !== undefined && + staged.orchestrationProtocol !== ORCHESTRATION_PROTOCOL_VERSION + ) { + // Connected clients would be refused until they update; leave this one to a person. + skippedTargets.add(targetVersion); + yield* Effect.logInfo("Skipping a background update that changes the client protocol", { + targetVersion, + }); + return; + } + yield* updateWindow.runWhenOpen( + { closesWindows: false }, + Effect.gen(function* () { + // The setting may have been turned off while waiting. + if (!(yield* automaticUpdatesEnabled)) return; + skippedTargets.add(targetVersion); + yield* Effect.logInfo("Installing a background server update", { targetVersion }); + yield* selfUpdate.update({ targetVersion }); + }), + ); + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("Background server update failed", { cause })), + Effect.withSpan("updates.ServerAutoUpdater.bootService"), + ); + + const updateDesktopApp = (downloadedVersion: string) => + Effect.gen(function* () { + if (!(yield* automaticUpdatesEnabled) || skippedTargets.has(downloadedVersion)) return; + yield* updateWindow.runWhenOpen( + { closesWindows: true }, + Effect.gen(function* () { + if (!(yield* automaticUpdatesEnabled)) return; + skippedTargets.add(downloadedVersion); + yield* Effect.logInfo("Relaunching the desktop app into a downloaded update", { + downloadedVersion, + }); + // The download is already on disk, so preparing it is quick. + const prepared = yield* selfUpdate.update({ targetVersion: downloadedVersion }); + if (prepared.desktopUpdateToken === undefined) return; + return yield* selfUpdate.commitDesktopUpdate(prepared.desktopUpdateToken); + }), + ); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Background desktop update failed", { cause }), + ), + Effect.withSpan("updates.ServerAutoUpdater.desktopApp"), + ); + + if (selfUpdate.capability === "boot-service") { + yield* forkParked( + Effect.sleep(timings.firstPassDelay).pipe( + Effect.andThen( + updateBootService.pipe(Effect.repeat(Schedule.spaced(timings.releaseCheckInterval))), + ), + ), + ); + } + + if (selfUpdate.capability === "desktop-managed") { + // The desktop app reports its updater state on attach and on every change. + yield* forkParked( + Effect.scoped( + Effect.gen(function* () { + const { latest, changes } = yield* receiver.desktopUpdates; + const reports = Option.match(latest, { + onNone: () => changes, + onSome: (report) => Stream.concat(Stream.make(report), changes), + }); + yield* reports.pipe( + Stream.map((report) => + report.state.status === "downloaded" ? report.state.downloadedVersion : null, + ), + Stream.filter((version): version is string => version !== null), + Stream.changes, + // A newer download replaces a wait for an older one. + Stream.switchMap((version) => Stream.fromEffect(updateDesktopApp(version))), + Stream.runDrain, + ); + }), + ), + ); + } + }), +); diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index be9e91055e840..a4ea69fcdff98 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -38,6 +38,8 @@ export const updateTimings = Config.Boolean("T3CODE_UPDATE_LAB").pipe( recheck: lab ? Duration.seconds(5) : Duration.seconds(30), firstPassDelay: lab ? Duration.seconds(10) : Duration.minutes(2), passInterval: lab ? Duration.seconds(20) : Duration.minutes(10), + /** T3 releases are checked this often; the desktop app polls its own feed. */ + releaseCheckInterval: lab ? Duration.seconds(20) : Duration.hours(1), })), ); diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index f5b07cfc8beee..bd955729620ae 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -2924,7 +2924,7 @@ export function GeneralSettingsPanel() { serverScoped settingKeys={["automaticUpdates"]} {...searchableSetting("automatic-updates")} - description="Install provider CLI updates in the background on the selected environments while nothing is running." + description="Install T3 Code and provider CLI updates in the background on the selected environments while nothing is running." status={ !supportsAutomaticUpdates ? "All selected connected environments must support automatic updates." diff --git a/docs/user/updating.md b/docs/user/updating.md index 8c2bf919fa350..2d24d3a45e0b3 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -72,13 +72,25 @@ update can roll back to the previous version. If the update still fails: 2. Check that you updated the server's machine, not only the device you are using. 3. For a command-line server, stop it and relaunch the exact version shown in the notice. -## Update providers +## Automatic updates **Settings → General → Update automatically** is on by default. Each environment -installs provider CLI updates in the background once no thread is running or -waiting for you, no client has been used for 15 minutes, and no scheduled task -or usage-limit resume is due within 5 minutes. A failed update is retried -after 6 hours or when a newer version is published. +waits until no thread is running or waiting for you, no client has been used for +15 minutes, and no scheduled task or usage-limit resume is due within 5 minutes. +Then it installs: + +- provider CLI updates that have a one-click update; +- the newest T3 Code release on its channel, when it runs as a background service + (`t3 service install`). It downloads first and restarts only inside that window; +- a desktop app update that has finished downloading, when no desktop window is + focused or the machine has been idle for 15 minutes. + +Servers started from a terminal or with `npx` are never replaced automatically. +A failed provider update is retried after 6 hours or when a newer version is +published. A failed or rolled-back T3 Code update is not retried until a newer +version is published. An update that needs newer clients waits for you to install it. + +## Update providers **Settings → Providers** shows provider updates for the selected environment. **Update all** updates every outdated provider on every connected environment From 08b23cc7072ef6f336cb91854719868c4415af65 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:10:47 +0000 Subject: [PATCH 004/149] feat(desktop): download updates in the background and install on quit The desktop app downloads an update as soon as a check finds one, instead of waiting for a click. Its server relaunches the app into it once nothing is running and no window is in use; quitting the app installs it too. --- .../src/updates/DesktopUpdates.test.ts | 5 +-- apps/desktop/src/updates/DesktopUpdates.ts | 34 ++++++++++++++++++- docs/user/updating.md | 5 +-- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index 9745662c1209a..c97c22e622870 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -205,7 +205,7 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect("checks for newer releases after an update has been downloaded", () => { + it.effect("downloads newer releases found after an update has been downloaded", () => { const harness = makeHarness(); return Effect.scoped( @@ -241,8 +241,9 @@ describe("DesktopUpdates", () => { harness.emit("update-available", { version: "1.2.5" }); yield* flushCallbacks; + // A release the background poller finds starts downloading on its own. const state = yield* updates.getState; - assert.equal(state.status, "available"); + assert.equal(state.status, "downloading"); assert.equal(state.availableVersion, "1.2.5"); assert.isNull(state.downloadedVersion); }), diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 245925a6e7319..4c49f3c39d8fc 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -50,6 +50,7 @@ import { const AUTO_UPDATE_STARTUP_DELAY = "15 seconds"; const AUTO_UPDATE_POLL_INTERVAL = "4 minutes"; const PREPARED_INSTALL_CHECK_WAIT = Duration.seconds(90); +const BACKGROUND_DOWNLOAD_START_WAIT = Duration.minutes(2); type UpdateAction = "check" | "download" | "install" | "install-recovery" | "channel"; @@ -289,6 +290,7 @@ export const make = Effect.gen(function* () { const finishedUpdateActions = yield* PubSub.unbounded(); const updaterConfiguredRef = yield* Ref.make(false); const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1); + const lastCheckReasonRef = yield* Ref.make(null); const updateStateRef = yield* Ref.make( createInitialDesktopUpdateState( environment.appVersion, @@ -425,6 +427,7 @@ export const make = Effect.gen(function* () { if (actionReservation === "acquire" && !(yield* tryStartUpdateAction("check"))) return false; + yield* Ref.set(lastCheckReasonRef, reason); const check = Effect.gen(function* () { const checkedAt = yield* currentIsoTimestamp; yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); @@ -516,6 +519,28 @@ export const make = Effect.gen(function* () { ); }).pipe(Effect.withSpan("desktop.updates.downloadAvailableUpdate")); + // update-available fires inside the check that found it, so the download + // waits for that check to release the updater. Installing stays gated: the + // app's server relaunches into it when idle, and quitting installs it. + const downloadInBackground = Effect.scoped( + Effect.gen(function* () { + const completions = yield* PubSub.subscribe(finishedUpdateActions); + while (Option.isSome(yield* activeUpdateAction)) { + yield* PubSub.take(completions); + } + if (yield* Ref.get(desktopState.quitting)) return; + yield* downloadAvailableUpdate; + }), + ).pipe( + Effect.timeout(BACKGROUND_DOWNLOAD_START_WAIT), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : logUpdaterWarning("background update download did not start"), + ), + Effect.withSpan("desktop.updates.downloadInBackground"), + ); + // Tells the primary backend that the coming stop is an update restart, so it // keeps its managed tunnel for the backend the updated app starts. Best // effort: without the marker the backend only re-provisions its tunnel. @@ -773,6 +798,12 @@ export const make = Effect.gen(function* () { releaseNoteGroups: releaseNotes.length, omittedReleaseCount, }); + // Updates found by the background pollers download on their own; a + // check someone started keeps its Download button. + const reason = yield* Ref.get(lastCheckReasonRef); + if (reason === "startup" || reason === "poll") { + yield* Effect.forkDetach(downloadInBackground); + } }), ), Effect.catchCause((cause) => { @@ -931,7 +962,8 @@ export const make = Effect.gen(function* () { yield* Ref.set(updaterConfiguredRef, true); yield* electronUpdater.setAutoDownload(false); - yield* electronUpdater.setAutoInstallOnAppQuit(false); + // A download that is never relaunched into installs when the user quits. + yield* electronUpdater.setAutoInstallOnAppQuit(true); yield* applyAutoUpdaterChannel(settings.updateChannel); yield* electronUpdater.setDisableDifferentialDownload( isArm64HostRunningIntelBuild(environment.runtimeInfo), diff --git a/docs/user/updating.md b/docs/user/updating.md index 2d24d3a45e0b3..a7b4dda394aae 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -82,8 +82,9 @@ Then it installs: - provider CLI updates that have a one-click update; - the newest T3 Code release on its channel, when it runs as a background service (`t3 service install`). It downloads first and restarts only inside that window; -- a desktop app update that has finished downloading, when no desktop window is - focused or the machine has been idle for 15 minutes. +- a desktop app update, which downloads in the background as soon as it is + published and installs when no desktop window is focused or the machine has + been idle for 15 minutes. Quitting the app also installs a downloaded update. Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours or when a newer version is From 51fc7141533589a3ac7811fb4a4699c027f438ef Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:11:20 +0000 Subject: [PATCH 005/149] feat(settings): continue threads after restarts by default Background updates restart environments without asking, so threads they interrupt now resume unless the user turns this off. --- docs/internals/server-updates.md | 2 +- docs/user/updating.md | 6 ++---- packages/contracts/src/settings.ts | 5 +++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 34fa5aa47b094..0c8d084656092 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -58,7 +58,7 @@ stopped backends and replays the failure for the same token. ## Recovering interrupted threads -Restart continuation is an environment-owned preference, off by default. The +Restart continuation is an environment-owned preference, on by default. The [v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts) requires matching durable run, provider thread, session, and native resume identity. Queued runs never started, so recovery holds them and continues the run they wait diff --git a/docs/user/updating.md b/docs/user/updating.md index a7b4dda394aae..6b9c5094bcbcb 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -10,16 +10,14 @@ notice. Server updates restart the connection and can interrupt active agents and terminal commands. Saved threads, settings, and project files remain. -**Settings → General → Continue threads after restarts** is off by default. -Enable it to resume supported active threads after an update, crash, or machine +**Settings → General → Continue threads after restarts** is on by default. +It resumes supported active threads after an update, crash, or machine restart. Changes are saved to connected environments that support this setting; update older servers first. If a supported environment was offline or has a different value, use **Apply to all** in Settings after it connects. T3 Code must start again on that machine; the setting does not enable automatic startup. Terminal commands may still be interrupted, and threads without saved provider resume state need a new message. -If you previously enabled continuation for updates, enable this setting once -to allow recovery without a connected client. Updates from the previous orchestration system preserve conversation transcripts but cannot carry every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md) diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 1ad8bf48a9957..71bd431b9d134 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1069,9 +1069,10 @@ export const ServerSettings = Schema.Struct({ enableProviderUpdateChecks: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), /** Install updates in the background once nothing is running and nobody is using the app. */ automaticUpdates: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), - // Retain the update-era key; recovery now needs an environment-owned opt-in. + // Retain the update-era key. On by default so background updates and + // crashes resume the work they interrupted. continueThreadsAfterServerUpdate: Schema.Boolean.pipe( - Schema.withDecodingDefault(Effect.succeed(false)), + Schema.withDecodingDefault(Effect.succeed(true)), ), /** * Whether agents may drive the in-app preview browser. Turning this off From 26af088ce1f376657bbe4c9416f6a196372f68ec Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:45:28 +0000 Subject: [PATCH 006/149] build: register the update lab as a package script Knip treats package scripts as entry points. --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 55869fc4b3911..2b839155031ab 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "start:desktop": "vp run --filter @t3tools/desktop start", "start:marketing": "vp run --filter @t3tools/marketing preview", "start:mock-update-server": "node scripts/mock-update-server.ts", + "update-lab": "node scripts/update-lab.ts", "screenshots:mobile": "node scripts/mobile-showcase.ts", "icons:export": "node scripts/export-brand-icons.ts", "icons:check": "node scripts/export-brand-icons.ts --check", From 08d378e1c1060c6d6495e473d89369bfbe7db07d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:11:18 +0000 Subject: [PATCH 007/149] fix(server): keep background updates behind pending work --- .../Adapters/ClaudeAdapterV2.test.ts | 8 +++++ .../Adapters/ClaudeAdapterV2.ts | 2 +- .../orchestration-v2/ProjectionStore.test.ts | 7 ++++ .../src/orchestration-v2/ProjectionStore.ts | 14 ++++++++ apps/server/src/updates/UpdateWindow.ts | 35 ++++++++++--------- 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index bfef7f9e9a104..5e31ce93bac4b 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3132,6 +3132,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { cacheCreationTokens: 0, }); } + const settledThread = harness.events.findLast( + (event) => event.type === "provider_thread.updated", + ); + assert.equal(settledThread?.type, "provider_thread.updated"); + if (settledThread?.type === "provider_thread.updated") { + assert.equal(settledThread.providerThread.status, "idle"); + assert.isEmpty(settledThread.providerThread.pendingBackgroundTasks ?? []); + } }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index de747802d3490..848897d78cffe 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5217,7 +5217,7 @@ export function makeClaudeAdapterV2( input.context.input.runOrdinal, lastRunOrdinal: input.context.input.runOrdinal, pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster), - status: input.status === "completed" ? "active" : "idle", + status: input.status === "completed" && roster.size > 0 ? "active" : "idle", updatedAt: input.completedAt, }; yield* rememberProviderThread(providerThread); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index e8a3c1bcfc08e..f51e2f3ecde06 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -337,6 +337,7 @@ it.effect("memory recovery selection ignores unfinished items from rolled-back r const threadId = yield* addRolledBackRecoveryCandidate("memory-recovery-candidates"); assert.notInclude(yield* projectionStore.getRecoveryThreadIds("runtime"), threadId); + assert.notInclude(yield* projectionStore.getRecoveryThreadIds("active-work"), threadId); }).pipe(Effect.provide(ProjectionStore.layerMemory)), ); @@ -346,6 +347,7 @@ it.effect("memory recovery selection includes unfinished items from missing runs const threadId = yield* addOrphanedRecoveryCandidate("memory-recovery-candidates"); assert.include(yield* projectionStore.getRecoveryThreadIds("runtime"), threadId); + assert.include(yield* projectionStore.getRecoveryThreadIds("active-work"), threadId); }).pipe(Effect.provide(ProjectionStore.layerMemory)), ); @@ -2416,6 +2418,11 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { assert.include(recoveryThreadIds, orphanedThreadId); assert.notInclude(recoveryThreadIds, settledThreadId); assert.notInclude(recoveryThreadIds, rolledBackThreadId); + const activeThreadIds = yield* projectionStore.getRecoveryThreadIds("active-work"); + assert.include(activeThreadIds, runningThreadId); + assert.include(activeThreadIds, orphanedThreadId); + assert.notInclude(activeThreadIds, settledThreadId); + assert.notInclude(activeThreadIds, rolledBackThreadId); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8119dba91fc95..abc736ec8e94d 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -578,6 +578,12 @@ function needsRecovery( projection.providerThreads.some( (thread) => thread.status === "active" || (thread.pendingBackgroundTasks?.length ?? 0) > 0, + ) || + projection.turnItems.some( + (item) => + ["command_execution", "dynamic_tool", "subagent"].includes(item.type) && + ["pending", "running", "waiting"].includes(item.status) && + !projection.runs.some((run) => run.id === item.runId && run.status === "rolled_back"), ) ); case "runtime": @@ -3627,6 +3633,14 @@ export const layer: Layer.Layer = OR CASE WHEN json_valid(payload_json) THEN json_array_length(payload_json, '$.pendingBackgroundTasks') > 0 ELSE 0 END + UNION + SELECT item.thread_id FROM orchestration_v2_projection_turn_items AS item + WHERE NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs AS run + WHERE run.run_id = item.run_id AND run.status = 'rolled_back' + ) + AND type IN ('command_execution', 'dynamic_tool', 'subagent') + AND status IN ('pending', 'running', 'waiting') `; case "runtime": return sql` diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index be9e91055e840..023794a69e900 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -16,11 +16,6 @@ import * as ServerSettings from "../serverSettings.ts"; /** Work due this soon keeps the window closed, so it does not start late. */ const UPDATE_LOOKAHEAD = Duration.minutes(5); -/** - * Tasks repeating at least this often always have a run due soon. Missed runs - * catch up after a restart, so they only block while running. - */ -const FREQUENT_TASK_INTERVAL = Duration.minutes(15); /** * Background update pacing. `T3CODE_UPDATE_LAB=1` (scripts/update-lab.ts) @@ -93,7 +88,7 @@ export class UpdateWindow extends Context.Service< } >()("t3/updates/UpdateWindow") {} -export const make = Effect.fn("updates.UpdateWindow.make")(function* () { +const make = Effect.fn("updates.UpdateWindow.make")(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; @@ -171,12 +166,6 @@ export const make = Effect.fn("updates.UpdateWindow.make")(function* () { continue; } if (!task.enabled || task.nextRunAt === null) continue; - if ( - task.schedule.type === "interval" && - task.schedule.everyMs < Duration.toMillis(FREQUENT_TASK_INTERVAL) - ) { - continue; - } const runsAt = DateTime.make(task.nextRunAt); if (Option.isSome(runsAt) && !DateTime.isGreaterThan(runsAt.value, horizon)) { blockers.push({ @@ -188,12 +177,26 @@ export const make = Effect.fn("updates.UpdateWindow.make")(function* () { } } - const settings = yield* serverSettings.getSettings.pipe(Effect.option); - if (Option.isSome(settings) && settings.value.autoResumeLimitedThreads) { + const settings = yield* serverSettings.getSettings.pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read settings", { cause }), + ), + Effect.option, + ); + if (Option.isNone(settings) || !settings.value.automaticUpdates) { + return { open: false, blockers }; + } + if (settings.value.autoResumeLimitedThreads) { const limited = yield* projections .getLimitRecoveryCandidates({ now: horizon, autoResume: true, snooze: false }) - .pipe(Effect.orElseSucceed(() => [])); - for (const thread of limited) { + .pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read usage-limit resumes", { cause }), + ), + Effect.option, + ); + if (Option.isNone(limited)) return { open: false, blockers }; + for (const thread of limited.value) { if (thread.limitRecovery?.autoResume !== true) continue; const resumesAt = DateTime.make(thread.limitRecovery.resetAt); if (Option.isSome(resumesAt)) { From 47f38cacf164751c978009a95ef2d2570abaa8bf Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:12:20 +0000 Subject: [PATCH 008/149] fix(server): refresh staged updates without relaunching active apps --- apps/server/src/updates/ServerAutoUpdater.ts | 134 +++++++------------ docs/user/updating.md | 7 +- 2 files changed, 51 insertions(+), 90 deletions(-) diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index e5945feb463b6..6b23b5555348c 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -1,11 +1,12 @@ import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; import { CLI_RELEASE_BASE_URL_ENV, cliReleaseChannelOf } from "@t3tools/shared/cliRelease"; +import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; -import * as Stream from "effect/Stream"; import { HttpClient } from "effect/http"; import packageJson from "../../package.json" with { type: "json" }; @@ -14,7 +15,6 @@ import * as ServerSelfUpdate from "../cloud/selfUpdate.ts"; import * as ServerConfig from "../config.ts"; import * as ServiceLauncherClient from "../cloud/serviceLauncherClient.ts"; import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts"; -import * as DesktopTelemetryReceiver from "../resourceTelemetry/DesktopTelemetryReceiver.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as UpdateWindow from "./UpdateWindow.ts"; @@ -24,9 +24,8 @@ import * as UpdateWindow from "./UpdateWindow.ts"; * * - Background service installs stage the newest release on their channel, * then hand off to the launcher once the update window opens. - * - Desktop-hosted servers wait for the app to finish its background - * download, then relaunch it through the existing prepare/commit handoff - * once no window is in use. + * - Desktop-hosted servers update when the user quits the app. One backend + * cannot establish that every other backend the app hosts is idle. * - Foreground and `npx` servers cannot replace themselves and are skipped. */ export const layer = Layer.effectDiscard( @@ -34,8 +33,8 @@ export const layer = Layer.effectDiscard( // A dev server must not relaunch the desktop app or service it runs beside. if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return; const selfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; + if (selfUpdate.capability !== "boot-service") return; const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; - const receiver = yield* DesktopTelemetryReceiver.DesktopTelemetryReceiver; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; const timings = yield* UpdateWindow.updateTimings; @@ -45,6 +44,8 @@ export const layer = Layer.effectDiscard( yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), ); const currentVersion = packageJson.version; + let pendingTarget: string | undefined; + let nextReleaseCheckAt = 0; // Targets this process already gave up on; the launcher remembers rollbacks across restarts. const skippedTargets = new Set(); if (launcher.lastOutcome !== undefined && launcher.lastOutcome.status !== "committed") { @@ -58,34 +59,46 @@ export const layer = Layer.effectDiscard( const updateBootService = Effect.gen(function* () { if (!(yield* automaticUpdatesEnabled)) return; - const targetVersion = yield* resolveNewestVersion( - cliReleaseChannelOf(currentVersion), - releaseBaseUrl, - ).pipe(Effect.provideService(HttpClient.HttpClient, httpClient)); - if ( - compareExactServiceVersions(targetVersion, currentVersion) <= 0 || - skippedTargets.has(targetVersion) - ) { - return; - } - yield* Effect.logInfo("Staging a background server update", { targetVersion }); - const staged = yield* selfUpdate.stage(targetVersion); - if ( - staged.orchestrationProtocol !== undefined && - staged.orchestrationProtocol !== ORCHESTRATION_PROTOCOL_VERSION - ) { - // Connected clients would be refused until they update; leave this one to a person. - skippedTargets.add(targetVersion); - yield* Effect.logInfo("Skipping a background update that changes the client protocol", { - targetVersion, - }); - return; + const nowMs = yield* Clock.currentTimeMillis; + if (nowMs >= nextReleaseCheckAt) { + nextReleaseCheckAt = nowMs + Duration.toMillis(timings.releaseCheckInterval); + const targetVersion = yield* resolveNewestVersion( + cliReleaseChannelOf(currentVersion), + releaseBaseUrl, + ).pipe(Effect.provideService(HttpClient.HttpClient, httpClient)); + if ( + compareExactServiceVersions(targetVersion, currentVersion) > 0 && + !skippedTargets.has(targetVersion) && + targetVersion !== pendingTarget + ) { + // A newer release replaces the pending target even while the window stays closed. + pendingTarget = undefined; + yield* Effect.logInfo("Staging a background server update", { targetVersion }); + const staged = yield* selfUpdate + .stage(targetVersion) + .pipe(Effect.tapError(() => Effect.sync(() => skippedTargets.add(targetVersion)))); + if (staged.orchestrationProtocol !== ORCHESTRATION_PROTOCOL_VERSION) { + // Without a matching protocol, connected clients could be refused after restart. + skippedTargets.add(targetVersion); + yield* Effect.logInfo( + "Skipping a background update without a matching client protocol", + { + targetVersion, + }, + ); + } else { + pendingTarget = targetVersion; + } + } } - yield* updateWindow.runWhenOpen( + const targetVersion = pendingTarget; + if (targetVersion === undefined) return; + yield* updateWindow.runIfOpen( { closesWindows: false }, Effect.gen(function* () { // The setting may have been turned off while waiting. if (!(yield* automaticUpdatesEnabled)) return; + pendingTarget = undefined; skippedTargets.add(targetVersion); yield* Effect.logInfo("Installing a background server update", { targetVersion }); yield* selfUpdate.update({ targetVersion }); @@ -96,63 +109,10 @@ export const layer = Layer.effectDiscard( Effect.withSpan("updates.ServerAutoUpdater.bootService"), ); - const updateDesktopApp = (downloadedVersion: string) => - Effect.gen(function* () { - if (!(yield* automaticUpdatesEnabled) || skippedTargets.has(downloadedVersion)) return; - yield* updateWindow.runWhenOpen( - { closesWindows: true }, - Effect.gen(function* () { - if (!(yield* automaticUpdatesEnabled)) return; - skippedTargets.add(downloadedVersion); - yield* Effect.logInfo("Relaunching the desktop app into a downloaded update", { - downloadedVersion, - }); - // The download is already on disk, so preparing it is quick. - const prepared = yield* selfUpdate.update({ targetVersion: downloadedVersion }); - if (prepared.desktopUpdateToken === undefined) return; - return yield* selfUpdate.commitDesktopUpdate(prepared.desktopUpdateToken); - }), - ); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Background desktop update failed", { cause }), - ), - Effect.withSpan("updates.ServerAutoUpdater.desktopApp"), - ); - - if (selfUpdate.capability === "boot-service") { - yield* forkParked( - Effect.sleep(timings.firstPassDelay).pipe( - Effect.andThen( - updateBootService.pipe(Effect.repeat(Schedule.spaced(timings.releaseCheckInterval))), - ), - ), - ); - } - - if (selfUpdate.capability === "desktop-managed") { - // The desktop app reports its updater state on attach and on every change. - yield* forkParked( - Effect.scoped( - Effect.gen(function* () { - const { latest, changes } = yield* receiver.desktopUpdates; - const reports = Option.match(latest, { - onNone: () => changes, - onSome: (report) => Stream.concat(Stream.make(report), changes), - }); - yield* reports.pipe( - Stream.map((report) => - report.state.status === "downloaded" ? report.state.downloadedVersion : null, - ), - Stream.filter((version): version is string => version !== null), - Stream.changes, - // A newer download replaces a wait for an older one. - Stream.switchMap((version) => Stream.fromEffect(updateDesktopApp(version))), - Stream.runDrain, - ); - }), - ), - ); - } + yield* forkParked( + Effect.sleep(timings.firstPassDelay).pipe( + Effect.andThen(updateBootService.pipe(Effect.repeat(Schedule.spaced(timings.recheck)))), + ), + ); }), ); diff --git a/docs/user/updating.md b/docs/user/updating.md index 2d24d3a45e0b3..c1c566d3b758f 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -81,9 +81,10 @@ Then it installs: - provider CLI updates that have a one-click update; - the newest T3 Code release on its channel, when it runs as a background service - (`t3 service install`). It downloads first and restarts only inside that window; -- a desktop app update that has finished downloading, when no desktop window is - focused or the machine has been idle for 15 minutes. + (`t3 service install`). It downloads first and restarts only inside that window. + +Desktop updates download in the background and install when you quit the app. +The app stays open while you work, including when it hosts multiple backends. Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours or when a newer version is From bd677e13c028187bce41ceac55576bfc3ab95df6 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:13:08 +0000 Subject: [PATCH 009/149] fix(desktop): keep slow background downloads reserved --- .../src/updates/DesktopUpdates.test.ts | 60 +++++++++++++++++++ apps/desktop/src/updates/DesktopUpdates.ts | 11 ++-- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index c97c22e622870..ce5935fe2fb3e 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -250,6 +250,66 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); + it.effect("keeps slow background downloads active after waiting for a check", () => + Effect.gen(function* () { + const checkStarted = yield* Deferred.make(); + const releaseCheck = yield* Deferred.make(); + const downloadStarted = yield* Deferred.make(); + const releaseDownload = yield* Deferred.make(); + const harness = makeHarness({ + checkForUpdates: Deferred.succeed(checkStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseCheck)), + ), + downloadUpdate: Deferred.succeed(downloadStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseDownload)), + ), + }); + + yield* Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + + const checkFiber = yield* updates.check("poll").pipe(Effect.forkScoped); + yield* Deferred.await(checkStarted); + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + + yield* TestClock.adjust(Duration.minutes(1)); + assert.equal(harness.downloadCount(), 0); + + yield* Deferred.succeed(releaseCheck, undefined); + yield* Fiber.join(checkFiber); + yield* Deferred.await(downloadStarted); + yield* TestClock.adjust(Duration.minutes(3)); + + assert.equal((yield* updates.getState).status, "downloading"); + assert.isTrue(yield* updates.isActionActive); + assert.isFalse((yield* updates.check("manual")).checked); + assert.isFalse((yield* updates.download).accepted); + const channelError = yield* updates.setChannel("nightly").pipe(Effect.flip); + assert.equal(channelError._tag, "DesktopUpdateActionInProgressError"); + if (channelError._tag === "DesktopUpdateActionInProgressError") { + assert.equal(channelError.action, "download"); + } + assert.equal(harness.downloadCount(), 1); + + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* Deferred.succeed(releaseDownload, undefined); + yield* flushCallbacks; + + const state = yield* updates.getState; + assert.equal(state.status, "downloaded"); + assert.equal(state.downloadedVersion, "1.2.4"); + assert.isFalse(yield* updates.isActionActive); + }), + ).pipe( + Effect.ensuring(Deferred.succeed(releaseDownload, undefined)), + Effect.provide(Layer.merge(TestClock.layer(), harness.layer)), + ); + }), + ); + it.effect("preserves a queued installer when the feed has no update", () => { const harness = makeHarness(); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 4c49f3c39d8fc..251c66ca9b71c 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -520,19 +520,22 @@ export const make = Effect.gen(function* () { }).pipe(Effect.withSpan("desktop.updates.downloadAvailableUpdate")); // update-available fires inside the check that found it, so the download - // waits for that check to release the updater. Installing stays gated: the - // app's server relaunches into it when idle, and quitting installs it. + // waits for that check to release the updater. Quitting installs the download. const downloadInBackground = Effect.scoped( Effect.gen(function* () { const completions = yield* PubSub.subscribe(finishedUpdateActions); while (Option.isSome(yield* activeUpdateAction)) { yield* PubSub.take(completions); } - if (yield* Ref.get(desktopState.quitting)) return; - yield* downloadAvailableUpdate; }), ).pipe( Effect.timeout(BACKGROUND_DOWNLOAD_START_WAIT), + Effect.andThen( + Effect.gen(function* () { + if (yield* Ref.get(desktopState.quitting)) return; + yield* downloadAvailableUpdate; + }), + ), Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Effect.void From 4ff07873e093952aa6c59a374dfaf088c0d690b1 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:13:08 +0000 Subject: [PATCH 010/149] fix(dev): preserve lab state and feed origins across restarts --- scripts/update-lab.ts | 169 ++++++++++++++++++++++++++++-------------- 1 file changed, 115 insertions(+), 54 deletions(-) diff --git a/scripts/update-lab.ts b/scripts/update-lab.ts index 5f70c4f84f9e7..29f8d91327581 100644 --- a/scripts/update-lab.ts +++ b/scripts/update-lab.ts @@ -191,18 +191,40 @@ function requireState(): LabState { return state; } -/** Fake installers and the CLI write concurrently, so writes are atomic renames. */ +/** Keep the descriptor open so the kernel releases the lock even if this process dies. */ +function acquireLock(name: "state" | "processes"): () => void { + NodeFS.mkdirSync(NodePath.dirname(LAB), { recursive: true }); + // Keep lock files outside LAB so reset cannot unlink a lock another process holds. + const fd = NodeFS.openSync(`${LAB}.${name}.lock`, "a"); + try { + NodeChildProcess.execFileSync("flock", ["-x", "3"], { + stdio: ["ignore", "ignore", "inherit", fd], + }); + } catch (error) { + NodeFS.closeSync(fd); + throw error; + } + return () => NodeFS.closeSync(fd); +} + +/** Atomic replacement keeps readers from observing a partially written state. */ function writeState(state: LabState): void { const temp = `${STATE_PATH}.${process.pid}.tmp`; NodeFS.writeFileSync(temp, `${JSON.stringify(state, null, 2)}\n`); NodeFS.renameSync(temp, STATE_PATH); } -function updateState(mutate: (state: LabState) => void): LabState { - const state = requireState(); - mutate(state); - writeState(state); - return state; +function updateState(mutate: (state: LabState) => void, initial?: LabState): LabState { + const release = acquireLock("state"); + try { + const state = readState() ?? initial ?? requireState(); + mutate(state); + NodeFS.mkdirSync(LAB, { recursive: true }); + writeState(state); + return state; + } finally { + release(); + } } function requireEnv(state: LabState, id: string | undefined): LabEnv { @@ -725,11 +747,16 @@ async function freePort(host: string, taken: Set): Promise { fail("Could not find a free port."); } -/** True only for a live process whose command line is still one the lab started. */ -function isLabProcess(pid: number | null, marker: string): pid is number { +/** Check the recorded process's command and working directory before signaling it. */ +function isLabProcess(pid: number | null, marker: string, cwd = REPO): pid is number { if (!pid) return false; try { - return NodeFS.readFileSync(`/proc/${pid}/cmdline`, "utf8").includes(marker); + const args = NodeFS.readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0"); + return ( + args.includes(marker) && + args.includes(marker === "__feed" ? SELF : SERVER_BIN) && + NodeFS.realpathSync(`/proc/${pid}/cwd`) === NodeFS.realpathSync(cwd) + ); } catch { return false; } @@ -827,7 +854,7 @@ async function waitForPairingUrl(env: LabEnv): Promise { const log = NodeFS.readFileSync(logPath(env.id), "utf8"); const url = /Pairing URL: (\S+)/.exec(log)?.[1]; if (url) return url; - if (!isLabProcess(env.pid, serverMarker(env))) { + if (!isLabProcess(env.pid, serverMarker(env), projectDir(env.id))) { fail(`${env.id} exited during startup; see ${logPath(env.id)}\n${log.slice(-2000)}`); } await new Promise((resolve) => setTimeout(resolve, 500)); @@ -836,11 +863,11 @@ async function waitForPairingUrl(env: LabEnv): Promise { } /** SIGTERM the process group we created, then SIGKILL it if it lingers. */ -async function stopGroup(pid: number, marker: string): Promise { - if (!isLabProcess(pid, marker)) return; +async function stopGroup(pid: number, marker: string, cwd = REPO): Promise { + if (!isLabProcess(pid, marker, cwd)) return; process.kill(-pid, "SIGTERM"); const deadline = Date.now() + 15_000; - while (Date.now() < deadline && isLabProcess(pid, marker)) { + while (Date.now() < deadline && isLabProcess(pid, marker, cwd)) { await new Promise((resolve) => setTimeout(resolve, 250)); } try { @@ -950,7 +977,7 @@ async function up(args: string[]): Promise { const envCount = Number(option(args, "envs") ?? previous?.envs.length ?? 3); const launcherCount = Number(option(args, "launcher") ?? 1); const [major, minor, patch] = REPO_VERSION.split(/[.-]/).map(Number); - const state: LabState = previous ?? { + const initial: LabState = { host, feedPort: 0, feedPid: null, @@ -962,56 +989,76 @@ async function up(args: string[]): Promise { ) as LabState["latest"], envs: [], }; - state.host = host; - state.busySeconds = Number(option(args, "busy-seconds") ?? state.busySeconds); - state.fromVersion = option(args, "from-version") ?? state.fromVersion; - NodeFS.mkdirSync(LAB, { recursive: true }); + let state = updateState((next) => { + next.host = host; + next.busySeconds = Number(option(args, "busy-seconds") ?? next.busySeconds); + next.fromVersion = option(args, "from-version") ?? next.fromVersion; + for (let index = next.envs.length; index < envCount; index++) { + const row = INSTALL_MATRIX[index % INSTALL_MATRIX.length]!; + next.envs.push({ + id: `env${index + 1}`, + port: 0, + pid: null, + launcher: index >= envCount - launcherCount, + projectId: null, + installs: Object.fromEntries( + PROVIDER_KEYS.map((key) => [ + key, + { method: row[key], version: PROVIDERS[key].installed, failing: false }, + ]), + ) as LabEnv["installs"], + }); + } + }, initial); const taken = new Set(state.envs.map((env) => env.port).concat(state.feedPort)); if (!isLabProcess(state.feedPid, "__feed")) { - state.feedPort = await freePort("127.0.0.1", taken); - state.feedPid = spawnDetached(NODE, [SELF, "__feed", String(state.feedPort)], { + const feedPort = + state.feedPort > 0 && !listeningPorts().has(state.feedPort) + ? state.feedPort + : await freePort("127.0.0.1", taken); + // Running servers retain the old feed origin in their environment until restarted. + if (feedPort !== state.feedPort) { + for (const env of state.envs) { + if (env.pid) await stopGroup(env.pid, serverMarker(env), projectDir(env.id)); + state = updateState((next) => { + requireEnv(next, env.id).pid = null; + }); + } + } + const feedPid = spawnDetached(NODE, [SELF, "__feed", String(feedPort)], { env: { PATH: process.env.PATH }, cwd: REPO, log: NodePath.join(LAB, "feed.log"), }); + state = updateState((next) => { + next.feedPort = feedPort; + next.feedPid = feedPid; + }); } // The feed always offers this checkout's version, the update a launcher env is behind on. if (!publishedRuntimes().includes(REPO_VERSION)) buildRuntime(REPO_VERSION, undefined); - for (let index = state.envs.length; index < envCount; index++) { - const row = INSTALL_MATRIX[index % INSTALL_MATRIX.length]!; - state.envs.push({ - id: `env${index + 1}`, - port: 0, - pid: null, - launcher: index >= envCount - launcherCount, - projectId: null, - installs: Object.fromEntries( - PROVIDER_KEYS.map((key) => [ - key, - { method: row[key], version: PROVIDERS[key].installed, failing: false }, - ]), - ) as LabEnv["installs"], - }); - } - writeState(state); - const started: LabEnv[] = []; for (const env of state.envs) { - if (isLabProcess(env.pid, serverMarker(env))) continue; + if (isLabProcess(env.pid, serverMarker(env), projectDir(env.id))) continue; seedEnvFiles(state, env); - if (!env.port || listeningPorts().has(env.port)) env.port = await freePort(host, taken); - env.pid = startServer(state, env); - started.push(env); - writeState(state); + const port = + !env.port || listeningPorts().has(env.port) ? await freePort(host, taken) : env.port; + const pid = startServer(state, { ...env, port }); + state = updateState((next) => { + const current = requireEnv(next, env.id); + current.port = port; + current.pid = pid; + }); + started.push(requireEnv(state, env.id)); } console.log( `feed http://127.0.0.1:${state.feedPort} (npm registry /npm, releases /releases)`, ); const urls = await Promise.all(started.map(waitForPairingUrl)); for (const env of state.envs) { - const index = started.indexOf(env); + const index = started.findIndex((candidate) => candidate.id === env.id); console.log(`${env.id.padEnd(9)} ${originOf(state, env)} pid ${env.pid}${launcherLabel(env)}`); console.log(` ${index >= 0 ? urls[index] : "(already running; `pair` mints a URL)"}`); } @@ -1021,12 +1068,15 @@ async function down(): Promise { const state = readState(); if (!state) return; for (const env of state.envs) { - if (env.pid) await stopGroup(env.pid, serverMarker(env)); - env.pid = null; + if (env.pid) await stopGroup(env.pid, serverMarker(env), projectDir(env.id)); + updateState((next) => { + requireEnv(next, env.id).pid = null; + }); } if (state.feedPid) await stopGroup(state.feedPid, "__feed"); - state.feedPid = null; - writeState(state); + updateState((next) => { + next.feedPid = null; + }); console.log("Lab stopped."); } @@ -1037,7 +1087,7 @@ function status(): void { `feed http://127.0.0.1:${state.feedPort} ${feed}; delay ${state.delayMs}ms; runtimes ${publishedRuntimes().join(", ") || "-"}`, ); for (const env of state.envs) { - const alive = isLabProcess(env.pid, serverMarker(env)); + const alive = isLabProcess(env.pid, serverMarker(env), projectDir(env.id)); console.log( `${env.id} ${originOf(state, env)} pid ${env.pid ?? "-"} ${alive ? "up" : "down"}${launcherLabel(env)}`, ); @@ -1182,11 +1232,16 @@ async function main(argv: string[]): Promise { return void console.log(`Published ${buildRuntime(args[0], option(args, "archive"))}`); } case "reset": { - const state = readState(); - if (state && [state.feedPid, ...state.envs.map((e) => e.pid)].some((pid) => pid !== null)) { - fail("Run `down` first."); + const release = acquireLock("state"); + try { + const state = readState(); + if (state && [state.feedPid, ...state.envs.map((e) => e.pid)].some((pid) => pid !== null)) { + fail("Run `down` first."); + } + NodeFS.rmSync(LAB, { recursive: true, force: true }); + } finally { + release(); } - NodeFS.rmSync(LAB, { recursive: true, force: true }); return void console.log(`Removed ${LAB}.`); } default: @@ -1195,4 +1250,10 @@ async function main(argv: string[]): Promise { } } -await main(process.argv.slice(2)); +const argv = process.argv.slice(2); +const release = ["up", "down", "reset"].includes(argv[0] ?? "") ? acquireLock("processes") : null; +try { + await main(argv); +} finally { + release?.(); +} From bd670c71c75cc158fe17f8002a2dc92ec0cd2535 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:19:25 +0000 Subject: [PATCH 011/149] fix(server): wait for background installs before provider startup --- .../ProviderTurnStartService.ts | 8 +++++++- .../providerMaintenanceCommandCoordinator.ts | 19 +++++++++++++++++++ apps/server/src/updates/UpdateWindow.ts | 10 ++++++---- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index a928680b24024..1d54dffaea777 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -25,6 +25,7 @@ import * as Schema from "effect/Schema"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ProviderAuthService from "../provider/ProviderAuthService.ts"; +import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; import * as EventSink from "./EventSink.ts"; import * as ContextHandoffService from "./ContextHandoffService.ts"; import { @@ -116,6 +117,7 @@ export const layer: Layer.Layer< const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const runExecution = yield* RunExecutionService.RunExecutionServiceV2; const runtimePolicy = yield* RuntimePolicy.RuntimePolicyV2; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; // These callbacks outlive startup while a run drains background work. Build // them outside start's scope so they cannot retain its full thread history. @@ -226,6 +228,10 @@ export const layer: Layer.Layer< readonly willRetry?: boolean; }) { const { runId } = input; + // The run is durably starting before this effect is dispatched. Wait for + // an admitted install, then reread: active-work blocks later installs, + // without holding the permit through provider commands such as /compact. + yield* admission.withPermit(Effect.void); const projection = yield* projectionStore.getTurnStartContext(input.threadId, runId); const run = projection.runs.find((candidate) => candidate.id === runId); if (run === undefined) { @@ -1291,4 +1297,4 @@ export const layer: Layer.Layer< ), }); }), -); +).pipe(Layer.provide(ProviderMaintenanceCoordinator.admissionLayer)); diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index f2fc03659be0c..f544028735512 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -1,6 +1,25 @@ import * as KeyedLock from "@t3tools/shared/KeyedLock"; +import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as Semaphore from "effect/Semaphore"; + +/** Shares background install admission with provider startup, before either touches a CLI. */ +export class ProviderMaintenanceAdmission extends Context.Service< + ProviderMaintenanceAdmission, + { + readonly withPermit: (effect: Effect.Effect) => Effect.Effect; + } +>()("t3/provider/ProviderMaintenanceAdmission") {} + +// Keep one layer reference so the update window and turn startup share its permit. +export const admissionLayer = Layer.effect( + ProviderMaintenanceAdmission, + Effect.map(Semaphore.make(1), (permit) => + ProviderMaintenanceAdmission.of({ withPermit: permit.withPermits(1) }), + ), +); export interface ProviderMaintenanceCommandCoordinatorShape { readonly withCommandLock: (input: { diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 023794a69e900..680cbf4ebfa36 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -6,10 +6,10 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Semaphore from "effect/Semaphore"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -95,6 +95,7 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const providers = yield* ProviderRegistry.ProviderRegistry; const timings = yield* updateTimings; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const check: UpdateWindow["Service"]["check"] = Effect.fn("updates.UpdateWindow.check")( function* (options) { @@ -213,9 +214,8 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { }, ); - const installLock = yield* Semaphore.make(1); const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (options, install) => - installLock.withPermits(1)( + admission.withPermit( check(options).pipe( Effect.flatMap((status) => (status.open ? Effect.asSome(install) : Effect.succeedNone)), ), @@ -232,4 +232,6 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { return UpdateWindow.of({ check, runIfOpen, runWhenOpen }); }); -export const layer = Layer.effect(UpdateWindow, make()); +export const layer = Layer.effect(UpdateWindow, make()).pipe( + Layer.provide(ProviderMaintenanceCoordinator.admissionLayer), +); From dedb32e66993fb8c7c2db20a7d7841db30936570 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:19:26 +0000 Subject: [PATCH 012/149] fix(server): hold startup admission through launcher handoff --- apps/server/src/updates/ServerAutoUpdater.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 6b23b5555348c..43c960dec60c9 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -102,6 +102,9 @@ export const layer = Layer.effectDiscard( skippedTargets.add(targetVersion); yield* Effect.logInfo("Installing a background server update", { targetVersion }); yield* selfUpdate.update({ targetVersion }); + // The launcher stops us after accepting the handoff. Keep new provider + // starts behind the install permit until this runtime is shut down. + yield* Effect.never; }), ); }).pipe( From 82a96439c3d5c7d08ef05856462060f3d7e3b676 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:26:46 +0000 Subject: [PATCH 013/149] fix(server): serialize manual installs with background updates --- .../providerMaintenanceCommandCoordinator.ts | 4 +- .../src/provider/providerMaintenanceRunner.ts | 24 +++++++-- apps/server/src/serverSettings.test.ts | 51 +++++++++++++------ apps/server/src/serverSettings.ts | 8 ++- .../server/src/updates/ProviderAutoUpdater.ts | 23 +++++++-- apps/server/src/updates/UpdateWindow.ts | 42 +++++++-------- 6 files changed, 106 insertions(+), 46 deletions(-) diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index f544028735512..3dec5a7981291 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -26,6 +26,7 @@ export interface ProviderMaintenanceCommandCoordinatorShape { readonly targetKey: string; readonly lockKey: string; readonly onQueued?: Effect.Effect; + readonly admit?: (effect: Effect.Effect) => Effect.Effect; readonly run: Effect.Effect; }) => Effect.Effect; } @@ -58,6 +59,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( targetKey, lockKey, onQueued, + admit, run, }) => Effect.gen(function* () { @@ -67,7 +69,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( } return yield* (onQueued ?? Effect.void).pipe( - Effect.andThen(locks.withLock(lockKey, run)), + Effect.andThen(admit ? admit(locks.withLock(lockKey, run)) : locks.withLock(lockKey, run)), Effect.ensuring(releaseTarget(targetKey)), ); }); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 5ef57660e01fe..109a554a6cb1e 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -27,7 +27,11 @@ import * as NodeUtil from "node:util"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; -import { makeProviderMaintenanceCommandCoordinator } from "./providerMaintenanceCommandCoordinator.ts"; +import { + admissionLayer, + makeProviderMaintenanceCommandCoordinator, + ProviderMaintenanceAdmission, +} from "./providerMaintenanceCommandCoordinator.ts"; import { enrichProviderSnapshotWithVersionAdvisory, makeTargetedProviderUpdateAction, @@ -67,6 +71,8 @@ export interface ProviderMaintenanceRunnerShape { readonly targetVersion?: string | undefined; }, ) => Effect.Effect; + /** Only for callers already holding the background update admission permit. */ + readonly updateProviderWhileAdmitted: ProviderMaintenanceRunnerShape["updateProvider"]; } export class ProviderMaintenanceRunner extends Context.Service< @@ -292,6 +298,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const httpClient = yield* HttpClient.HttpClient; const versionCache = yield* ProviderVersionCache; + const admission = yield* ProviderMaintenanceAdmission; const runMaintenanceCommand = ( update: ProviderMaintenanceCommandAction, onProgress: (line: string) => Effect.Effect, @@ -381,9 +388,10 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { }), ); - const updateProvider: ProviderMaintenanceRunnerShape["updateProvider"] = Effect.fn( - "ProviderMaintenanceRunner.updateProvider", - )(function* (target) { + const updateProvider = Effect.fn("ProviderMaintenanceRunner.updateProvider")(function* ( + target: Parameters[0], + alreadyAdmitted = false, + ) { const provider = typeof target === "string" ? target : target.provider; const instanceId = typeof target === "string" @@ -578,6 +586,9 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { targetKey, lockKey: update.lockKey, onQueued: setQueuedState, + // Admit before the installer lock: a manual update must not hold that + // lock while a background pass holding admission waits for it. + ...(alreadyAdmitted ? {} : { admit: admission.withPermit }), run: runProviderUpdate(), }) .pipe( @@ -594,7 +605,10 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { return ProviderMaintenanceRunner.of({ updateProvider, + updateProviderWhileAdmitted: (target) => updateProvider(target, true), }); }); -export const layer = Layer.effect(ProviderMaintenanceRunner, make()); +export const layer = Layer.effect(ProviderMaintenanceRunner, make()).pipe( + Layer.provide(admissionLayer), +); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 4444a7851dffc..469adc22c87fd 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1030,23 +1030,44 @@ it.layer(NodeServices.layer)("server settings", (it) => { }).pipe(Effect.provide(layerServerSettings())), ); - it.effect("migrates legacy providers from an invalid file without rewriting it", () => - Effect.gen(function* () { - const serverConfig = yield* ServerConfig.ServerConfig; - const fileSystem = yield* FileSystem.FileSystem; - const serverSettings = yield* ServerSettingsModule.ServerSettingsService; - const raw = '{"addProjectBaseDirectory":42,"providers":{"codex":{"binaryPath":"/x"}}}'; - yield* fileSystem.writeFileString(serverConfig.settingsPath, raw); + it.effect("disables automatic recovery and updates without rewriting an invalid file", () => + Effect.forEach( + [ + { + raw: '{"automaticUpdates":false,"continueThreadsAfterServerUpdate":false,"addProjectBaseDirectory":42,"providers":{"codex":{"binaryPath":"/x"}}}', + migratedProvider: { + driver: ProviderDriverKind.make("codex"), + config: { binaryPath: "/x" }, + }, + }, + { + raw: '{"automaticUpdates":true,"continueThreadsAfterServerUpdate":true,"providers":42}', + migratedProvider: undefined, + }, + { + raw: '{"automaticUpdates":false,"continueThreadsAfterServerUpdate":false', + migratedProvider: undefined, + }, + ], + ({ raw, migratedProvider }) => + Effect.gen(function* () { + const serverConfig = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const serverSettings = yield* ServerSettingsModule.ServerSettingsService; + yield* fileSystem.writeFileString(serverConfig.settingsPath, raw); - const settings = yield* serverSettings.getSettings; + const settings = yield* serverSettings.getSettings; - // The readable provider settings still apply, but the file stays for the user to repair. - assert.deepEqual(settings.providerInstances[ProviderInstanceId.make("codex")], { - driver: ProviderDriverKind.make("codex"), - config: { binaryPath: "/x" }, - }); - assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); - }).pipe(Effect.provide(layerServerSettings())), + // Readable legacy provider settings still apply while the file stays for repair. + assert.deepEqual( + settings.providerInstances[ProviderInstanceId.make("codex")], + migratedProvider, + ); + assert.isFalse(settings.automaticUpdates); + assert.isFalse(settings.continueThreadsAfterServerUpdate); + assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); + }).pipe(Effect.provide(layerServerSettings())), + ), ); it.effect("skips a disabled provider instance when picking the text generation fallback", () => diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 7536a74d6e02c..d05ddeafbc739 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -786,8 +786,14 @@ const make = Effect.gen(function* () { if (decoded._tag === "Failure" || legacySettings._tag === "Failure") { const failure = decoded._tag === "Failure" ? decoded : legacySettings; settingsFileTrusted = false; + // Unreadable persisted choices cannot authorize automatic updates or recovery. + settings = { + ...DEFAULT_SERVER_SETTINGS, + automaticUpdates: false, + continueThreadsAfterServerUpdate: false, + }; if (failure._tag === "Failure") { - yield* Effect.logWarning("failed to parse settings.json, using defaults", { + yield* Effect.logWarning("failed to parse settings.json, using safe defaults", { path: settingsPath, issues: Cause.pretty(failure.cause), cause: failure.cause, diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index 2c0c38a54b670..c6350733b69cf 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -58,16 +58,31 @@ export const layer = Layer.effectDiscard( return attemptedAt === undefined || nowMs - attemptedAt >= Duration.toMillis(RETRY_AFTER); }); for (const provider of candidates) { - const targetVersion = provider.versionAdvisory?.latestVersion; const install = Effect.gen(function* () { - attempts.set(`${provider.instanceId}@${targetVersion}`, yield* Clock.currentTimeMillis); + const currentSettings = yield* serverSettings.getSettings; + if (!currentSettings.automaticUpdates || !currentSettings.enableProviderUpdateChecks) + return; + const current = (yield* providers.getProviders).find( + (candidate) => candidate.instanceId === provider.instanceId, + ); + if (!current || !isAutoUpdatable(current)) return; + const targetVersion = current.versionAdvisory?.latestVersion; + const attemptKey = `${current.instanceId}@${targetVersion}`; + const nowMs = yield* Clock.currentTimeMillis; + const attemptedAt = attempts.get(attemptKey); + if (attemptedAt !== undefined && nowMs - attemptedAt < Duration.toMillis(RETRY_AFTER)) + return; + attempts.set(attemptKey, nowMs); yield* Effect.logInfo("Updating provider in the background", { instanceId: provider.instanceId, - fromVersion: provider.version, + fromVersion: current.version, targetVersion, }); yield* runner - .updateProvider({ provider: provider.driver, instanceId: provider.instanceId }) + .updateProviderWhileAdmitted({ + provider: current.driver, + instanceId: current.instanceId, + }) .pipe( Effect.catchCause((cause) => Effect.logWarning("Background provider update failed", { diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 680cbf4ebfa36..430baea17c4f1 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -187,26 +187,28 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { if (Option.isNone(settings) || !settings.value.automaticUpdates) { return { open: false, blockers }; } - if (settings.value.autoResumeLimitedThreads) { - const limited = yield* projections - .getLimitRecoveryCandidates({ now: horizon, autoResume: true, snooze: false }) - .pipe( - Effect.tapError((cause) => - Effect.logWarning("Update window could not read usage-limit resumes", { cause }), - ), - Effect.option, - ); - if (Option.isNone(limited)) return { open: false, blockers }; - for (const thread of limited.value) { - if (thread.limitRecovery?.autoResume !== true) continue; - const resumesAt = DateTime.make(thread.limitRecovery.resetAt); - if (Option.isSome(resumesAt)) { - blockers.push({ - type: "usage-limit-resume", - threadId: thread.id, - resumesAt: resumesAt.value, - }); - } + const limited = yield* projections + .getLimitRecoveryCandidates({ + now: horizon, + autoResume: settings.value.autoResumeLimitedThreads, + snooze: false, + }) + .pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read usage-limit resumes", { cause }), + ), + Effect.option, + ); + if (Option.isNone(limited)) return { open: false, blockers }; + for (const thread of limited.value) { + if (thread.usageLimitResetAt === null || thread.usageLimitResetAt === undefined) continue; + const resumesAt = DateTime.make(thread.usageLimitResetAt); + if (Option.isSome(resumesAt) && !DateTime.isGreaterThan(resumesAt.value, horizon)) { + blockers.push({ + type: "usage-limit-resume", + threadId: thread.id, + resumesAt: resumesAt.value, + }); } } From 4ce604c83f881cb2e6e52d90fb806e1b8800daef Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:31:29 +0000 Subject: [PATCH 014/149] fix(server): protect terminal commands during automatic updates --- apps/server/src/terminal/Manager.test.ts | 156 +++++++++++++++++++ apps/server/src/terminal/Manager.ts | 128 ++++++++++++--- apps/server/src/updates/ServerAutoUpdater.ts | 2 +- apps/server/src/updates/UpdateWindow.ts | 20 ++- docs/user/updating.md | 2 + 5 files changed, 277 insertions(+), 31 deletions(-) diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index 5a40f7bbb9334..75d6e8a949e2f 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -1297,7 +1297,15 @@ it.layer( yield* manager.open(openInput({ terminalId: "subshell" })); yield* manager.open(openInput({ threadId: "thread-2" })); + assert.equal(yield* manager.hasBusyTerminals, true); + expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([ + false, + false, + false, + false, + ]); yield* manager.closeIdle({ threadId: "thread-1" }); + assert.equal(yield* manager.hasBusyTerminals, true); expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([ true, @@ -1342,6 +1350,154 @@ it.layer( }), ); + it.effect("checks fresh work across threads without changing idle shells or metadata", () => + Effect.gen(function* () { + let snapshotCalls = 0; + const shells = [ + { pid: 9000, ppid: 1, name: "zsh" }, + { pid: 100, ppid: 9000, name: "zsh" }, + { pid: 9001, ppid: 1, name: "zsh" }, + ]; + let processes = shells; + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + subprocessPollIntervalMs: 60_000, + processTable: Effect.sync(() => { + snapshotCalls += 1; + return processes; + }), + }).pipe(Effect.provide(layerWithHostPlatform("linux"))); + + assert.equal(yield* manager.hasBusyTerminals, false); + assert.equal(snapshotCalls, 0); + yield* manager.open(openInput()); + yield* manager.open(openInput({ threadId: "thread-2" })); + const events = yield* getEvents; + assert.equal(yield* manager.hasBusyTerminals, false); + assert.equal(snapshotCalls, 1); + + processes = [...shells, { pid: 200, ppid: 9001, name: "node" }]; + assert.equal(yield* manager.hasBusyTerminals, true); + processes = [ + ...shells, + { pid: 200, ppid: 9001, name: "zsh" }, + { pid: 201, ppid: 200, name: "sleep" }, + ]; + assert.equal(yield* manager.hasBusyTerminals, true); + processes = shells; + assert.equal(yield* manager.hasBusyTerminals, false); + expect(yield* getEvents).toEqual(events); + expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([false, false]); + }), + ); + + it.effect("treats failed, truncated, timed out, and missing-shell inspections as busy", () => + Effect.gen(function* () { + let stdout = "9000 1 zsh"; + let code = 0; + let timedOut = false; + let stdoutTruncated = false; + const processRunner: ProcessRunner.ProcessRunner["Service"] = { + run: () => + Effect.sync(() => ({ + stdout, + stderr: "", + code: ChildProcessSpawner.ExitCode(code), + timedOut, + stdoutTruncated, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + })), + }; + const { manager, ptyAdapter } = yield* createManager(5, { + subprocessPollIntervalMs: 60_000, + }).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, processRunner), + Effect.provide(layerWithHostPlatform("linux")), + ); + yield* manager.open(openInput()); + assert.equal(yield* manager.hasBusyTerminals, false); + code = 1; + assert.equal(yield* manager.hasBusyTerminals, true); + code = 0; + stdoutTruncated = true; + assert.equal(yield* manager.hasBusyTerminals, true); + stdoutTruncated = false; + timedOut = true; + assert.equal(yield* manager.hasBusyTerminals, true); + timedOut = false; + stdout = ""; + assert.equal(yield* manager.hasBusyTerminals, true); + stdout = "9000 1 zsh"; + assert.equal(yield* manager.hasBusyTerminals, false); + expect(ptyAdapter.processes[0]!.killed).toBe(false); + }), + ); + + it.effect("treats input, output, and new terminals during inspection as busy", () => + Effect.gen(function* () { + let duringCheck: Effect.Effect = Effect.void; + const { manager, ptyAdapter } = yield* createManager(5, { + subprocessPollIntervalMs: 60_000, + subprocessInspector: () => + duringCheck.pipe( + Effect.as({ hasRunningSubprocess: false, childCommand: null, processIds: [] }), + ), + }); + yield* manager.open(openInput()); + assert.equal(yield* manager.hasBusyTerminals, false); + duringCheck = manager + .write({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, data: "make build\r" }) + .pipe(Effect.orDie); + assert.equal(yield* manager.hasBusyTerminals, true); + duringCheck = Effect.sync(() => ptyAdapter.processes[0]!.emitData("make build\r\n")); + assert.equal(yield* manager.hasBusyTerminals, true); + duringCheck = manager + .open(openInput({ threadId: "thread-2" })) + .pipe(Effect.asVoid, Effect.orDie); + assert.equal(yield* manager.hasBusyTerminals, true); + expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([false, false]); + }), + ); + + it.effect("holds terminal launches and writes behind a server restart permit", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + subprocessPollIntervalMs: 60_000, + subprocessInspector: () => + Effect.succeed({ hasRunningSubprocess: false, childCommand: null, processIds: [] }), + }); + yield* manager.open(openInput()); + const held = yield* Deferred.make(); + const release = yield* Deferred.make(); + const restartWindow = yield* manager + .withRestartPermit( + Deferred.succeed(held, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(held); + assert.equal(yield* manager.hasBusyTerminals, false); + + const write = yield* manager + .write({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, data: "make build\r" }) + .pipe(Effect.forkScoped); + const open = yield* manager.open(openInput({ threadId: "thread-2" })).pipe(Effect.forkScoped); + const restart = yield* manager.restart(restartInput()).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(ptyAdapter.spawnInputs).toHaveLength(1); + expect(ptyAdapter.processes[0]!.writes).toEqual([]); + expect(ptyAdapter.processes[0]!.killed).toBe(false); + + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(restartWindow); + yield* Fiber.join(write); + yield* Fiber.join(open); + yield* Fiber.join(restart); + expect(ptyAdapter.spawnInputs).toHaveLength(3); + expect(ptyAdapter.processes.flatMap((process) => process.writes)).toEqual(["make build\r"]); + }), + ); + it.effect("backs off the spawned fallback when the resource monitor snapshot fails", () => Effect.gen(function* () { const fallbackCalls: Array = []; diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 562307477dcf7..ee294307a243f 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -60,6 +60,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as SynchronizedRef from "effect/SynchronizedRef"; import * as ServerConfig from "../config.ts"; @@ -219,6 +220,15 @@ export class TerminalManager extends Context.Service< readonly terminalId?: string; }) => Effect.Effect; + /** Inspect current terminal work without changing sessions. Unknown activity is busy. */ + readonly hasBusyTerminals: Effect.Effect; + + /** + * Hold terminal launches and input while checking idle state and handing off a server restart. + * Read-only activity checks stay available; do not call open, restart, or write inside this permit. + */ + readonly withRestartPermit: (effect: Effect.Effect) => Effect.Effect; + /** * Subscribe to terminal runtime events with a direct callback. * @@ -243,6 +253,8 @@ interface TerminalSubprocessInspectResult { readonly hasRunningSubprocess: boolean; readonly childCommand: string | null; readonly processIds: ReadonlyArray; + /** False when an otherwise successful process snapshot omitted the shell. */ + readonly terminalExists?: boolean; } interface TerminalSubprocessInspector { @@ -727,7 +739,12 @@ function deriveSubprocessInspectResult( (snapshot.childrenByParent.get(pid)?.length ?? 0) > 0, ); if (childPid === undefined) { - return { hasRunningSubprocess: false, childCommand: null, processIds: [] }; + return { + hasRunningSubprocess: false, + childCommand: null, + processIds: [], + terminalExists: snapshot.commandById.has(terminalPid), + }; } const processIds = new Set([terminalPid]); const pending = [terminalPid]; @@ -745,6 +762,7 @@ function deriveSubprocessInspectResult( hasRunningSubprocess: true, childCommand: normalized ? truncateTerminalWireLabel(normalized) : null, processIds: [...processIds], + terminalExists: snapshot.commandById.has(terminalPid), }; } @@ -1577,6 +1595,8 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func killFibers: new Map(), }); const threadLocks = yield* KeyedLock.make(); + const terminalAdmission = yield* Semaphore.make(1); + const withRestartPermit = terminalAdmission.withPermits(1); const terminalEventListeners = new Set<(event: TerminalEvent) => Effect.Effect>(); const workerScope = yield* Scope.make("sequential"); yield* Effect.addFinalizer(() => Scope.close(workerScope, Exit.void)); @@ -2701,9 +2721,11 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }); const openLocked = (input: TerminalOpenInput) => - withWorkspaceLease( - path.resolve(input.worktreePath ?? input.cwd), - openWithWorkspaceLease(input), + withRestartPermit( + withWorkspaceLease( + path.resolve(input.worktreePath ?? input.cwd), + openWithWorkspaceLease(input), + ), ); const open: TerminalManager["Service"]["open"] = (input) => @@ -2950,29 +2972,34 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); }; - const write: TerminalManager["Service"]["write"] = Effect.fn("terminal.write")(function* (input) { - const terminalId = input.terminalId; - const session = yield* requireSession(input.threadId, terminalId); - const process = session.process; - if (!process || session.status !== "running") { - if (session.status === "exited") return; - return yield* new TerminalNotRunningError({ - threadId: input.threadId, - terminalId, - }); - } - session.inputCount += 1; - yield* Effect.try({ - try: () => process.write(input.data), - catch: (cause) => - new TerminalWriteError({ + const writeUnlocked: TerminalManager["Service"]["write"] = Effect.fn("terminal.write")( + function* (input) { + const terminalId = input.terminalId; + const session = yield* requireSession(input.threadId, terminalId); + const process = session.process; + if (!process || session.status !== "running") { + if (session.status === "exited") return; + return yield* new TerminalNotRunningError({ threadId: input.threadId, terminalId, - terminalPid: process.pid, - cause, - }), - }); - }); + }); + } + session.inputCount += 1; + yield* Effect.try({ + try: () => process.write(input.data), + catch: (cause) => + new TerminalWriteError({ + threadId: input.threadId, + terminalId, + terminalPid: process.pid, + cause, + }), + }); + }, + ); + + const write: TerminalManager["Service"]["write"] = (input) => + withRestartPermit(writeUnlocked(input)); const resizeLocked = Effect.fn("terminal.resize")(function* (input: TerminalResizeInput) { const session = yield* getSession(input.threadId, input.terminalId); @@ -3103,6 +3130,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func restartResolved(resolved), ), ), + withRestartPermit, ), ); @@ -3174,6 +3202,54 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ), ); + const hasBusyTerminals: TerminalManager["Service"]["hasBusyTerminals"] = Effect.gen(function* () { + const state = yield* readManagerState; + const running = new Map( + [...state.sessions] + .filter(([, session]) => session.status === "starting" || session.status === "running") + .map( + ([key, session]) => + [ + key, + { + status: session.status, + pid: session.pid, + process: session.process, + activityMark: session.eventSequence + session.inputCount, + }, + ] as const, + ), + ); + if (running.size === 0) return false; + if ([...running.values()].some((session) => session.status === "starting")) return true; + const { inspector } = yield* acquireSubprocessInspector; + for (const session of running.values()) { + if (session.pid === null || !Number.isInteger(session.pid) || session.pid <= 0) return true; + const result = yield* inspector(session.pid); + if (result.hasRunningSubprocess || result.terminalExists === false) return true; + } + // Input, output, or a new process can arrive after the shared snapshot. + const latest = yield* readManagerState; + return [...latest.sessions].some(([key, session]) => { + if (session.status === "starting") return true; + if (session.status !== "running") return false; + const inspected = running.get(key); + return ( + inspected === undefined || + inspected.pid !== session.pid || + inspected.process !== session.process || + inspected.activityMark !== session.eventSequence + session.inputCount || + session.pendingProcessEvents.length > session.pendingProcessEventIndex + ); + }); + }).pipe( + Effect.catch((error) => + Effect.logWarning("failed to inspect busy terminals", { error: error.message }).pipe( + Effect.as(true), + ), + ), + ); + return TerminalManager.of({ open, attachStream, @@ -3184,6 +3260,8 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func restart, close, closeIdle, + hasBusyTerminals, + withRestartPermit, subscribe, subscribeMetadata, }); diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 43c960dec60c9..946a489d4e5da 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -94,7 +94,7 @@ export const layer = Layer.effectDiscard( const targetVersion = pendingTarget; if (targetVersion === undefined) return; yield* updateWindow.runIfOpen( - { closesWindows: false }, + { closesWindows: false, restartsServer: true }, Effect.gen(function* () { // The setting may have been turned off while waiting. if (!(yield* automaticUpdatesEnabled)) return; diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 9fd06ed2b37d4..51ae4778a800c 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -13,6 +13,7 @@ import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenance import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as ServerSettings from "../serverSettings.ts"; +import * as TerminalManager from "../terminal/Manager.ts"; /** Work due this soon keeps the window closed, so it does not start late. */ const UPDATE_LOOKAHEAD = Duration.minutes(5); @@ -43,6 +44,7 @@ export type UpdateWindowBlocker = | { readonly type: "recent-interaction"; readonly quietAt: DateTime.Utc } | { readonly type: "focused-window" } | { readonly type: "provider-update" } + | { readonly type: "terminal-work" } | { readonly type: "scheduled-task"; readonly taskId: ScheduledTaskId; @@ -59,6 +61,8 @@ export type UpdateWindowBlocker = export interface UpdateWindowOptions { /** The update closes the app windows (a desktop install), not just a reconnect. */ readonly closesWindows: boolean; + /** Server shutdown also terminates integrated terminals. */ + readonly restartsServer?: boolean; } export interface UpdateWindowStatus { @@ -98,6 +102,7 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const providers = yield* ProviderRegistry.ProviderRegistry; const timings = yield* updateTimings; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; + const terminals = yield* TerminalManager.TerminalManager; const check: UpdateWindow["Service"]["check"] = Effect.fn("updates.UpdateWindow.check")( function* (options) { @@ -127,6 +132,9 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { if (providerUpdating) { blockers.push({ type: "provider-update" }); } + if (options.restartsServer && (yield* terminals.hasBusyTerminals)) { + blockers.push({ type: "terminal-work" }); + } const lastInteraction = yield* backgroundPolicy.lastClientInteractionAt; if (Option.isSome(lastInteraction)) { @@ -218,12 +226,14 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { }, ); - const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (options, install) => - admission.withPermit( - check(options).pipe( - Effect.flatMap((status) => (status.open ? Effect.asSome(install) : Effect.succeedNone)), - ), + const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (options, install) => { + const attempt = check(options).pipe( + Effect.flatMap((status) => (status.open ? Effect.asSome(install) : Effect.succeedNone)), + ); + return admission.withPermit( + options.restartsServer ? terminals.withRestartPermit(attempt) : attempt, ); + }; const runWhenOpen: UpdateWindow["Service"]["runWhenOpen"] = (options, install) => Effect.gen(function* () { while (true) { diff --git a/docs/user/updating.md b/docs/user/updating.md index c1c566d3b758f..cb0de606f90c2 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -83,6 +83,8 @@ Then it installs: - the newest T3 Code release on its channel, when it runs as a background service (`t3 service install`). It downloads first and restarts only inside that window. +Background services also wait for integrated terminal commands to finish. + Desktop updates download in the background and install when you quit the app. The app stays open while you work, including when it hosts multiple backends. From bc2441fed3bd1372f20f5ac6b8b94e3a35dae5d3 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:34:21 +0000 Subject: [PATCH 015/149] fix(server): clear canceled provider install queues --- .../providerMaintenanceCommandCoordinator.ts | 3 +++ .../src/provider/providerMaintenanceRunner.test.ts | 3 +++ .../src/provider/providerMaintenanceRunner.ts | 14 ++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index 3dec5a7981291..9d5c953a85c88 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -26,6 +26,7 @@ export interface ProviderMaintenanceCommandCoordinatorShape { readonly targetKey: string; readonly lockKey: string; readonly onQueued?: Effect.Effect; + readonly onInterrupted?: Effect.Effect; readonly admit?: (effect: Effect.Effect) => Effect.Effect; readonly run: Effect.Effect; }) => Effect.Effect; @@ -59,6 +60,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( targetKey, lockKey, onQueued, + onInterrupted, admit, run, }) => @@ -70,6 +72,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( return yield* (onQueued ?? Effect.void).pipe( Effect.andThen(admit ? admit(locks.withLock(lockKey, run)) : locks.withLock(lockKey, run)), + Effect.onInterrupt(() => onInterrupted ?? Effect.void), Effect.ensuring(releaseTarget(targetKey)), ); }); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index a3c494e504b03..b1e936ece257c 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -913,6 +913,9 @@ describe("providerMaintenanceRunner", () => { yield* Fiber.interrupt(first); releaseQueuedStateLatch.resolve(); + const canceled = (yield* registry.getProviders)[0]?.updateState; + assert.strictEqual(canceled?.status, "failed"); + assert.strictEqual(canceled?.message, "Update canceled before installation started."); const second = yield* updater.updateProvider(CODEX_DRIVER).pipe(Effect.exit); assert.strictEqual(Exit.isSuccess(second), true); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 109a554a6cb1e..90a8144fe5dff 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -586,6 +586,20 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { targetKey, lockKey: update.lockKey, onQueued: setQueuedState, + onInterrupted: Effect.gen(function* () { + const current = (yield* providerRegistry.getProviders).find( + (candidate) => candidate.instanceId === instanceId, + ); + if (current?.updateState?.status !== "queued") return; + yield* setUpdateState( + makeUpdateState({ + status: "failed", + startedAt: null, + finishedAt: yield* nowIso, + message: "Update canceled before installation started.", + }), + ); + }), // Admit before the installer lock: a manual update must not hold that // lock while a background pass holding admission waits for it. ...(alreadyAdmitted ? {} : { admit: admission.withPermit }), From 0b1a0a43fb9b2ce6a6dd37461fdc0ba8c7e4feb2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:35:51 +0000 Subject: [PATCH 016/149] fix(server): keep replaced terminal shells busy during updates --- apps/server/src/terminal/Manager.test.ts | 29 ++++++++++++++++++++++-- apps/server/src/terminal/Manager.ts | 27 ++++++++++++++++++---- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index 75d6e8a949e2f..e5483e9ad78e7 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -227,6 +227,7 @@ interface CreateManagerOptions { readonly hasRunningSubprocess: boolean; readonly childCommand: string | null; readonly processIds: ReadonlyArray; + readonly terminalCommand?: string | null; }>; processTable?: Effect.Effect< ReadonlyArray<{ readonly pid: number; readonly ppid: number; readonly name: string }>, @@ -1279,6 +1280,7 @@ it.layer( Effect.gen(function* () { // FakePtyAdapter assigns pids from 9000 in open order. const { manager, ptyAdapter } = yield* createManager(5, { + shellResolver: () => "/bin/zsh", processTable: Effect.succeed([ { pid: 9000, ppid: 1, name: "zsh" }, // An async prompt worker: a copy of the shell with no children. @@ -1360,6 +1362,7 @@ it.layer( ]; let processes = shells; const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "/bin/zsh", subprocessPollIntervalMs: 60_000, processTable: Effect.sync(() => { snapshotCalls += 1; @@ -1383,6 +1386,15 @@ it.layer( { pid: 201, ppid: 200, name: "sleep" }, ]; assert.equal(yield* manager.hasBusyTerminals, true); + // `exec` replaces the shell at its PID and leaves no child to inspect. + processes = shells.map((entry) => + entry.pid === 9001 ? { ...entry, name: "/usr/bin/sleep" } : entry, + ); + assert.equal(yield* manager.hasBusyTerminals, true); + processes = shells.map((entry) => (entry.pid === 9001 ? { ...entry, name: "node" } : entry)); + assert.equal(yield* manager.hasBusyTerminals, true); + processes = shells.map((entry) => (entry.pid === 9001 ? { ...entry, name: "" } : entry)); + assert.equal(yield* manager.hasBusyTerminals, true); processes = shells; assert.equal(yield* manager.hasBusyTerminals, false); expect(yield* getEvents).toEqual(events); @@ -1410,6 +1422,7 @@ it.layer( })), }; const { manager, ptyAdapter } = yield* createManager(5, { + shellResolver: () => "/bin/zsh", subprocessPollIntervalMs: 60_000, }).pipe( Effect.provideService(ProcessRunner.ProcessRunner, processRunner), @@ -1438,10 +1451,16 @@ it.layer( Effect.gen(function* () { let duringCheck: Effect.Effect = Effect.void; const { manager, ptyAdapter } = yield* createManager(5, { + shellResolver: () => "/bin/zsh", subprocessPollIntervalMs: 60_000, subprocessInspector: () => duringCheck.pipe( - Effect.as({ hasRunningSubprocess: false, childCommand: null, processIds: [] }), + Effect.as({ + hasRunningSubprocess: false, + childCommand: null, + processIds: [], + terminalCommand: "zsh", + }), ), }); yield* manager.open(openInput()); @@ -1463,9 +1482,15 @@ it.layer( it.effect("holds terminal launches and writes behind a server restart permit", () => Effect.gen(function* () { const { manager, ptyAdapter } = yield* createManager(5, { + shellResolver: () => "/bin/zsh", subprocessPollIntervalMs: 60_000, subprocessInspector: () => - Effect.succeed({ hasRunningSubprocess: false, childCommand: null, processIds: [] }), + Effect.succeed({ + hasRunningSubprocess: false, + childCommand: null, + processIds: [], + terminalCommand: "zsh", + }), }); yield* manager.open(openInput()); const held = yield* Deferred.make(); diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index ee294307a243f..98f2202792ae9 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -253,8 +253,8 @@ interface TerminalSubprocessInspectResult { readonly hasRunningSubprocess: boolean; readonly childCommand: string | null; readonly processIds: ReadonlyArray; - /** False when an otherwise successful process snapshot omitted the shell. */ - readonly terminalExists?: boolean; + /** The normalized command currently running at the terminal's root PID. */ + readonly terminalCommand?: string | null; } interface TerminalSubprocessInspector { @@ -318,6 +318,8 @@ interface TerminalSessionState { hasRunningSubprocess: boolean; /** Normalized child command name when `hasRunningSubprocess`; cleared when idle. */ childCommandLabel: string | null; + /** The shell originally spawned, so `exec` at its PID still counts as work. */ + launchedShellCommand?: string | null; runtimeEnv: Record | null; } @@ -730,6 +732,7 @@ function deriveSubprocessInspectResult( const commandName = (pid: number) => normalizeChildCommandName(snapshot.commandById.get(pid) ?? "", platform); const shellName = commandName(terminalPid); + const terminalCommand = platform === "win32" ? (shellName?.toLowerCase() ?? null) : shellName; // Async prompt themes fork the shell into a helper that waits with no // children of its own. That copy is not a command the user started. const childPid = (snapshot.childrenByParent.get(terminalPid) ?? []).find( @@ -743,7 +746,7 @@ function deriveSubprocessInspectResult( hasRunningSubprocess: false, childCommand: null, processIds: [], - terminalExists: snapshot.commandById.has(terminalPid), + terminalCommand, }; } const processIds = new Set([terminalPid]); @@ -762,7 +765,7 @@ function deriveSubprocessInspectResult( hasRunningSubprocess: true, childCommand: normalized ? truncateTerminalWireLabel(normalized) : null, processIds: [...processIds], - terminalExists: snapshot.commandById.has(terminalPid), + terminalCommand, }; } @@ -2211,6 +2214,12 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); if (attempt._tag === "Success") { + const shellCommand = normalizeChildCommandName( + basenameForPlatform(candidate.shell, platform), + platform, + ); + session.launchedShellCommand = + platform === "win32" ? (shellCommand?.toLowerCase() ?? null) : shellCommand; return { process: attempt.success, shellLabel: formatShellCandidate(candidate), @@ -3215,6 +3224,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func status: session.status, pid: session.pid, process: session.process, + launchedShellCommand: session.launchedShellCommand, activityMark: session.eventSequence + session.inputCount, }, ] as const, @@ -3226,7 +3236,14 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func for (const session of running.values()) { if (session.pid === null || !Number.isInteger(session.pid) || session.pid <= 0) return true; const result = yield* inspector(session.pid); - if (result.hasRunningSubprocess || result.terminalExists === false) return true; + if ( + result.hasRunningSubprocess || + !result.terminalCommand || + !session.launchedShellCommand || + result.terminalCommand !== session.launchedShellCommand + ) { + return true; + } } // Input, output, or a new process can arrive after the shared snapshot. const latest = yield* readManagerState; From 0df12e8170ac317b49232aa8c5dfaf2553c07f3e Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:39:54 +0000 Subject: [PATCH 017/149] fix(server): mark launcher handoff as a terminal effect --- apps/server/src/updates/ServerAutoUpdater.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 946a489d4e5da..8c367e699d1a2 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -104,7 +104,7 @@ export const layer = Layer.effectDiscard( yield* selfUpdate.update({ targetVersion }); // The launcher stops us after accepting the handoff. Keep new provider // starts behind the install permit until this runtime is shut down. - yield* Effect.never; + return yield* Effect.never; }), ); }).pipe( From aac65cfc630499d469872c2dec7fabeb9a15152d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:39:54 +0000 Subject: [PATCH 018/149] fix(server): preserve maintenance admission effect types --- .../src/provider/providerMaintenanceCommandCoordinator.ts | 4 ++-- apps/server/src/provider/providerMaintenanceRunner.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index 9d5c953a85c88..49cdae4ff995f 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -11,7 +11,7 @@ export class ProviderMaintenanceAdmission extends Context.Service< { readonly withPermit: (effect: Effect.Effect) => Effect.Effect; } ->()("t3/provider/ProviderMaintenanceAdmission") {} +>()("t3/provider/providerMaintenanceCommandCoordinator/ProviderMaintenanceAdmission") {} // Keep one layer reference so the update window and turn startup share its permit. export const admissionLayer = Layer.effect( @@ -27,7 +27,7 @@ export interface ProviderMaintenanceCommandCoordinatorShape { readonly lockKey: string; readonly onQueued?: Effect.Effect; readonly onInterrupted?: Effect.Effect; - readonly admit?: (effect: Effect.Effect) => Effect.Effect; + readonly admit?: ProviderMaintenanceAdmission["Service"]["withPermit"]; readonly run: Effect.Effect; }) => Effect.Effect; } diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 90a8144fe5dff..d31aec3c3f00a 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -390,7 +390,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const updateProvider = Effect.fn("ProviderMaintenanceRunner.updateProvider")(function* ( target: Parameters[0], - alreadyAdmitted = false, + alreadyAdmitted: boolean = false, ) { const provider = typeof target === "string" ? target : target.provider; const instanceId = From 526820e43770b3693d44f0b3d1e919a6c351da78 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:50:27 +0000 Subject: [PATCH 019/149] fix(dev): serve web assets and settle closed lab sockets --- scripts/update-lab.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/update-lab.ts b/scripts/update-lab.ts index 29f8d91327581..71dafb213ea10 100644 --- a/scripts/update-lab.ts +++ b/scripts/update-lab.ts @@ -54,7 +54,8 @@ t3 release feed, all under .t3/update-lab (state survives down/up). build-runtime [--archive file] Publish a t3 runtime to the release feed: a source runtime (copy of apps/server stamped with , runs on this checkout's deps) or - a real CLI archive built by the release pipeline. + a real CLI archive built by the release pipeline. Source runtimes + include apps/web/dist when present; otherwise they serve APIs only. reset Delete the lab (only after down). Providers: codex claude opencode pi grok. Install methods rotate across envs: @@ -644,7 +645,8 @@ function compareVersions(a: string, b: string): number { * Publish `t3--.tar.gz` + SHA256SUMS. Without `--archive` * this is a source runtime: a copy of apps/server whose package.json carries * `version` (every version check reads it) and whose node_modules links back - * to this checkout, started by a `t3` wrapper. + * to this checkout, started by a `t3` wrapper. An existing web build is copied + * beside the server; publishing does not build the client. */ function buildRuntime(version: string, archive: string | undefined): string { if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) @@ -664,6 +666,12 @@ function buildRuntime(version: string, archive: string | undefined): string { filter: (source) => !["node_modules", "dist", "dist-exe"].includes(NodePath.relative(serverSource, source)), }); + const webDist = NodePath.join(REPO, "apps/web/dist"); + if (NodeFS.existsSync(NodePath.join(webDist, "index.html"))) { + NodeFS.cpSync(webDist, NodePath.join(stage, "web/dist"), { recursive: true }); + } else { + console.warn("No apps/web/dist/index.html; publishing an API-only source runtime."); + } const packageJsonPath = NodePath.join(stage, "server/package.json"); const packageJson = JSON.parse(NodeFS.readFileSync(packageJsonPath, "utf8")); packageJson.version = version; @@ -923,8 +931,14 @@ async function rpc(state: LabState, env: LabEnv, tag: string, payload: unknown): `${origin.replace("http", "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL}&wsTicket=${encodeURIComponent(ticket)}`, ); return await new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(`${tag} timed out`)), 10 * 60_000); + let settled = false; + const timer = setTimeout( + () => finish(() => reject(new Error(`${tag} timed out`))), + 10 * 60_000, + ); const finish = (settle: () => void) => { + if (settled) return; + settled = true; clearTimeout(timer); socket.close(); settle(); @@ -933,6 +947,9 @@ async function rpc(state: LabState, env: LabEnv, tag: string, payload: unknown): socket.send(JSON.stringify({ _tag: "Request", id: "1", tag, payload, headers: [] })), ); socket.addEventListener("error", () => finish(() => reject(new Error(`${tag}: socket error`)))); + socket.addEventListener("close", () => + finish(() => reject(new Error(`${tag}: socket closed before response`))), + ); socket.addEventListener("message", (event) => { const decoded = JSON.parse(String(event.data)); for (const message of Array.isArray(decoded) ? decoded : [decoded]) { From 22149e999e7e5d365cc41674a17d2c71bf5cc347 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:59:27 +0000 Subject: [PATCH 020/149] fix(dev): reject live update lab host changes --- scripts/update-lab.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/update-lab.ts b/scripts/update-lab.ts index 71dafb213ea10..13ab1e3bf2de7 100644 --- a/scripts/update-lab.ts +++ b/scripts/update-lab.ts @@ -1007,6 +1007,14 @@ async function up(args: string[]): Promise { envs: [], }; let state = updateState((next) => { + if ( + next.host !== host && + next.envs.some((env) => isLabProcess(env.pid, serverMarker(env), projectDir(env.id))) + ) { + fail( + "Cannot change --host while lab servers are running. Run `node scripts/update-lab.ts down` first.", + ); + } next.host = host; next.busySeconds = Number(option(args, "busy-seconds") ?? next.busySeconds); next.fromVersion = option(args, "from-version") ?? next.fromVersion; From 65a7cb779b2eddb07ff0e9912a76ae44d5bd3881 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:03:16 +0000 Subject: [PATCH 021/149] fix(dev): reuse existing projects for lab turns --- scripts/update-lab.ts | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/scripts/update-lab.ts b/scripts/update-lab.ts index 13ab1e3bf2de7..7267d56422ccb 100644 --- a/scripts/update-lab.ts +++ b/scripts/update-lab.ts @@ -1129,19 +1129,29 @@ function status(): void { async function busy(state: LabState, env: LabEnv): Promise { const headers = await authHeaders(env); if (!env.projectId) { - const projectId = NodeCrypto.randomUUID(); - const created = await fetch(`${originOf(state, env)}/api/projects/mutate`, { - method: "POST", - headers, - body: JSON.stringify({ - type: "project.create", - commandId: NodeCrypto.randomUUID(), - projectId, - title: `${env.id} lab project`, - workspaceRoot: projectDir(env.id), - }), - }); - if (!created.ok) fail(`Project creation failed: ${created.status} ${await created.text()}`); + const response = await fetch(`${originOf(state, env)}/api/projects`, { headers }); + if (!response.ok) fail(`Project snapshot failed: ${response.status} ${await response.text()}`); + const snapshot = (await response.json()) as { + projects: Array<{ id: string; workspaceRoot: string; deletedAt: string | null }>; + }; + const existing = snapshot.projects.find( + (project) => project.workspaceRoot === projectDir(env.id) && project.deletedAt === null, + ); + const projectId = existing?.id ?? NodeCrypto.randomUUID(); + if (!existing) { + const created = await fetch(`${originOf(state, env)}/api/projects/mutate`, { + method: "POST", + headers, + body: JSON.stringify({ + type: "project.create", + commandId: NodeCrypto.randomUUID(), + projectId, + title: `${env.id} lab project`, + workspaceRoot: projectDir(env.id), + }), + }); + if (!created.ok) fail(`Project creation failed: ${created.status} ${await created.text()}`); + } updateState((next) => { requireEnv(next, env.id).projectId = projectId; }); From c1a98f405e3d403d5de6a095e226c742b4dc1af6 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:11:05 +0000 Subject: [PATCH 022/149] fix(server): keep terminal input available during update checks --- apps/server/src/updates/UpdateWindow.ts | 15 +++++++++++---- docs/user/updating.md | 3 --- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 51ae4778a800c..ba63c4e72cc2a 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -228,11 +228,18 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (options, install) => { const attempt = check(options).pipe( - Effect.flatMap((status) => (status.open ? Effect.asSome(install) : Effect.succeedNone)), - ); - return admission.withPermit( - options.restartsServer ? terminals.withRestartPermit(attempt) : attempt, + Effect.flatMap((status) => { + if (!status.open) return Effect.succeedNone; + if (!options.restartsServer) return Effect.asSome(install); + // Keep input available during the full check, then close the terminal race before restart. + return terminals.withRestartPermit( + terminals.hasBusyTerminals.pipe( + Effect.flatMap((busy) => (busy ? Effect.succeedNone : Effect.asSome(install))), + ), + ); + }), ); + return admission.withPermit(attempt); }; const runWhenOpen: UpdateWindow["Service"]["runWhenOpen"] = (options, install) => Effect.gen(function* () { diff --git a/docs/user/updating.md b/docs/user/updating.md index cb0de606f90c2..2c27589a717ff 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -85,9 +85,6 @@ Then it installs: Background services also wait for integrated terminal commands to finish. -Desktop updates download in the background and install when you quit the app. -The app stays open while you work, including when it hosts multiple backends. - Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours or when a newer version is published. A failed or rolled-back T3 Code update is not retried until a newer From 37abc15e670593acee77eb885dc9cb4333e61ddf Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:18:49 +0000 Subject: [PATCH 023/149] chore(updates): keep simulation tooling local --- apps/server/src/updates/UpdateWindow.ts | 31 +- package.json | 1 - .../src/server/maintenanceResolver.ts | 2 +- scripts/update-lab.ts | 1294 ----------------- 4 files changed, 13 insertions(+), 1315 deletions(-) delete mode 100644 scripts/update-lab.ts diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 430baea17c4f1..4f9051ec7602a 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -1,5 +1,4 @@ import type { ScheduledTaskId, ThreadId } from "@t3tools/contracts"; -import * as Config from "effect/Config"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -17,24 +16,18 @@ import * as ServerSettings from "../serverSettings.ts"; /** Work due this soon keeps the window closed, so it does not start late. */ const UPDATE_LOOKAHEAD = Duration.minutes(5); -/** - * Background update pacing. `T3CODE_UPDATE_LAB=1` (scripts/update-lab.ts) - * shortens the waits so update flows can be exercised in seconds. - */ -export const updateTimings = Config.Boolean("T3CODE_UPDATE_LAB").pipe( - Config.withDefault(false), - Config.map((lab) => ({ - /** - * How long every client must be untouched before an update may interrupt - * it. Clients report activity about every 25s, so this must stay well above that. - */ - quietPeriod: lab ? Duration.seconds(60) : Duration.minutes(15), - /** How often a waiting update rechecks the window or looks for new versions. */ - recheck: lab ? Duration.seconds(5) : Duration.seconds(30), - firstPassDelay: lab ? Duration.seconds(10) : Duration.minutes(2), - passInterval: lab ? Duration.seconds(20) : Duration.minutes(10), - })), -); +/** Background update pacing. */ +export const updateTimings = Effect.succeed({ + /** + * How long every client must be untouched before an update may interrupt + * it. Clients report activity about every 25s, so this must stay well above that. + */ + quietPeriod: Duration.minutes(15), + /** How often a waiting update rechecks the window or looks for new versions. */ + recheck: Duration.seconds(30), + firstPassDelay: Duration.minutes(2), + passInterval: Duration.minutes(10), +}); export type UpdateWindowBlocker = | { readonly type: "active-threads"; readonly threadIds: ReadonlyArray } diff --git a/package.json b/package.json index 2b839155031ab..55869fc4b3911 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,6 @@ "start:desktop": "vp run --filter @t3tools/desktop start", "start:marketing": "vp run --filter @t3tools/marketing preview", "start:mock-update-server": "node scripts/mock-update-server.ts", - "update-lab": "node scripts/update-lab.ts", "screenshots:mobile": "node scripts/mobile-showcase.ts", "icons:export": "node scripts/export-brand-icons.ts", "icons:check": "node scripts/export-brand-icons.ts --check", diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index 794385b3755a1..8fcee193960d7 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -742,7 +742,7 @@ export function createProviderVersionAdvisory(input: { const DEFAULT_NPM_REGISTRY_URL = "https://registry.npmjs.org"; -/** npm's own registry override, so mirrors (and local fake registries) answer "latest". */ +/** Respect npm's registry override when checking mirrors for newer provider versions. */ const NpmRegistryUrlConfig = Config.String("npm_config_registry").pipe( Config.withDefault(DEFAULT_NPM_REGISTRY_URL), Config.map((url) => url.trim().replace(/\/+$/, "") || DEFAULT_NPM_REGISTRY_URL), diff --git a/scripts/update-lab.ts b/scripts/update-lab.ts deleted file mode 100644 index 7267d56422ccb..0000000000000 --- a/scripts/update-lab.ts +++ /dev/null @@ -1,1294 +0,0 @@ -#!/usr/bin/env node -// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off globalFetch:off globalConsole:off preferSchemaOverJson:off - Host-side dev lab that spawns isolated servers and fake CLIs with Node APIs directly. -/** - * Update lab: several isolated T3 servers ("machines") with fake provider - * CLIs, a fake npm registry and a fake t3 release feed, for exercising - * provider, server and launcher update flows end to end. State lives in the - * gitignored `/.t3/update-lab`. Fake CLIs and package managers re-enter - * this file as `__tool`, and the registry/feed server as `__feed`, so the lab - * stays one file. Run `node scripts/update-lab.ts --help`. - */ -import * as NodeChildProcess from "node:child_process"; -import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; -import * as NodeHttp from "node:http"; -import * as NodeNet from "node:net"; -import * as NodePath from "node:path"; -import * as NodeReadline from "node:readline"; - -const REPO = NodePath.resolve(import.meta.dirname, ".."); -const LAB = NodePath.join(REPO, ".t3", "update-lab"); -const STATE_PATH = NodePath.join(LAB, "state.json"); -const RELEASES = NodePath.join(LAB, "releases"); -const SERVER_BIN = NodePath.join(REPO, "apps/server/src/bin.ts"); -const MOCK_ACP_AGENT = NodePath.join(REPO, "apps/server/scripts/acp-mock-agent.ts"); -const NODE = process.execPath; -const SELF = import.meta.filename; -const REPO_VERSION: string = JSON.parse( - NodeFS.readFileSync(NodePath.join(REPO, "apps/server/package.json"), "utf8"), -).version; -// oxlint-disable-next-line t3code/no-global-process-runtime -- Plain Node dev script; the lab only targets this host. -const PLATFORM_KEY = `${process.platform}-${process.arch}`; -const ORCHESTRATION_PROTOCOL = "2"; - -const HELP = `Usage: node scripts/update-lab.ts - -Isolated T3 servers with fake provider CLIs, a fake npm registry and a fake -t3 release feed, all under .t3/update-lab (state survives down/up). - - up [--envs N] [--host IP] [--launcher N] [--from-version V] [--busy-seconds S] - Start the feed and N servers (default 3; host 127.0.0.1). The last N - envs (default 1) run under the real service launcher on runtime V - (default: repo version minus one patch) so self-update can be tried. - Prints each env's origin and one-time pairing URL. Rerun to resume. - down Stop only the processes the lab recorded. - status Envs, ports, liveness, installed vs latest. - pair Mint a fresh pairing URL for env (e.g. env1). - publish Set the registry's latest for a provider. - fail on|off Make that env's installer/updater exit 1. - delay Installer/updater duration (default 3000). - providers Print the provider list clients see for env. - update [v] Run server.updateProvider like the UI does. - busy Start a mock-acp turn that runs --busy-seconds. - self-update Run server.updateServer (launcher envs only). - build-runtime [--archive file] - Publish a t3 runtime to the release feed: a source runtime (copy of - apps/server stamped with , runs on this checkout's deps) or - a real CLI archive built by the release pipeline. Source runtimes - include apps/web/dist when present; otherwise they serve APIs only. - reset Delete the lab (only after down). - -Providers: codex claude opencode pi grok. Install methods rotate across envs: -npm prefix, pnpm, bun, yarn, volta, vite-plus, Homebrew formula and cask, -native installers (claude update, codex standalone, opencode upgrade, -grok update, pi update --self) and mise (manual-only). Each env also has a -"mock-acp" provider (apps/server/scripts/acp-mock-agent.ts) for busy turns. -Servers cache npm "latest" for an hour: down && up after publish. -Not covered: cursor, antigravity, muse (no npm advisory or network-only).`; - -type ProviderKey = "codex" | "claude" | "opencode" | "pi" | "grok"; -type Method = - | "npm" - | "pnpm" - | "bun" - | "yarn" - | "volta" - | "vite-plus" - | "homebrew" - | "native" - | "mise"; - -const PROVIDERS: Record< - ProviderKey, - { - readonly command: string; - readonly pkg: string; - readonly driver: string; - readonly installed: string; - readonly latest: string; - readonly brew: { readonly kind: "formula" | "cask"; readonly name: string } | null; - } -> = { - codex: { - command: "codex", - pkg: "@openai/codex", - driver: "codex", - installed: "0.158.0", - latest: "0.160.0", - brew: { kind: "formula", name: "codex" }, - }, - claude: { - command: "claude", - pkg: "@anthropic-ai/claude-code", - driver: "claudeAgent", - installed: "2.1.285", - latest: "2.1.290", - brew: { kind: "cask", name: "claude-code" }, - }, - opencode: { - command: "opencode", - pkg: "opencode-ai", - driver: "opencode", - installed: "1.14.19", - latest: "1.14.25", - brew: { kind: "formula", name: "opencode" }, - }, - pi: { - command: "pi", - pkg: "@earendil-works/pi-coding-agent", - driver: "pi", - installed: "0.80.6", - latest: "0.81.0", - brew: null, - }, - grok: { - command: "grok", - pkg: "@xai-official/grok", - driver: "grok", - installed: "1.0.13", - latest: "1.0.20", - brew: null, - }, -}; -const PROVIDER_KEYS = Object.keys(PROVIDERS) as ProviderKey[]; - -// Env N uses row (N-1) % rows; three envs already reach every resolver branch. -const INSTALL_MATRIX: ReadonlyArray> = [ - { codex: "npm", claude: "native", opencode: "homebrew", pi: "pnpm", grok: "native" }, - { codex: "native", claude: "homebrew", opencode: "bun", pi: "native", grok: "npm" }, - { codex: "volta", claude: "yarn", opencode: "native", pi: "vite-plus", grok: "bun" }, - { codex: "mise", claude: "pnpm", opencode: "npm", pi: "yarn", grok: "native" }, - { codex: "vite-plus", claude: "bun", opencode: "volta", pi: "npm", grok: "pnpm" }, -]; - -// The package manager each fake tool stands in for. -const TOOL_METHOD: Record = { - npm: "npm", - pnpm: "pnpm", - bun: "bun", - yarn: "yarn", - volta: "volta", - vp: "vite-plus", - brew: "homebrew", -}; - -interface Install { - method: Method; - version: string; - failing: boolean; -} - -interface LabEnv { - id: string; - port: number; - pid: number | null; - launcher: boolean; - projectId: string | null; - installs: Record; -} - -interface LabState { - host: string; - feedPort: number; - feedPid: number | null; - delayMs: number; - busySeconds: number; - fromVersion: string; - latest: Record; - envs: LabEnv[]; -} - -// --------------------------------------------------------------------------- -// State - -function readState(): LabState | null { - if (!NodeFS.existsSync(STATE_PATH)) return null; - return JSON.parse(NodeFS.readFileSync(STATE_PATH, "utf8")); -} - -function requireState(): LabState { - const state = readState(); - if (!state) fail("No lab yet. Run `node scripts/update-lab.ts up` first."); - return state; -} - -/** Keep the descriptor open so the kernel releases the lock even if this process dies. */ -function acquireLock(name: "state" | "processes"): () => void { - NodeFS.mkdirSync(NodePath.dirname(LAB), { recursive: true }); - // Keep lock files outside LAB so reset cannot unlink a lock another process holds. - const fd = NodeFS.openSync(`${LAB}.${name}.lock`, "a"); - try { - NodeChildProcess.execFileSync("flock", ["-x", "3"], { - stdio: ["ignore", "ignore", "inherit", fd], - }); - } catch (error) { - NodeFS.closeSync(fd); - throw error; - } - return () => NodeFS.closeSync(fd); -} - -/** Atomic replacement keeps readers from observing a partially written state. */ -function writeState(state: LabState): void { - const temp = `${STATE_PATH}.${process.pid}.tmp`; - NodeFS.writeFileSync(temp, `${JSON.stringify(state, null, 2)}\n`); - NodeFS.renameSync(temp, STATE_PATH); -} - -function updateState(mutate: (state: LabState) => void, initial?: LabState): LabState { - const release = acquireLock("state"); - try { - const state = readState() ?? initial ?? requireState(); - mutate(state); - NodeFS.mkdirSync(LAB, { recursive: true }); - writeState(state); - return state; - } finally { - release(); - } -} - -function requireEnv(state: LabState, id: string | undefined): LabEnv { - const env = state.envs.find((candidate) => candidate.id === id); - if (!env) fail(`Unknown env ${id ?? "(none)"}. Envs: ${state.envs.map((e) => e.id).join(", ")}`); - return env; -} - -function requireProvider(name: string | undefined): ProviderKey { - if (name && name in PROVIDERS) return name as ProviderKey; - fail(`Unknown provider ${name ?? "(none)"}. Providers: ${PROVIDER_KEYS.join(", ")}`); -} - -function fail(message: string): never { - process.stderr.write(`${message}\n`); - process.exit(1); -} - -const envDir = (id: string) => NodePath.join(LAB, "envs", id); -const homeDir = (id: string) => NodePath.join(envDir(id), "home"); -const t3Home = (id: string) => NodePath.join(envDir(id), "t3"); -const projectDir = (id: string) => NodePath.join(envDir(id), "project"); -const logPath = (id: string) => NodePath.join(envDir(id), "server.log"); -const originOf = (state: LabState, env: LabEnv) => `http://${state.host}:${env.port}`; - -// --------------------------------------------------------------------------- -// Fake installs - -function writeExecutable(path: string, content: string): void { - NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true }); - NodeFS.writeFileSync(path, content, { mode: 0o755 }); -} - -function link(target: string, path: string): void { - NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true }); - NodeFS.rmSync(path, { force: true }); - NodeFS.symlinkSync(target, path); -} - -/** A shell shim that re-enters this file as a fake tool; `$0` names a shared volta shim. */ -function toolShim(envId: string, tool: string | null): string { - const name = tool === null ? `"$(basename "$0")"` : `'${tool}'`; - return `#!/bin/sh\nexec '${NODE}' '${SELF}' __tool '${envId}' ${name} "$@"\n`; -} - -/** Search order shared by the server's PATH and the fake login shell profile. */ -function envPath(envId: string): string { - const home = homeDir(envId); - return [ - ".local/bin", - ".npm-global/bin", - ".local/share/pnpm", - ".bun/bin", - ".yarn/bin", - ".volta/bin", - ".vite-plus/bin", - "homebrew/bin", - ".opencode/bin", - ".grok/bin", - ".pi/bin", - ".local/share/mise/shims", - ] - .map((dir) => NodePath.join(home, dir)) - .concat([NodePath.join(envDir(envId), "tools"), "/usr/local/bin", "/usr/bin", "/bin"]) - .join(":"); -} - -/** Lay a provider out the way its installer would, so the server's resolver sees real paths. */ -function layOutInstall(envId: string, provider: ProviderKey, method: Method): void { - const home = homeDir(envId); - const { command, pkg, brew } = PROVIDERS[provider]; - const shim = toolShim(envId, command); - const at = (...parts: string[]) => NodePath.join(home, ...parts); - switch (method) { - case "npm": { - const real = at(".npm-global/lib/node_modules", pkg, "bin", `${command}.js`); - writeExecutable(real, shim); - link(real, at(".npm-global/bin", command)); - return; - } - case "yarn": { - const real = at(".config/yarn/global/node_modules", pkg, "bin", `${command}.js`); - writeExecutable(real, shim); - link(real, at(".yarn/bin", command)); - return; - } - case "pnpm": - return writeExecutable(at(".local/share/pnpm", command), shim); - case "bun": - return writeExecutable(at(".bun/bin", command), shim); - case "vite-plus": - return writeExecutable(at(".vite-plus/bin", command), shim); - case "volta": - writeExecutable(at(".volta/bin/volta-shim"), toolShim(envId, null)); - writeExecutable(at(".volta/bin/volta"), toolShim(envId, "volta")); - NodeFS.mkdirSync(at(".volta/tools/image/packages", ...pkg.split("/")), { recursive: true }); - link(at(".volta/bin/volta-shim"), at(".volta/bin", command)); - return; - case "homebrew": { - if (!brew) fail(`${provider} has no Homebrew package.`); - const keg = - brew.kind === "formula" - ? at("homebrew/Cellar", brew.name, PROVIDERS[provider].installed, "bin", command) - : at("homebrew/Caskroom", brew.name, PROVIDERS[provider].installed, command); - writeExecutable(keg, shim); - link(keg, at("homebrew/bin", command)); - writeExecutable(at("homebrew/bin/brew"), toolShim(envId, "brew")); - return; - } - case "mise": { - const real = at(".local/share/mise/installs", command, "latest", "bin", command); - writeExecutable(real, shim); - link(real, at(".local/share/mise/shims", command)); - return; - } - case "native": - switch (provider) { - case "claude": - return writeExecutable(at(".local/bin/claude"), shim); - case "codex": { - const real = at(".codex/packages/standalone/current/bin/codex"); - writeExecutable(real, shim); - link(real, at(".local/bin/codex")); - return; - } - case "opencode": - return writeExecutable(at(".opencode/bin/opencode"), shim); - case "grok": - return writeExecutable(at(".grok/bin/grok"), shim); - case "pi": - return writeExecutable(at(".pi/bin/pi"), shim); - } - } -} - -function seedEnvFiles(state: LabState, env: LabEnv): void { - const home = homeDir(env.id); - NodeFS.mkdirSync(home, { recursive: true }); - // The server reads PATH from a login shell first; these dotfiles keep it on the lab PATH. - const profile = `export PATH='${envPath(env.id)}'\n`; - for (const file of [".bash_profile", ".bashrc", ".profile", ".zprofile", ".zshrc"]) { - NodeFS.writeFileSync(NodePath.join(home, file), profile); - } - const tools = NodePath.join(envDir(env.id), "tools"); - for (const tool of ["npm", "pnpm", "bun", "yarn", "vp"]) { - writeExecutable(NodePath.join(tools, tool), toolShim(env.id, tool)); - } - link(NODE, NodePath.join(tools, "node")); - for (const provider of PROVIDER_KEYS) { - layOutInstall(env.id, provider, env.installs[provider].method); - } - - const project = projectDir(env.id); - if (!NodeFS.existsSync(NodePath.join(project, ".git"))) { - NodeFS.mkdirSync(project, { recursive: true }); - NodeFS.writeFileSync(NodePath.join(project, "README.md"), `# ${env.id} lab project\n`); - const git = (...args: string[]) => - NodeChildProcess.execFileSync("git", ["-C", project, ...args], { stdio: "ignore" }); - git("init", "-q"); - git("add", "."); - git("-c", "user.name=Update Lab", "-c", "user.email=lab@localhost", "commit", "-qm", "init"); - } - - const settingsPath = NodePath.join(t3Home(env.id), "userdata", "settings.json"); - NodeFS.mkdirSync(NodePath.dirname(settingsPath), { recursive: true }); - const settings = NodeFS.existsSync(settingsPath) - ? JSON.parse(NodeFS.readFileSync(settingsPath, "utf8")) - : {}; - settings.providerInstances = { - ...settings.providerInstances, - opencode: { driver: "opencode", enabled: true }, - pi: { driver: "pi", enabled: true }, - grok: { driver: "grok", enabled: true }, - "mock-acp": { - driver: "acpRegistry", - displayName: "Mock ACP", - enabled: true, - environment: [ - { name: "T3_ACP_PROMPT_DELAY_MS", value: String(state.busySeconds * 1000) }, - { name: "T3_ACP_PROMPT_RESPONSE_TEXT", value: `${env.id} finished its busy turn.` }, - ], - config: { source: "local", commandPath: NODE, commandArgs: [MOCK_ACP_AGENT] }, - }, - }; - settings.defaultModelSelection ??= { instanceId: "mock-acp", model: "default" }; - NodeFS.writeFileSync(settingsPath, `${JSON.stringify(settings, null, 2)}\n`); -} - -// --------------------------------------------------------------------------- -// Fake CLIs and package managers (`__tool ...args`) - -async function runTool(envId: string, tool: string, args: string[]): Promise { - const state = requireState(); - const env = requireEnv(state, envId); - const provider = PROVIDER_KEYS.find((key) => PROVIDERS[key].command === tool); - if (provider) return runProviderCli(state, env, provider, args); - const method = TOOL_METHOD[tool]; - if (!method) fail(`update-lab: unknown fake tool ${tool}`); - if (args[0] === "--version" || args[0] === "-v") return void console.log("0.0.0-update-lab"); - - if (tool === "brew") { - const prefix = NodePath.join(homeDir(env.id), "homebrew"); - if (args[0] === "--prefix") return void console.log(prefix); - const name = args.findLast((arg) => !arg.startsWith("-")); - const owner = PROVIDER_KEYS.find((key) => PROVIDERS[key].brew?.name === name); - if (!owner) fail(`Error: No available formula or cask with the name "${name}".`); - if (args[0] === "info") { - const latest = state.latest[owner]; - return void console.log( - JSON.stringify( - PROVIDERS[owner].brew!.kind === "formula" - ? { formulae: [{ versions: { stable: latest } }], casks: [] } - : { formulae: [], casks: [{ version: `${latest},lab` }] }, - ), - ); - } - if (args[0] === "upgrade") return install(env.id, owner, tool, undefined); - fail(`update-lab brew: unsupported ${args.join(" ")}`); - } - - // npm/pnpm/bun/yarn/volta/vp install commands name ``, `@latest` or `@x.y.z`. - for (const arg of args) { - for (const key of PROVIDER_KEYS) { - const { pkg } = PROVIDERS[key]; - if (arg !== pkg && !arg.startsWith(`${pkg}@`)) continue; - const requested = arg.slice(pkg.length + 1); - if (method === "npm") { - const prefix = args[args.indexOf("--prefix") + 1]; - if ( - env.installs[key].method !== "npm" || - prefix !== NodePath.join(homeDir(env.id), ".npm-global") - ) { - fail(`npm error ${pkg} is not installed under --prefix ${prefix ?? "(default)"}`); - } - } - return install( - env.id, - key, - tool, - requested && requested !== "latest" ? requested : undefined, - ); - } - } - fail(`update-lab ${tool}: unsupported ${args.join(" ")}`); -} - -async function runProviderCli( - state: LabState, - env: LabEnv, - provider: ProviderKey, - args: string[], -): Promise { - const version = env.installs[provider].version; - const [first, second] = args; - if (first === "--version" || first === "-v") { - const banner = { - codex: `codex-cli ${version}`, - claude: `${version} (Claude Code)`, - opencode: version, - pi: version, - grok: `grok ${version}`, - }[provider]; - return void console.log(banner); - } - const native = - (provider === "claude" && first === "update") || - (provider === "codex" && first === "update") || - (provider === "opencode" && first === "upgrade") || - (provider === "grok" && first === "update") || - (provider === "pi" && first === "update" && second === "--self"); - if (native) return install(env.id, provider, PROVIDERS[provider].command, undefined); - if (provider === "grok" && first === "models") return void console.log("You are logged in"); - if (provider === "codex" && first === "app-server") return codexAppServer(env, version); - // Session probes (claude stream-json, opencode serve, pi --mode rpc) are out of - // scope: a quiet failure keeps installed + version, so advisories still show. - process.stderr.write(`update-lab fake ${provider}: ${args.join(" ")} is not simulated\n`); - process.exit(1); -} - -/** Just enough of `codex app-server` for the status probe to read the version. */ -async function codexAppServer(env: LabEnv, version: string): Promise { - const codexHome = process.env.CODEX_HOME || NodePath.join(homeDir(env.id), ".codex"); - const results: Record = { - initialize: { - codexHome, - platformFamily: "unix", - platformOs: "linux", - userAgent: `codex_cli_rs/${version}`, - }, - "account/read": { account: null, requiresOpenaiAuth: false }, - "skills/list": { data: [] }, - "model/list": { data: [] }, - }; - const lines = NodeReadline.createInterface({ input: process.stdin }); - for await (const line of lines) { - let message: { id?: unknown; method?: string }; - try { - message = JSON.parse(line); - } catch { - continue; - } - if (message.id === undefined || message.method === undefined) continue; - const result = results[message.method]; - process.stdout.write( - `${JSON.stringify( - result === undefined - ? { id: message.id, error: { code: -32601, message: `${message.method} not simulated` } } - : { id: message.id, result }, - )}\n`, - ); - } -} - -/** Shared install path: progress output for `delayMs`, injected failure, then the new version. */ -async function install( - envId: string, - provider: ProviderKey, - tool: string, - requested: string | undefined, -): Promise { - const state = requireState(); - const env = requireEnv(state, envId); - const current = env.installs[provider]; - const method = TOOL_METHOD[tool]; - if (method && method !== current.method) { - fail(`${tool}: ${PROVIDERS[provider].pkg} is not managed by ${tool} here (${current.method}).`); - } - const target = requested ?? state.latest[provider]; - const steps = 5; - for (let step = 1; step <= steps; step++) { - console.log(`${tool}: fetching ${PROVIDERS[provider].pkg}@${target} [${step}/${steps}]`); - await new Promise((resolve) => setTimeout(resolve, state.delayMs / steps)); - } - if (requireState().envs.find((e) => e.id === envId)!.installs[provider].failing) { - process.stderr.write(`${tool}: injected failure installing ${PROVIDERS[provider].pkg}\n`); - process.exit(1); - } - updateState((next) => { - requireEnv(next, envId).installs[provider].version = target; - }); - console.log(`${tool}: installed ${PROVIDERS[provider].pkg}@${target} (was ${current.version})`); -} - -// --------------------------------------------------------------------------- -// Fake npm registry + t3 release feed (`__feed `) - -function runFeed(port: number): void { - const server = NodeHttp.createServer((request, response) => { - const url = new URL(request.url ?? "/", "http://lab"); - const send = (status: number, body: unknown) => { - response.writeHead(status, { "content-type": "application/json" }); - response.end(JSON.stringify(body)); - }; - const state = readState(); - if (!state) return send(503, { error: "no lab state" }); - - // GET /npm//latest and /npm/; the server URI-encodes scoped names. - const npm = /^\/npm\/(.+?)(\/latest)?$/.exec(url.pathname); - if (npm) { - const name = decodeURIComponent(npm[1]!); - const key = PROVIDER_KEYS.find((candidate) => PROVIDERS[candidate].pkg === name); - if (!key) return send(404, { error: "Not found" }); - const version = state.latest[key]; - return npm[2] - ? send(200, { name, version }) - : send(200, { - name, - "dist-tags": { latest: version }, - versions: { [version]: { name, version } }, - }); - } - - // GET /releases/index.json?page=N: GitHub's list-releases shape, newest first. - if (url.pathname === "/releases/index.json") { - const page = Number(url.searchParams.get("page") ?? "1"); - return send( - 200, - page === 1 ? publishedRuntimes().map((v) => ({ tag_name: `v${v}`, draft: false })) : [], - ); - } - - // GET /releases/v/: SHA256SUMS and archives. - const asset = /^\/releases\/(v[^/]+)\/([^/]+)$/.exec(url.pathname); - if (asset) { - const file = NodePath.join(RELEASES, asset[1]!, asset[2]!); - if (!NodeFS.existsSync(file)) return send(404, { error: "Not found" }); - response.writeHead(200, { "content-length": NodeFS.statSync(file).size }); - return void NodeFS.createReadStream(file).pipe(response); - } - send(404, { error: "Not found" }); - }); - server.listen(port, "127.0.0.1"); -} - -function publishedRuntimes(): string[] { - if (!NodeFS.existsSync(RELEASES)) return []; - return NodeFS.readdirSync(RELEASES) - .filter((dir) => NodeFS.existsSync(NodePath.join(RELEASES, dir, "SHA256SUMS"))) - .map((dir) => dir.slice(1)) - .sort((a, b) => compareVersions(b, a)); -} - -function compareVersions(a: string, b: string): number { - const parse = (v: string) => v.split(/[.-]/).map((part) => Number(part) || 0); - const [left, right] = [parse(a), parse(b)]; - for (let i = 0; i < Math.max(left.length, right.length); i++) { - const diff = (left[i] ?? 0) - (right[i] ?? 0); - if (diff !== 0) return diff; - } - return 0; -} - -// --------------------------------------------------------------------------- -// Runtimes for the release feed and the service launcher - -/** - * Publish `t3--.tar.gz` + SHA256SUMS. Without `--archive` - * this is a source runtime: a copy of apps/server whose package.json carries - * `version` (every version check reads it) and whose node_modules links back - * to this checkout, started by a `t3` wrapper. An existing web build is copied - * beside the server; publishing does not build the client. - */ -function buildRuntime(version: string, archive: string | undefined): string { - if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) - fail(`Not an exact version: ${version}`); - const releaseDir = NodePath.join(RELEASES, `v${version}`); - const fileName = `t3-${version}-${PLATFORM_KEY}.tar.gz`; - const archivePath = NodePath.join(releaseDir, fileName); - NodeFS.mkdirSync(releaseDir, { recursive: true }); - if (archive) { - NodeFS.copyFileSync(NodePath.resolve(archive), archivePath); - } else { - const stage = NodePath.join(LAB, "build", `t3-${version}-${PLATFORM_KEY}`); - NodeFS.rmSync(stage, { recursive: true, force: true }); - const serverSource = NodePath.join(REPO, "apps/server"); - NodeFS.cpSync(serverSource, NodePath.join(stage, "server"), { - recursive: true, - filter: (source) => - !["node_modules", "dist", "dist-exe"].includes(NodePath.relative(serverSource, source)), - }); - const webDist = NodePath.join(REPO, "apps/web/dist"); - if (NodeFS.existsSync(NodePath.join(webDist, "index.html"))) { - NodeFS.cpSync(webDist, NodePath.join(stage, "web/dist"), { recursive: true }); - } else { - console.warn("No apps/web/dist/index.html; publishing an API-only source runtime."); - } - const packageJsonPath = NodePath.join(stage, "server/package.json"); - const packageJson = JSON.parse(NodeFS.readFileSync(packageJsonPath, "utf8")); - packageJson.version = version; - NodeFS.writeFileSync(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`); - NodeFS.symlinkSync( - NodePath.join(serverSource, "node_modules"), - NodePath.join(stage, "server/node_modules"), - ); - writeExecutable( - NodePath.join(stage, "t3"), - `#!/bin/sh\nexec '${NODE}' "$(dirname "$0")/server/src/bin.ts" "$@"\n`, - ); - NodeChildProcess.execFileSync("tar", [ - "-czf", - archivePath, - "-C", - NodePath.dirname(stage), - NodePath.basename(stage), - ]); - NodeFS.rmSync(stage, { recursive: true, force: true }); - } - const digest = NodeCrypto.createHash("sha256") - .update(NodeFS.readFileSync(archivePath)) - .digest("hex"); - NodeFS.writeFileSync(NodePath.join(releaseDir, "SHA256SUMS"), `${digest} ${fileName}\n`); - return archivePath; -} - -/** Install a published runtime the way pinnedRuntime does, and make it the launcher's active version. */ -function installLauncherRuntime(envId: string, version: string): void { - const runtimeDir = NodePath.join(t3Home(envId), "runtime"); - const versionDir = NodePath.join(runtimeDir, "versions", version); - if (!NodeFS.existsSync(NodePath.join(versionDir, ".install-complete"))) { - const archive = NodePath.join(RELEASES, `v${version}`, `t3-${version}-${PLATFORM_KEY}.tar.gz`); - if (!NodeFS.existsSync(archive)) buildRuntime(version, undefined); - NodeFS.mkdirSync(versionDir, { recursive: true }); - NodeChildProcess.execFileSync("tar", [ - "-xzf", - archive, - "-C", - versionDir, - "--strip-components=1", - ]); - NodeFS.writeFileSync(NodePath.join(versionDir, ".install-complete"), `${version}\n`); - } - const statePath = NodePath.join(runtimeDir, "service-state.json"); - if (!NodeFS.existsSync(statePath)) { - NodeFS.writeFileSync(statePath, `${JSON.stringify({ protocol: 3, activeVersion: version })}\n`); - } -} - -// --------------------------------------------------------------------------- -// Processes - -function listeningPorts(): Set { - const output = NodeChildProcess.execFileSync("ss", ["-H", "-ltn"], { encoding: "utf8" }); - return new Set( - output - .split("\n") - .map((line) => Number(line.trim().split(/\s+/)[3]?.split(":").pop())) - .filter((port) => Number.isInteger(port) && port > 0), - ); -} - -/** An OS-assigned port that `ss` does not already list and nobody else in the lab holds. */ -async function freePort(host: string, taken: Set): Promise { - for (let attempt = 0; attempt < 20; attempt++) { - const port = await new Promise((resolve, reject) => { - const probe = NodeNet.createServer(); - probe.once("error", reject); - probe.listen(0, host, () => { - const address = probe.address(); - probe.close(() => resolve(typeof address === "object" && address ? address.port : 0)); - }); - }); - if (port > 0 && !taken.has(port) && !listeningPorts().has(port)) { - taken.add(port); - return port; - } - } - fail("Could not find a free port."); -} - -/** Check the recorded process's command and working directory before signaling it. */ -function isLabProcess(pid: number | null, marker: string, cwd = REPO): pid is number { - if (!pid) return false; - try { - const args = NodeFS.readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0"); - return ( - args.includes(marker) && - args.includes(marker === "__feed" ? SELF : SERVER_BIN) && - NodeFS.realpathSync(`/proc/${pid}/cwd`) === NodeFS.realpathSync(cwd) - ); - } catch { - return false; - } -} - -const serverMarker = (env: LabEnv) => (env.launcher ? "__service-launcher" : t3Home(env.id)); - -/** The runtime the service launcher runs now, which moves after a self-update. */ -function launcherLabel(env: LabEnv): string { - if (!env.launcher) return ""; - const statePath = NodePath.join(t3Home(env.id), "runtime", "service-state.json"); - const active = NodeFS.existsSync(statePath) - ? JSON.parse(NodeFS.readFileSync(statePath, "utf8")).activeVersion - : "?"; - return ` launcher@${active}`; -} - -/** The minimal environment a lab server sees; nothing from this shell leaks in. */ -function serverEnvironment(state: LabState, env: LabEnv): NodeJS.ProcessEnv { - const home = homeDir(env.id); - const keep = ["LANG", "LC_ALL", "TZ", "USER", "LOGNAME", "TMPDIR"]; - return { - ...Object.fromEntries( - keep.flatMap((key) => (process.env[key] ? [[key, process.env[key]]] : [])), - ), - HOME: home, - PATH: envPath(env.id), - SHELL: "/bin/bash", - XDG_CONFIG_HOME: NodePath.join(home, ".config"), - XDG_DATA_HOME: NodePath.join(home, ".local/share"), - XDG_CACHE_HOME: NodePath.join(home, ".cache"), - XDG_STATE_HOME: NodePath.join(home, ".local/state"), - npm_config_registry: `http://127.0.0.1:${state.feedPort}/npm`, - T3CODE_RELEASE_BASE_URL: `http://127.0.0.1:${state.feedPort}/releases`, - T3CODE_UPDATE_LAB: "1", - T3CODE_NO_BROWSER: "1", - }; -} - -function spawnDetached( - command: string, - args: string[], - options: { env: NodeJS.ProcessEnv; cwd: string; log: string }, -): number { - const fd = NodeFS.openSync(options.log, "a"); - const child = NodeChildProcess.spawn(command, args, { - cwd: options.cwd, - env: options.env, - detached: true, - stdio: ["ignore", fd, fd], - }); - NodeFS.closeSync(fd); - child.unref(); - if (!child.pid) fail(`Failed to start ${command} ${args.join(" ")}`); - return child.pid; -} - -function startServer(state: LabState, env: LabEnv): number { - NodeFS.writeFileSync(logPath(env.id), ""); - const environment = serverEnvironment(state, env); - if (env.launcher) { - installLauncherRuntime(env.id, state.fromVersion); - // The launcher runs `/t3 serve` and passes its environment through. - return spawnDetached(NODE, [SERVER_BIN, "__service-launcher"], { - env: { - ...environment, - T3CODE_HOME: t3Home(env.id), - T3CODE_PORT: String(env.port), - T3CODE_HOST: state.host, - }, - cwd: projectDir(env.id), - log: logPath(env.id), - }); - } - return spawnDetached( - NODE, - [ - SERVER_BIN, - "serve", - "--base-dir", - t3Home(env.id), - "--port", - String(env.port), - "--host", - state.host, - projectDir(env.id), - ], - { env: environment, cwd: projectDir(env.id), log: logPath(env.id) }, - ); -} - -async function waitForPairingUrl(env: LabEnv): Promise { - const deadline = Date.now() + 120_000; - while (Date.now() < deadline) { - const log = NodeFS.readFileSync(logPath(env.id), "utf8"); - const url = /Pairing URL: (\S+)/.exec(log)?.[1]; - if (url) return url; - if (!isLabProcess(env.pid, serverMarker(env), projectDir(env.id))) { - fail(`${env.id} exited during startup; see ${logPath(env.id)}\n${log.slice(-2000)}`); - } - await new Promise((resolve) => setTimeout(resolve, 500)); - } - fail(`${env.id} did not print a pairing URL within 120s; see ${logPath(env.id)}`); -} - -/** SIGTERM the process group we created, then SIGKILL it if it lingers. */ -async function stopGroup(pid: number, marker: string, cwd = REPO): Promise { - if (!isLabProcess(pid, marker, cwd)) return; - process.kill(-pid, "SIGTERM"); - const deadline = Date.now() + 15_000; - while (Date.now() < deadline && isLabProcess(pid, marker, cwd)) { - await new Promise((resolve) => setTimeout(resolve, 250)); - } - try { - process.kill(-pid, "SIGKILL"); - } catch { - // The group is already gone. - } -} - -// --------------------------------------------------------------------------- -// Talking to a lab server like a client does - -async function sessionToken(env: LabEnv): Promise { - const tokenPath = NodePath.join(envDir(env.id), "session-token"); - if (NodeFS.existsSync(tokenPath)) return NodeFS.readFileSync(tokenPath, "utf8").trim(); - const token = NodeChildProcess.execFileSync( - NODE, - [ - SERVER_BIN, - "auth", - "session", - "issue", - "--base-dir", - t3Home(env.id), - "--label", - "update-lab", - "--token-only", - ], - { cwd: REPO, encoding: "utf8", env: { PATH: process.env.PATH, HOME: homeDir(env.id) } }, - ).trim(); - NodeFS.writeFileSync(tokenPath, token, { mode: 0o600 }); - return token; -} - -async function authHeaders(env: LabEnv): Promise> { - return { - Authorization: `Bearer ${await sessionToken(env)}`, - "Content-Type": "application/json", - "x-t3-orchestration-protocol": ORCHESTRATION_PROTOCOL, - }; -} - -/** One unary call over the server's Effect RPC WebSocket protocol (JSON frames). */ -async function rpc(state: LabState, env: LabEnv, tag: string, payload: unknown): Promise { - const origin = originOf(state, env); - const ticketResponse = await fetch(`${origin}/api/auth/websocket-ticket`, { - method: "POST", - headers: await authHeaders(env), - }); - if (!ticketResponse.ok) fail(`${env.id}: websocket ticket failed (${ticketResponse.status})`); - const { ticket } = (await ticketResponse.json()) as { ticket: string }; - const socket = new WebSocket( - `${origin.replace("http", "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL}&wsTicket=${encodeURIComponent(ticket)}`, - ); - return await new Promise((resolve, reject) => { - let settled = false; - const timer = setTimeout( - () => finish(() => reject(new Error(`${tag} timed out`))), - 10 * 60_000, - ); - const finish = (settle: () => void) => { - if (settled) return; - settled = true; - clearTimeout(timer); - socket.close(); - settle(); - }; - socket.addEventListener("open", () => - socket.send(JSON.stringify({ _tag: "Request", id: "1", tag, payload, headers: [] })), - ); - socket.addEventListener("error", () => finish(() => reject(new Error(`${tag}: socket error`)))); - socket.addEventListener("close", () => - finish(() => reject(new Error(`${tag}: socket closed before response`))), - ); - socket.addEventListener("message", (event) => { - const decoded = JSON.parse(String(event.data)); - for (const message of Array.isArray(decoded) ? decoded : [decoded]) { - if (message._tag !== "Exit" || message.requestId !== "1") continue; - const exit = message.exit; - finish(() => - exit._tag === "Success" - ? resolve(exit.value) - : reject(new Error(`${tag} failed: ${JSON.stringify(exit.cause)}`)), - ); - } - }); - }); -} - -function printProviders(providers: ReadonlyArray): void { - for (const provider of providers) { - const advisory = provider.versionAdvisory ?? {}; - const update = provider.updateState ?? null; - console.log( - [ - ` ${provider.instanceId.padEnd(10)} ${String(provider.status).padEnd(8)}`, - `installed=${provider.version ?? "-"} latest=${advisory.latestVersion ?? "-"} advisory=${advisory.status ?? "-"}`, - `update=${advisory.canUpdate ? JSON.stringify(advisory.updateCommand) : "manual"}`, - update ? `last=${update.status}: ${update.message ?? ""}` : "", - ].join(" "), - ); - } -} - -// --------------------------------------------------------------------------- -// Commands - -function option(args: string[], name: string): string | undefined { - const index = args.indexOf(`--${name}`); - return index >= 0 ? args[index + 1] : undefined; -} - -async function up(args: string[]): Promise { - const previous = readState(); - const host = option(args, "host") ?? previous?.host ?? "127.0.0.1"; - const envCount = Number(option(args, "envs") ?? previous?.envs.length ?? 3); - const launcherCount = Number(option(args, "launcher") ?? 1); - const [major, minor, patch] = REPO_VERSION.split(/[.-]/).map(Number); - const initial: LabState = { - host, - feedPort: 0, - feedPid: null, - delayMs: 3000, - busySeconds: 120, - fromVersion: `${major}.${minor}.${Math.max(0, patch! - 1)}`, - latest: Object.fromEntries( - PROVIDER_KEYS.map((key) => [key, PROVIDERS[key].latest]), - ) as LabState["latest"], - envs: [], - }; - let state = updateState((next) => { - if ( - next.host !== host && - next.envs.some((env) => isLabProcess(env.pid, serverMarker(env), projectDir(env.id))) - ) { - fail( - "Cannot change --host while lab servers are running. Run `node scripts/update-lab.ts down` first.", - ); - } - next.host = host; - next.busySeconds = Number(option(args, "busy-seconds") ?? next.busySeconds); - next.fromVersion = option(args, "from-version") ?? next.fromVersion; - for (let index = next.envs.length; index < envCount; index++) { - const row = INSTALL_MATRIX[index % INSTALL_MATRIX.length]!; - next.envs.push({ - id: `env${index + 1}`, - port: 0, - pid: null, - launcher: index >= envCount - launcherCount, - projectId: null, - installs: Object.fromEntries( - PROVIDER_KEYS.map((key) => [ - key, - { method: row[key], version: PROVIDERS[key].installed, failing: false }, - ]), - ) as LabEnv["installs"], - }); - } - }, initial); - const taken = new Set(state.envs.map((env) => env.port).concat(state.feedPort)); - - if (!isLabProcess(state.feedPid, "__feed")) { - const feedPort = - state.feedPort > 0 && !listeningPorts().has(state.feedPort) - ? state.feedPort - : await freePort("127.0.0.1", taken); - // Running servers retain the old feed origin in their environment until restarted. - if (feedPort !== state.feedPort) { - for (const env of state.envs) { - if (env.pid) await stopGroup(env.pid, serverMarker(env), projectDir(env.id)); - state = updateState((next) => { - requireEnv(next, env.id).pid = null; - }); - } - } - const feedPid = spawnDetached(NODE, [SELF, "__feed", String(feedPort)], { - env: { PATH: process.env.PATH }, - cwd: REPO, - log: NodePath.join(LAB, "feed.log"), - }); - state = updateState((next) => { - next.feedPort = feedPort; - next.feedPid = feedPid; - }); - } - // The feed always offers this checkout's version, the update a launcher env is behind on. - if (!publishedRuntimes().includes(REPO_VERSION)) buildRuntime(REPO_VERSION, undefined); - - const started: LabEnv[] = []; - for (const env of state.envs) { - if (isLabProcess(env.pid, serverMarker(env), projectDir(env.id))) continue; - seedEnvFiles(state, env); - const port = - !env.port || listeningPorts().has(env.port) ? await freePort(host, taken) : env.port; - const pid = startServer(state, { ...env, port }); - state = updateState((next) => { - const current = requireEnv(next, env.id); - current.port = port; - current.pid = pid; - }); - started.push(requireEnv(state, env.id)); - } - console.log( - `feed http://127.0.0.1:${state.feedPort} (npm registry /npm, releases /releases)`, - ); - const urls = await Promise.all(started.map(waitForPairingUrl)); - for (const env of state.envs) { - const index = started.findIndex((candidate) => candidate.id === env.id); - console.log(`${env.id.padEnd(9)} ${originOf(state, env)} pid ${env.pid}${launcherLabel(env)}`); - console.log(` ${index >= 0 ? urls[index] : "(already running; `pair` mints a URL)"}`); - } -} - -async function down(): Promise { - const state = readState(); - if (!state) return; - for (const env of state.envs) { - if (env.pid) await stopGroup(env.pid, serverMarker(env), projectDir(env.id)); - updateState((next) => { - requireEnv(next, env.id).pid = null; - }); - } - if (state.feedPid) await stopGroup(state.feedPid, "__feed"); - updateState((next) => { - next.feedPid = null; - }); - console.log("Lab stopped."); -} - -function status(): void { - const state = requireState(); - const feed = isLabProcess(state.feedPid, "__feed") ? "up" : "down"; - console.log( - `feed http://127.0.0.1:${state.feedPort} ${feed}; delay ${state.delayMs}ms; runtimes ${publishedRuntimes().join(", ") || "-"}`, - ); - for (const env of state.envs) { - const alive = isLabProcess(env.pid, serverMarker(env), projectDir(env.id)); - console.log( - `${env.id} ${originOf(state, env)} pid ${env.pid ?? "-"} ${alive ? "up" : "down"}${launcherLabel(env)}`, - ); - for (const key of PROVIDER_KEYS) { - const install = env.installs[key]; - const behind = compareVersions(install.version, state.latest[key]) < 0 ? "behind" : "current"; - console.log( - ` ${key.padEnd(9)} ${install.method.padEnd(9)} installed ${install.version.padEnd(8)} latest ${state.latest[key].padEnd(8)} ${behind}${install.failing ? " FAILING" : ""}`, - ); - } - } -} - -async function busy(state: LabState, env: LabEnv): Promise { - const headers = await authHeaders(env); - if (!env.projectId) { - const response = await fetch(`${originOf(state, env)}/api/projects`, { headers }); - if (!response.ok) fail(`Project snapshot failed: ${response.status} ${await response.text()}`); - const snapshot = (await response.json()) as { - projects: Array<{ id: string; workspaceRoot: string; deletedAt: string | null }>; - }; - const existing = snapshot.projects.find( - (project) => project.workspaceRoot === projectDir(env.id) && project.deletedAt === null, - ); - const projectId = existing?.id ?? NodeCrypto.randomUUID(); - if (!existing) { - const created = await fetch(`${originOf(state, env)}/api/projects/mutate`, { - method: "POST", - headers, - body: JSON.stringify({ - type: "project.create", - commandId: NodeCrypto.randomUUID(), - projectId, - title: `${env.id} lab project`, - workspaceRoot: projectDir(env.id), - }), - }); - if (!created.ok) fail(`Project creation failed: ${created.status} ${await created.text()}`); - } - updateState((next) => { - requireEnv(next, env.id).projectId = projectId; - }); - env.projectId = projectId; - } - const threadId = NodeCrypto.randomUUID(); - await rpc(state, env, "orchestration.launchThread", { - commandId: NodeCrypto.randomUUID(), - threadId, - projectId: env.projectId, - title: `Busy for ${state.busySeconds}s`, - modelSelection: { instanceId: "mock-acp", model: "default" }, - runtimeMode: "full-access", - interactionMode: "default", - workspaceStrategy: { type: "root" }, - initialMessage: { text: "Stay busy so updates have to wait.", attachments: [] }, - }); - console.log(`${env.id}: thread ${threadId} runs for ~${state.busySeconds}s on mock-acp.`); -} - -async function main(argv: string[]): Promise { - const [command, ...args] = argv; - switch (command) { - case "__tool": - return runTool(args[0]!, args[1]!, args.slice(2)); - case "__feed": - return runFeed(Number(args[0])); - case "up": - return up(args); - case "down": - return down(); - case "status": - return status(); - case "pair": { - const state = requireState(); - const env = requireEnv(state, args[0]); - const output = NodeChildProcess.execFileSync( - NODE, - [ - SERVER_BIN, - "auth", - "pairing", - "create", - "--base-dir", - t3Home(env.id), - "--base-url", - originOf(state, env), - ], - { cwd: REPO, encoding: "utf8", env: { PATH: process.env.PATH, HOME: homeDir(env.id) } }, - ); - return void console.log(/(http\S+\/pair#token=\S+)/.exec(output)?.[1] ?? output); - } - case "publish": { - const provider = requireProvider(args[0]); - if (!args[1]) fail("Usage: publish "); - updateState((state) => { - state.latest[provider] = args[1]!; - }); - return void console.log(`${PROVIDERS[provider].pkg} latest is now ${args[1]}.`); - } - case "fail": { - const provider = requireProvider(args[1]); - if (args[2] !== "on" && args[2] !== "off") fail("Usage: fail on|off"); - updateState((state) => { - requireEnv(state, args[0]).installs[provider].failing = args[2] === "on"; - }); - return void console.log( - `${args[0]} ${provider} installs will ${args[2] === "on" ? "fail" : "succeed"}.`, - ); - } - case "delay": { - const ms = Number(args[0]); - if (!Number.isInteger(ms) || ms < 0) fail("Usage: delay "); - updateState((state) => { - state.delayMs = ms; - }); - return void console.log(`Installers now take ${ms}ms.`); - } - case "providers": { - const state = requireState(); - const env = requireEnv(state, args[0]); - const result = await rpc(state, env, "server.getConfig", {}); - console.log(`${env.id} ${originOf(state, env)}`); - return printProviders(result.providers); - } - case "update": { - const state = requireState(); - const env = requireEnv(state, args[0]); - const instanceId = args[1] ?? fail("Usage: update [version]"); - const driver = - PROVIDER_KEYS.map((key) => PROVIDERS[key].driver).find((d) => d === instanceId) ?? - instanceId; - const result = await rpc(state, env, "server.updateProvider", { - provider: driver, - instanceId, - ...(args[2] ? { targetVersion: args[2] } : {}), - }); - return printProviders(result.providers.filter((p: any) => p.instanceId === instanceId)); - } - case "busy": { - const state = requireState(); - return busy(state, requireEnv(state, args[0])); - } - case "self-update": { - const state = requireState(); - const env = requireEnv(state, args[0]); - if (!args[1]) fail("Usage: self-update "); - const result = await rpc(state, env, "server.updateServer", { targetVersion: args[1] }); - return void console.log(JSON.stringify(result)); - } - case "build-runtime": { - if (!args[0]) fail("Usage: build-runtime [--archive file]"); - return void console.log(`Published ${buildRuntime(args[0], option(args, "archive"))}`); - } - case "reset": { - const release = acquireLock("state"); - try { - const state = readState(); - if (state && [state.feedPid, ...state.envs.map((e) => e.pid)].some((pid) => pid !== null)) { - fail("Run `down` first."); - } - NodeFS.rmSync(LAB, { recursive: true, force: true }); - } finally { - release(); - } - return void console.log(`Removed ${LAB}.`); - } - default: - console.log(HELP); - if (command && command !== "--help" && command !== "help") process.exit(1); - } -} - -const argv = process.argv.slice(2); -const release = ["up", "down", "reset"].includes(argv[0] ?? "") ? acquireLock("processes") : null; -try { - await main(argv); -} finally { - release?.(); -} From 1f5eb23f03e7aee624ff3911f2b6824534f5cd2e Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:29:57 +0000 Subject: [PATCH 024/149] docs(updates): explain pending background work --- docs/user/updating.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user/updating.md b/docs/user/updating.md index 2c27589a717ff..15f5d48b4c193 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -75,8 +75,9 @@ update can roll back to the previous version. If the update still fails: ## Automatic updates **Settings → General → Update automatically** is on by default. Each environment -waits until no thread is running or waiting for you, no client has been used for -15 minutes, and no scheduled task or usage-limit resume is due within 5 minutes. +waits until no thread is running or waiting for you, no background work is pending, +no client has been used for 15 minutes, and no scheduled task or usage-limit resume +is due within 5 minutes. Then it installs: - provider CLI updates that have a one-click update; From f3963cc9079dc69634662cf7c4571b0b81e6dc13 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:38:56 +0000 Subject: [PATCH 025/149] docs(updates): clarify terminal wait during server restarts --- docs/user/updating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/updating.md b/docs/user/updating.md index 15f5d48b4c193..777fe42c9a0d5 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -84,7 +84,7 @@ Then it installs: - the newest T3 Code release on its channel, when it runs as a background service (`t3 service install`). It downloads first and restarts only inside that window. -Background services also wait for integrated terminal commands to finish. +T3 Code server restarts also wait for integrated terminal commands to finish. Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours or when a newer version is From fc5a32193549438e86d098fdd6b7005c90e8c9d2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:48:25 +0000 Subject: [PATCH 026/149] fix(desktop): prevent stale updates after channel changes --- .../src/updates/DesktopUpdates.test.ts | 104 +++++++++++++++++- apps/desktop/src/updates/DesktopUpdates.ts | 41 +++++++ .../desktop/src/updates/updatesTestHarness.ts | 7 +- docs/user/updating.md | 2 + 4 files changed, 152 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index ce5935fe2fb3e..3cf5361eecdfd 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -614,6 +614,7 @@ describe("DesktopUpdates", () => { it.effect("clears quitting state after an unexpected install setup failure", () => { const harness = makeHarness({ + platform: "win32", stopBackend: Effect.die(new Error("backend stop failed")), }); @@ -825,6 +826,95 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); + it.effect("disarms a previous channel's download until a matching download completes", () => { + const harness = makeHarness({ platform: "win32" }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + assert.isTrue(harness.autoInstallOnAppQuit()); + + yield* updates.setChannel("nightly"); + assert.isFalse(harness.autoInstallOnAppQuit()); + const beforeStaleDownload = yield* updates.getState; + harness.emit("update-downloaded", { version: "1.2.4" }); + // electron-updater decides whether to stage on macOS as the callback returns. + assert.isFalse(harness.autoInstallOnAppQuit()); + yield* flushCallbacks; + assert.deepEqual(yield* updates.getState, beforeStaleDownload); + assert.isFalse((yield* updates.install).accepted); + + harness.emit("update-downloaded", { version: "1.2.4-nightly.20260709.766" }); + assert.isTrue(harness.autoInstallOnAppQuit()); + yield* flushCallbacks; + assert.equal((yield* updates.getState).status, "downloaded"); + assert.isTrue((yield* updates.install).accepted); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + + it.effect( + "keeps a staged macOS download's channel even when a newer release is available", + () => { + const harness = makeHarness(); + + return Effect.scoped( + Effect.gen(function* () { + const settings = yield* DesktopAppSettings.DesktopAppSettings; + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + harness.emit("update-available", { version: "1.2.5" }); + yield* flushCallbacks; + const beforeChannelChange = yield* updates.getState; + assert.isNull(beforeChannelChange.downloadedVersion); + + const exit = yield* updates.setChannel("nightly").pipe(Effect.exit); + assert.equal(exit._tag, "Failure"); + if (exit._tag === "Failure") { + const error = Cause.squash(exit.cause); + assert.instanceOf(error, DesktopUpdates.DesktopUpdateChannelStagedError); + assert.include(error.message, "Install the downloaded update"); + } + assert.deepEqual(yield* updates.getState, beforeChannelChange); + assert.equal((yield* settings.get).updateChannel, "latest"); + assert.isTrue(harness.autoInstallOnAppQuit()); + assert.equal(harness.checkCount(), 0); + assert.deepEqual(yield* updates.setChannel("latest"), beforeChannelChange); + assert.isTrue((yield* updates.check("manual")).checked); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }, + ); + + it.effect("refuses a stale payload through both automatic and explicit install paths", () => { + const harness = makeHarness(); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + const beforeStaleDownload = yield* updates.getState; + + harness.emit("update-downloaded", { version: "1.2.5-nightly.20260710.1" }); + assert.isFalse(harness.autoInstallOnAppQuit()); + yield* flushCallbacks; + assert.deepEqual(yield* updates.getState, beforeStaleDownload); + assert.isFalse((yield* updates.install).accepted); + assert.isFalse((yield* updates.installPrepared("1.2.4")).accepted); + assert.equal(harness.quitAndInstalls(), 0); + const error = yield* updates.setChannel("nightly").pipe(Effect.flip); + assert.equal(error._tag, "DesktopUpdateChannelStagedError"); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + it.effect("does not persist an unchanged update channel as a user preference", () => { const harness = makeHarness(); @@ -896,6 +986,12 @@ describe("DesktopUpdates", () => { const channelFiber = yield* updates.setChannel("nightly").pipe(Effect.forkScoped); yield* Deferred.await(channelChangeStarted); + harness.emit("update-downloaded", { version: "1.2.4" }); + assert.isFalse(harness.autoInstallOnAppQuit()); + yield* flushCallbacks; + assert.isNull((yield* updates.getState).downloadedVersion); + assert.isFalse((yield* updates.install).accepted); + const checkResult = yield* updates.check("manual"); assert.isFalse(checkResult.checked); assert.equal(harness.checkCount(), 0); @@ -917,13 +1013,17 @@ describe("DesktopUpdates", () => { path: "/tmp/settings.json", cause: diskFailure, }); - const harness = makeHarness({ setUpdateChannelError: settingsFailure }); + const harness = makeHarness({ platform: "win32", setUpdateChannelError: settingsFailure }); return Effect.scoped( Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + const beforeChannelChange = yield* updates.getState; + const error = yield* updates.setChannel("nightly").pipe(Effect.flip); assert.instanceOf(error, DesktopUpdates.DesktopUpdateChannelPersistenceError); @@ -932,6 +1032,8 @@ describe("DesktopUpdates", () => { assert.strictEqual(error.cause.cause, diskFailure); assert.equal(error.message, "Failed to persist the nightly desktop update channel."); assert.notInclude(error.message, diskFailure.message); + assert.deepEqual(yield* updates.getState, beforeChannelChange); + assert.isTrue(harness.autoInstallOnAppQuit()); const checkResult = yield* updates.check("manual"); assert.isTrue(checkResult.checked); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 251c66ca9b71c..6c40d27d726da 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -102,6 +102,15 @@ export class DesktopUpdateChannelPersistenceError extends Schema.TaggedError()( + "DesktopUpdateChannelStagedError", + { requestedChannel: DesktopUpdateChannelSchema }, +) { + override get message(): string { + return "Install the downloaded update by quitting and reopening the app before changing channels on macOS."; + } +} + export class DesktopUpdatePollerError extends Schema.TaggedError()( "DesktopUpdatePollerError", { @@ -155,6 +164,7 @@ export type DesktopUpdateConfigureError = never; export const DesktopUpdateSetChannelError = Schema.Union([ DesktopUpdateActionInProgressError, DesktopUpdateChannelPersistenceError, + DesktopUpdateChannelStagedError, ]); export type DesktopUpdateSetChannelError = typeof DesktopUpdateSetChannelError.Type; @@ -291,6 +301,10 @@ export const make = Effect.gen(function* () { const updaterConfiguredRef = yield* Ref.make(false); const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1); const lastCheckReasonRef = yield* Ref.make(null); + // Feed checks can clear the UI's downloadedVersion without clearing the + // updater's payload. Native macOS staging cannot be cancelled afterward. + const downloadedPayloadVersionRef = yield* Ref.make(null); + const nativeUpdateStagedRef = yield* Ref.make(false); const updateStateRef = yield* Ref.make( createInitialDesktopUpdateState( environment.appVersion, @@ -618,6 +632,8 @@ export const make = Effect.gen(function* () { const activeAction = yield* Ref.get(activeUpdateActionRef); const hasExpectedDownload = state.downloadedVersion !== null && + state.downloadedVersion === (yield* Ref.get(downloadedPayloadVersionRef)) && + resolveDefaultDesktopUpdateChannel(state.downloadedVersion) === state.channel && (expectedVersion === undefined || state.downloadedVersion === expectedVersion); if ( (yield* Ref.get(desktopState.quitting)) || @@ -904,10 +920,30 @@ export const make = Effect.gen(function* () { const handleUpdateDownloaded = Effect.fn("desktop.updates.handleUpdateDownloaded")(function* ( raw: unknown, ) { + // Keep this prelude synchronous: MacUpdater checks the flag immediately + // after emitting this event, before it hands the download to Squirrel. + yield* electronUpdater.setAutoInstallOnAppQuit(false); + yield* Ref.set(downloadedPayloadVersionRef, null); yield* decodeUpdateInfo(raw).pipe( Effect.flatMap( Effect.fn("desktop.updates.applyUpdateDownloaded")(function* (info) { const state = yield* Ref.get(updateStateRef); + yield* Ref.set(downloadedPayloadVersionRef, info.version); + const action = yield* activeUpdateAction; + if ( + resolveDefaultDesktopUpdateChannel(info.version) !== state.channel || + (Option.isSome(action) && action.value === "channel") + ) { + yield* logUpdaterInfo("ignoring download outside the selected update channel", { + version: info.version, + channel: state.channel, + }); + return; + } + if (environment.platform === "darwin") { + yield* Ref.set(nativeUpdateStagedRef, true); + } + yield* electronUpdater.setAutoInstallOnAppQuit(true); yield* setState(reduceDesktopUpdateStateOnDownloadComplete(state, info.version)); yield* logUpdaterInfo("update downloaded", { version: info.version }); }), @@ -1021,6 +1057,10 @@ export const make = Effect.gen(function* () { return state; } + if (environment.platform === "darwin" && (yield* Ref.get(nativeUpdateStagedRef))) { + return yield* new DesktopUpdateChannelStagedError({ requestedChannel: nextChannel }); + } + yield* desktopSettings .setUpdateChannel(nextChannel) .pipe( @@ -1029,6 +1069,7 @@ export const make = Effect.gen(function* () { ), ); + yield* electronUpdater.setAutoInstallOnAppQuit(false); const enabled = yield* shouldEnableAutoUpdates; yield* setState(createBaseUpdateState(nextChannel, enabled, environment)); diff --git a/apps/desktop/src/updates/updatesTestHarness.ts b/apps/desktop/src/updates/updatesTestHarness.ts index 9d4556af42b47..d843552a065ec 100644 --- a/apps/desktop/src/updates/updatesTestHarness.ts +++ b/apps/desktop/src/updates/updatesTestHarness.ts @@ -45,6 +45,7 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { let downloadCount = 0; let allowDowngrade = false; let fullChangelog = false; + let autoInstallOnAppQuit = true; const feedUrls: ElectronUpdater.ElectronUpdaterFeedUrl[] = []; const listeners = new Map void>>(); const sentStates: DesktopUpdateState[] = []; @@ -73,7 +74,10 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { feedUrls.push(options); }), setAutoDownload: () => Effect.void, - setAutoInstallOnAppQuit: () => Effect.void, + setAutoInstallOnAppQuit: (value) => + Effect.sync(() => { + autoInstallOnAppQuit = value; + }), setChannel: () => Effect.void, setAllowPrerelease: () => Effect.void, allowDowngrade: Effect.sync(() => allowDowngrade), @@ -262,6 +266,7 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { downloadCount: () => downloadCount, feedUrls: (): ElectronUpdater.ElectronUpdaterFeedUrl[] => feedUrls, fullChangelog: () => fullChangelog, + autoInstallOnAppQuit: () => autoInstallOnAppQuit, listenerCount: () => Array.from(listeners.values()).reduce( (total, eventListeners) => total + eventListeners.size, diff --git a/docs/user/updating.md b/docs/user/updating.md index 59857b3e4782e..eb2cbe00f8378 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -89,6 +89,8 @@ T3 Code server restarts also wait for integrated terminal commands to finish. Desktop updates download in the background when the app detects an available release and install when you quit the app. The app stays open while you work, including when it hosts multiple backends. +On macOS, quit and reopen the app to install a downloaded update before switching +update channels. Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours or when a newer version is From ce07c8670e26f504e9555e6f6a5935894bb139cc Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:18:21 +0000 Subject: [PATCH 027/149] refactor(server): simplify background provider updates --- .../server/src/background/BackgroundPolicy.ts | 9 +- apps/server/src/cli/update.ts | 8 +- .../Adapters/ClaudeAdapterV2.test.ts | 8 - .../Adapters/ClaudeAdapterV2.ts | 2 +- .../orchestration-v2/ProjectionStore.test.ts | 7 +- .../src/orchestration-v2/ProjectionStore.ts | 28 +- .../ProviderTurnStartService.ts | 2 +- .../providerMaintenanceCommandCoordinator.ts | 26 +- .../src/provider/providerMaintenanceRunner.ts | 24 +- apps/server/src/serverSettings.test.ts | 52 ++-- apps/server/src/serverSettings.ts | 10 +- .../server/src/updates/ProviderAutoUpdater.ts | 46 +--- apps/server/src/updates/UpdateWindow.ts | 258 ++++++------------ .../src/server/maintenanceResolver.ts | 13 +- packages/shared/src/cliRelease.ts | 11 +- 15 files changed, 147 insertions(+), 357 deletions(-) diff --git a/apps/server/src/background/BackgroundPolicy.ts b/apps/server/src/background/BackgroundPolicy.ts index 207c2454feede..48701d0fcaf03 100644 --- a/apps/server/src/background/BackgroundPolicy.ts +++ b/apps/server/src/background/BackgroundPolicy.ts @@ -16,7 +16,6 @@ import * as DateTime from "effect/DateTime"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Scope from "effect/Scope"; @@ -41,8 +40,8 @@ export class BackgroundPolicy extends Context.Service< ) => Effect.Effect; readonly reportHostPowerState: (snapshot: HostPowerSnapshot) => Effect.Effect; readonly snapshot: Effect.Effect; - /** When any client last reported recent interaction, so updates can wait for quiet. */ - readonly lastClientInteractionAt: Effect.Effect>; + /** When any client last reported recent interaction (boot time until one does), so updates can wait for quiet. */ + readonly lastClientInteractionAt: Effect.Effect; readonly streamChanges: Stream.Stream; readonly subscribe: Effect.Effect< { @@ -216,7 +215,7 @@ export const make = Effect.fn("background.policy.make")(function* () { const hostPowerMonitor = yield* HostPowerMonitor.HostPowerMonitor; const serverSettings = yield* ServerSettings.ServerSettingsService; const leasesRef = yield* Ref.make(new Map()); - const lastClientInteractionAtRef = yield* Ref.make(Option.none()); + const lastClientInteractionAtRef = yield* Ref.make(yield* DateTime.now); const changes = yield* PubSub.sliding(1); const publishMutex = yield* Semaphore.make(1); @@ -271,7 +270,7 @@ export const make = Effect.fn("background.policy.make")(function* () { }; yield* Ref.update(leasesRef, (leases) => upsertClientActivityLease(leases, lease, now)); if (input.recentlyInteracted) { - yield* Ref.set(lastClientInteractionAtRef, Option.some(now)); + yield* Ref.set(lastClientInteractionAtRef, now); } yield* publishSnapshotUnlocked; }), diff --git a/apps/server/src/cli/update.ts b/apps/server/src/cli/update.ts index fa167e3bdeeb7..576158e316bc2 100644 --- a/apps/server/src/cli/update.ts +++ b/apps/server/src/cli/update.ts @@ -64,13 +64,12 @@ const RELEASE_INDEX_MAX_PAGES = 10; /** Asks GitHub for the newest published version on a channel, page by page. */ const resolveNewestVersion = Effect.fn("cli.update.resolve_newest")(function* ( channel: CliReleaseChannel, - releaseBaseUrl: string | undefined, ) { const httpClient = yield* HttpClient.HttpClient; for (let page = 1; page <= RELEASE_INDEX_MAX_PAGES; page += 1) { const body = yield* httpClient .execute( - HttpClientRequest.get(cliReleaseIndexPageUrl(page, releaseBaseUrl)).pipe( + HttpClientRequest.get(cliReleaseIndexPageUrl(page)).pipe( HttpClientRequest.setHeader("Accept", "application/vnd.github+json"), ), ) @@ -348,7 +347,6 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: { const service = yield* BootService.BootService; const currentVersion = packageJson.version; - const releaseBaseUrl = environment[CLI_RELEASE_BASE_URL_ENV]?.trim() || undefined; const channel = input.channel ?? cliReleaseChannelOf(currentVersion); if (input.requestedVersion !== undefined && !isExactServiceVersion(input.requestedVersion)) { return yield* new CliUpdateError({ @@ -359,7 +357,7 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: { progress.status("Checking for updates..."); const targetVersion = yield* ( input.requestedVersion === undefined - ? resolveNewestVersion(channel, releaseBaseUrl) + ? resolveNewestVersion(channel) : Effect.succeed(input.requestedVersion) ).pipe(Effect.ensuring(Effect.sync(progress.finish))); const targetChannel = cliReleaseChannelOf(targetVersion); @@ -493,7 +491,7 @@ const runUpdate = Effect.fn("cli.update.run")(function* (input: { httpClient, platform, arch, - releaseBaseUrl, + releaseBaseUrl: environment[CLI_RELEASE_BASE_URL_ENV]?.trim() || undefined, validate: (paths) => runner .run({ diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 437de43fb14d5..9c4e336815f76 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3198,14 +3198,6 @@ describe("ClaudeAdapterV2 background wake turns", () => { cacheCreationTokens: 0, }); } - const settledThread = harness.events.findLast( - (event) => event.type === "provider_thread.updated", - ); - assert.equal(settledThread?.type, "provider_thread.updated"); - if (settledThread?.type === "provider_thread.updated") { - assert.equal(settledThread.providerThread.status, "idle"); - assert.isEmpty(settledThread.providerThread.pendingBackgroundTasks ?? []); - } }).pipe( Effect.scoped, Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 859d7a0fe82b7..c7b1fb0d58190 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5216,7 +5216,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( input.context.input.runOrdinal, lastRunOrdinal: input.context.input.runOrdinal, pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster), - status: input.status === "completed" && roster.size > 0 ? "active" : "idle", + status: input.status === "completed" ? "active" : "idle", updatedAt: input.completedAt, }; yield* rememberProviderThread(providerThread); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index f51e2f3ecde06..6b771ecced700 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -337,7 +337,6 @@ it.effect("memory recovery selection ignores unfinished items from rolled-back r const threadId = yield* addRolledBackRecoveryCandidate("memory-recovery-candidates"); assert.notInclude(yield* projectionStore.getRecoveryThreadIds("runtime"), threadId); - assert.notInclude(yield* projectionStore.getRecoveryThreadIds("active-work"), threadId); }).pipe(Effect.provide(ProjectionStore.layerMemory)), ); @@ -347,7 +346,8 @@ it.effect("memory recovery selection includes unfinished items from missing runs const threadId = yield* addOrphanedRecoveryCandidate("memory-recovery-candidates"); assert.include(yield* projectionStore.getRecoveryThreadIds("runtime"), threadId); - assert.include(yield* projectionStore.getRecoveryThreadIds("active-work"), threadId); + // An orphaned item must not hold background updates back forever. + assert.notInclude(yield* projectionStore.getRecoveryThreadIds("active-work"), threadId); }).pipe(Effect.provide(ProjectionStore.layerMemory)), ); @@ -2420,9 +2420,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { assert.notInclude(recoveryThreadIds, rolledBackThreadId); const activeThreadIds = yield* projectionStore.getRecoveryThreadIds("active-work"); assert.include(activeThreadIds, runningThreadId); - assert.include(activeThreadIds, orphanedThreadId); + assert.notInclude(activeThreadIds, orphanedThreadId); assert.notInclude(activeThreadIds, settledThreadId); - assert.notInclude(activeThreadIds, rolledBackThreadId); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8f55f052f1062..d377738547a13 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -577,16 +577,8 @@ function needsRecovery( ["preparing", "starting", "running", "waiting"].includes(run.status) || (run.status === "queued" && run.queueHeld !== true), ) || - projection.runtimeRequests.some((request) => request.status === "pending") || projection.providerThreads.some( - (thread) => - thread.status === "active" || (thread.pendingBackgroundTasks?.length ?? 0) > 0, - ) || - projection.turnItems.some( - (item) => - ["command_execution", "dynamic_tool", "subagent"].includes(item.type) && - ["pending", "running", "waiting"].includes(item.status) && - !projection.runs.some((run) => run.id === item.runId && run.status === "rolled_back"), + (thread) => (thread.pendingBackgroundTasks?.length ?? 0) > 0, ) ); case "runtime": @@ -3628,22 +3620,10 @@ export const layer: Layer.Layer = THEN json_extract(payload_json, '$.queueHeld') IS NOT 1 ELSE 1 END UNION - SELECT thread_id FROM orchestration_v2_projection_runtime_requests - WHERE status = 'pending' - UNION SELECT thread_id FROM orchestration_v2_projection_provider_threads - WHERE status = 'active' - OR CASE WHEN json_valid(payload_json) - THEN json_array_length(payload_json, '$.pendingBackgroundTasks') > 0 - ELSE 0 END - UNION - SELECT item.thread_id FROM orchestration_v2_projection_turn_items AS item - WHERE NOT EXISTS ( - SELECT 1 FROM orchestration_v2_projection_runs AS run - WHERE run.run_id = item.run_id AND run.status = 'rolled_back' - ) - AND type IN ('command_execution', 'dynamic_tool', 'subagent') - AND status IN ('pending', 'running', 'waiting') + WHERE CASE WHEN json_valid(payload_json) + THEN json_array_length(payload_json, '$.pendingBackgroundTasks') > 0 + ELSE 0 END `; case "runtime": return sql` diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index ebd12b4776a63..eca11701dab10 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1292,4 +1292,4 @@ export const layer: Layer.Layer< ), }); }), -).pipe(Layer.provide(ProviderMaintenanceCoordinator.admissionLayer)); +); diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index 49cdae4ff995f..28cac15d4b23b 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -1,25 +1,15 @@ import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; import * as Semaphore from "effect/Semaphore"; -/** Shares background install admission with provider startup, before either touches a CLI. */ -export class ProviderMaintenanceAdmission extends Context.Service< - ProviderMaintenanceAdmission, - { - readonly withPermit: (effect: Effect.Effect) => Effect.Effect; - } ->()("t3/provider/providerMaintenanceCommandCoordinator/ProviderMaintenanceAdmission") {} - -// Keep one layer reference so the update window and turn startup share its permit. -export const admissionLayer = Layer.effect( - ProviderMaintenanceAdmission, - Effect.map(Semaphore.make(1), (permit) => - ProviderMaintenanceAdmission.of({ withPermit: permit.withPermits(1) }), - ), -); +/** One permit per process: background installs hold it, provider turn starts wait for it. */ +export class ProviderMaintenanceAdmission extends Context.Reference<{ + readonly withPermit: (effect: Effect.Effect) => Effect.Effect; +}>("t3/provider/providerMaintenanceCommandCoordinator/ProviderMaintenanceAdmission", { + defaultValue: () => ({ withPermit: Semaphore.makeUnsafe(1).withPermits(1) }), +}) {} export interface ProviderMaintenanceCommandCoordinatorShape { readonly withCommandLock: (input: { @@ -27,7 +17,6 @@ export interface ProviderMaintenanceCommandCoordinatorShape { readonly lockKey: string; readonly onQueued?: Effect.Effect; readonly onInterrupted?: Effect.Effect; - readonly admit?: ProviderMaintenanceAdmission["Service"]["withPermit"]; readonly run: Effect.Effect; }) => Effect.Effect; } @@ -61,7 +50,6 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( lockKey, onQueued, onInterrupted, - admit, run, }) => Effect.gen(function* () { @@ -71,7 +59,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( } return yield* (onQueued ?? Effect.void).pipe( - Effect.andThen(admit ? admit(locks.withLock(lockKey, run)) : locks.withLock(lockKey, run)), + Effect.andThen(locks.withLock(lockKey, run)), Effect.onInterrupt(() => onInterrupted ?? Effect.void), Effect.ensuring(releaseTarget(targetKey)), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index b874c5d897e94..ce0dc5ff7bd2a 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -27,11 +27,7 @@ import * as NodeUtil from "node:util"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; -import { - admissionLayer, - makeProviderMaintenanceCommandCoordinator, - ProviderMaintenanceAdmission, -} from "./providerMaintenanceCommandCoordinator.ts"; +import { makeProviderMaintenanceCommandCoordinator } from "./providerMaintenanceCommandCoordinator.ts"; import { enrichProviderSnapshotWithVersionAdvisory, makeTargetedProviderUpdateAction, @@ -71,8 +67,6 @@ export interface ProviderMaintenanceRunnerShape { readonly targetVersion?: string | undefined; }, ) => Effect.Effect; - /** Only for callers already holding the background update admission permit. */ - readonly updateProviderWhileAdmitted: ProviderMaintenanceRunnerShape["updateProvider"]; } export class ProviderMaintenanceRunner extends Context.Service< @@ -298,7 +292,6 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const httpClient = yield* HttpClient.HttpClient; const latestVersions = yield* ProviderLatestVersions.ProviderLatestVersions; - const admission = yield* ProviderMaintenanceAdmission; const runMaintenanceCommand = ( update: ProviderMaintenanceCommandAction, onProgress: (line: string) => Effect.Effect, @@ -388,10 +381,9 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { }), ); - const updateProvider = Effect.fn("ProviderMaintenanceRunner.updateProvider")(function* ( - target: Parameters[0], - alreadyAdmitted: boolean = false, - ) { + const updateProvider: ProviderMaintenanceRunnerShape["updateProvider"] = Effect.fn( + "ProviderMaintenanceRunner.updateProvider", + )(function* (target) { const provider = typeof target === "string" ? target : target.provider; const instanceId = typeof target === "string" @@ -603,9 +595,6 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { }), ); }), - // Admit before the installer lock: a manual update must not hold that - // lock while a background pass holding admission waits for it. - ...(alreadyAdmitted ? {} : { admit: admission.withPermit }), run: runProviderUpdate(), }) .pipe( @@ -622,10 +611,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { return ProviderMaintenanceRunner.of({ updateProvider, - updateProviderWhileAdmitted: (target) => updateProvider(target, true), }); }); -export const layer = Layer.effect(ProviderMaintenanceRunner, make()).pipe( - Layer.provide(admissionLayer), -); +export const layer = Layer.effect(ProviderMaintenanceRunner, make()); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 469adc22c87fd..f9bc173b91849 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1030,44 +1030,24 @@ it.layer(NodeServices.layer)("server settings", (it) => { }).pipe(Effect.provide(layerServerSettings())), ); - it.effect("disables automatic recovery and updates without rewriting an invalid file", () => - Effect.forEach( - [ - { - raw: '{"automaticUpdates":false,"continueThreadsAfterServerUpdate":false,"addProjectBaseDirectory":42,"providers":{"codex":{"binaryPath":"/x"}}}', - migratedProvider: { - driver: ProviderDriverKind.make("codex"), - config: { binaryPath: "/x" }, - }, - }, - { - raw: '{"automaticUpdates":true,"continueThreadsAfterServerUpdate":true,"providers":42}', - migratedProvider: undefined, - }, - { - raw: '{"automaticUpdates":false,"continueThreadsAfterServerUpdate":false', - migratedProvider: undefined, - }, - ], - ({ raw, migratedProvider }) => - Effect.gen(function* () { - const serverConfig = yield* ServerConfig.ServerConfig; - const fileSystem = yield* FileSystem.FileSystem; - const serverSettings = yield* ServerSettingsModule.ServerSettingsService; - yield* fileSystem.writeFileString(serverConfig.settingsPath, raw); + it.effect("migrates legacy providers from an invalid file without rewriting it", () => + Effect.gen(function* () { + const serverConfig = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const serverSettings = yield* ServerSettingsModule.ServerSettingsService; + const raw = '{"addProjectBaseDirectory":42,"providers":{"codex":{"binaryPath":"/x"}}}'; + yield* fileSystem.writeFileString(serverConfig.settingsPath, raw); - const settings = yield* serverSettings.getSettings; + const settings = yield* serverSettings.getSettings; - // Readable legacy provider settings still apply while the file stays for repair. - assert.deepEqual( - settings.providerInstances[ProviderInstanceId.make("codex")], - migratedProvider, - ); - assert.isFalse(settings.automaticUpdates); - assert.isFalse(settings.continueThreadsAfterServerUpdate); - assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); - }).pipe(Effect.provide(layerServerSettings())), - ), + // The readable provider settings still apply, but the file stays for the user to repair. + assert.deepEqual(settings.providerInstances[ProviderInstanceId.make("codex")], { + driver: ProviderDriverKind.make("codex"), + config: { binaryPath: "/x" }, + }); + assert.isFalse(settings.automaticUpdates); + assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); + }).pipe(Effect.provide(layerServerSettings())), ); it.effect("skips a disabled provider instance when picking the text generation fallback", () => diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index d05ddeafbc739..61cc0f29a0b3d 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -786,14 +786,10 @@ const make = Effect.gen(function* () { if (decoded._tag === "Failure" || legacySettings._tag === "Failure") { const failure = decoded._tag === "Failure" ? decoded : legacySettings; settingsFileTrusted = false; - // Unreadable persisted choices cannot authorize automatic updates or recovery. - settings = { - ...DEFAULT_SERVER_SETTINGS, - automaticUpdates: false, - continueThreadsAfterServerUpdate: false, - }; + // An unreadable file cannot authorize unattended updates. + settings = { ...DEFAULT_SERVER_SETTINGS, automaticUpdates: false }; if (failure._tag === "Failure") { - yield* Effect.logWarning("failed to parse settings.json, using safe defaults", { + yield* Effect.logWarning("failed to parse settings.json, using defaults", { path: settingsPath, issues: Cause.pretty(failure.cause), cause: failure.cause, diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index c6350733b69cf..db06bbcd95f64 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -15,6 +15,8 @@ import * as UpdateWindow from "./UpdateWindow.ts"; /** A failed install is retried for the same version only after this long. */ const RETRY_AFTER = Duration.hours(6); +const FIRST_PASS_DELAY = Duration.minutes(2); +const PASS_INTERVAL = Duration.minutes(10); function isAutoUpdatable(provider: ServerProvider): boolean { const advisory = provider.versionAdvisory; @@ -42,62 +44,40 @@ export const layer = Layer.effectDiscard( const runner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; - const timings = yield* UpdateWindow.updateTimings; // Keyed by instance and target version, so a newer release retries at once. const attempts = new Map(); const pass = Effect.gen(function* () { - const settings = yield* serverSettings.getSettings; - if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; - const nowMs = yield* Clock.currentTimeMillis; - const candidates = (yield* providers.getProviders).filter((provider) => { - if (!isAutoUpdatable(provider)) return false; - const attemptedAt = attempts.get( - `${provider.instanceId}@${provider.versionAdvisory?.latestVersion}`, - ); - return attemptedAt === undefined || nowMs - attemptedAt >= Duration.toMillis(RETRY_AFTER); - }); - for (const provider of candidates) { + for (const { instanceId } of yield* providers.getProviders) { const install = Effect.gen(function* () { - const currentSettings = yield* serverSettings.getSettings; - if (!currentSettings.automaticUpdates || !currentSettings.enableProviderUpdateChecks) - return; + const settings = yield* serverSettings.getSettings; + if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; const current = (yield* providers.getProviders).find( - (candidate) => candidate.instanceId === provider.instanceId, + (candidate) => candidate.instanceId === instanceId, ); if (!current || !isAutoUpdatable(current)) return; const targetVersion = current.versionAdvisory?.latestVersion; - const attemptKey = `${current.instanceId}@${targetVersion}`; + const attemptKey = `${instanceId}@${targetVersion}`; const nowMs = yield* Clock.currentTimeMillis; const attemptedAt = attempts.get(attemptKey); if (attemptedAt !== undefined && nowMs - attemptedAt < Duration.toMillis(RETRY_AFTER)) return; attempts.set(attemptKey, nowMs); yield* Effect.logInfo("Updating provider in the background", { - instanceId: provider.instanceId, + instanceId, fromVersion: current.version, targetVersion, }); yield* runner - .updateProviderWhileAdmitted({ - provider: current.driver, - instanceId: current.instanceId, - }) + .updateProvider({ provider: current.driver, instanceId }) .pipe( Effect.catchCause((cause) => - Effect.logWarning("Background provider update failed", { - instanceId: provider.instanceId, - cause, - }), + Effect.logWarning("Background provider update failed", { instanceId, cause }), ), ); }); // Rechecked per install: the previous one may have run for minutes. - const installed = yield* updateWindow.runIfOpen({ closesWindows: false }, install); - if (Option.isNone(installed)) { - yield* Effect.logDebug("Provider auto-update waiting for the update window"); - return; - } + if (Option.isNone(yield* updateWindow.runIfOpen(install))) return; } }).pipe( Effect.catchCause((cause) => @@ -107,8 +87,8 @@ export const layer = Layer.effectDiscard( ); yield* forkParked( - Effect.sleep(timings.firstPassDelay).pipe( - Effect.andThen(pass.pipe(Effect.repeat(Schedule.spaced(timings.passInterval)))), + Effect.sleep(FIRST_PASS_DELAY).pipe( + Effect.andThen(pass.pipe(Effect.repeat(Schedule.spaced(PASS_INTERVAL)))), ), ); }), diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 4f9051ec7602a..5c88feb1e41b5 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -1,4 +1,3 @@ -import type { ScheduledTaskId, ThreadId } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -15,69 +14,26 @@ import * as ServerSettings from "../serverSettings.ts"; /** Work due this soon keeps the window closed, so it does not start late. */ const UPDATE_LOOKAHEAD = Duration.minutes(5); - -/** Background update pacing. */ -export const updateTimings = Effect.succeed({ - /** - * How long every client must be untouched before an update may interrupt - * it. Clients report activity about every 25s, so this must stay well above that. - */ - quietPeriod: Duration.minutes(15), - /** How often a waiting update rechecks the window or looks for new versions. */ - recheck: Duration.seconds(30), - firstPassDelay: Duration.minutes(2), - passInterval: Duration.minutes(10), -}); - -export type UpdateWindowBlocker = - | { readonly type: "active-threads"; readonly threadIds: ReadonlyArray } - | { readonly type: "recent-interaction"; readonly quietAt: DateTime.Utc } - | { readonly type: "focused-window" } - | { readonly type: "provider-update" } - | { - readonly type: "scheduled-task"; - readonly taskId: ScheduledTaskId; - readonly title: string; - /** Null while the task is running. */ - readonly runsAt: DateTime.Utc | null; - } - | { - readonly type: "usage-limit-resume"; - readonly threadId: ThreadId; - readonly resumesAt: DateTime.Utc; - }; - -export interface UpdateWindowOptions { - /** The update closes the app windows (a desktop install), not just a reconnect. */ - readonly closesWindows: boolean; -} - -export interface UpdateWindowStatus { - readonly open: boolean; - readonly blockers: ReadonlyArray; -} +/** + * How long every client must be untouched before an update may interrupt it. + * Clients report activity about every 25s, so this must stay well above that. + */ +const QUIET_PERIOD = Duration.minutes(15); /** - * Decides when a background update may restart the server, replace a provider - * CLI, or relaunch the desktop app without cutting off someone's work. + * Decides when a background update may restart the server or replace a + * provider CLI without cutting off someone's work. */ export class UpdateWindow extends Context.Service< UpdateWindow, { - readonly check: (options: UpdateWindowOptions) => Effect.Effect; /** - * Runs `install` if the window is open, holding a lock every background - * install shares, so a server restart never cuts off a CLI install. + * Runs `install` if the window is open, holding the admission permit that + * provider turn starts wait for, so a CLI is never replaced as a turn starts. */ readonly runIfOpen: ( - options: UpdateWindowOptions, install: Effect.Effect, ) => Effect.Effect, E, R>; - /** Waits for the window, then runs `install` like `runIfOpen`. */ - readonly runWhenOpen: ( - options: UpdateWindowOptions, - install: Effect.Effect, - ) => Effect.Effect; } >()("t3/updates/UpdateWindow") {} @@ -87,146 +43,100 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; const serverSettings = yield* ServerSettings.ServerSettingsService; const providers = yield* ProviderRegistry.ProviderRegistry; - const timings = yield* updateTimings; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; - const check: UpdateWindow["Service"]["check"] = Effect.fn("updates.UpdateWindow.check")( - function* (options) { - const now = yield* DateTime.now; - const horizon = DateTime.addDuration(now, UPDATE_LOOKAHEAD); - const blockers: Array = []; + /** Returns what keeps the window closed; empty means open. */ + const blockers = Effect.fn("updates.UpdateWindow.blockers")(function* () { + const now = yield* DateTime.now; + const horizon = DateTime.addDuration(now, UPDATE_LOOKAHEAD); + const found: Array = []; - // A failed read keeps the window closed: guessing "idle" could cut off a turn. - const activeThreadIds = yield* projections.getRecoveryThreadIds("active-work").pipe( - Effect.tapError((cause) => - Effect.logWarning("Update window could not read threads", { cause }), - ), - Effect.option, - ); - if (Option.isNone(activeThreadIds)) { - return { open: false, blockers: [{ type: "active-threads", threadIds: [] }] }; - } - if (activeThreadIds.value.length > 0) { - blockers.push({ type: "active-threads", threadIds: activeThreadIds.value }); - } - - // Includes installs someone started by hand. - const providerUpdating = (yield* providers.getProviders).some( - (provider) => - provider.updateState?.status === "queued" || provider.updateState?.status === "running", - ); - if (providerUpdating) { - blockers.push({ type: "provider-update" }); - } + // A failed read keeps the window closed: guessing "idle" could cut off a turn. + const activeThreadIds = yield* projections.getRecoveryThreadIds("active-work").pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read threads", { cause }), + ), + Effect.option, + ); + if (Option.isNone(activeThreadIds)) return ["active-threads"]; + if (activeThreadIds.value.length > 0) found.push("active-threads"); - const lastInteraction = yield* backgroundPolicy.lastClientInteractionAt; - if (Option.isSome(lastInteraction)) { - const quietAt = DateTime.addDuration(lastInteraction.value, timings.quietPeriod); - if (DateTime.isGreaterThan(quietAt, now)) { - blockers.push({ type: "recent-interaction", quietAt }); - } - } + // Includes installs someone started by hand. + const providerUpdating = (yield* providers.getProviders).some( + (provider) => + provider.updateState?.status === "queued" || provider.updateState?.status === "running", + ); + if (providerUpdating) found.push("provider-update"); - if (options.closesWindows) { - const policy = yield* backgroundPolicy.snapshot; - const hostIdleSeconds = policy.hostPower.stale ? null : policy.hostPower.idleSeconds; - const hostAway = - hostIdleSeconds !== null && hostIdleSeconds >= Duration.toSeconds(timings.quietPeriod); - const windowFocused = policy.leases.some( - (lease) => lease.clientKind === "desktop-renderer" && lease.focused, - ); - if (windowFocused && !hostAway) { - blockers.push({ type: "focused-window" }); - } - } + const lastInteraction = yield* backgroundPolicy.lastClientInteractionAt; + if (DateTime.isGreaterThan(DateTime.addDuration(lastInteraction, QUIET_PERIOD), now)) { + found.push("recent-interaction"); + } - const tasks = yield* scheduledTasks.list().pipe( - Effect.tapError((cause) => - Effect.logWarning("Update window could not read scheduled tasks", { cause }), - ), - Effect.option, - ); - if (Option.isNone(tasks)) { - return { open: false, blockers }; + const tasks = yield* scheduledTasks.list().pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read scheduled tasks", { cause }), + ), + Effect.option, + ); + if (Option.isNone(tasks)) return [...found, "scheduled-task"]; + for (const task of tasks.value.tasks) { + if (task.lastRunStatus === "running") { + found.push("scheduled-task"); + continue; } - for (const task of tasks.value.tasks) { - if (task.lastRunStatus === "running") { - blockers.push({ - type: "scheduled-task", - taskId: task.id, - title: task.title, - runsAt: null, - }); - continue; - } - if (!task.enabled || task.nextRunAt === null) continue; - const runsAt = DateTime.make(task.nextRunAt); - if (Option.isSome(runsAt) && !DateTime.isGreaterThan(runsAt.value, horizon)) { - blockers.push({ - type: "scheduled-task", - taskId: task.id, - title: task.title, - runsAt: runsAt.value, - }); - } + if (!task.enabled || task.nextRunAt === null) continue; + const runsAt = DateTime.make(task.nextRunAt); + if (Option.isSome(runsAt) && !DateTime.isGreaterThan(runsAt.value, horizon)) { + found.push("scheduled-task"); } + } - const settings = yield* serverSettings.getSettings.pipe( + const settings = yield* serverSettings.getSettings.pipe( + Effect.tapError((cause) => + Effect.logWarning("Update window could not read settings", { cause }), + ), + Effect.option, + ); + if (Option.isNone(settings)) return [...found, "settings"]; + const limited = yield* projections + .getLimitRecoveryCandidates({ + now: horizon, + autoResume: settings.value.autoResumeLimitedThreads, + snooze: false, + }) + .pipe( Effect.tapError((cause) => - Effect.logWarning("Update window could not read settings", { cause }), + Effect.logWarning("Update window could not read usage-limit resumes", { cause }), ), Effect.option, ); - if (Option.isNone(settings) || !settings.value.automaticUpdates) { - return { open: false, blockers }; - } - const limited = yield* projections - .getLimitRecoveryCandidates({ - now: horizon, - autoResume: settings.value.autoResumeLimitedThreads, - snooze: false, - }) - .pipe( - Effect.tapError((cause) => - Effect.logWarning("Update window could not read usage-limit resumes", { cause }), - ), - Effect.option, - ); - if (Option.isNone(limited)) return { open: false, blockers }; - for (const thread of limited.value) { - if (thread.usageLimitResetAt === null || thread.usageLimitResetAt === undefined) continue; - const resumesAt = DateTime.make(thread.usageLimitResetAt); - if (Option.isSome(resumesAt) && !DateTime.isGreaterThan(resumesAt.value, horizon)) { - blockers.push({ - type: "usage-limit-resume", - threadId: thread.id, - resumesAt: resumesAt.value, - }); - } + if (Option.isNone(limited)) return [...found, "usage-limit-resume"]; + for (const thread of limited.value) { + if (thread.usageLimitResetAt === null || thread.usageLimitResetAt === undefined) continue; + const resumesAt = DateTime.make(thread.usageLimitResetAt); + if (Option.isSome(resumesAt) && !DateTime.isGreaterThan(resumesAt.value, horizon)) { + found.push("usage-limit-resume"); } + } - return { open: blockers.length === 0, blockers }; - }, - ); + return found; + }); - const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (options, install) => + const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (install) => admission.withPermit( - check(options).pipe( - Effect.flatMap((status) => (status.open ? Effect.asSome(install) : Effect.succeedNone)), + blockers().pipe( + Effect.flatMap((found) => + found.length === 0 + ? Effect.asSome(install) + : Effect.logDebug("Background update waiting for the update window", { + blockers: [...new Set(found)], + }).pipe(Effect.as(Option.none())), + ), ), ); - const runWhenOpen: UpdateWindow["Service"]["runWhenOpen"] = (options, install) => - Effect.gen(function* () { - while (true) { - const result = yield* runIfOpen(options, install); - if (Option.isSome(result)) return result.value; - yield* Effect.sleep(timings.recheck); - } - }); - return UpdateWindow.of({ check, runIfOpen, runWhenOpen }); + return UpdateWindow.of({ runIfOpen }); }); -export const layer = Layer.effect(UpdateWindow, make()).pipe( - Layer.provide(ProviderMaintenanceCoordinator.admissionLayer), -); +export const layer = Layer.effect(UpdateWindow, make()); diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index 8fcee193960d7..fb985bd2011af 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -740,21 +740,10 @@ export function createProviderVersionAdvisory(input: { }; } -const DEFAULT_NPM_REGISTRY_URL = "https://registry.npmjs.org"; - -/** Respect npm's registry override when checking mirrors for newer provider versions. */ -const NpmRegistryUrlConfig = Config.String("npm_config_registry").pipe( - Config.withDefault(DEFAULT_NPM_REGISTRY_URL), - Config.map((url) => url.trim().replace(/\/+$/, "") || DEFAULT_NPM_REGISTRY_URL), -); - const fetchNpmLatestVersion = Effect.fn("fetchNpmLatestVersion")(function* (packageName: string) { const client = yield* HttpClient.HttpClient; - const registryUrl = yield* NpmRegistryUrlConfig.pipe( - Effect.orElseSucceed(() => DEFAULT_NPM_REGISTRY_URL), - ); const request = HttpClientRequest.get( - `${registryUrl}/${encodeURIComponent(packageName)}/latest`, + `https://registry.npmjs.org/${encodeURIComponent(packageName)}/latest`, ).pipe(HttpClientRequest.setHeader("accept", "application/json")); const response = yield* client.execute(request).pipe( Effect.timeoutOption(LATEST_VERSION_TIMEOUT_MS), diff --git a/packages/shared/src/cliRelease.ts b/packages/shared/src/cliRelease.ts index 69d0daddb4fc0..28f0d530bb297 100644 --- a/packages/shared/src/cliRelease.ts +++ b/packages/shared/src/cliRelease.ts @@ -96,16 +96,9 @@ export function cliReleaseChannelOf(version: string): CliReleaseChannel { * One page of GitHub's list-releases endpoint, newest first. Callers walk pages * until a channel match turns up; a busy nightly train can push the newest * preview or stable release past any single page. - * - * A mirror set through `T3CODE_RELEASE_BASE_URL` serves the same JSON (an - * array of `{ tag_name, draft? }`, newest first, empty past the last page) at - * `/index.json?per_page=100&page=N`. */ -export function cliReleaseIndexPageUrl(page: number, baseUrl?: string): string { - const mirror = baseUrl?.trim().replace(/\/+$/, ""); - return mirror - ? `${mirror}/index.json?per_page=100&page=${page}` - : `https://api.github.com/repos/${CLI_RELEASE_REPOSITORY}/releases?per_page=100&page=${page}`; +export function cliReleaseIndexPageUrl(page: number): string { + return `https://api.github.com/repos/${CLI_RELEASE_REPOSITORY}/releases?per_page=100&page=${page}`; } /** From 8429186d492a48a268dfc875c84d47db7526e768 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:20:39 +0000 Subject: [PATCH 028/149] refactor(server): simplify background server updates --- apps/server/src/cloud/selfUpdate.ts | 18 +- apps/server/src/terminal/Manager.test.ts | 181 ------------------- apps/server/src/terminal/Manager.ts | 145 ++++----------- apps/server/src/updates/ServerAutoUpdater.ts | 33 ++-- 4 files changed, 51 insertions(+), 326 deletions(-) diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index a6d8c70740cc2..0e73689b98d33 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -15,7 +15,6 @@ import * as Effect from "effect/Effect"; import * as HashSet from "effect/HashSet"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; -import * as Semaphore from "effect/Semaphore"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -62,7 +61,7 @@ export class ServerSelfUpdate extends Context.Service< ) => Effect.Effect; /** How this server can update itself, or null when it cannot. */ readonly capability: ServerSelfUpdateCapability | null; - /** Downloads and preflights a boot-service runtime without restarting. */ + /** Downloads and preflights a runtime without restarting. Only for `capability === "boot-service"`. */ readonly stage: ( targetVersion: string, ) => Effect.Effect< @@ -196,8 +195,6 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), ); const inFlight = yield* Ref.make(false); - // A background stage and a manual update may target the same version. - const stageLock = yield* Semaphore.make(1); const capability: ServerSelfUpdateCapability | null = serverConfig.mode === "desktop" ? "desktop-managed" : launcher.managed ? "boot-service" : null; @@ -210,11 +207,6 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { update can stage first and restart later. */ const stageRuntime = Effect.fn("cloud.server_self_update.stage")(function* ( targetVersion: string, - ) { - return yield* stageLock.withPermits(1)(stageRuntimeUnlocked(targetVersion)); - }); - const stageRuntimeUnlocked = Effect.fn("cloud.server_self_update.stageUnlocked")(function* ( - targetVersion: string, ) { let orchestrationProtocol: number | undefined; const paths = yield* ensurePinnedRuntimeInstalled({ @@ -371,11 +363,9 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { desktopAppUpdate.commit(requestId, onHandoffAccepted), capability, stage: (targetVersion) => - capability === "boot-service" - ? stageRuntime(targetVersion).pipe( - Effect.map(({ orchestrationProtocol }) => ({ orchestrationProtocol })), - ) - : Effect.fail(failWith("Only background service installs can stage an update.")), + stageRuntime(targetVersion).pipe( + Effect.map(({ orchestrationProtocol }) => ({ orchestrationProtocol })), + ), }); }); diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index e5483e9ad78e7..5a40f7bbb9334 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -227,7 +227,6 @@ interface CreateManagerOptions { readonly hasRunningSubprocess: boolean; readonly childCommand: string | null; readonly processIds: ReadonlyArray; - readonly terminalCommand?: string | null; }>; processTable?: Effect.Effect< ReadonlyArray<{ readonly pid: number; readonly ppid: number; readonly name: string }>, @@ -1280,7 +1279,6 @@ it.layer( Effect.gen(function* () { // FakePtyAdapter assigns pids from 9000 in open order. const { manager, ptyAdapter } = yield* createManager(5, { - shellResolver: () => "/bin/zsh", processTable: Effect.succeed([ { pid: 9000, ppid: 1, name: "zsh" }, // An async prompt worker: a copy of the shell with no children. @@ -1299,15 +1297,7 @@ it.layer( yield* manager.open(openInput({ terminalId: "subshell" })); yield* manager.open(openInput({ threadId: "thread-2" })); - assert.equal(yield* manager.hasBusyTerminals, true); - expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([ - false, - false, - false, - false, - ]); yield* manager.closeIdle({ threadId: "thread-1" }); - assert.equal(yield* manager.hasBusyTerminals, true); expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([ true, @@ -1352,177 +1342,6 @@ it.layer( }), ); - it.effect("checks fresh work across threads without changing idle shells or metadata", () => - Effect.gen(function* () { - let snapshotCalls = 0; - const shells = [ - { pid: 9000, ppid: 1, name: "zsh" }, - { pid: 100, ppid: 9000, name: "zsh" }, - { pid: 9001, ppid: 1, name: "zsh" }, - ]; - let processes = shells; - const { manager, ptyAdapter, getEvents } = yield* createManager(5, { - shellResolver: () => "/bin/zsh", - subprocessPollIntervalMs: 60_000, - processTable: Effect.sync(() => { - snapshotCalls += 1; - return processes; - }), - }).pipe(Effect.provide(layerWithHostPlatform("linux"))); - - assert.equal(yield* manager.hasBusyTerminals, false); - assert.equal(snapshotCalls, 0); - yield* manager.open(openInput()); - yield* manager.open(openInput({ threadId: "thread-2" })); - const events = yield* getEvents; - assert.equal(yield* manager.hasBusyTerminals, false); - assert.equal(snapshotCalls, 1); - - processes = [...shells, { pid: 200, ppid: 9001, name: "node" }]; - assert.equal(yield* manager.hasBusyTerminals, true); - processes = [ - ...shells, - { pid: 200, ppid: 9001, name: "zsh" }, - { pid: 201, ppid: 200, name: "sleep" }, - ]; - assert.equal(yield* manager.hasBusyTerminals, true); - // `exec` replaces the shell at its PID and leaves no child to inspect. - processes = shells.map((entry) => - entry.pid === 9001 ? { ...entry, name: "/usr/bin/sleep" } : entry, - ); - assert.equal(yield* manager.hasBusyTerminals, true); - processes = shells.map((entry) => (entry.pid === 9001 ? { ...entry, name: "node" } : entry)); - assert.equal(yield* manager.hasBusyTerminals, true); - processes = shells.map((entry) => (entry.pid === 9001 ? { ...entry, name: "" } : entry)); - assert.equal(yield* manager.hasBusyTerminals, true); - processes = shells; - assert.equal(yield* manager.hasBusyTerminals, false); - expect(yield* getEvents).toEqual(events); - expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([false, false]); - }), - ); - - it.effect("treats failed, truncated, timed out, and missing-shell inspections as busy", () => - Effect.gen(function* () { - let stdout = "9000 1 zsh"; - let code = 0; - let timedOut = false; - let stdoutTruncated = false; - const processRunner: ProcessRunner.ProcessRunner["Service"] = { - run: () => - Effect.sync(() => ({ - stdout, - stderr: "", - code: ChildProcessSpawner.ExitCode(code), - timedOut, - stdoutTruncated, - stderrTruncated: false, - stdoutInvalidUtf8: false, - stderrInvalidUtf8: false, - })), - }; - const { manager, ptyAdapter } = yield* createManager(5, { - shellResolver: () => "/bin/zsh", - subprocessPollIntervalMs: 60_000, - }).pipe( - Effect.provideService(ProcessRunner.ProcessRunner, processRunner), - Effect.provide(layerWithHostPlatform("linux")), - ); - yield* manager.open(openInput()); - assert.equal(yield* manager.hasBusyTerminals, false); - code = 1; - assert.equal(yield* manager.hasBusyTerminals, true); - code = 0; - stdoutTruncated = true; - assert.equal(yield* manager.hasBusyTerminals, true); - stdoutTruncated = false; - timedOut = true; - assert.equal(yield* manager.hasBusyTerminals, true); - timedOut = false; - stdout = ""; - assert.equal(yield* manager.hasBusyTerminals, true); - stdout = "9000 1 zsh"; - assert.equal(yield* manager.hasBusyTerminals, false); - expect(ptyAdapter.processes[0]!.killed).toBe(false); - }), - ); - - it.effect("treats input, output, and new terminals during inspection as busy", () => - Effect.gen(function* () { - let duringCheck: Effect.Effect = Effect.void; - const { manager, ptyAdapter } = yield* createManager(5, { - shellResolver: () => "/bin/zsh", - subprocessPollIntervalMs: 60_000, - subprocessInspector: () => - duringCheck.pipe( - Effect.as({ - hasRunningSubprocess: false, - childCommand: null, - processIds: [], - terminalCommand: "zsh", - }), - ), - }); - yield* manager.open(openInput()); - assert.equal(yield* manager.hasBusyTerminals, false); - duringCheck = manager - .write({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, data: "make build\r" }) - .pipe(Effect.orDie); - assert.equal(yield* manager.hasBusyTerminals, true); - duringCheck = Effect.sync(() => ptyAdapter.processes[0]!.emitData("make build\r\n")); - assert.equal(yield* manager.hasBusyTerminals, true); - duringCheck = manager - .open(openInput({ threadId: "thread-2" })) - .pipe(Effect.asVoid, Effect.orDie); - assert.equal(yield* manager.hasBusyTerminals, true); - expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([false, false]); - }), - ); - - it.effect("holds terminal launches and writes behind a server restart permit", () => - Effect.gen(function* () { - const { manager, ptyAdapter } = yield* createManager(5, { - shellResolver: () => "/bin/zsh", - subprocessPollIntervalMs: 60_000, - subprocessInspector: () => - Effect.succeed({ - hasRunningSubprocess: false, - childCommand: null, - processIds: [], - terminalCommand: "zsh", - }), - }); - yield* manager.open(openInput()); - const held = yield* Deferred.make(); - const release = yield* Deferred.make(); - const restartWindow = yield* manager - .withRestartPermit( - Deferred.succeed(held, undefined).pipe(Effect.andThen(Deferred.await(release))), - ) - .pipe(Effect.forkScoped); - yield* Deferred.await(held); - assert.equal(yield* manager.hasBusyTerminals, false); - - const write = yield* manager - .write({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, data: "make build\r" }) - .pipe(Effect.forkScoped); - const open = yield* manager.open(openInput({ threadId: "thread-2" })).pipe(Effect.forkScoped); - const restart = yield* manager.restart(restartInput()).pipe(Effect.forkScoped); - yield* Effect.yieldNow; - expect(ptyAdapter.spawnInputs).toHaveLength(1); - expect(ptyAdapter.processes[0]!.writes).toEqual([]); - expect(ptyAdapter.processes[0]!.killed).toBe(false); - - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(restartWindow); - yield* Fiber.join(write); - yield* Fiber.join(open); - yield* Fiber.join(restart); - expect(ptyAdapter.spawnInputs).toHaveLength(3); - expect(ptyAdapter.processes.flatMap((process) => process.writes)).toEqual(["make build\r"]); - }), - ); - it.effect("backs off the spawned fallback when the resource monitor snapshot fails", () => Effect.gen(function* () { const fallbackCalls: Array = []; diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index bc6137878deae..82b820ec2e645 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -60,7 +60,6 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; -import * as Semaphore from "effect/Semaphore"; import * as SynchronizedRef from "effect/SynchronizedRef"; import * as ServerConfig from "../config.ts"; @@ -220,15 +219,9 @@ export class TerminalManager extends Context.Service< readonly terminalId?: string; }) => Effect.Effect; - /** Inspect current terminal work without changing sessions. Unknown activity is busy. */ + /** True while a terminal starts or the last subprocess poll saw a command running in one. */ readonly hasBusyTerminals: Effect.Effect; - /** - * Hold terminal launches and input while checking idle state and handing off a server restart. - * Read-only activity checks stay available; do not call open, restart, or write inside this permit. - */ - readonly withRestartPermit: (effect: Effect.Effect) => Effect.Effect; - /** * Subscribe to terminal runtime events with a direct callback. * @@ -253,8 +246,6 @@ interface TerminalSubprocessInspectResult { readonly hasRunningSubprocess: boolean; readonly childCommand: string | null; readonly processIds: ReadonlyArray; - /** The normalized command currently running at the terminal's root PID. */ - readonly terminalCommand?: string | null; } interface TerminalSubprocessInspector { @@ -318,8 +309,6 @@ interface TerminalSessionState { hasRunningSubprocess: boolean; /** Normalized child command name when `hasRunningSubprocess`; cleared when idle. */ childCommandLabel: string | null; - /** The shell originally spawned, so `exec` at its PID still counts as work. */ - launchedShellCommand?: string | null; runtimeEnv: Record | null; } @@ -732,7 +721,6 @@ function deriveSubprocessInspectResult( const commandName = (pid: number) => normalizeChildCommandName(snapshot.commandById.get(pid) ?? "", platform); const shellName = commandName(terminalPid); - const terminalCommand = platform === "win32" ? (shellName?.toLowerCase() ?? null) : shellName; // Async prompt themes fork the shell into a helper that waits with no // children of its own. That copy is not a command the user started. const childPid = (snapshot.childrenByParent.get(terminalPid) ?? []).find( @@ -742,12 +730,7 @@ function deriveSubprocessInspectResult( (snapshot.childrenByParent.get(pid)?.length ?? 0) > 0, ); if (childPid === undefined) { - return { - hasRunningSubprocess: false, - childCommand: null, - processIds: [], - terminalCommand, - }; + return { hasRunningSubprocess: false, childCommand: null, processIds: [] }; } const processIds = new Set([terminalPid]); const pending = [terminalPid]; @@ -765,7 +748,6 @@ function deriveSubprocessInspectResult( hasRunningSubprocess: true, childCommand: normalized ? truncateTerminalWireLabel(normalized) : null, processIds: [...processIds], - terminalCommand, }; } @@ -1598,8 +1580,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func killFibers: new Map(), }); const threadLocks = yield* KeyedLock.make(); - const terminalAdmission = yield* Semaphore.make(1); - const withRestartPermit = terminalAdmission.withPermits(1); const terminalEventListeners = new Set<(event: TerminalEvent) => Effect.Effect>(); const workerScope = yield* Scope.make("sequential"); yield* Effect.addFinalizer(() => Scope.close(workerScope, Exit.void)); @@ -2214,12 +2194,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); if (attempt._tag === "Success") { - const shellCommand = normalizeChildCommandName( - basenameForPlatform(candidate.shell, platform), - platform, - ); - session.launchedShellCommand = - platform === "win32" ? (shellCommand?.toLowerCase() ?? null) : shellCommand; return { process: attempt.success, shellLabel: formatShellCandidate(candidate), @@ -2731,11 +2705,9 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }); const openLocked = (input: TerminalOpenInput) => - withRestartPermit( - withWorkspaceLease( - path.resolve(input.worktreePath ?? input.cwd), - openWithWorkspaceLease(input), - ), + withWorkspaceLease( + path.resolve(input.worktreePath ?? input.cwd), + openWithWorkspaceLease(input), ); const open: TerminalManager["Service"]["open"] = (input) => @@ -2982,34 +2954,29 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); }; - const writeUnlocked: TerminalManager["Service"]["write"] = Effect.fn("terminal.write")( - function* (input) { - const terminalId = input.terminalId; - const session = yield* requireSession(input.threadId, terminalId); - const process = session.process; - if (!process || session.status !== "running") { - if (session.status === "exited") return; - return yield* new TerminalNotRunningError({ + const write: TerminalManager["Service"]["write"] = Effect.fn("terminal.write")(function* (input) { + const terminalId = input.terminalId; + const session = yield* requireSession(input.threadId, terminalId); + const process = session.process; + if (!process || session.status !== "running") { + if (session.status === "exited") return; + return yield* new TerminalNotRunningError({ + threadId: input.threadId, + terminalId, + }); + } + session.inputCount += 1; + yield* Effect.try({ + try: () => process.write(input.data), + catch: (cause) => + new TerminalWriteError({ threadId: input.threadId, terminalId, - }); - } - session.inputCount += 1; - yield* Effect.try({ - try: () => process.write(input.data), - catch: (cause) => - new TerminalWriteError({ - threadId: input.threadId, - terminalId, - terminalPid: process.pid, - cause, - }), - }); - }, - ); - - const write: TerminalManager["Service"]["write"] = (input) => - withRestartPermit(writeUnlocked(input)); + terminalPid: process.pid, + cause, + }), + }); + }); const resizeLocked = Effect.fn("terminal.resize")(function* (input: TerminalResizeInput) { const session = yield* getSession(input.threadId, input.terminalId); @@ -3140,7 +3107,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func restartResolved(resolved), ), ), - withRestartPermit, ), ); @@ -3212,58 +3178,12 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ), ); - const hasBusyTerminals: TerminalManager["Service"]["hasBusyTerminals"] = Effect.gen(function* () { - const state = yield* readManagerState; - const running = new Map( - [...state.sessions] - .filter(([, session]) => session.status === "starting" || session.status === "running") - .map( - ([key, session]) => - [ - key, - { - status: session.status, - pid: session.pid, - process: session.process, - launchedShellCommand: session.launchedShellCommand, - activityMark: session.eventSequence + session.inputCount, - }, - ] as const, - ), - ); - if (running.size === 0) return false; - if ([...running.values()].some((session) => session.status === "starting")) return true; - const { inspector } = yield* acquireSubprocessInspector; - for (const session of running.values()) { - if (session.pid === null || !Number.isInteger(session.pid) || session.pid <= 0) return true; - const result = yield* inspector(session.pid); - if ( - result.hasRunningSubprocess || - !result.terminalCommand || - !session.launchedShellCommand || - result.terminalCommand !== session.launchedShellCommand - ) { - return true; - } - } - // Input, output, or a new process can arrive after the shared snapshot. - const latest = yield* readManagerState; - return [...latest.sessions].some(([key, session]) => { - if (session.status === "starting") return true; - if (session.status !== "running") return false; - const inspected = running.get(key); - return ( - inspected === undefined || - inspected.pid !== session.pid || - inspected.process !== session.process || - inspected.activityMark !== session.eventSequence + session.inputCount || - session.pendingProcessEvents.length > session.pendingProcessEventIndex - ); - }); - }).pipe( - Effect.catch((error) => - Effect.logWarning("failed to inspect busy terminals", { error: error.message }).pipe( - Effect.as(true), + const hasBusyTerminals: TerminalManager["Service"]["hasBusyTerminals"] = readManagerState.pipe( + Effect.map((state) => + [...state.sessions.values()].some( + (session) => + session.status === "starting" || + (session.status === "running" && session.hasRunningSubprocess), ), ), ); @@ -3279,7 +3199,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func close, closeIdle, hasBusyTerminals, - withRestartPermit, subscribe, subscribeMetadata, }); diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 8c367e699d1a2..b8f84df9a9ae8 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -1,11 +1,9 @@ import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; -import { CLI_RELEASE_BASE_URL_ENV, cliReleaseChannelOf } from "@t3tools/shared/cliRelease"; +import { cliReleaseChannelOf } from "@t3tools/shared/cliRelease"; import * as Clock from "effect/Clock"; -import * as Config from "effect/Config"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import { HttpClient } from "effect/http"; @@ -19,6 +17,12 @@ import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as UpdateWindow from "./UpdateWindow.ts"; +const FIRST_PASS_DELAY = Duration.minutes(2); +/** How often a staged update rechecks the update window. */ +const RECHECK = Duration.seconds(30); +/** T3 releases are checked this often; the desktop app polls its own feed. */ +const RELEASE_CHECK_INTERVAL = Duration.hours(1); + /** * Updates this environment in the background when `automaticUpdates` is on. * @@ -37,12 +41,7 @@ export const layer = Layer.effectDiscard( const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; - const timings = yield* UpdateWindow.updateTimings; const httpClient = yield* HttpClient.HttpClient; - // Same mirror the staged download uses, so a mirror lists what it serves. - const releaseBaseUrl = Option.getOrUndefined( - yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), - ); const currentVersion = packageJson.version; let pendingTarget: string | undefined; let nextReleaseCheckAt = 0; @@ -61,18 +60,16 @@ export const layer = Layer.effectDiscard( if (!(yield* automaticUpdatesEnabled)) return; const nowMs = yield* Clock.currentTimeMillis; if (nowMs >= nextReleaseCheckAt) { - nextReleaseCheckAt = nowMs + Duration.toMillis(timings.releaseCheckInterval); - const targetVersion = yield* resolveNewestVersion( - cliReleaseChannelOf(currentVersion), - releaseBaseUrl, - ).pipe(Effect.provideService(HttpClient.HttpClient, httpClient)); + nextReleaseCheckAt = nowMs + Duration.toMillis(RELEASE_CHECK_INTERVAL); + const targetVersion = yield* resolveNewestVersion(cliReleaseChannelOf(currentVersion)).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + ); if ( compareExactServiceVersions(targetVersion, currentVersion) > 0 && !skippedTargets.has(targetVersion) && targetVersion !== pendingTarget ) { - // A newer release replaces the pending target even while the window stays closed. - pendingTarget = undefined; + // A newer release replaces the pending target only once it stages. yield* Effect.logInfo("Staging a background server update", { targetVersion }); const staged = yield* selfUpdate .stage(targetVersion) @@ -94,7 +91,6 @@ export const layer = Layer.effectDiscard( const targetVersion = pendingTarget; if (targetVersion === undefined) return; yield* updateWindow.runIfOpen( - { closesWindows: false, restartsServer: true }, Effect.gen(function* () { // The setting may have been turned off while waiting. if (!(yield* automaticUpdatesEnabled)) return; @@ -106,6 +102,7 @@ export const layer = Layer.effectDiscard( // starts behind the install permit until this runtime is shut down. return yield* Effect.never; }), + { restartsServer: true }, ); }).pipe( Effect.catchCause((cause) => Effect.logWarning("Background server update failed", { cause })), @@ -113,8 +110,8 @@ export const layer = Layer.effectDiscard( ); yield* forkParked( - Effect.sleep(timings.firstPassDelay).pipe( - Effect.andThen(updateBootService.pipe(Effect.repeat(Schedule.spaced(timings.recheck)))), + Effect.sleep(FIRST_PASS_DELAY).pipe( + Effect.andThen(updateBootService.pipe(Effect.repeat(Schedule.spaced(RECHECK)))), ), ); }), From e67c3f8bb19859521a835ed4be9eb12507dc775c Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:21:03 +0000 Subject: [PATCH 029/149] refactor(desktop): simplify background update downloads --- apps/desktop/src/app/DesktopLifecycle.ts | 13 +- .../src/ipc/methods/localEnvironment.test.ts | 2 + apps/desktop/src/ipc/methods/wsl.test.ts | 2 + .../src/updates/DesktopUpdates.test.ts | 122 ++++++++---------- apps/desktop/src/updates/DesktopUpdates.ts | 65 +++------- docs/user/updating.md | 3 +- 6 files changed, 93 insertions(+), 114 deletions(-) diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.ts index 924341781aa6b..c84b14f01032c 100644 --- a/apps/desktop/src/app/DesktopLifecycle.ts +++ b/apps/desktop/src/app/DesktopLifecycle.ts @@ -12,6 +12,7 @@ import { makeComponentLogger } from "./DesktopObservability.ts"; import * as DesktopShutdown from "./DesktopShutdown.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronTheme from "../electron/ElectronTheme.ts"; +import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as DesktopState from "./DesktopState.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; @@ -41,14 +42,18 @@ type DesktopLifecycleRegistrationServices = | ElectronWindow.ElectronWindow; /** - * @effect-expect-leaking DesktopEnvironment | DesktopShutdown | DesktopState | DesktopWindow | ElectronApp | ElectronTheme | ElectronWindow + * @effect-expect-leaking DesktopEnvironment | DesktopShutdown | DesktopState | DesktopWindow | ElectronApp | ElectronTheme | ElectronUpdater | ElectronWindow */ export class DesktopLifecycle extends Context.Service< DesktopLifecycle, { readonly relaunch: ( reason: string, - ) => Effect.Effect; + ) => Effect.Effect< + void, + never, + DesktopLifecycleRuntimeServices | ElectronUpdater.ElectronUpdater + >; readonly register: Effect.Effect< void, never, @@ -165,6 +170,7 @@ export const make = DesktopLifecycle.of({ const electronApp = yield* ElectronApp.ElectronApp; const environment = yield* DesktopEnvironment.DesktopEnvironment; const state = yield* DesktopState.DesktopState; + const electronUpdater = yield* ElectronUpdater.ElectronUpdater; yield* logLifecycleInfo("desktop relaunch requested", { reason }); yield* Effect.gen(function* () { yield* Effect.yieldNow; @@ -174,6 +180,9 @@ export const make = DesktopLifecycle.of({ yield* electronApp.exit(75); return; } + // app.exit still emits "quit", where electron-updater would install a + // downloaded update under the relaunched app. Keep it for a real quit. + yield* electronUpdater.setAutoInstallOnAppQuit(false); yield* electronApp.relaunch({ execPath: process.execPath, args: process.argv.slice(1), diff --git a/apps/desktop/src/ipc/methods/localEnvironment.test.ts b/apps/desktop/src/ipc/methods/localEnvironment.test.ts index c6376554f99af..c1cfc1a8940ce 100644 --- a/apps/desktop/src/ipc/methods/localEnvironment.test.ts +++ b/apps/desktop/src/ipc/methods/localEnvironment.test.ts @@ -8,6 +8,7 @@ import * as DesktopShutdown from "../../app/DesktopShutdown.ts"; import * as DesktopState from "../../app/DesktopState.ts"; import * as ElectronApp from "../../electron/ElectronApp.ts"; import * as ElectronTheme from "../../electron/ElectronTheme.ts"; +import * as ElectronUpdater from "../../electron/ElectronUpdater.ts"; import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopWindow from "../../window/DesktopWindow.ts"; import { getLocalEnvironmentEnabled, setLocalEnvironmentEnabled } from "./localEnvironment.ts"; @@ -26,6 +27,7 @@ const layerUnusedLifecycleRuntime = Layer.mergeAll( Layer.mock(DesktopWindow.DesktopWindow, {}), Layer.mock(ElectronApp.ElectronApp, {}), Layer.mock(ElectronTheme.ElectronTheme, {}), + Layer.mock(ElectronUpdater.ElectronUpdater, {}), ); describe("local environment IPC", () => { diff --git a/apps/desktop/src/ipc/methods/wsl.test.ts b/apps/desktop/src/ipc/methods/wsl.test.ts index 644597eaf451d..e3c0086b5e6c6 100644 --- a/apps/desktop/src/ipc/methods/wsl.test.ts +++ b/apps/desktop/src/ipc/methods/wsl.test.ts @@ -11,6 +11,7 @@ import * as DesktopState from "../../app/DesktopState.ts"; import * as ElectronApp from "../../electron/ElectronApp.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; import * as ElectronTheme from "../../electron/ElectronTheme.ts"; +import * as ElectronUpdater from "../../electron/ElectronUpdater.ts"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts"; @@ -72,6 +73,7 @@ const layerUnusedLifecycleRuntime = Layer.mergeAll( ElectronTheme.ElectronTheme, ElectronTheme.ElectronTheme.of({} as ElectronTheme.ElectronTheme["Service"]), ), + Layer.mock(ElectronUpdater.ElectronUpdater, {}), Layer.succeed( ElectronDialog.ElectronDialog, ElectronDialog.ElectronDialog.of({} as ElectronDialog.ElectronDialog["Service"]), diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index 3cf5361eecdfd..d193453f9f0eb 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -205,7 +205,7 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect("downloads newer releases found after an update has been downloaded", () => { + it.effect("checks for newer releases after an update has been downloaded", () => { const harness = makeHarness(); return Effect.scoped( @@ -241,74 +241,70 @@ describe("DesktopUpdates", () => { harness.emit("update-available", { version: "1.2.5" }); yield* flushCallbacks; - // A release the background poller finds starts downloading on its own. const state = yield* updates.getState; - assert.equal(state.status, "downloading"); + assert.equal(state.status, "available"); assert.equal(state.availableVersion, "1.2.5"); assert.isNull(state.downloadedVersion); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect("keeps slow background downloads active after waiting for a check", () => - Effect.gen(function* () { - const checkStarted = yield* Deferred.make(); - const releaseCheck = yield* Deferred.make(); - const downloadStarted = yield* Deferred.make(); - const releaseDownload = yield* Deferred.make(); - const harness = makeHarness({ - checkForUpdates: Deferred.succeed(checkStarted, undefined).pipe( - Effect.andThen(Deferred.await(releaseCheck)), - ), - downloadUpdate: Deferred.succeed(downloadStarted, undefined).pipe( - Effect.andThen(Deferred.await(releaseDownload)), - ), - }); + it.effect("downloads releases the pollers find and leaves manual checks alone", () => { + const harness = makeHarness({ + checkForUpdates: Effect.sync(() => { + harness.emit("update-available", { version: "1.2.4" }); + }), + }); - yield* Effect.scoped( - Effect.gen(function* () { - const updates = yield* DesktopUpdates.DesktopUpdates; - yield* updates.configure; + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; - const checkFiber = yield* updates.check("poll").pipe(Effect.forkScoped); - yield* Deferred.await(checkStarted); - harness.emit("update-available", { version: "1.2.4" }); - yield* flushCallbacks; + assert.isTrue((yield* updates.check("manual")).checked); + assert.equal((yield* updates.getState).status, "available"); + assert.equal(harness.downloadCount(), 0); - yield* TestClock.adjust(Duration.minutes(1)); - assert.equal(harness.downloadCount(), 0); + yield* TestClock.adjust(Duration.seconds(15)); + assert.equal((yield* updates.getState).status, "downloading"); + assert.equal(harness.downloadCount(), 1); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); - yield* Deferred.succeed(releaseCheck, undefined); - yield* Fiber.join(checkFiber); - yield* Deferred.await(downloadStarted); - yield* TestClock.adjust(Duration.minutes(3)); - - assert.equal((yield* updates.getState).status, "downloading"); - assert.isTrue(yield* updates.isActionActive); - assert.isFalse((yield* updates.check("manual")).checked); - assert.isFalse((yield* updates.download).accepted); - const channelError = yield* updates.setChannel("nightly").pipe(Effect.flip); - assert.equal(channelError._tag, "DesktopUpdateActionInProgressError"); - if (channelError._tag === "DesktopUpdateActionInProgressError") { - assert.equal(channelError.action, "download"); - } - assert.equal(harness.downloadCount(), 1); + it.effect("retries a failed background download only for a newer release", () => { + let offeredVersion = "1.2.4"; + let downloadSetups = 0; + const harness = makeHarness({ + checkForUpdates: Effect.sync(() => { + harness.emit("update-available", { version: offeredVersion }); + }), + // configure makes the first call; the first download makes the second. + setDisableDifferentialDownload: Effect.suspend(() => { + downloadSetups += 1; + return downloadSetups === 2 ? Effect.die(new Error("download failed")) : Effect.void; + }), + }); - harness.emit("update-downloaded", { version: "1.2.4" }); - yield* Deferred.succeed(releaseDownload, undefined); - yield* flushCallbacks; + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; - const state = yield* updates.getState; - assert.equal(state.status, "downloaded"); - assert.equal(state.downloadedVersion, "1.2.4"); - assert.isFalse(yield* updates.isActionActive); - }), - ).pipe( - Effect.ensuring(Deferred.succeed(releaseDownload, undefined)), - Effect.provide(Layer.merge(TestClock.layer(), harness.layer)), - ); - }), - ); + yield* TestClock.adjust(Duration.seconds(15)); + assert.equal((yield* updates.getState).errorContext, "download"); + + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.checkCount(), 2); + assert.equal(downloadSetups, 2); + + offeredVersion = "1.2.5"; + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.checkCount(), 3); + assert.equal(harness.downloadCount(), 1); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); it.effect("preserves a queued installer when the feed has no update", () => { const harness = makeHarness(); @@ -891,8 +887,8 @@ describe("DesktopUpdates", () => { }, ); - it.effect("refuses a stale payload through both automatic and explicit install paths", () => { - const harness = makeHarness(); + it.effect("never installs .deb updates on quit because dpkg asks for a password", () => { + const harness = makeHarness({ platform: "linux", packageType: "deb" }); return Effect.scoped( Effect.gen(function* () { @@ -900,17 +896,9 @@ describe("DesktopUpdates", () => { yield* updates.configure; harness.emit("update-downloaded", { version: "1.2.4" }); yield* flushCallbacks; - const beforeStaleDownload = yield* updates.getState; - harness.emit("update-downloaded", { version: "1.2.5-nightly.20260710.1" }); + assert.equal((yield* updates.getState).status, "downloaded"); assert.isFalse(harness.autoInstallOnAppQuit()); - yield* flushCallbacks; - assert.deepEqual(yield* updates.getState, beforeStaleDownload); - assert.isFalse((yield* updates.install).accepted); - assert.isFalse((yield* updates.installPrepared("1.2.4")).accepted); - assert.equal(harness.quitAndInstalls(), 0); - const error = yield* updates.setChannel("nightly").pipe(Effect.flip); - assert.equal(error._tag, "DesktopUpdateChannelStagedError"); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 6c40d27d726da..781a824e714fa 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -50,7 +50,6 @@ import { const AUTO_UPDATE_STARTUP_DELAY = "15 seconds"; const AUTO_UPDATE_POLL_INTERVAL = "4 minutes"; const PREPARED_INSTALL_CHECK_WAIT = Duration.seconds(90); -const BACKGROUND_DOWNLOAD_START_WAIT = Duration.minutes(2); type UpdateAction = "check" | "download" | "install" | "install-recovery" | "channel"; @@ -300,10 +299,8 @@ export const make = Effect.gen(function* () { const finishedUpdateActions = yield* PubSub.unbounded(); const updaterConfiguredRef = yield* Ref.make(false); const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1); - const lastCheckReasonRef = yield* Ref.make(null); - // Feed checks can clear the UI's downloadedVersion without clearing the - // updater's payload. Native macOS staging cannot be cancelled afterward. - const downloadedPayloadVersionRef = yield* Ref.make(null); + const failedBackgroundDownloadRef = yield* Ref.make(null); + // Native macOS staging cannot be cancelled once Squirrel has the download. const nativeUpdateStagedRef = yield* Ref.make(false); const updateStateRef = yield* Ref.make( createInitialDesktopUpdateState( @@ -441,7 +438,6 @@ export const make = Effect.gen(function* () { if (actionReservation === "acquire" && !(yield* tryStartUpdateAction("check"))) return false; - yield* Ref.set(lastCheckReasonRef, reason); const check = Effect.gen(function* () { const checkedAt = yield* currentIsoTimestamp; yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); @@ -533,29 +529,18 @@ export const make = Effect.gen(function* () { ); }).pipe(Effect.withSpan("desktop.updates.downloadAvailableUpdate")); - // update-available fires inside the check that found it, so the download - // waits for that check to release the updater. Quitting installs the download. - const downloadInBackground = Effect.scoped( - Effect.gen(function* () { - const completions = yield* PubSub.subscribe(finishedUpdateActions); - while (Option.isSome(yield* activeUpdateAction)) { - yield* PubSub.take(completions); + // Updates the background pollers find download on their own; a check someone + // started keeps its Download button. A failed download waits for a newer release. + const checkAndDownloadInBackground = Effect.fn("desktop.updates.checkAndDownloadInBackground")( + function* (reason: "startup" | "poll") { + if (!(yield* checkForUpdates(reason)) || (yield* Ref.get(desktopState.quitting))) return; + const version = (yield* Ref.get(updateStateRef)).availableVersion; + if (version === null || version === (yield* Ref.get(failedBackgroundDownloadRef))) return; + const result = yield* downloadAvailableUpdate; + if (result.accepted && !result.completed) { + yield* Ref.set(failedBackgroundDownloadRef, version); } - }), - ).pipe( - Effect.timeout(BACKGROUND_DOWNLOAD_START_WAIT), - Effect.andThen( - Effect.gen(function* () { - if (yield* Ref.get(desktopState.quitting)) return; - yield* downloadAvailableUpdate; - }), - ), - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) - ? Effect.void - : logUpdaterWarning("background update download did not start"), - ), - Effect.withSpan("desktop.updates.downloadInBackground"), + }, ); // Tells the primary backend that the coming stop is an update restart, so it @@ -632,8 +617,6 @@ export const make = Effect.gen(function* () { const activeAction = yield* Ref.get(activeUpdateActionRef); const hasExpectedDownload = state.downloadedVersion !== null && - state.downloadedVersion === (yield* Ref.get(downloadedPayloadVersionRef)) && - resolveDefaultDesktopUpdateChannel(state.downloadedVersion) === state.channel && (expectedVersion === undefined || state.downloadedVersion === expectedVersion); if ( (yield* Ref.get(desktopState.quitting)) || @@ -748,7 +731,7 @@ export const make = Effect.gen(function* () { const startUpdatePollers: Effect.Effect = Effect.gen(function* () { yield* Effect.sleep(AUTO_UPDATE_STARTUP_DELAY).pipe( - Effect.andThen(checkForUpdates("startup")), + Effect.andThen(checkAndDownloadInBackground("startup")), Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { return Effect.void; @@ -762,7 +745,7 @@ export const make = Effect.gen(function* () { Effect.forkScoped, ); yield* Effect.sleep(AUTO_UPDATE_POLL_INTERVAL).pipe( - Effect.andThen(checkForUpdates("poll")), + Effect.andThen(checkAndDownloadInBackground("poll")), Effect.forever, Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { @@ -817,12 +800,6 @@ export const make = Effect.gen(function* () { releaseNoteGroups: releaseNotes.length, omittedReleaseCount, }); - // Updates found by the background pollers download on their own; a - // check someone started keeps its Download button. - const reason = yield* Ref.get(lastCheckReasonRef); - if (reason === "startup" || reason === "poll") { - yield* Effect.forkDetach(downloadInBackground); - } }), ), Effect.catchCause((cause) => { @@ -923,12 +900,10 @@ export const make = Effect.gen(function* () { // Keep this prelude synchronous: MacUpdater checks the flag immediately // after emitting this event, before it hands the download to Squirrel. yield* electronUpdater.setAutoInstallOnAppQuit(false); - yield* Ref.set(downloadedPayloadVersionRef, null); yield* decodeUpdateInfo(raw).pipe( Effect.flatMap( Effect.fn("desktop.updates.applyUpdateDownloaded")(function* (info) { const state = yield* Ref.get(updateStateRef); - yield* Ref.set(downloadedPayloadVersionRef, info.version); const action = yield* activeUpdateAction; if ( resolveDefaultDesktopUpdateChannel(info.version) !== state.channel || @@ -943,7 +918,10 @@ export const make = Effect.gen(function* () { if (environment.platform === "darwin") { yield* Ref.set(nativeUpdateStagedRef, true); } - yield* electronUpdater.setAutoInstallOnAppQuit(true); + // Installing a .deb asks for a password, which must not appear on quit. + if (!isDebPackage) { + yield* electronUpdater.setAutoInstallOnAppQuit(true); + } yield* setState(reduceDesktopUpdateStateOnDownloadComplete(state, info.version)); yield* logUpdaterInfo("update downloaded", { version: info.version }); }), @@ -1001,8 +979,7 @@ export const make = Effect.gen(function* () { yield* Ref.set(updaterConfiguredRef, true); yield* electronUpdater.setAutoDownload(false); - // A download that is never relaunched into installs when the user quits. - yield* electronUpdater.setAutoInstallOnAppQuit(true); + yield* electronUpdater.setAutoInstallOnAppQuit(false); yield* applyAutoUpdaterChannel(settings.updateChannel); yield* electronUpdater.setDisableDifferentialDownload( isArm64HostRunningIntelBuild(environment.runtimeInfo), @@ -1057,7 +1034,7 @@ export const make = Effect.gen(function* () { return state; } - if (environment.platform === "darwin" && (yield* Ref.get(nativeUpdateStagedRef))) { + if (yield* Ref.get(nativeUpdateStagedRef)) { return yield* new DesktopUpdateChannelStagedError({ requestedChannel: nextChannel }); } diff --git a/docs/user/updating.md b/docs/user/updating.md index eb2cbe00f8378..fcebfe61d131c 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -87,7 +87,8 @@ Then it installs: T3 Code server restarts also wait for integrated terminal commands to finish. Desktop updates download in the background when the app detects an available -release and install when you quit the app. +release and install when you quit the app. Linux `.deb` updates ask for your +password, so they wait for you to choose **Install**. The app stays open while you work, including when it hosts multiple backends. On macOS, quit and reopen the app to install a downloaded update before switching update channels. From e2fec1d844b4cf229a91374c475b7e412d02443f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:21:34 +0000 Subject: [PATCH 030/149] fix(server): frequent scheduled tasks no longer hold back background updates A task repeating more often than every 15 minutes always has a run due soon, so it would keep the update window closed forever. Its runs wait for the admission permit or catch up after a restart. --- apps/server/src/updates/UpdateWindow.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 5c88feb1e41b5..0999bf1e2f761 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -19,6 +19,12 @@ const UPDATE_LOOKAHEAD = Duration.minutes(5); * Clients report activity about every 25s, so this must stay well above that. */ const QUIET_PERIOD = Duration.minutes(15); +/** + * Tasks repeating more often than this always have a run due soon. A run that + * starts mid-install waits for the admission permit, and a run missed during a + * restart catches up afterwards, so these only block while running. + */ +const FREQUENT_TASK_INTERVAL = Duration.minutes(15); /** * Decides when a background update may restart the server or replace a @@ -86,6 +92,12 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { continue; } if (!task.enabled || task.nextRunAt === null) continue; + if ( + task.schedule.type === "interval" && + task.schedule.everyMs < Duration.toMillis(FREQUENT_TASK_INTERVAL) + ) { + continue; + } const runsAt = DateTime.make(task.nextRunAt); if (Option.isSome(runsAt) && !DateTime.isGreaterThan(runsAt.value, horizon)) { found.push("scheduled-task"); From 53fc18dfa8dc9706d0fe8e4cbba75e27df7eb98c Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:22:22 +0000 Subject: [PATCH 031/149] fix(server): retry a background server update after a failed download Only a rollback or a protocol change should skip a release for good; a network failure while staging now retries at the next hourly check. --- apps/server/src/updates/ServerAutoUpdater.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index b8f84df9a9ae8..efc52129f7b7f 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -71,9 +71,8 @@ export const layer = Layer.effectDiscard( ) { // A newer release replaces the pending target only once it stages. yield* Effect.logInfo("Staging a background server update", { targetVersion }); - const staged = yield* selfUpdate - .stage(targetVersion) - .pipe(Effect.tapError(() => Effect.sync(() => skippedTargets.add(targetVersion)))); + // A failed download or preflight is retried at the next release check. + const staged = yield* selfUpdate.stage(targetVersion); if (staged.orchestrationProtocol !== ORCHESTRATION_PROTOCOL_VERSION) { // Without a matching protocol, connected clients could be refused after restart. skippedTargets.add(targetVersion); From 1bec6f306621932fe9f25e9488bbbbe7946357a7 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:34:25 +0000 Subject: [PATCH 032/149] fix(server): a canceled provider update no longer blocks background updates An interrupted install left its state running, which kept the update window closed until restart. Background passes also skip all work while automatic updates are off, and an unreadable settings file falls back to defaults like every other setting. --- apps/server/src/provider/providerMaintenanceRunner.ts | 11 ++++++++--- apps/server/src/serverSettings.test.ts | 1 - apps/server/src/serverSettings.ts | 2 -- apps/server/src/updates/ProviderAutoUpdater.ts | 2 ++ 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index ce0dc5ff7bd2a..fd4e23aee3701 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -585,13 +585,18 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const current = (yield* providerRegistry.getProviders).find( (candidate) => candidate.instanceId === instanceId, ); - if (current?.updateState?.status !== "queued") return; + // A state left queued or running would block background updates until restart. + const status = current?.updateState?.status; + if (status !== "queued" && status !== "running") return; yield* setUpdateState( makeUpdateState({ status: "failed", - startedAt: null, + startedAt: current?.updateState?.startedAt ?? null, finishedAt: yield* nowIso, - message: "Update canceled before installation started.", + message: + status === "queued" + ? "Update canceled before installation started." + : "Update canceled.", }), ); }), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index f9bc173b91849..4444a7851dffc 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1045,7 +1045,6 @@ it.layer(NodeServices.layer)("server settings", (it) => { driver: ProviderDriverKind.make("codex"), config: { binaryPath: "/x" }, }); - assert.isFalse(settings.automaticUpdates); assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); }).pipe(Effect.provide(layerServerSettings())), ); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 61cc0f29a0b3d..7536a74d6e02c 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -786,8 +786,6 @@ const make = Effect.gen(function* () { if (decoded._tag === "Failure" || legacySettings._tag === "Failure") { const failure = decoded._tag === "Failure" ? decoded : legacySettings; settingsFileTrusted = false; - // An unreadable file cannot authorize unattended updates. - settings = { ...DEFAULT_SERVER_SETTINGS, automaticUpdates: false }; if (failure._tag === "Failure") { yield* Effect.logWarning("failed to parse settings.json, using defaults", { path: settingsPath, diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index db06bbcd95f64..6cfbcceefe456 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -48,6 +48,8 @@ export const layer = Layer.effectDiscard( const attempts = new Map(); const pass = Effect.gen(function* () { + const settings = yield* serverSettings.getSettings; + if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; for (const { instanceId } of yield* providers.getProviders) { const install = Effect.gen(function* () { const settings = yield* serverSettings.getSettings; From 4102d587b7698297e68d711d6f3480a86cf10538 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:34:43 +0000 Subject: [PATCH 033/149] docs(updates): describe when background updates retry and what they install --- .../settings/SettingsServerControlsRouteScreen.tsx | 2 +- apps/web/src/components/settings/SettingsPanels.tsx | 2 +- docs/user/updating.md | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index df70d72ad7be7..8d86eeed06218 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -473,7 +473,7 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { projectSelected ? "Environment-wide setting. Select All projects to change it." : supportsAutomaticUpdates - ? "Install T3 Code and provider updates while nothing is running." + ? "Install provider and T3 Code service updates while nothing is running." : "Update older servers to control automatic updates." } value={uniform("automaticUpdates")} diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index bd955729620ae..6f5c7bd265bc2 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -2924,7 +2924,7 @@ export function GeneralSettingsPanel() { serverScoped settingKeys={["automaticUpdates"]} {...searchableSetting("automatic-updates")} - description="Install T3 Code and provider CLI updates in the background on the selected environments while nothing is running." + description="Install provider CLI and T3 Code service updates in the background on the selected environments while nothing is running." status={ !supportsAutomaticUpdates ? "All selected connected environments must support automatic updates." diff --git a/docs/user/updating.md b/docs/user/updating.md index 777fe42c9a0d5..c6d2b11514652 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -77,10 +77,11 @@ update can roll back to the previous version. If the update still fails: **Settings → General → Update automatically** is on by default. Each environment waits until no thread is running or waiting for you, no background work is pending, no client has been used for 15 minutes, and no scheduled task or usage-limit resume -is due within 5 minutes. -Then it installs: +is due within 5 minutes. Tasks that repeat more often than every 15 minutes only +hold updates back while they run. Then it installs: -- provider CLI updates that have a one-click update; +- provider CLI updates that have a one-click update, when **Provider update + checks** is also on; - the newest T3 Code release on its channel, when it runs as a background service (`t3 service install`). It downloads first and restarts only inside that window. @@ -88,8 +89,9 @@ T3 Code server restarts also wait for integrated terminal commands to finish. Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours or when a newer version is -published. A failed or rolled-back T3 Code update is not retried until a newer -version is published. An update that needs newer clients waits for you to install it. +published. A T3 Code release that fails to download is retried at the next hourly +check. One that rolls back is skipped until a newer version is published. An +update that needs newer clients waits for you to install it. ## Update providers From b759d5c66711b15cb8b8a7038b83a5a35df79aba Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:34:58 +0000 Subject: [PATCH 034/149] docs(desktop): note that desktop updates install on quit regardless of the environment setting --- apps/desktop/src/app/DesktopLifecycle.ts | 3 ++- docs/user/updating.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.ts index c84b14f01032c..3b1ca6c820484 100644 --- a/apps/desktop/src/app/DesktopLifecycle.ts +++ b/apps/desktop/src/app/DesktopLifecycle.ts @@ -181,7 +181,8 @@ export const make = DesktopLifecycle.of({ return; } // app.exit still emits "quit", where electron-updater would install a - // downloaded update under the relaunched app. Keep it for a real quit. + // downloaded update under the relaunched app on Windows and Linux. Keep + // it for a real quit. A macOS update Squirrel already staged still applies. yield* electronUpdater.setAutoInstallOnAppQuit(false); yield* electronApp.relaunch({ execPath: process.execPath, diff --git a/docs/user/updating.md b/docs/user/updating.md index a2d1535ddf052..a1d8a94957eb2 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -87,8 +87,8 @@ hold updates back while they run. Then it installs: T3 Code server restarts also wait for integrated terminal commands to finish. -Desktop updates download in the background when the app detects an available -release and install when you quit the app. Linux `.deb` updates ask for your +The desktop app downloads its updates in the background and installs them when +you quit, whatever **Update automatically** is set to. Linux `.deb` updates ask for your password, so they wait for you to choose **Install**. The app stays open while you work, including when it hosts multiple backends. On macOS, quit and reopen the app to install a downloaded update before switching From 33b55065b7d3bfde8a654183f26622a6897f4152 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:35:23 +0000 Subject: [PATCH 035/149] fix(server): stop continuing a run restarted three times in a row With continuation on by default, a turn that crashes the server would be continued after every restart. Three consecutive automatic continuations now need a person to send the next message. --- .../orchestration-v2/RestartContinuation.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 013e7a07fbc45..077cd458d02c9 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -87,6 +87,9 @@ export function restartContinuationRun( return run; } +/** Consecutive automatic continuations allowed before a person has to step in. */ +const MAX_CONTINUATION_CHAIN = 3; + export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")( function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) { const settings = yield* ServerSettings.ServerSettingsService; @@ -125,6 +128,22 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta ) return; if (projection.thread.providerInstanceId !== source.providerInstanceId) return; + // A turn that keeps crashing the server would otherwise restart it forever. + let chainLength = 0; + for ( + let run: (typeof projection.runs)[number] | undefined = source; + run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN; + run = projection.runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId) + ) { + chainLength += 1; + } + if (chainLength >= MAX_CONTINUATION_CHAIN) { + yield* Effect.logWarning("Not continuing a run restarted too many times in a row", { + threadId: input.threadId, + sourceRunId: input.sourceRunId, + }); + return; + } const sourceRecords = yield* threads.getThreadRecords( input.threadId, ["messages", "turnItems"], From 973dbe5f1b2d649e2a80bec30c4ce9287ed30cfa Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:36:38 +0000 Subject: [PATCH 036/149] fix(server): keep background updates from stalling --- .../src/orchestration-v2/ProjectionStore.ts | 23 ++++++++++++++----- apps/server/src/updates/UpdateWindow.ts | 7 +++++- docs/user/updating.md | 12 ++++++---- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index d377738547a13..b99f1a483e5be 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -577,9 +577,13 @@ function needsRecovery( ["preparing", "starting", "running", "waiting"].includes(run.status) || (run.status === "queued" && run.queueHeld !== true), ) || - projection.providerThreads.some( + // A roster left by a crashed provider or archived thread has no live session. + (projection.providerThreads.some( (thread) => (thread.pendingBackgroundTasks?.length ?? 0) > 0, - ) + ) && + projection.providerSessions.some( + (session) => session.status !== "stopped" && session.status !== "error", + )) ); case "runtime": return ( @@ -3620,10 +3624,17 @@ export const layer: Layer.Layer = THEN json_extract(payload_json, '$.queueHeld') IS NOT 1 ELSE 1 END UNION - SELECT thread_id FROM orchestration_v2_projection_provider_threads - WHERE CASE WHEN json_valid(payload_json) - THEN json_array_length(payload_json, '$.pendingBackgroundTasks') > 0 - ELSE 0 END + SELECT threads.thread_id FROM orchestration_v2_projection_provider_threads AS threads + WHERE CASE WHEN json_valid(threads.payload_json) + THEN json_array_length(threads.payload_json, '$.pendingBackgroundTasks') > 0 + ELSE 0 END + AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings + CROSS JOIN orchestration_v2_projection_provider_sessions AS sessions + ON sessions.provider_session_id = bindings.provider_session_id + WHERE bindings.thread_id = threads.thread_id + AND sessions.status NOT IN ('stopped', 'error') + ) `; case "runtime": return sql` diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 0999bf1e2f761..184475a92cbb4 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -127,7 +127,12 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { for (const thread of limited.value) { if (thread.usageLimitResetAt === null || thread.usageLimitResetAt === undefined) continue; const resumesAt = DateTime.make(thread.usageLimitResetAt); - if (Option.isSome(resumesAt) && !DateTime.isGreaterThan(resumesAt.value, horizon)) { + // A reset already past (say, a resume that keeps failing) must not block forever. + if ( + Option.isSome(resumesAt) && + DateTime.isGreaterThan(resumesAt.value, now) && + !DateTime.isGreaterThan(resumesAt.value, horizon) + ) { found.push("usage-limit-resume"); } } diff --git a/docs/user/updating.md b/docs/user/updating.md index 8c2bf919fa350..349a1bc400127 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -74,11 +74,13 @@ update can roll back to the previous version. If the update still fails: ## Update providers -**Settings → General → Update automatically** is on by default. Each environment -installs provider CLI updates in the background once no thread is running or -waiting for you, no client has been used for 15 minutes, and no scheduled task -or usage-limit resume is due within 5 minutes. A failed update is retried -after 6 hours or when a newer version is published. +**Settings → General → Update automatically** is on by default. With +**Provider update checks** also on, each environment installs provider CLI +updates in the background once no thread is running or waiting for you, no +client has been used for 15 minutes, and no scheduled task or usage-limit resume +is due within 5 minutes. Tasks repeating more often than every 15 minutes only +hold updates back while they run. A failed update is retried after 6 hours or +when a newer version is published. **Settings → Providers** shows provider updates for the selected environment. **Update all** updates every outdated provider on every connected environment From f58dce43e48a2bca87d85ae929b26494fa7ff134 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:38:17 +0000 Subject: [PATCH 037/149] fix(server): bound background server update handoffs --- .../SettingsServerControlsRouteScreen.tsx | 2 +- apps/server/src/server.ts | 1 - apps/server/src/updates/ServerAutoUpdater.ts | 52 +++++++++++++++++-- .../components/settings/SettingsPanels.tsx | 2 +- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 8d86eeed06218..2289b4eb351b7 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -473,7 +473,7 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { projectSelected ? "Environment-wide setting. Select All projects to change it." : supportsAutomaticUpdates - ? "Install provider and T3 Code service updates while nothing is running." + ? "Install provider and service updates while nothing is running." : "Update older servers to control automatic updates." } value={uniform("automaticUpdates")} diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 97a92debbbf4d..e1d2416008858 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -742,7 +742,6 @@ const layerMakeRoutes = Layer.mergeAll( // Shares the routes' ServerSelfUpdate, so a manual and a background update never overlap. ServerAutoUpdater.layer.pipe( Layer.provide(UpdateWindow.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), - Layer.provide(layerDesktopTelemetryReceiver), ), ).pipe( // Both transports consume the same service instance, so caches single-flight across clients diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index efc52129f7b7f..885f0f62f356e 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -3,16 +3,23 @@ import { cliReleaseChannelOf } from "@t3tools/shared/cliRelease"; import * as Clock from "effect/Clock"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as Schedule from "effect/Schedule"; import { HttpClient } from "effect/http"; import packageJson from "../../package.json" with { type: "json" }; import { resolveNewestVersion } from "../cli/update.ts"; +import { pinnedRuntimeVersionsDir } from "../cloud/pinnedRuntime.ts"; import * as ServerSelfUpdate from "../cloud/selfUpdate.ts"; import * as ServerConfig from "../config.ts"; import * as ServiceLauncherClient from "../cloud/serviceLauncherClient.ts"; -import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts"; +import { + compareExactServiceVersions, + SERVICE_STATE_FILE, + serviceStateActiveVersion, +} from "../cloud/serviceProtocol.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as UpdateWindow from "./UpdateWindow.ts"; @@ -22,6 +29,8 @@ const FIRST_PASS_DELAY = Duration.minutes(2); const RECHECK = Duration.seconds(30); /** T3 releases are checked this often; the desktop app polls its own feed. */ const RELEASE_CHECK_INTERVAL = Duration.hours(1); +/** How long a handed-off install waits for the launcher to stop this server. */ +const HANDOFF_TIMEOUT = Duration.minutes(5); /** * Updates this environment in the background when `automaticUpdates` is on. @@ -35,13 +44,16 @@ const RELEASE_CHECK_INTERVAL = Duration.hours(1); export const layer = Layer.effectDiscard( Effect.gen(function* () { // A dev server must not relaunch the desktop app or service it runs beside. - if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return; + const { baseDir, devUrl } = yield* ServerConfig.ServerConfig; + if (devUrl !== undefined) return; const selfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; if (selfUpdate.capability !== "boot-service") return; const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; const httpClient = yield* HttpClient.HttpClient; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const currentVersion = packageJson.version; let pendingTarget: string | undefined; let nextReleaseCheckAt = 0; @@ -56,6 +68,29 @@ export const layer = Layer.effectDiscard( Effect.orElseSucceed(() => false), ); + /** Deletes a staged runtime nothing will run: never this version or the launcher's active one. */ + const removeStagedRuntime = Effect.fn("updates.ServerAutoUpdater.removeStagedRuntime")( + function* (version: string) { + if (version === currentVersion) return; + const activeVersion = yield* fs + .readFileString(path.join(baseDir, "runtime", SERVICE_STATE_FILE)) + .pipe(Effect.map(serviceStateActiveVersion), Effect.option); + // Without readable launcher state, the version might be active; keep it. + if (activeVersion._tag === "None" || activeVersion.value === undefined) return; + if (activeVersion.value === version) return; + yield* fs + .remove(path.join(pinnedRuntimeVersionsDir(path, baseDir), version), { + recursive: true, + force: true, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Could not remove a staged server runtime", { version, cause }), + ), + ); + }, + ); + const updateBootService = Effect.gen(function* () { if (!(yield* automaticUpdatesEnabled)) return; const nowMs = yield* Clock.currentTimeMillis; @@ -82,8 +117,11 @@ export const layer = Layer.effectDiscard( targetVersion, }, ); + yield* removeStagedRuntime(targetVersion); } else { + const replaced = pendingTarget; pendingTarget = targetVersion; + if (replaced !== undefined) yield* removeStagedRuntime(replaced); } } } @@ -98,8 +136,14 @@ export const layer = Layer.effectDiscard( yield* Effect.logInfo("Installing a background server update", { targetVersion }); yield* selfUpdate.update({ targetVersion }); // The launcher stops us after accepting the handoff. Keep new provider - // starts behind the install permit until this runtime is shut down. - return yield* Effect.never; + // starts behind the install permit until then, but not forever. + yield* Effect.sleep(HANDOFF_TIMEOUT); + yield* Effect.logWarning( + "The service launcher did not stop this server after an update", + { + targetVersion, + }, + ); }), { restartsServer: true }, ); diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 6f5c7bd265bc2..dee396f4f72a0 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -2924,7 +2924,7 @@ export function GeneralSettingsPanel() { serverScoped settingKeys={["automaticUpdates"]} {...searchableSetting("automatic-updates")} - description="Install provider CLI and T3 Code service updates in the background on the selected environments while nothing is running." + description="Install provider CLI and background service updates on the selected environments while nothing is running." status={ !supportsAutomaticUpdates ? "All selected connected environments must support automatic updates." From 66920cb49a217ba56a03d2d9e0c1d3202ec4ab6f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:44:27 +0000 Subject: [PATCH 038/149] fix(server): a failed terminal poll no longer reads as idle before a restart A failed subprocess inspection left the last cached activity in place, so a running command could look idle to a background server update. --- apps/server/src/terminal/Manager.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 82b820ec2e645..5ccb38c6ed459 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -219,7 +219,7 @@ export class TerminalManager extends Context.Service< readonly terminalId?: string; }) => Effect.Effect; - /** True while a terminal starts or the last subprocess poll saw a command running in one. */ + /** True while a terminal starts, runs a command, or its last subprocess poll failed. */ readonly hasBusyTerminals: Effect.Effect; /** @@ -2416,6 +2416,8 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func } }); + // A failed inspection leaves cached activity stale, so busy checks distrust it. + let lastSubprocessPollSucceeded = true; const pollSubprocessActivity = Effect.fn("terminal.pollSubprocessActivity")(function* () { const state = yield* readManagerState; const runningSessions = [...state.sessions.values()].filter( @@ -2444,6 +2446,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); if (Option.isNone(inspectorOption)) { + lastSubprocessPollSucceeded = false; return false; } @@ -2466,6 +2469,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); if (Option.isNone(inspectResult)) { + lastSubprocessPollSucceeded = false; return; } @@ -2512,6 +2516,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func } }); + lastSubprocessPollSucceeded = snapshotSucceeded; yield* Effect.forEach(runningSessions, checkSubprocessActivity, { concurrency: "unbounded", discard: true, @@ -3183,7 +3188,8 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func [...state.sessions.values()].some( (session) => session.status === "starting" || - (session.status === "running" && session.hasRunningSubprocess), + (session.status === "running" && + (session.hasRunningSubprocess || !lastSubprocessPollSucceeded)), ), ), ); From 7b9e2f9e3bcb7aadb7601e5c2ba0ce3d0cb71bc0 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:55:52 +0000 Subject: [PATCH 039/149] fix(server): queued runs and incompatible releases no longer stall background updates A queued run held by a usage limit kept the update window closed until the limit reset; queued runs are covered by their live run or by the admission permit a starting turn waits for. A latest version marked broken or unsupported is no longer attempted in the background. --- .../src/orchestration-v2/ProjectionStore.ts | 15 +++++---------- apps/server/src/updates/ProviderAutoUpdater.ts | 3 +++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index b99f1a483e5be..91030ab58795e 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -571,11 +571,12 @@ function needsRecovery( ); } case "active-work": + // Queued runs don't count: one behind a live run is covered by that run, + // one about to start waits for the update admission permit, and one held + // by a usage limit would otherwise block updates until the limit resets. return ( - projection.runs.some( - (run) => - ["preparing", "starting", "running", "waiting"].includes(run.status) || - (run.status === "queued" && run.queueHeld !== true), + projection.runs.some((run) => + ["preparing", "starting", "running", "waiting"].includes(run.status), ) || // A roster left by a crashed provider or archived thread has no live session. (projection.providerThreads.some( @@ -3618,12 +3619,6 @@ export const layer: Layer.Layer = SELECT thread_id FROM orchestration_v2_projection_runs WHERE status IN ('preparing', 'starting', 'running', 'waiting') UNION - SELECT thread_id FROM orchestration_v2_projection_runs - WHERE status = 'queued' - AND CASE WHEN json_valid(payload_json) - THEN json_extract(payload_json, '$.queueHeld') IS NOT 1 - ELSE 1 END - UNION SELECT threads.thread_id FROM orchestration_v2_projection_provider_threads AS threads WHERE CASE WHEN json_valid(threads.payload_json) THEN json_array_length(threads.payload_json, '$.pendingBackgroundTasks') > 0 diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index 6cfbcceefe456..e70a5acc32533 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -27,6 +27,9 @@ function isAutoUpdatable(provider: ServerProvider): boolean { advisory?.status === "behind_latest" && advisory.canUpdate && advisory.latestVersion !== null && + // The runner refuses these, which would leave a failure nobody asked for. + provider.compatibilityAdvisory?.latestVersionStatus !== "broken" && + provider.compatibilityAdvisory?.latestVersionStatus !== "unsupported" && updateStatus !== "queued" && updateStatus !== "running" ); From 05a6980c0c03d1451b35ff74ac69ca5ccd990950 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:56:33 +0000 Subject: [PATCH 040/149] fix(server): background server updates no longer delete runtimes or retry blocked releases Deleting staged runtimes could remove one the service unit, the t3 launcher, or a pending launcher update still needed, so staged versions now stay on disk. A release whose preflight needs a newer launcher is skipped instead of re-downloaded every hour, and a handoff refused before the launcher accepts it is retried. --- apps/server/src/updates/ServerAutoUpdater.ts | 64 +++++++------------- 1 file changed, 23 insertions(+), 41 deletions(-) diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 885f0f62f356e..0e164ca6e250c 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -3,23 +3,17 @@ import { cliReleaseChannelOf } from "@t3tools/shared/cliRelease"; import * as Clock from "effect/Clock"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as Path from "effect/Path"; +import * as Predicate from "effect/Predicate"; import * as Schedule from "effect/Schedule"; import { HttpClient } from "effect/http"; import packageJson from "../../package.json" with { type: "json" }; import { resolveNewestVersion } from "../cli/update.ts"; -import { pinnedRuntimeVersionsDir } from "../cloud/pinnedRuntime.ts"; import * as ServerSelfUpdate from "../cloud/selfUpdate.ts"; import * as ServerConfig from "../config.ts"; import * as ServiceLauncherClient from "../cloud/serviceLauncherClient.ts"; -import { - compareExactServiceVersions, - SERVICE_STATE_FILE, - serviceStateActiveVersion, -} from "../cloud/serviceProtocol.ts"; +import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as UpdateWindow from "./UpdateWindow.ts"; @@ -44,16 +38,13 @@ const HANDOFF_TIMEOUT = Duration.minutes(5); export const layer = Layer.effectDiscard( Effect.gen(function* () { // A dev server must not relaunch the desktop app or service it runs beside. - const { baseDir, devUrl } = yield* ServerConfig.ServerConfig; - if (devUrl !== undefined) return; + if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return; const selfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; if (selfUpdate.capability !== "boot-service") return; const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; const httpClient = yield* HttpClient.HttpClient; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const currentVersion = packageJson.version; let pendingTarget: string | undefined; let nextReleaseCheckAt = 0; @@ -68,29 +59,6 @@ export const layer = Layer.effectDiscard( Effect.orElseSucceed(() => false), ); - /** Deletes a staged runtime nothing will run: never this version or the launcher's active one. */ - const removeStagedRuntime = Effect.fn("updates.ServerAutoUpdater.removeStagedRuntime")( - function* (version: string) { - if (version === currentVersion) return; - const activeVersion = yield* fs - .readFileString(path.join(baseDir, "runtime", SERVICE_STATE_FILE)) - .pipe(Effect.map(serviceStateActiveVersion), Effect.option); - // Without readable launcher state, the version might be active; keep it. - if (activeVersion._tag === "None" || activeVersion.value === undefined) return; - if (activeVersion.value === version) return; - yield* fs - .remove(path.join(pinnedRuntimeVersionsDir(path, baseDir), version), { - recursive: true, - force: true, - }) - .pipe( - Effect.catch((cause) => - Effect.logWarning("Could not remove a staged server runtime", { version, cause }), - ), - ); - }, - ); - const updateBootService = Effect.gen(function* () { if (!(yield* automaticUpdatesEnabled)) return; const nowMs = yield* Clock.currentTimeMillis; @@ -106,8 +74,17 @@ export const layer = Layer.effectDiscard( ) { // A newer release replaces the pending target only once it stages. yield* Effect.logInfo("Staging a background server update", { targetVersion }); - // A failed download or preflight is retried at the next release check. - const staged = yield* selfUpdate.stage(targetVersion); + // A failed download is retried at the next release check; a release + // this launcher cannot run waits for a manual service update. + const staged = yield* selfUpdate.stage(targetVersion).pipe( + Effect.tapError((error) => + Effect.sync(() => { + if (Predicate.isTagged(error.cause, "PinnedRuntimePreflightBlockedError")) { + skippedTargets.add(targetVersion); + } + }), + ), + ); if (staged.orchestrationProtocol !== ORCHESTRATION_PROTOCOL_VERSION) { // Without a matching protocol, connected clients could be refused after restart. skippedTargets.add(targetVersion); @@ -117,11 +94,8 @@ export const layer = Layer.effectDiscard( targetVersion, }, ); - yield* removeStagedRuntime(targetVersion); } else { - const replaced = pendingTarget; pendingTarget = targetVersion; - if (replaced !== undefined) yield* removeStagedRuntime(replaced); } } } @@ -134,7 +108,15 @@ export const layer = Layer.effectDiscard( pendingTarget = undefined; skippedTargets.add(targetVersion); yield* Effect.logInfo("Installing a background server update", { targetVersion }); - yield* selfUpdate.update({ targetVersion }); + yield* selfUpdate.update({ targetVersion }).pipe( + // Refused before the handoff (say, a manual update in progress): try again later. + Effect.tapError(() => + Effect.sync(() => { + skippedTargets.delete(targetVersion); + pendingTarget ??= targetVersion; + }), + ), + ); // The launcher stops us after accepting the handoff. Keep new provider // starts behind the install permit until then, but not forever. yield* Effect.sleep(HANDOFF_TIMEOUT); From d36093c71e9cc442eddb9acd3aca2f32db3b6012 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:16:38 +0000 Subject: [PATCH 041/149] fix(server): only a live owning session keeps background tasks blocking updates A task roster left by a stopped session kept its thread active whenever any other session on the thread was still live. --- .../src/orchestration-v2/ProjectionStore.ts | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 91030ab58795e..b88a828a9db8f 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -578,13 +578,18 @@ function needsRecovery( projection.runs.some((run) => ["preparing", "starting", "running", "waiting"].includes(run.status), ) || - // A roster left by a crashed provider or archived thread has no live session. - (projection.providerThreads.some( - (thread) => (thread.pendingBackgroundTasks?.length ?? 0) > 0, - ) && - projection.providerSessions.some( - (session) => session.status !== "stopped" && session.status !== "error", - )) + // A roster counts only while the session that owns it is live; one left + // by a crashed provider or archived thread would block updates forever. + projection.providerThreads.some( + (thread) => + (thread.pendingBackgroundTasks?.length ?? 0) > 0 && + projection.providerSessions.some( + (session) => + session.id === thread.providerSessionId && + session.status !== "stopped" && + session.status !== "error", + ), + ) ); case "runtime": return ( @@ -3624,10 +3629,8 @@ export const layer: Layer.Layer = THEN json_array_length(threads.payload_json, '$.pendingBackgroundTasks') > 0 ELSE 0 END AND EXISTS ( - SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings - CROSS JOIN orchestration_v2_projection_provider_sessions AS sessions - ON sessions.provider_session_id = bindings.provider_session_id - WHERE bindings.thread_id = threads.thread_id + SELECT 1 FROM orchestration_v2_projection_provider_sessions AS sessions + WHERE sessions.provider_session_id = threads.provider_session_id AND sessions.status NOT IN ('stopped', 'error') ) `; From 27c2d3d034097140efdab2689afe817e884dc1ba Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:27:36 +0000 Subject: [PATCH 042/149] fix(server): background server updates back off and stage one release at a time A refused handoff retried the preflight every 30 seconds; it now waits for the next hourly release check. Newer releases are not downloaded while one is staged, so a long-closed update window cannot fill the disk. A ps fallback after a failed native process snapshot no longer marks every terminal busy. --- apps/server/src/terminal/Manager.ts | 3 ++- apps/server/src/updates/ServerAutoUpdater.ts | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 5ccb38c6ed459..e47f10f32244f 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -2516,7 +2516,8 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func } }); - lastSubprocessPollSucceeded = snapshotSucceeded; + // `snapshotSucceeded` only drives backoff: the ps fallback still reads a fresh table. + lastSubprocessPollSucceeded = true; yield* Effect.forEach(runningSessions, checkSubprocessActivity, { concurrency: "unbounded", discard: true, diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 0e164ca6e250c..23b3d92ae1923 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -62,17 +62,16 @@ export const layer = Layer.effectDiscard( const updateBootService = Effect.gen(function* () { if (!(yield* automaticUpdatesEnabled)) return; const nowMs = yield* Clock.currentTimeMillis; - if (nowMs >= nextReleaseCheckAt) { + // One staged release at a time, so a closed window cannot pile up downloads. + if (nowMs >= nextReleaseCheckAt && pendingTarget === undefined) { nextReleaseCheckAt = nowMs + Duration.toMillis(RELEASE_CHECK_INTERVAL); const targetVersion = yield* resolveNewestVersion(cliReleaseChannelOf(currentVersion)).pipe( Effect.provideService(HttpClient.HttpClient, httpClient), ); if ( compareExactServiceVersions(targetVersion, currentVersion) > 0 && - !skippedTargets.has(targetVersion) && - targetVersion !== pendingTarget + !skippedTargets.has(targetVersion) ) { - // A newer release replaces the pending target only once it stages. yield* Effect.logInfo("Staging a background server update", { targetVersion }); // A failed download is retried at the next release check; a release // this launcher cannot run waits for a manual service update. @@ -109,12 +108,15 @@ export const layer = Layer.effectDiscard( skippedTargets.add(targetVersion); yield* Effect.logInfo("Installing a background server update", { targetVersion }); yield* selfUpdate.update({ targetVersion }).pipe( - // Refused before the handoff (say, a manual update in progress): try again later. + // Refused before the handoff (say, a manual update in progress): the + // next hourly release check stages it again. Effect.tapError(() => - Effect.sync(() => { - skippedTargets.delete(targetVersion); - pendingTarget ??= targetVersion; - }), + Clock.currentTimeMillis.pipe( + Effect.map((failedAtMs) => { + skippedTargets.delete(targetVersion); + nextReleaseCheckAt = failedAtMs + Duration.toMillis(RELEASE_CHECK_INTERVAL); + }), + ), ), ); // The launcher stops us after accepting the handoff. Keep new provider From d0fb53f7d7d2e0b873c4855b287839a746ae7298 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:38:50 +0000 Subject: [PATCH 043/149] fix(server): a corrupt scheduled task no longer closes the update window The window listed scheduled tasks with the strict API decode, so one undecodable row kept every background update blocked. It now reads only running or due tasks and skips corrupt rows, like the scheduler does. --- .../scheduledTasks/ScheduledTaskService.ts | 66 ++++++++++++------- apps/server/src/updates/UpdateWindow.ts | 30 ++++----- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 51403c124a982..2ac47208e99e7 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -352,6 +352,35 @@ const decodeRow = (row: ScheduledTaskRow, origin: WebhookOrigin | null = null) = ); /** Select poll candidates before decoding their schedules or other JSON payloads. */ +/** Decodes rows for background readers, skipping corrupt ones so one bad row cannot stall them. */ +const decodeTaskRowsLeniently = Effect.fn("ScheduledTaskService.decodeTaskRowsLeniently")( + function* (rows: ReadonlyArray) { + const tasks: ScheduledTask[] = []; + for (const row of rows) { + const decoded = yield* Effect.result(decodeRow(row)); + if (Result.isSuccess(decoded)) { + const task = decoded.success; + // next_run_at is a freeform string at the schema level; a stored value + // that cannot parse as a DateTime would defect the poll below, so the + // row is skipped here like any other corrupt row. + if (task.nextRunAt === null || Option.isSome(DateTime.make(task.nextRunAt))) { + tasks.push(task); + } else { + yield* Effect.logWarning("Skipping schedule task row with invalid next_run_at", { + taskId: row.task_id, + }); + } + } else { + yield* Effect.logWarning("Skipping undecodable schedule task row", { + taskId: row.task_id, + cause: decoded.failure, + }); + } + } + return tasks; + }, +); + export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(function* ( now: DateTime.DateTime, ) { @@ -362,31 +391,22 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct AND next_run_at <= ${iso(now)} AND last_run_status <> 'running' ORDER BY next_run_at ASC, task_id ASC `; - const tasks: ScheduledTask[] = []; - for (const row of rows) { - const decoded = yield* Effect.result(decodeRow(row)); - if (Result.isSuccess(decoded)) { - const task = decoded.success; - // next_run_at is a freeform string at the schema level; a stored value - // that cannot parse as a DateTime would defect the poll below, so the - // row is skipped here like any other corrupt row. - if (task.nextRunAt === null || Option.isSome(DateTime.make(task.nextRunAt))) { - tasks.push(task); - } else { - yield* Effect.logWarning("Skipping schedule task row with invalid next_run_at", { - taskId: row.task_id, - }); - } - } else { - yield* Effect.logWarning("Skipping undecodable schedule task row", { - taskId: row.task_id, - cause: decoded.failure, - }); - } - } - return tasks; + return yield* decodeTaskRowsLeniently(rows); }); +/** Tasks running now or due by `by`, for background work that must not start under them. */ +export const listRunningOrDueTasks = Effect.fn("ScheduledTaskService.listRunningOrDueTasks")( + function* (by: DateTime.DateTime) { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql` + SELECT * FROM scheduled_tasks + WHERE last_run_status = 'running' + OR (enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ${iso(by)}) + `; + return yield* decodeTaskRowsLeniently(rows); + }, +); + export const layer = Layer.effect( ScheduledTaskService, Effect.gen(function* () { diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 184475a92cbb4..c549bc4a56b2d 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -4,6 +4,7 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as SqlClient from "effect/sql/SqlClient"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; @@ -20,9 +21,10 @@ const UPDATE_LOOKAHEAD = Duration.minutes(5); */ const QUIET_PERIOD = Duration.minutes(15); /** - * Tasks repeating more often than this always have a run due soon. A run that - * starts mid-install waits for the admission permit, and a run missed during a - * restart catches up afterwards, so these only block while running. + * Tasks repeating more often than this always have a run due soon, so they + * only block while running. A run that starts mid-install waits for the + * admission permit; one that fires during a server handoff is recorded as + * interrupted and the task runs again on its next interval. */ const FREQUENT_TASK_INTERVAL = Duration.minutes(15); @@ -46,7 +48,7 @@ export class UpdateWindow extends Context.Service< const make = Effect.fn("updates.UpdateWindow.make")(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; - const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; + const sql = yield* SqlClient.SqlClient; const serverSettings = yield* ServerSettings.ServerSettingsService; const providers = yield* ProviderRegistry.ProviderRegistry; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; @@ -79,27 +81,21 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { found.push("recent-interaction"); } - const tasks = yield* scheduledTasks.list().pipe( + // Corrupt task rows are skipped, so one bad row cannot close the window for good. + const tasks = yield* ScheduledTaskService.listRunningOrDueTasks(horizon).pipe( + Effect.provideService(SqlClient.SqlClient, sql), Effect.tapError((cause) => Effect.logWarning("Update window could not read scheduled tasks", { cause }), ), Effect.option, ); if (Option.isNone(tasks)) return [...found, "scheduled-task"]; - for (const task of tasks.value.tasks) { - if (task.lastRunStatus === "running") { - found.push("scheduled-task"); - continue; - } - if (!task.enabled || task.nextRunAt === null) continue; + for (const task of tasks.value) { if ( - task.schedule.type === "interval" && - task.schedule.everyMs < Duration.toMillis(FREQUENT_TASK_INTERVAL) + task.lastRunStatus === "running" || + task.schedule.type !== "interval" || + task.schedule.everyMs >= Duration.toMillis(FREQUENT_TASK_INTERVAL) ) { - continue; - } - const runsAt = DateTime.make(task.nextRunAt); - if (Option.isSome(runsAt) && !DateTime.isGreaterThan(runsAt.value, horizon)) { found.push("scheduled-task"); } } From 104cd8706e7eced0bea977c1fc31bef5891ac0a4 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:39:06 +0000 Subject: [PATCH 044/149] docs(updates): note that newer releases wait for a staged one to install --- docs/user/updating.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user/updating.md b/docs/user/updating.md index c6d2b11514652..786a22398015d 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -84,6 +84,7 @@ hold updates back while they run. Then it installs: checks** is also on; - the newest T3 Code release on its channel, when it runs as a background service (`t3 service install`). It downloads first and restarts only inside that window. + While one release waits for the window, newer releases wait for it to install. T3 Code server restarts also wait for integrated terminal commands to finish. From 1005be90a16f215558d329cd367e53fceae42a10 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:41:37 +0000 Subject: [PATCH 045/149] test(server): cover the restart continuation chain limit --- .../RestartContinuation.test.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index dcbf4aa637284..3dcafae687171 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -643,6 +643,31 @@ const cutMidTurn = (extra: Record = {}) => { } as unknown as OrchestrationV2ThreadProjection; }; +it.effect("stops continuing after three automatic restart continuations in a row", () => + Effect.gen(function* () { + const first = RunId.make("run:first-continuation"); + const second = RunId.make("run:second-continuation"); + const cut = cutMidTurn({ restartContinuationOfRunId: second }); + const withTwoLinks = { + ...cut, + runs: [ + ...cut.runs, + { ...cut.runs[0]!, id: second, restartContinuationOfRunId: first }, + { ...cut.runs[0]!, id: first, restartContinuationOfRunId: undefined }, + ], + } as unknown as OrchestrationV2ThreadProjection; + assert.lengthOf(yield* continuationTexts(withTwoLinks), 1); + + const withThreeLinks = { + ...withTwoLinks, + runs: withTwoLinks.runs.map((run) => + run.id === first ? { ...run, restartContinuationOfRunId: RunId.make("run:root") } : run, + ), + } as unknown as OrchestrationV2ThreadProjection; + assert.lengthOf(yield* continuationTexts(withThreeLinks), 0); + }), +); + const queuedFollowUp = { id: RunId.make("run:queued"), ordinal: 2, From f38785969bd0a45c3cdd562b9bd815a126bac2c8 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:13:13 +0000 Subject: [PATCH 046/149] docs(server): keep the scheduled task poll comment on listDueTasks --- apps/server/src/scheduledTasks/ScheduledTaskService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 2ac47208e99e7..749dace7ef464 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -351,7 +351,6 @@ const decodeRow = (row: ScheduledTaskRow, origin: WebhookOrigin | null = null) = }), ); -/** Select poll candidates before decoding their schedules or other JSON payloads. */ /** Decodes rows for background readers, skipping corrupt ones so one bad row cannot stall them. */ const decodeTaskRowsLeniently = Effect.fn("ScheduledTaskService.decodeTaskRowsLeniently")( function* (rows: ReadonlyArray) { @@ -381,6 +380,7 @@ const decodeTaskRowsLeniently = Effect.fn("ScheduledTaskService.decodeTaskRowsLe }, ); +/** Select poll candidates before decoding their schedules or other JSON payloads. */ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(function* ( now: DateTime.DateTime, ) { From c1f1861eaad2f4948d99c245769bc4b09b63081b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:00:21 +0000 Subject: [PATCH 047/149] fix(desktop): keep install on quit honest after failed downloads A failed macOS Squirrel staging no longer leaves the channel locked, a failed download disarms install on quit, and the background poller no longer replaces an update that is already downloaded. --- .../src/updates/DesktopUpdates.test.ts | 51 +++++++++++++++++++ apps/desktop/src/updates/DesktopUpdates.ts | 17 +++++++ apps/desktop/src/updates/updateMachine.ts | 2 + docs/operations/release.md | 4 +- 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index d193453f9f0eb..e4e5a3b6b054d 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -887,6 +887,57 @@ describe("DesktopUpdates", () => { }, ); + it.effect("releases the channel and install on quit when macOS staging fails", () => { + const harness = makeHarness({ + // MacUpdater emits update-downloaded before Squirrel staging rejects the download. + downloadUpdate: Effect.sync(() => + harness.emit("update-downloaded", { version: "1.2.4" }), + ).pipe( + Effect.andThen(flushCallbacks), + Effect.andThen(Effect.die(new Error("Squirrel staging failed"))), + ), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + + const result = yield* updates.download; + assert.isFalse(result.completed); + assert.isNull(result.state.downloadedVersion); + assert.isFalse(harness.autoInstallOnAppQuit()); + assert.equal((yield* updates.setChannel("nightly")).channel, "nightly"); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + + it.effect("keeps a downloaded update for quit instead of polling for a newer one", () => { + const harness = makeHarness({ + platform: "win32", + checkForUpdates: Effect.sync(() => { + harness.emit("update-available", { version: "1.2.5" }); + }), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + + yield* TestClock.adjust(Duration.minutes(5)); + assert.equal(harness.checkCount(), 0); + assert.equal(harness.downloadCount(), 0); + assert.equal((yield* updates.getState).downloadedVersion, "1.2.4"); + assert.isTrue(harness.autoInstallOnAppQuit()); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + it.effect("never installs .deb updates on quit because dpkg asks for a password", () => { const harness = makeHarness({ platform: "linux", packageType: "deb" }); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 781a824e714fa..5b083a2a22640 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -481,6 +481,18 @@ export const make = Effect.gen(function* () { return { accepted: false, completed: false }; } + // A failed download leaves nothing to install on quit: electron-updater drops + // the previous installer, and on macOS update-downloaded fires before Squirrel + // stages the download, so a later failure means nothing was staged. + const stagedBefore = yield* Ref.get(nativeUpdateStagedRef); + const releaseFailedDownload = Effect.all( + [ + Ref.set(nativeUpdateStagedRef, stagedBefore), + electronUpdater.setAutoInstallOnAppQuit(false), + ], + { discard: true }, + ); + return yield* Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); yield* electronUpdater.setDisableDifferentialDownload( @@ -493,6 +505,7 @@ export const make = Effect.gen(function* () { Effect.catchTags({ ElectronUpdaterDownloadUpdateError: Effect.fn("desktop.updates.handleDownloadFailure")( function* (error) { + yield* releaseFailedDownload; yield* updateState((current) => reduceDesktopUpdateStateOnDownloadFailure(current, error.message), ); @@ -515,6 +528,7 @@ export const make = Effect.gen(function* () { } const error = new DesktopUpdateUnexpectedActionError({ action: "download", cause }); return Effect.gen(function* () { + yield* releaseFailedDownload; yield* updateState((current) => reduceDesktopUpdateStateOnDownloadFailure(current, error.message), ); @@ -531,8 +545,11 @@ export const make = Effect.gen(function* () { // Updates the background pollers find download on their own; a check someone // started keeps its Download button. A failed download waits for a newer release. + // A downloaded update installs on quit, so the pollers leave it alone: a new + // download would discard its installer, and the next launch finds newer releases. const checkAndDownloadInBackground = Effect.fn("desktop.updates.checkAndDownloadInBackground")( function* (reason: "startup" | "poll") { + if ((yield* Ref.get(updateStateRef)).downloadedVersion !== null) return; if (!(yield* checkForUpdates(reason)) || (yield* Ref.get(desktopState.quitting))) return; const version = (yield* Ref.get(updateStateRef)).availableVersion; if (version === null || version === (yield* Ref.get(failedBackgroundDownloadRef))) return; diff --git a/apps/desktop/src/updates/updateMachine.ts b/apps/desktop/src/updates/updateMachine.ts index 85d8782ea6688..b07bd46f1fd9f 100644 --- a/apps/desktop/src/updates/updateMachine.ts +++ b/apps/desktop/src/updates/updateMachine.ts @@ -162,6 +162,8 @@ export function reduceDesktopUpdateStateOnDownloadFailure( return { ...state, status: nextStatusAfterDownloadFailure(state), + // A failed download leaves no installer, including a failed macOS staging. + downloadedVersion: null, message, downloadPercent: null, errorContext: "download", diff --git a/docs/operations/release.md b/docs/operations/release.md index 6be0d8fd90fea..82088face2676 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -399,8 +399,8 @@ available. - `apps/desktop/src/main.ts` only wires the updater layers into the desktop runtime. - Update UX: - Background checks run on startup delay + interval. - - No automatic download or install. - - The desktop UI shows a rocket update button when an update is available; click once to download, click again after download to restart/install. + - Updates found by background checks download automatically and install when the app quits (except `.deb`, where dpkg asks for a password). + - The desktop UI shows a rocket update button when an update is available; click once to download, click again after download to restart/install right away. - Provider: GitHub Releases (`provider: github`) configured at build time. - Repository slug source: - `T3CODE_DESKTOP_UPDATE_REPOSITORY` (format `owner/repo`), if set. From 2b347172546c0465442c1bb3d80fc16a37760a05 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:10:52 +0000 Subject: [PATCH 048/149] fix(server): an unreadable settings file keeps automatic updates off settings.json that fails to decode falls back to defaults, which turned automatic updates on even for someone who had turned them off. The fallback now keeps them off until the file can be read. --- apps/server/src/serverSettings.test.ts | 2 ++ apps/server/src/serverSettings.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 4444a7851dffc..b32457f71305f 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1045,6 +1045,8 @@ it.layer(NodeServices.layer)("server settings", (it) => { driver: ProviderDriverKind.make("codex"), config: { binaryPath: "/x" }, }); + // An unreadable file may hold an opt-out, so background updates stay off. + assert.equal(settings.automaticUpdates, false); assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); }).pipe(Effect.provide(layerServerSettings())), ); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 7536a74d6e02c..da3f0a53b3751 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -786,6 +786,8 @@ const make = Effect.gen(function* () { if (decoded._tag === "Failure" || legacySettings._tag === "Failure") { const failure = decoded._tag === "Failure" ? decoded : legacySettings; settingsFileTrusted = false; + // The file may hold an opt-out we can no longer read; never install unattended on a guess. + settings = { ...DEFAULT_SERVER_SETTINGS, automaticUpdates: false }; if (failure._tag === "Failure") { yield* Effect.logWarning("failed to parse settings.json, using defaults", { path: settingsPath, From 4fc9c54dc38ee716ff0b936993a7503d90041bbb Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:10:52 +0000 Subject: [PATCH 049/149] fix(server): background updates wait for agent background work and returning users The update window now stays closed while a live provider session still runs background commands or subagents, and while a snoozed usage-limit resume is about to fire. The quiet period uses the monotonic clock, so time asleep and clock jumps never count as idle, and the web client counts returning to the window and keys inside controls that stop propagation as interaction. --- .../server/src/background/BackgroundPolicy.ts | 20 +++++++++--- .../src/orchestration-v2/EffectWorker.test.ts | 1 + .../ProviderSessionManager.ts | 18 +++++++++++ .../ProviderTurnControlService.test.ts | 1 + apps/server/src/updates/UpdateWindow.ts | 17 +++++++--- .../web/src/lib/backgroundActivityReporter.ts | 32 +++++++++++-------- docs/user/updating.md | 10 +++--- 7 files changed, 73 insertions(+), 26 deletions(-) diff --git a/apps/server/src/background/BackgroundPolicy.ts b/apps/server/src/background/BackgroundPolicy.ts index 48701d0fcaf03..d89026d811c91 100644 --- a/apps/server/src/background/BackgroundPolicy.ts +++ b/apps/server/src/background/BackgroundPolicy.ts @@ -12,8 +12,10 @@ import { resolveServerBackgroundActivitySettings, type ResolvedBackgroundActivitySettings, } from "@t3tools/shared/backgroundActivitySettings"; +import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; @@ -40,8 +42,12 @@ export class BackgroundPolicy extends Context.Service< ) => Effect.Effect; readonly reportHostPowerState: (snapshot: HostPowerSnapshot) => Effect.Effect; readonly snapshot: Effect.Effect; - /** When any client last reported recent interaction (boot time until one does), so updates can wait for quiet. */ - readonly lastClientInteractionAt: Effect.Effect; + /** + * Time since any client last reported recent interaction (since boot until + * one does), so updates can wait for quiet. Measured on the monotonic + * clock: time asleep and wall-clock jumps never count as quiet. + */ + readonly sinceLastClientInteraction: Effect.Effect; readonly streamChanges: Stream.Stream; readonly subscribe: Effect.Effect< { @@ -215,7 +221,7 @@ export const make = Effect.fn("background.policy.make")(function* () { const hostPowerMonitor = yield* HostPowerMonitor.HostPowerMonitor; const serverSettings = yield* ServerSettings.ServerSettingsService; const leasesRef = yield* Ref.make(new Map()); - const lastClientInteractionAtRef = yield* Ref.make(yield* DateTime.now); + const lastClientInteractionNanosRef = yield* Ref.make(yield* Clock.monotonicTimeNanos); const changes = yield* PubSub.sliding(1); const publishMutex = yield* Semaphore.make(1); @@ -270,7 +276,7 @@ export const make = Effect.fn("background.policy.make")(function* () { }; yield* Ref.update(leasesRef, (leases) => upsertClientActivityLease(leases, lease, now)); if (input.recentlyInteracted) { - yield* Ref.set(lastClientInteractionAtRef, now); + yield* Ref.set(lastClientInteractionNanosRef, yield* Clock.monotonicTimeNanos); } yield* publishSnapshotUnlocked; }), @@ -345,7 +351,11 @@ export const make = Effect.fn("background.policy.make")(function* () { removeRpcClient, reportHostPowerState: hostPowerMonitor.report, snapshot, - lastClientInteractionAt: Ref.get(lastClientInteractionAtRef), + sinceLastClientInteraction: Effect.zipWith( + Clock.monotonicTimeNanos, + Ref.get(lastClientInteractionNanosRef), + (now, last) => Duration.nanos(now - last), + ), streamChanges: Stream.fromPubSub(changes), subscribe: subscribeBeforeSnapshot(changes, snapshot, publishMutex), hasDemand, diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts index d9bd8689f0922..4e18e5d344257 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.test.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts @@ -102,6 +102,7 @@ function layerExecutorFor(input: { ProviderSessionManager.ProviderSessionManagerV2, ProviderSessionManager.ProviderSessionManagerV2.of({ shutdown: Effect.void, + hasPendingBackgroundWork: Effect.succeed(false), open: () => Effect.die("unused open"), get: () => Effect.succeed(Option.none()), close: () => Effect.void, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 53c1c06f13ef0..b32cc9f9fb98e 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -191,6 +191,11 @@ export interface ProviderSessionManagerV2Shape { */ readonly revokeMcpCredential?: boolean; }) => Effect.Effect; + /** + * True while any live session still runs provider work outside a turn, such + * as background commands or subagents, so updates can wait for it. + */ + readonly hasPendingBackgroundWork: Effect.Effect; } export class ProviderSessionManagerV2 extends Context.Service< @@ -2015,8 +2020,21 @@ export const layerWithOptions = ( ), ); + const hasPendingBackgroundWork = Effect.gen(function* () { + for (const entry of (yield* Ref.get(sessions)).values()) { + if (entry.runtime.hasPendingBackgroundWork === undefined) continue; + // An unreadable session counts as busy: guessing idle could cut off its work. + const pending = yield* entry.runtime.hasPendingBackgroundWork.pipe( + Effect.catchCause(() => Effect.succeed(true)), + ); + if (pending) return true; + } + return false; + }); + return ProviderSessionManagerV2.of({ shutdown, + hasPendingBackgroundWork, open: (input) => sessionOpen.withLock( input.providerSessionId, diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 2ac00838018b4..492a93a1654bb 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -266,6 +266,7 @@ it.effect( ProviderSessionManager.ProviderSessionManagerV2, ProviderSessionManager.ProviderSessionManagerV2.of({ shutdown: Effect.void, + hasPendingBackgroundWork: Effect.succeed(false), open: () => Effect.die("unused open"), get: (providerSessionId) => Effect.succeed( diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index c549bc4a56b2d..9f29e92a70ee6 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -8,6 +8,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; @@ -47,6 +48,7 @@ export class UpdateWindow extends Context.Service< const make = Effect.fn("updates.UpdateWindow.make")(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const sql = yield* SqlClient.SqlClient; const serverSettings = yield* ServerSettings.ServerSettingsService; @@ -68,6 +70,8 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { ); if (Option.isNone(activeThreadIds)) return ["active-threads"]; if (activeThreadIds.value.length > 0) found.push("active-threads"); + // Work a provider keeps running after its turn ends, such as background commands. + if (yield* sessions.hasPendingBackgroundWork) found.push("background-work"); // Includes installs someone started by hand. const providerUpdating = (yield* providers.getProviders).some( @@ -76,8 +80,7 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { ); if (providerUpdating) found.push("provider-update"); - const lastInteraction = yield* backgroundPolicy.lastClientInteractionAt; - if (DateTime.isGreaterThan(DateTime.addDuration(lastInteraction, QUIET_PERIOD), now)) { + if (Duration.isLessThan(yield* backgroundPolicy.sinceLastClientInteraction, QUIET_PERIOD)) { found.push("recent-interaction"); } @@ -122,8 +125,14 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { if (Option.isNone(limited)) return [...found, "usage-limit-resume"]; for (const thread of limited.value) { if (thread.usageLimitResetAt === null || thread.usageLimitResetAt === undefined) continue; - const resumesAt = DateTime.make(thread.usageLimitResetAt); - // A reset already past (say, a resume that keeps failing) must not block forever. + const resetAt = DateTime.make(thread.usageLimitResetAt); + // A snoozed thread resumes once both its reset and its snooze have passed. + const resumesAt = Option.map(resetAt, (reset) => + thread.snoozedUntil != null && DateTime.isGreaterThan(thread.snoozedUntil, reset) + ? thread.snoozedUntil + : reset, + ); + // A resume time already past (say, a resume that keeps failing) must not block forever. if ( Option.isSome(resumesAt) && DateTime.isGreaterThan(resumesAt.value, now) && diff --git a/apps/web/src/lib/backgroundActivityReporter.ts b/apps/web/src/lib/backgroundActivityReporter.ts index 5a65dab380950..adae5198ac7b0 100644 --- a/apps/web/src/lib/backgroundActivityReporter.ts +++ b/apps/web/src/lib/backgroundActivityReporter.ts @@ -191,7 +191,13 @@ export const layer = Layer.effectDiscard( requestReport(); } }; - const passiveListenerOptions = { passive: true } as const; + // Captured, so a control that stops propagation (say, an editor's keydown) still counts. + const interactionListenerOptions = { capture: true, passive: true } as const; + // Coming back to the window means someone is about to use it. + const onForeground = () => { + if (document.visibilityState === "visible") recordInteraction(); + requestReport(); + }; const report = Effect.gen(function* () { const observedAtMs = yield* Clock.currentTimeMillis; @@ -215,26 +221,26 @@ export const layer = Layer.effectDiscard( yield* Effect.acquireRelease( Effect.sync(() => { retainedScopeListeners.add(requestReport); - document.addEventListener("visibilitychange", requestReport); - window.addEventListener("focus", requestReport); + document.addEventListener("visibilitychange", onForeground); + window.addEventListener("focus", onForeground); window.addEventListener("blur", requestReport); window.addEventListener("online", requestReport); - window.addEventListener("pointermove", recordInteraction); - window.addEventListener("keydown", recordInteraction); - window.addEventListener("wheel", recordInteraction, passiveListenerOptions); - window.addEventListener("touchstart", recordInteraction, passiveListenerOptions); + window.addEventListener("pointermove", recordInteraction, interactionListenerOptions); + window.addEventListener("keydown", recordInteraction, interactionListenerOptions); + window.addEventListener("wheel", recordInteraction, interactionListenerOptions); + window.addEventListener("touchstart", recordInteraction, interactionListenerOptions); }), () => Effect.sync(() => { retainedScopeListeners.delete(requestReport); - document.removeEventListener("visibilitychange", requestReport); - window.removeEventListener("focus", requestReport); + document.removeEventListener("visibilitychange", onForeground); + window.removeEventListener("focus", onForeground); window.removeEventListener("blur", requestReport); window.removeEventListener("online", requestReport); - window.removeEventListener("pointermove", recordInteraction); - window.removeEventListener("keydown", recordInteraction); - window.removeEventListener("wheel", recordInteraction); - window.removeEventListener("touchstart", recordInteraction); + window.removeEventListener("pointermove", recordInteraction, interactionListenerOptions); + window.removeEventListener("keydown", recordInteraction, interactionListenerOptions); + window.removeEventListener("wheel", recordInteraction, interactionListenerOptions); + window.removeEventListener("touchstart", recordInteraction, interactionListenerOptions); }), ); diff --git a/docs/user/updating.md b/docs/user/updating.md index 349a1bc400127..76e25ec164595 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -77,10 +77,12 @@ update can roll back to the previous version. If the update still fails: **Settings → General → Update automatically** is on by default. With **Provider update checks** also on, each environment installs provider CLI updates in the background once no thread is running or waiting for you, no -client has been used for 15 minutes, and no scheduled task or usage-limit resume -is due within 5 minutes. Tasks repeating more often than every 15 minutes only -hold updates back while they run. A failed update is retried after 6 hours or -when a newer version is published. +agent is still running background work, no client has been used for 15 minutes, +and no scheduled task or usage-limit resume is due within 5 minutes. Tasks +repeating more often than every 15 minutes only hold updates back while they +run. A failed update is retried after 6 hours, when a newer version is +published, or after the server restarts. While `settings.json` cannot be read, +automatic updates stay off. **Settings → Providers** shows provider updates for the selected environment. **Update all** updates every outdated provider on every connected environment From c4511883ab87dc0b072b929fbae7668852f1ea6e Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:10:52 +0000 Subject: [PATCH 050/149] fix(server): background provider installs never stall or race a provider launch The npm version lookup timeout now covers the response body, an installer that ignores SIGTERM is killed after 5 seconds, and background installs stop streaming installer output to every client. Text generation and checkpoint rollback wait for a running install like turn starts do. --- .../CheckpointRollbackService.ts | 4 ++++ .../src/provider/providerMaintenanceRunner.ts | 15 +++++++++++-- .../src/textGeneration/TextGeneration.ts | 15 +++++++++---- .../server/src/updates/ProviderAutoUpdater.ts | 2 +- .../src/server/maintenanceResolver.ts | 21 +++++++------------ 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index acacfed521c6f..8efe1b732d20a 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -23,6 +23,7 @@ import { EventSinkV2 } from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; +import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; @@ -101,6 +102,7 @@ export const layer: Layer.Layer< const ids = yield* IdAllocator.IdAllocatorV2; const projections = yield* ProjectionStoreV2; const sessions = yield* ProviderSessionManagerV2; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const runtimePolicy = yield* RuntimePolicyV2; const fileSystem = yield* FileSystem.FileSystem; const projects = yield* ProjectStore.ProjectStoreV2; @@ -183,6 +185,8 @@ export const layer: Layer.Layer< const existingSession = projection.providerSessions.find( (candidate) => candidate.id === providerThread.providerSessionId, ); + // Opening a session can launch the provider CLI; wait out a background install. + yield* admission.withPermit(Effect.void); const session = yield* sessions.open({ threadId: input.threadId, providerSessionId: providerThread.providerSessionId, diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index fd4e23aee3701..6dca9f2603095 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -40,6 +40,7 @@ import { collectUint8StreamText } from "@t3tools/provider-core/server/collectStr const isServerProviderUpdateError = Schema.is(ServerProviderUpdateError); const UPDATE_TIMEOUT_MS = 5 * 60_000; +const UPDATE_FORCE_KILL_AFTER = Duration.seconds(5); const UPDATE_OUTPUT_MAX_BYTES = 10_000; // Every progress publish resends the provider list to each client, so the // installer's output is sampled rather than forwarded line by line. @@ -65,6 +66,8 @@ export interface ProviderMaintenanceRunnerShape { readonly provider: ProviderDriverKind; readonly instanceId?: ProviderInstanceId | undefined; readonly targetVersion?: string | undefined; + /** Skips installer output progress, for background installs nobody watches. */ + readonly quiet?: boolean | undefined; }, ) => Effect.Effect; } @@ -106,6 +109,8 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR .spawn( ChildProcess.make(resolved.command, resolved.args, { shell: resolved.shell, + // An installer that ignores SIGTERM must not hold up the update forever. + forceKillAfter: UPDATE_FORCE_KILL_AFTER, ...(input.env ? { env: input.env, extendEnv: true } : {}), }), ) @@ -118,7 +123,9 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR }), ), ); - yield* Effect.addFinalizer(() => child.kill().pipe(Effect.ignore)); + yield* Effect.addFinalizer(() => + child.kill({ forceKillAfter: UPDATE_FORCE_KILL_AFTER }).pipe(Effect.ignore), + ); // Holds the newest output line not yet reported; the sampler takes it. const pendingProgress = yield* Ref.make(null); @@ -390,6 +397,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { ? defaultInstanceIdForDriver(provider) : (target.instanceId ?? defaultInstanceIdForDriver(provider)); const targetVersion = typeof target === "string" ? undefined : target.targetVersion; + const quiet = typeof target !== "string" && target.quiet === true; const targetKey = `instance:${instanceId}`; const capabilities = yield* providerRegistry.getProviderMaintenanceCapabilitiesForInstance( instanceId, @@ -499,7 +507,10 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { ); } yield* setRunningMessage(`Running ${describeCommand(command)}`); - const result = yield* runMaintenanceCommand(command, setRunningMessage); + const result = yield* runMaintenanceCommand( + command, + quiet ? () => Effect.void : setRunningMessage, + ); const finishedAt = yield* nowIso; if (result.timedOut || result.exitCode !== 0) { return yield* finish( diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index ee6f18e7b217b..ce99d1812ceed 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -6,6 +6,7 @@ import type { ProviderTextGeneration } from "@t3tools/provider-core/server/textG import { TextGenerationError } from "@t3tools/contracts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; +import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; import * as ThreadTitleLinks from "./ThreadTitleLinks.ts"; @@ -56,21 +57,27 @@ const resolveInstance = ( export const make = Effect.gen(function* () { const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; const sourceControl = yield* SourceControlProviderRegistry.SourceControlProviderRegistry; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; + // Waits out a background install so a provider CLI is never launched mid-replacement. + const instanceFor = (operation: TextGenerationOp, instanceId: ProviderInstanceId) => + admission + .withPermit(Effect.void) + .pipe(Effect.andThen(resolveInstance(registry, operation, instanceId))); return TextGeneration.of({ generateCommitMessage: (input) => - resolveInstance(registry, "generateCommitMessage", input.modelSelection.instanceId).pipe( + instanceFor("generateCommitMessage", input.modelSelection.instanceId).pipe( Effect.flatMap((textGeneration) => textGeneration.generateCommitMessage(input)), ), generatePrContent: (input) => - resolveInstance(registry, "generatePrContent", input.modelSelection.instanceId).pipe( + instanceFor("generatePrContent", input.modelSelection.instanceId).pipe( Effect.flatMap((textGeneration) => textGeneration.generatePrContent(input)), ), generateBranchName: (input) => - resolveInstance(registry, "generateBranchName", input.modelSelection.instanceId).pipe( + instanceFor("generateBranchName", input.modelSelection.instanceId).pipe( Effect.flatMap((textGeneration) => textGeneration.generateBranchName(input)), ), generateThreadTitle: (input) => - resolveInstance(registry, "generateThreadTitle", input.modelSelection.instanceId).pipe( + instanceFor("generateThreadTitle", input.modelSelection.instanceId).pipe( Effect.flatMap((textGeneration) => Effect.gen(function* () { const linkedContext = diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index e70a5acc32533..7d04e6727611e 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -74,7 +74,7 @@ export const layer = Layer.effectDiscard( targetVersion, }); yield* runner - .updateProvider({ provider: current.driver, instanceId }) + .updateProvider({ provider: current.driver, instanceId, quiet: true }) .pipe( Effect.catchCause((cause) => Effect.logWarning("Background provider update failed", { instanceId, cause }), diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index fb985bd2011af..05ec18a4ce21d 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -745,22 +745,17 @@ const fetchNpmLatestVersion = Effect.fn("fetchNpmLatestVersion")(function* (pack const request = HttpClientRequest.get( `https://registry.npmjs.org/${encodeURIComponent(packageName)}/latest`, ).pipe(HttpClientRequest.setHeader("accept", "application/json")); - const response = yield* client.execute(request).pipe( + // The timeout covers the body too: a stalled response must not hold up an update. + const payload = yield* client.execute(request).pipe( + Effect.flatMap((response) => + response.status < 200 || response.status >= 300 + ? Effect.succeed(null) + : response.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(NpmLatestVersionResponse))), + ), Effect.timeoutOption(LATEST_VERSION_TIMEOUT_MS), Effect.orElseSucceed(() => Option.none()), ); - if (Option.isNone(response)) { - return null; - } - const httpResponse = response.value; - if (httpResponse.status < 200 || httpResponse.status >= 300) { - return null; - } - const payload = yield* httpResponse.json.pipe( - Effect.flatMap(Schema.decodeUnknownEffect(NpmLatestVersionResponse)), - Effect.orElseSucceed(() => null), - ); - return payload ? nonEmptyString(payload.version) : null; + return Option.isSome(payload) && payload.value ? nonEmptyString(payload.value.version) : null; }); export const resolveLatestProviderVersion = Effect.fn("resolveLatestProviderVersion")(function* ( From 660fa8467370520727297059c3d52abbe8e97c0b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:10:53 +0000 Subject: [PATCH 051/149] fix(opencode): a configured OpenCode server never updates the local CLI An instance pointed at serverUrl reports that server's version, so updating the local binary could never bring it current. Such instances now offer no update. --- packages/provider-opencode/src/server/driver.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/provider-opencode/src/server/driver.ts b/packages/provider-opencode/src/server/driver.ts index 3834ec73534bc..b3f911af21ef4 100644 --- a/packages/provider-opencode/src/server/driver.ts +++ b/packages/provider-opencode/src/server/driver.ts @@ -248,8 +248,9 @@ export const OpenCodeDriver: ProviderDriver ); // A fresh read (an update about to run, or a manual refresh) re-probes, so // a binary replaced by the other major version gets its own package. + // A configured server is someone else's install; updating the local binary would not touch it. const resolveMaintenance = (options?: { readonly fresh?: boolean }) => - !effectiveConfig.enabled + !effectiveConfig.enabled || effectiveConfig.serverUrl.trim().length > 0 ? Effect.succeed(noUpdate) : options?.fresh === true ? runtimeProbe.refresh.pipe( From 89593cb3f1cd6352852667632fc02c615eb2248d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:12:07 +0000 Subject: [PATCH 052/149] fix(server): background server restarts wait for first installs and stop at a protocol change A managed Codex or Antigravity install downloads inside the server, so a restart would throw it away; the restart now waits for it. Once a release needs a newer client protocol, later releases do too, so the updater stops downloading them instead of keeping a copy of each. --- apps/server/src/updates/ServerAutoUpdater.ts | 2 ++ apps/server/src/updates/UpdateWindow.ts | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 23b3d92ae1923..daa617acc655c 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -86,7 +86,9 @@ export const layer = Layer.effectDiscard( ); if (staged.orchestrationProtocol !== ORCHESTRATION_PROTOCOL_VERSION) { // Without a matching protocol, connected clients could be refused after restart. + // Later releases keep the newer protocol, so stop downloading them. skippedTargets.add(targetVersion); + nextReleaseCheckAt = Number.POSITIVE_INFINITY; yield* Effect.logInfo( "Skipping a background update without a matching client protocol", { diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 1db480db6d139..0e0d46bf0ba0d 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -9,6 +9,8 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; +import * as AntigravityInstallation from "../provider/AntigravityInstallation.ts"; +import * as CodexInstallation from "../provider/CodexInstallation.ts"; import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; @@ -58,6 +60,10 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const providers = yield* ProviderRegistry.ProviderRegistry; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const terminals = yield* TerminalManager.TerminalManager; + const managedInstalls = [ + yield* CodexInstallation.CodexInstallation, + yield* AntigravityInstallation.AntigravityInstallation, + ]; /** Returns what keeps the window closed; empty means open. */ const blockers = Effect.fn("updates.UpdateWindow.blockers")(function* (restartsServer: boolean) { @@ -84,6 +90,15 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { ); if (providerUpdating) found.push("provider-update"); if (restartsServer && (yield* terminals.hasBusyTerminals)) found.push("terminal-work"); + // A first install downloads inside this server, so a restart would discard it. + if (restartsServer) { + for (const installation of managedInstalls) { + const { phase } = yield* installation.state; + if (phase === "downloading" || phase === "extracting" || phase === "verifying") { + found.push("provider-install"); + } + } + } if (Duration.isLessThan(yield* backgroundPolicy.sinceLastClientInteraction, QUIET_PERIOD)) { found.push("recent-interaction"); From 2cf1f33fedd1528c336a240fc2ff3204ab875bfe Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:12:07 +0000 Subject: [PATCH 053/149] docs(updates): an exact version only stays while automatic updates are off --- docs/operations/release.md | 2 +- docs/user/background-service.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/operations/release.md b/docs/operations/release.md index 6be0d8fd90fea..fcf1ca60887d0 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -366,7 +366,7 @@ One-time Vercel dashboard setup: ## Server self-update release invariant -Connected servers update to the client's exact version, not to an npm dist-tag. Every released +A client that updates a connected server asks for its own exact version, not an npm dist-tag. Every released desktop or hosted client version must therefore have a matching `t3@` package available on npm before users can receive that client. diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 9964fba54f74c..75d7b48d2e8f2 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -29,11 +29,11 @@ to pick up the new version. Wait for any remote update already in progress before updating; to match a remote client's version, follow [Updating T3 Code](./updating.md). -Pass an exact version (`t3 update 0.0.42`) to pin one, `--channel nightly` to -switch trains, or `--allow-downgrade` to move backwards. `preview` is a -maintainers' test train: its builds can be broken and are never offered as -updates, so the installer and `t3 update` ask for confirmation before -installing one. +Pass an exact version (`t3 update 0.0.42`) to install one (turn off **Update +automatically** to keep it), `--channel nightly` to switch trains, or +`--allow-downgrade` to move backwards. `preview` is a maintainers' test train: +its builds can be broken and are never offered as updates, so the installer and +`t3 update` ask for confirmation before installing one. `t3 uninstall` removes the background service, the `t3` launcher, and the downloaded versions after showing you the list and asking once. Your projects, From 26a39af146e03750af175ff55439370376527efa Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:58 +0000 Subject: [PATCH 054/149] fix(opencode): restart continuation follows a turn a surviving server still runs --- .../Adapters/OpenCode2AdapterV2.test.ts | 58 ++++++++++++ .../src/server/adapter.test.ts | 88 +++++++++++++++++++ .../provider-opencode/src/server/adapter.ts | 59 +++++++++++++ .../src/server/v2/adapter.ts | 40 +++++++++ 4 files changed, 245 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 205f54da70442..3e0e76dda9ec3 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -404,6 +404,64 @@ const history = { }; it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { + it.effect.each([false, true])( + "follows a surviving execution after a T3 restart (buffered=%s) without another prompt", + (buffered) => + Effect.gen(function* () { + const duringRestart = { data: history.data.slice(0, 2).toReversed(), cursor: {} }; + const { runtime, thread } = yield* resumed( + [ + ...(buffered ? [event("session.execution.started", { sessionID: SESSION })] : []), + out("session.active"), + replyData("session.active", { [SESSION]: { type: "running" } }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", duringRestart), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", duringRestart), + event("session.execution.succeeded", { sessionID: SESSION }), + ], + { external: true }, + ); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn({ + ...turnInput(thread), + restartContinuationOfRunId: RunId.make("interrupted-run"), + }); + const collected = yield* Fiber.join(events); + assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); + assert.include( + collected.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "assistant_message" + ? [event.turnItem.text] + : [], + ), + "391 is not prime: it's the product 17 × 23.", + ); + }).pipe(Effect.scoped), + ); + + it.effect("prompts after a restart when the native execution did not survive", () => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.active"), + replyData("session.active", {}), + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); + yield* runtime.startTurn({ + ...turnInput(thread), + restartContinuationOfRunId: RunId.make("interrupted-run"), + }); + assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + }).pipe(Effect.scoped), + ); + it.effect("switches the session's model and variant before a turn that changed them", () => Effect.gen(function* () { const { runtime, thread } = yield* resumed([ diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index c2ec42bd16b5f..305a39b709de2 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -186,8 +186,10 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct const startTurn = ( text = "hello", startProviderThread: OrchestrationV2ProviderThread = providerThread, + restartContinuationOfRunId?: RunId, ) => runtime.startTurn({ + ...(restartContinuationOfRunId === undefined ? {} : { restartContinuationOfRunId }), appThread: { id: threadId, projectId: ProjectId.make(`project-opencode-${suffix}`), @@ -241,6 +243,92 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct }); describe("OpenCodeAdapterV2", () => { + it.effect.each(["busy", "retry", "idle"] as const)( + "does not prompt a surviving %s session again after a T3 restart", + (status) => + Effect.gen(function* () { + const nativeEvents = asyncEventStream(); + let prompts = 0; + const harness = yield* makeOpenCodeRuntimeHarness(`restart-${status}`, "root", { + event: { subscribe: async () => ({ stream: nativeEvents.stream }) }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + get: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + status: async () => ({ data: { root: { type: status } } }), + messages: async () => ({ + data: [ + { + info: { + id: "original-prompt", + role: "user", + sessionID: "root", + time: { created: 1 }, + }, + parts: [], + }, + { + info: { + id: "assistant-gap", + role: "assistant", + sessionID: "root", + parentID: "original-prompt", + time: { created: 2 }, + }, + parts: [ + { + type: "text", + id: "part-gap", + sessionID: "root", + messageID: "assistant-gap", + text: "work during restart", + }, + ], + }, + ], + }), + promptAsync: async () => { + prompts++; + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + }); + const thread = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + }); + const terminal = yield* harness.runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* harness.startTurn( + "Continue where you left off.", + thread, + RunId.make("interrupted-run"), + ); + assert.equal(prompts, status === "idle" ? 1 : 0); + if (status !== "idle") { + yield* Effect.promise(() => + nativeEvents.push({ type: "session.idle", properties: { sessionID: "root" } }), + ); + const collected = yield* Fiber.join(terminal); + assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); + assert.include( + collected.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "assistant_message" + ? [event.turnItem.text] + : [], + ), + "work during restart", + ); + } + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + it.effect.each(["completed", "failed", "unresolved", "unavailable", "reconnect"] as const)( "normalizes OpenCode step usage for %s turns", (ending) => diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 96d535070294d..1b38bf2966c9d 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -3231,6 +3231,65 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function lastRunOrdinal: turnInput.runOrdinal, }); yield* updateProviderSession("running", null); + if (turnInput.restartContinuationOfRunId !== undefined) { + const statuses = unwrapData( + "session.status", + yield* sdkCall("session.status", { sessionID: sessionId }, () => + client.session.status(), + ).pipe( + Effect.tapError((cause) => + finalizeTurn(state, turn, "failed", { + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ), + ); + if (statuses[sessionId] !== undefined && statuses[sessionId]?.type !== "idle") { + // A server that outlived T3 still owns this turn; follow it + // instead of adding another prompt to its running execution. + turn.admissionMessageId = null; + advanceOpenCodePromptAdmission(turn, "busy"); + const admissionAction = advanceOpenCodePromptAdmission(turn, "accepted"); + yield* Deferred.succeed(admissionSettled, undefined); + turn.admissionAbortController = null; + const history = unwrapData( + "session.messages", + yield* sdkCall("session.messages", { sessionID: sessionId }, () => + client.session.messages({ sessionID: sessionId }), + ).pipe( + Effect.tapError((cause) => + finalizeTurn(state, turn, "failed", { + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ), + ); + const userIndex = history.findLastIndex((entry) => entry.info.role === "user"); + for (const entry of userIndex < 0 ? [] : history.slice(userIndex)) { + if (turn.finalized) break; + yield* handleMessageUpdated({ + id: entry.info.id, + type: "message.updated", + properties: { sessionID: sessionId, info: entry.info }, + }); + for (const part of entry.parts) { + yield* handlePartUpdated({ + id: part.id, + type: "message.part.updated", + properties: { + sessionID: sessionId, + part, + time: DateTime.toEpochMillis(startedAt), + }, + }); + } + } + if (admissionAction === "reconcile-idle") { + yield* reconcilePromptAdmission(state, turn); + } + return; + } + } if (isCompaction) { yield* sdkCall( "session.summarize", diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 306836614d550..c4e7840cee0e1 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -3742,6 +3742,46 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } // OpenCode already ran this turn on its own; it prompts nothing. if (isContinuation(turnInput)) return yield* runWake(state, turnInput); + if (turnInput.restartContinuationOfRunId !== undefined && !state.unsettled) { + const followed = yield* lock.withPermit( + Effect.gen(function* () { + const active = yield* client.session + .active() + .pipe(Effect.timeout(ACTIVE_CHECK_TIMEOUT)); + if (!(sessionId in active)) return false; + // A surviving server is still executing the interrupted + // run. Attach its events without enqueuing another prompt. + const history = yield* OpenCode2Client.paginate( + { sessionID: Session.ID.make(sessionId), order: "desc" as const, limit: 50 }, + client.message.list, + ).pipe( + Stream.takeUntil((message) => message.type === "user"), + Stream.runCollect, + ); + const prompt = history.at(-1); + const turn = yield* beginTurn(state, turnInput, false); + yield* takeRunningWake(state); + if (state.active !== turn) return true; + if (prompt?.type === "user") { + turn.providerTurn = { + ...turn.providerTurn, + nativeTurnRef: ref(prompt.id, "weak"), + }; + yield* emitProviderTurn(state, turn, turn.providerTurn); + yield* backfill(sessionId, state).pipe( + Effect.tapError((cause) => + finishTurn(state, { + status: "failed", + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ); + } + return true; + }), + ); + if (followed) return; + } // After a timed-out Stop the server says whether that run is gone. A // run still going is stopped again and this turn fails so it can be // sent again; a run that is gone may still have its end on the From ab2450ef1fcfafe63f54470a55837f9162dd7148 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:58 +0000 Subject: [PATCH 055/149] fix(codex): restart continuation retries a goal hold and prompts a fresh thread --- .../Adapters/CodexAdapterV2.test.ts | 279 +++++++++++++----- .../Adapters/CodexAdapterV2.ts | 31 +- 2 files changed, 229 insertions(+), 81 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 5b18f436c26e3..d01e1e95aa150 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -3124,90 +3124,215 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); - it.effect("continues an interrupted native thread with empty input and reasoning summaries", () => - Effect.scoped( - Effect.gen(function* () { - const scenario = "codex-restart-promptless"; - const nativeThreadId = "native-restart-promptless"; - const nativeTurnId = "turn-restart-promptless"; - const preamble = codexReplayPreamble({ - nativeThreadId, - nativeTurnId, - prompt: "unused", - }).slice(0, 5); - const transcript = makeCodexReplayTranscript({ - scenario, - entries: [ - ...preamble, - { - type: "expect_outbound", - label: "resume", - frame: { - id: 3, - method: "thread/resume", - params: { - threadId: nativeThreadId, - excludeTurns: true, - config: CodexAdapterV2.CODEX_THREAD_CONFIG, + it.effect.each([ + "interrupted", + "goal-hold", + "fresh-thread", + "other-error", + "settled", + "paused-goal", + "limited-goal", + ] as const)( + "continues a restarted %s without dropping context or duplicating accepted turns", + (state) => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-restart-promptless"; + const nativeThreadId = "native-restart-promptless"; + const nativeTurnId = "turn-restart-promptless"; + const preamble = codexReplayPreamble({ + nativeThreadId, + nativeTurnId, + prompt: "unused", + }).slice(0, 5); + const prompt = + state === "fresh-thread" + ? "Recovered request and partial work.\n\nContinue where you left off." + : "Continue where you left off."; + const params = { + threadId: nativeThreadId, + input: state === "fresh-thread" ? [{ type: "text", text: prompt }] : [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + }; + const emptyInputRejected = [ + "goal-hold", + "settled", + "paused-goal", + "limited-goal", + ].includes(state); + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...preamble, + { + type: "expect_outbound", + label: "resume", + frame: { + id: 3, + method: "thread/resume", + params: { + threadId: nativeThreadId, + excludeTurns: true, + config: CodexAdapterV2.CODEX_THREAD_CONFIG, + }, }, }, - }, - { - type: "emit_inbound", - label: "resume", - frame: { id: 3, result: { thread: { id: nativeThreadId, updatedAt: 1782622450 } } }, - }, - { - type: "expect_outbound", - label: "continue", - frame: { - id: 4, - method: "turn/start", - params: { - threadId: nativeThreadId, - input: [], - cwd: "/workspace", - model: "gpt-5.4", - approvalPolicy: "never", - approvalsReviewer: "user", - sandboxPolicy: { type: "dangerFullAccess" }, - summary: "detailed", + { + type: "emit_inbound", + label: "resume", + frame: { id: 3, result: { thread: { id: nativeThreadId, updatedAt: 1782622450 } } }, + }, + { + type: "expect_outbound", + label: "continue", + frame: { + id: 4, + method: "turn/start", + params, }, }, - }, - { - type: "emit_inbound", - label: "continue", - frame: { - id: 4, - result: { turn: makeCodexReplayTurn({ id: nativeTurnId, status: "inProgress" }) }, + { + type: "emit_inbound", + label: "continue", + frame: + emptyInputRejected || state === "other-error" + ? { + id: 4, + error: { + code: -32603, + message: emptyInputRejected + ? "failed to submit turn input: EmptyInput" + : "failed to submit turn input: unexpected failure", + }, + } + : { + id: 4, + result: { + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "inProgress" }), + }, + }, }, - }, - ], - }); - const harness = yield* makeCodexReplayHarness(transcript); - const resumed = yield* harness.runtime.resumeThread({ - providerThread: harness.providerThread, - modelSelection: CODEX_TEST_MODEL_SELECTION, - runtimePolicy: CODEX_TEST_RUNTIME_POLICY, - }); - yield* harness.runtime.startTurn({ - ...makeCodexTestTurnInput({ - threadId: harness.threadId, - providerThread: resumed, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("attempt-restart-promptless"), - text: "Continue where you left off.", - }), - restartContinuationOfRunId: RunId.make("run-before-restart"), - }); - assert.equal(resumed.nativeThreadRef?.nativeId, nativeThreadId); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ...(emptyInputRejected + ? [ + { + type: "expect_outbound" as const, + label: "read current goal", + frame: { + id: 5, + method: "thread/goal/get", + params: { threadId: nativeThreadId }, + }, + }, + { + type: "emit_inbound" as const, + label: "read current goal", + frame: { + id: 5, + result: { + goal: + state === "settled" + ? null + : { + threadId: nativeThreadId, + objective: "Finish the work", + status: + state === "paused-goal" + ? "paused" + : state === "limited-goal" + ? "budgetLimited" + : "active", + tokenBudget: null, + tokensUsed: 10, + timeUsedSeconds: 3, + createdAt: 1782622440, + updatedAt: 1782622450, + }, + }, + }, + }, + ] + : []), + ...(state === "goal-hold" + ? [ + { + type: "expect_outbound" as const, + label: "prompted continuation", + frame: { + id: 6, + method: "turn/start", + params: { ...params, input: [{ type: "text", text: prompt }] }, + }, + }, + { + type: "emit_inbound" as const, + label: "prompted continuation", + frame: { + id: 6, + result: { + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "inProgress" }), + }, + }, + }, + ] + : []), + ], + }); + const registered = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness(transcript, (event) => + event.type === "provider_turn.updated" + ? Deferred.succeed(registered, undefined) + : Effect.void, + ); + const resumed = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }); + const started = yield* harness.runtime + .startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: resumed, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-restart-promptless"), + text: prompt, + }), + nativeThreadHasTurns: state !== "fresh-thread", + restartContinuationOfRunId: RunId.make("run-before-restart"), + }) + .pipe(Effect.result); + if (state === "other-error" || (emptyInputRejected && state !== "goal-hold")) { + assert.equal(started._tag, "Failure"); + if (started._tag === "Failure") { + assert.nestedPropertyVal( + started.failure, + "cause.errorMessage", + emptyInputRejected + ? "failed to submit turn input: EmptyInput" + : "failed to submit turn input: unexpected failure", + ); + } + assert.isEmpty( + harness.events.filter((event) => event.type === "provider_turn.updated"), + ); + return; + } + assert.equal(started._tag, "Success"); + yield* Deferred.await(registered); + const turns = harness.events.filter((event) => event.type === "provider_turn.updated"); + assert.lengthOf(turns, 1); + assert.equal(resumed.nativeThreadRef?.nativeId, nativeThreadId); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), ), ), - ), ); it.effect("resolves retryable app-server errors on resumed provider activity", () => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index dbdbab32a74da..61f078bb47cdb 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -6612,11 +6612,34 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( : null; if (goalCommand !== null) return runGoalCommand(turnInput, goalCommand); return Effect.gen(function* () { - yield* startNativeTurn( - turnInput, - turnInput.restartContinuationOfRunId === undefined + const input = + turnInput.restartContinuationOfRunId === undefined || + turnInput.nativeThreadHasTurns === false ? yield* toCodexInput(turnInput) - : [], + : []; + yield* startNativeTurn(turnInput, input).pipe( + Effect.catchTags({ + CodexAppServerRequestError: (cause) => { + // A completed native goal turn can still hold the T3 run. + // Codex rejects empty input there before starting a turn. + if ( + input.length !== 0 || + turnInput.restartContinuationOfRunId === undefined || + cause.code !== -32603 || + cause.errorMessage !== "failed to submit turn input: EmptyInput" + ) + return Effect.fail(cause); + return Effect.gen(function* () { + const current = yield* client + .request("thread/goal/get", { + threadId: yield* getNativeThreadId(turnInput.providerThread), + }) + .pipe(Effect.orElseSucceed(() => null)); + if (current?.goal?.status !== "active") return yield* Effect.fail(cause); + yield* startNativeTurn(turnInput, yield* toCodexInput(turnInput)); + }); + }, + }), ); }).pipe( Effect.mapError( From b2842df404eee422c23315046effd2a9f28f8804 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:58 +0000 Subject: [PATCH 056/149] fix(claude): restart resume skips a consumed rollback cursor and rejects a missing transcript --- .../Adapters/ClaudeAdapterV2.test.ts | 161 +++++++++++++++++- .../Adapters/ClaudeAdapterV2.testkit.ts | 1 + .../Adapters/ClaudeAdapterV2.ts | 77 ++++++++- ...laudeAutomaticDelivery.integration.test.ts | 1 + 4 files changed, 237 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 9c4e336815f76..8fc0543465128 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -62,7 +62,7 @@ import { WorktreeToolkit } from "../../mcp/toolkits/worktree/tools.ts"; import { ThreadToolkit } from "../../mcp/toolkits/thread/tools.ts"; import { OrchestratorToolkit } from "../../mcp/toolkits/orchestrator/tools.ts"; import { ClaudeExecutableFileCheck } from "../../provider/Drivers/ClaudeExecutable.ts"; -import type * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; +import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; @@ -951,6 +951,7 @@ describe("ClaudeAdapterV2 Auto-accept edits", () => { crypto: yield* Crypto.Crypto, idAllocator, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("native-thread-claude-accept-edits"), open: (input) => Effect.sync(() => { @@ -1106,6 +1107,7 @@ const captureSdkExecutablePaths = Effect.fn("captureSdkExecutablePaths")(functio }), ), Effect.provideService(ClaudeAdapterV2.ClaudeAgentSdkQueryRunner, { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("native-thread-claude-binary-path"), open: (input) => Effect.sync(() => { @@ -1206,6 +1208,7 @@ describe("ClaudeAdapterV2 resume compaction", () => { crypto: yield* Crypto.Crypto, idAllocator, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("native-thread-claude-resume"), open: (input) => Effect.sync(() => { @@ -1430,6 +1433,7 @@ describe("ClaudeAdapterV2 attachments", () => { crypto: yield* Crypto.Crypto, idAllocator, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("native-thread-claude-attachments"), open: () => Effect.succeed({ @@ -1575,6 +1579,7 @@ describe("ClaudeAdapterV2 attachments", () => { crypto: yield* Crypto.Crypto, idAllocator, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("native-thread-claude-unsupported-attachment"), open: () => Effect.sync(() => { @@ -1669,6 +1674,7 @@ describe("ClaudeAdapterV2 native fork", () => { crypto: yield* Crypto.Crypto, idAllocator, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("source-native-session"), open: (input) => Effect.sync(() => { @@ -1827,6 +1833,83 @@ describe("ClaudeAdapterV2 native fork", () => { }); describe("ClaudeAdapterV2 native session identity", () => { + it.effect("rejects a missing transcript before accepting a resume", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const configDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-missing-transcript-", + }); + const runner = yield* ClaudeAdapterV2.ClaudeAgentSdkQueryRunner; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + settings: { ...DEFAULT_CLAUDE_SETTINGS, homePath: configDir }, + environment: {}, + attachmentsDir: configDir, + fileSystem, + path, + crypto: yield* Crypto.Crypto, + idAllocator: yield* IdAllocator.IdAllocatorV2, + queryRunner: runner, + }); + const threadId = ThreadId.make("thread-claude-missing-transcript"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("session-claude-missing-transcript"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const resumeInput = { + providerThread, + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }; + const missing = yield* runtime.resumeThread(resumeInput).pipe(Effect.result); + assert.equal(missing._tag, "Failure"); + if (missing._tag === "Failure") + assert.equal(missing.failure._tag, "ProviderAdapterResumeThreadError"); + const directory = path.join(configDir, "projects", "-workspace"); + yield* fileSystem.makeDirectory(directory, { recursive: true }); + yield* fileSystem.writeFileString( + path.join(directory, `${providerThread.nativeThreadRef!.nativeId}.jsonl`), + '{"type":"user","message":{"role":"user","content":"previous prompt"}}\n', + ); + const resumed = yield* runtime.resumeThread(resumeInput); + assert.deepEqual(resumed.nativeThreadRef, providerThread.nativeThreadRef); + assert.equal(resumed.providerSessionId, runtime.providerSession.id); + assert.isFalse( + yield* runner.hasSession({ + sessionId: providerThread.nativeThreadRef!.nativeId!, + configDir: path.join(configDir, "other-home"), + }), + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + ClaudeAdapterV2.layerQueryRunner, + IdAllocator.layer, + McpProviderSessions.layer, + ).pipe( + Layer.provideMerge(NodeServices.layer), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ), + ), + ), + ), + ); + const openTurnWithOrdinal = (providerTurnOrdinal: number, nativeThreadHasTurns?: boolean) => Effect.scoped( Effect.gen(function* () { @@ -1846,6 +1929,7 @@ describe("ClaudeAdapterV2 native session identity", () => { crypto: yield* Crypto.Crypto, idAllocator, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed("native-session-identity"), open: (input) => Effect.sync(() => { @@ -2155,6 +2239,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: (input) => Effect.gen(function* () { @@ -2275,6 +2360,71 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); const makeWakeHarness = makeWakeHarnessWithOptions(); + it.effect.each(["assistant", "compaction"] as const)( + "drops a rollback cursor consumed by %s before a resumed turn is cut", + (progress) => + Effect.gen(function* () { + const harness = yield* makeWakeHarnessWithOptions({ freshQueueOnReopen: true }); + const providerThread = { + ...harness.providerThread, + nativeConversationHeadRef: { + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeId: "rollback-cursor", + strength: "weak" as const, + }, + }; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("rollback-restart-first"), + text: "Continue after rollback.", + attachments: [], + }), + ); + assert.equal(harness.getOpenedOptions()?.resumeSessionAt, "rollback-cursor"); + yield* harness.offerAndWait( + progress === "assistant" + ? makeAssistantTextFrame({ uuid: "post-rollback-progress", text: "New progress." }) + : claudeSdkFrame({ + type: "system", + subtype: "compact_boundary", + compact_metadata: { trigger: "auto", pre_tokens: 1500, post_tokens: 400 }, + uuid: "post-rollback-compaction", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.shutdown(harness.sdkMessages); + yield* Queue.take(harness.terminalReceipts); + const updates = harness.events.filter((event) => event.type === "provider_thread.updated"); + assert.lengthOf(updates, 2); + assert.isNull(updates[0]?.providerThread.nativeConversationHeadRef); + const saved = updates.at(-1); + assert.equal(saved?.type, "provider_thread.updated"); + if (saved?.type !== "provider_thread.updated") return; + assert.isNull(saved.providerThread.nativeConversationHeadRef); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: saved.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("rollback-restart-second"), + text: "Continue where you left off.", + attachments: [], + providerTurnOrdinal: 2, + }), + ); + assert.equal(harness.getOpenedOptions()?.resume, WAKE_NATIVE_SESSION); + assert.isUndefined(harness.getOpenedOptions()?.resumeSessionAt); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(NodeServices.layer, IdAllocator.layer, McpProviderSessions.layer), + ), + ), + ); + it.effect.each([ { isError: false, title: "Check weather" }, { isError: true, title: "Check weather" }, @@ -4065,6 +4215,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => Effect.gen(function* () { @@ -4316,6 +4467,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { offer: () => Effect.void, }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.sync(() => { const next = nativeIds[allocateIndex] ?? `native-thread-roster-extra-${allocateIndex}`; @@ -8608,6 +8760,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => Effect.gen(function* () { @@ -8797,6 +8950,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => Effect.gen(function* () { @@ -9038,6 +9192,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => Effect.gen(function* () { @@ -9236,6 +9391,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { idAllocator, continuationRequests: { offer: () => Effect.void }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => Effect.gen(function* () { @@ -9409,6 +9565,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { offer: () => Effect.void, }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => { openCount += 1; @@ -9547,6 +9704,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => { openCount += 1; @@ -9743,6 +9901,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { offer: () => Effect.void, }, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), open: () => Effect.fail( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts index cc8a5bef6f5a9..67cc46c6cc1ec 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts @@ -929,6 +929,7 @@ function replayQueryRunnerService( catch: (cause) => replayQueryRunnerError(transcript, cause), }); return ClaudeAdapterV2.ClaudeAgentSdkQueryRunner.of({ + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(nativeSessionIdFor(transcript)), open: (input) => replay(() => queryRunner.open(input)), forkSession: (input) => replay(() => queryRunner.forkSession(input)), diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index c7b1fb0d58190..e96aebdd201b2 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -94,6 +94,7 @@ import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import { claudeSignedOutMessage, makeClaudeEnvironment, + resolveClaudeHomePath, } from "../../provider/Drivers/ClaudeHome.ts"; import { BUNDLED_CLAUDE_MODEL_CATALOG, @@ -364,6 +365,10 @@ export class ClaudeBackgroundWorkBlocksQueryReplacementError extends Schema.Tagg export interface ClaudeAgentSdkQueryRunnerShape { readonly allocateSessionId: Effect.Effect; + readonly hasSession: (input: { + readonly sessionId: string; + readonly configDir: string; + }) => Effect.Effect; readonly open: ( input: ClaudeAgentSdkQueryOpenInput, ) => Effect.Effect; @@ -599,17 +604,41 @@ export function makeClaudeAgentSdkProtocolLogger(input: { export const layerQueryRunner: Layer.Layer< ClaudeAgentSdkQueryRunner, never, - Crypto.Crypto | ProviderEventLoggers.ProviderEventLoggers + Crypto.Crypto | FileSystem.FileSystem | Path.Path | ProviderEventLoggers.ProviderEventLoggers > = Layer.effect( ClaudeAgentSdkQueryRunner, Effect.gen(function* () { const crypto = yield* Crypto.Crypto; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers; return ClaudeAgentSdkQueryRunner.of({ allocateSessionId: crypto.randomUUIDv4.pipe( Effect.mapError((cause) => queryRunnerError(cause, "allocateSessionId")), ), + hasSession: Effect.fn("ClaudeAgentSdkQueryRunner.hasSession")( + function* (input: { readonly sessionId: string; readonly configDir: string }) { + if (path.basename(input.sessionId) !== input.sessionId) return false; + const projectsDir = path.join(input.configDir, "projects"); + const directories = yield* fileSystem.readDirectory(projectsDir).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed([]), + ), + ); + for (const directory of directories) { + if ( + yield* fileSystem.exists( + path.join(projectsDir, directory, `${input.sessionId}.jsonl`), + ) + ) + return true; + } + return false; + }, + (effect) => effect.pipe(Effect.mapError((cause) => queryRunnerError(cause, "hasSession"))), + ), open: Effect.fn("ClaudeAgentSdkQueryRunner.open")(function* ( input: ClaudeAgentSdkQueryOpenInput, ) { @@ -2721,6 +2750,7 @@ interface ActiveClaudeTurnContext { readonly nativeThreadId: string; readonly nativeTurnId: string; nativeMessageCursor: string | null; + nativeConversationHeadConsumed: boolean; readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; readonly providerTurnOrdinal: number; readonly startedAt: DateTime.Utc; @@ -5179,7 +5209,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeThreadId, ); const clearConversationHead = - input.status === "completed" && + (input.status === "completed" || input.context.nativeConversationHeadConsumed) && input.context.input.providerThread.nativeConversationHeadRef !== null; // While a goal is set, Claude ends a turn on its own only after the // goal's Stop hook passes. It defers that check while background @@ -5942,6 +5972,32 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } } + // A rollback cursor applies once. Persist its consumption before + // a restart can resume past it and discard this turn's progress. + if ( + input.replayed !== true && + !context.nativeConversationHeadConsumed && + context.input.providerThread.nativeConversationHeadRef !== null && + ((message.type === "assistant" && + message.parent_tool_use_id === null && + message.error === undefined) || + (message.type === "system" && message.subtype === "compact_boundary")) + ) { + context.nativeConversationHeadConsumed = true; + const providerThread: OrchestrationV2ProviderThread = { + ...((yield* Ref.get(lastProviderThreadByNativeThread)).get(context.nativeThreadId) ?? + context.input.providerThread), + nativeConversationHeadRef: null, + updatedAt: yield* DateTime.now, + }; + yield* rememberProviderThread(providerThread); + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CLAUDE_PROVIDER, + providerThread, + }); + } + if (message.type === "assistant" && input.replayed !== true) { context.nativeMessageCursor = message.uuid; if (message.parent_tool_use_id === null) { @@ -7523,6 +7579,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeThreadId, nativeTurnId, nativeMessageCursor: null, + nativeConversationHeadConsumed: false, providerTurnId, providerTurnOrdinal, startedAt, @@ -7928,6 +7985,22 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ), resumeThread: Effect.fn("ClaudeAdapterV2.resumeThread")( function* (threadInput: { readonly providerThread: OrchestrationV2ProviderThread }) { + const nativeThreadId = yield* getNativeThreadId(threadInput.providerThread); + const existing = yield* Ref.get(queryContext); + // query.open reports a missing transcript asynchronously, too late + // for ProviderTurnStartService's portable-context fallback. + if (existing?.nativeThreadId !== nativeThreadId || existing.stopping) { + const configDir = yield* resolveClaudeHomePath( + adapterOptions.settings, + adapterOptions.environment, + ).pipe(Effect.provideService(Path.Path, path)); + if (!(yield* queryRunner.hasSession({ sessionId: nativeThreadId, configDir }))) { + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CLAUDE_PROVIDER, + detail: "The saved Claude session transcript is missing.", + }); + } + } const updatedAt = yield* DateTime.now; return { ...threadInput.providerThread, diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index 15e175aa7c742..b648d2238c821 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -142,6 +142,7 @@ it.effect.each(["child completion", "scheduled message", "user steering"] as con crypto: yield* Crypto.Crypto, idAllocator: yield* IdAllocator.IdAllocatorV2, queryRunner: { + hasSession: () => Effect.succeed(true), allocateSessionId: Effect.succeed(sessionId), open: () => Effect.succeed({ From 1ef4518d25e0a0ff38cb607b7fa068ee722dcb60 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:58 +0000 Subject: [PATCH 057/149] fix(acp): an agent that cannot reload sessions falls back to portable recovery --- .../Adapters/AcpRegistryAdapterV2.test.ts | 120 ++++++++++++++++++ .../src/server/AcpSessionRuntime.ts | 1 + 2 files changed, 121 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts index 171f7ae7eedc7..afa712537be32 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts @@ -99,6 +99,126 @@ const layerTest = Layer.mergeAll( ); describe("AcpRegistryAdapterV2", () => { + it.effect("preserves portable recovery when the restarted agent cannot reload sessions", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const replayDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-acp-registry-no-reload-", + }); + const scriptPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-replay-agent.ts", import.meta.url), + ); + const transcripts = yield* Effect.forEach([false, true], (fresh) => + decodeAcpReplayTranscript( + { + provider: ACP_REGISTRY_PROVIDER, + protocol: "acp.ndjson-jsonrpc", + version: "1", + scenario: fresh ? "fresh-session" : "unsupported-reload", + entries: [ + { + type: "expect_outbound", + frame: { kind: "request", method: "initialize", params: "" }, + }, + { + type: "emit_inbound", + frame: { + kind: "response", + method: "initialize", + result: { protocolVersion: 1, agentCapabilities: { loadSession: false } }, + }, + }, + ...(fresh + ? [ + { + type: "expect_outbound", + frame: { kind: "request", method: "session/new", params: "" }, + }, + { + type: "emit_inbound", + frame: { + kind: "response", + method: "session/new", + result: { sessionId: "replacement-session" }, + }, + }, + ] + : []), + ] as never, + }, + ACP_REGISTRY_PROVIDER, + ), + ); + const instanceId = ProviderInstanceId.make("acp-registry-no-reload"); + const threadId = ThreadId.make("thread-acp-registry-no-reload"); + const modelSelection = { instanceId, model: "default" } as const; + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: replayDir, + }); + let runtimeOrdinal = 0; + const adapter = yield* makeAcpRegistryAdapterV2({ + selfInvocation: yield* resolveSelfInvocation(), + instanceId, + settings: yield* decodeAcpRegistryAdapterSettings({ agentId: "fixture-agent" }), + environment: {}, + makeRuntime: (input) => + makeAcpReplayRuntime({ + transcript: transcripts[runtimeOrdinal]!, + statusPath: path.join(replayDir, `${runtimeOrdinal++}.json`), + scriptPath, + childProcessSpawner, + fileSystem, + })(input), + }).pipe(Effect.provide(layerInjectedRuntimeCatalog)); + yield* Effect.gen(function* () { + const session = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-no-reload"), + modelSelection, + runtimePolicy, + initialNativeThreadId: "saved-session", + }); + const freshThread = yield* session.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const savedThread = { + ...freshThread, + nativeThreadRef: { + driver: ACP_REGISTRY_PROVIDER, + nativeId: "saved-session", + strength: "strong" as const, + }, + }; + const resumeError = yield* session + .resumeThread({ providerThread: savedThread, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(resumeError._tag, "ProviderAdapterResumeThreadError"); + const recovered = yield* session.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: { ...savedThread, nativeThreadRef: null }, + }); + assert.equal(recovered.id, savedThread.id); + assert.equal(recovered.nativeThreadRef?.nativeId, "replacement-session"); + }).pipe(Effect.scoped); + assert.equal(runtimeOrdinal, 2); + for (const [index, transcript] of transcripts.entries()) { + yield* makeAcpReplayCompletenessAssertion( + fileSystem, + path.join(replayDir, `${index}.json`), + transcript, + ); + } + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it("preserves and sanitizes structured ACP errors without exposing arbitrary defects", () => { const limit = new EffectAcpErrors.AcpRequestError({ code: -31001, diff --git a/packages/provider-acp/src/server/AcpSessionRuntime.ts b/packages/provider-acp/src/server/AcpSessionRuntime.ts index 9891ba50ded22..0bc6c35b20eec 100644 --- a/packages/provider-acp/src/server/AcpSessionRuntime.ts +++ b/packages/provider-acp/src/server/AcpSessionRuntime.ts @@ -2303,6 +2303,7 @@ export const make = ( } else { return yield* new EffectAcpErrors.AcpRequestError({ code: -32601, + method: "session/load", errorMessage: "ACP agent does not advertise session/load or session/resume support", }); } From ea78d7aeef3617206ab05059d72834a91d204e6c Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:58 +0000 Subject: [PATCH 058/149] fix(server): scheduled runs wait for restart continuation --- .../ScheduledTaskService.test.ts | 128 +++++++++++++++++- .../scheduledTasks/ScheduledTaskService.ts | 37 ++++- .../ScheduledTaskService.webhook.test.ts | 82 +++++++++++ 3 files changed, 241 insertions(+), 6 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 1c95fecdea795..83bf6902dca5e 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -58,6 +58,108 @@ const insertRow = ( run_count: 0, })}`; +it.effect.each([ + ["interval", "pending"], + ["interval", "running"], + ["fixed_time", "pending"], + ["fixed_time", "running"], +] as const)( + "defers a bound %s catch-up while its restart continuation is %s", + ([trigger, continuationStatus]) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-09-09T12:00:00.000Z"; + const scheduleJson = + trigger === "interval" + ? '{"type":"interval","everyMs":60000}' + : '{"type":"fixed_time","timeOfDay":"12:00"}'; + yield* TestClock.setTime(Date.parse(now) + 1_000); + for (const id of ["bound", "fresh"]) { + yield* insertRow( + sql, + { id, next: now, enabled: 1, status: "succeeded", scheduleJson }, + now, + ); + } + yield* sql`UPDATE scheduled_tasks SET thread_id = 'thread:restart', run_count = 1 + WHERE task_id = 'bound'`; + yield* sql`INSERT INTO orchestration_v2_effect_outbox ( + effect_id, command_id, thread_id, effect_type, payload_json, status, + available_at, created_at, updated_at + ) VALUES ( + 'effect:restart-continuation:cut', 'command:restart', 'thread:restart', + 'provider-runtime.continue', '{"type":"provider-runtime.continue","sourceRunId":"cut"}', + ${continuationStatus}, ${now}, ${now}, ${now} + )`; + const sends: Parameters< + ThreadManagementService.ThreadManagementService["Service"]["sendToThread"] + >[0][] = []; + let launches = 0; + let poll: Effect.Effect | undefined; + yield* Effect.scoped( + Effect.gen(function* () { + yield* Layer.build( + ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + NodeCrypto.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: () => { + launches += 1; + return Effect.succeed({} as never); + }, + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => { + sends.push(input); + return Effect.succeed({} as never); + }, + }), + Layer.succeed(Scheduler.Scheduler, { + register: (_name: string, work: Effect.Effect) => + Effect.context().pipe( + Effect.map((context) => { + poll = work.pipe(Effect.provideContext(context), Effect.orDie); + }), + ), + }), + ), + ), + ), + ); + assert.isDefined(poll); + yield* poll!; + assert.lengthOf(sends, 0); + assert.equal(launches, 1); + const deferred = yield* sql<{ next_run_at: string; run_count: number }>` + SELECT next_run_at, run_count FROM scheduled_tasks WHERE task_id = 'bound' + `; + assert.equal(deferred[0]?.next_run_at, now); + assert.equal(deferred[0]?.run_count, 1); + + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'succeeded'`; + yield* poll!; + assert.lengthOf(sends, 1); + assert.equal(sends[0]?.threadId, "thread:restart"); + assert.equal(sends[0]?.mode, "queue"); + const completed = yield* sql<{ + next_run_at: string; + run_count: number; + last_run_status: string; + }>`SELECT next_run_at, run_count, last_run_status FROM scheduled_tasks + WHERE task_id = 'bound'`; + assert.equal(completed[0]?.last_run_status, "succeeded"); + assert.equal(completed[0]?.run_count, 2); + assert.isAbove(Date.parse(completed[0]!.next_run_at), Date.parse(now) + 1_000); + yield* poll!; + assert.lengthOf(sends, 1); + assert.equal(launches, 1); + }), + ); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("loads only due tasks and skips corrupt due rows without decoding settled tasks", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -168,6 +270,18 @@ it.effect( const hugeInterval = '{"type":"interval","everyMs":9000000000000000}'; for (const row of [ { id: "stuck-valid", next: now, status: "running" }, + { + id: "stuck-fixed", + next: now, + status: "running", + scheduleJson: '{"type":"fixed_time","timeOfDay":"12:00"}', + }, + { + id: "stuck-webhook", + next: null, + status: "running", + scheduleJson: '{"type":"webhook","signature":null}', + }, { id: "stuck-corrupt", next: now, status: "running", scheduleJson: "broken" }, { id: "stuck-huge", next: now, status: "running", scheduleJson: hugeInterval }, { id: null, next: now, status: "running" }, @@ -241,19 +355,29 @@ it.effect( }>`SELECT task_id, last_run_status, last_run_error, next_run_at, run_count FROM scheduled_tasks ORDER BY task_id`; const byId = new Map(rows.map((row) => [row.task_id, row])); - assert.equal(rows.length, 9); + assert.equal(rows.length, 11); // The decodable stuck run is rescheduled for its next occurrence; the // undecodable and unrepresentable rows are released without a next run. // A NULL id must still match its own row — `= NULL` never does. assert.equal(byId.get("stuck-valid")?.last_run_status, "failed"); assert.isNotNull(byId.get("stuck-valid")?.next_run_at); + assert.isAbove(Date.parse(byId.get("stuck-fixed")!.next_run_at!), Date.parse(now) + 7_000); + assert.isNull(byId.get("stuck-webhook")?.next_run_at); assert.equal(byId.get("stuck-corrupt")?.last_run_status, "failed"); assert.equal(byId.get(null)?.last_run_status, "failed"); const stuckHuge = byId.get("stuck-huge"); assert.equal(stuckHuge?.last_run_status, "failed"); assert.isNull(stuckHuge?.next_run_at); - for (const id of ["stuck-valid", "stuck-corrupt", "stuck-huge", null]) { + for (const id of [ + "stuck-valid", + "stuck-fixed", + "stuck-webhook", + "stuck-corrupt", + "stuck-huge", + null, + ]) { const row = byId.get(id); + assert.equal(row?.last_run_status, "failed"); assert.equal(row?.last_run_error, "Run was interrupted by a server restart."); assert.equal(row?.run_count, 1); } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 749dace7ef464..e2bf272e7ab37 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -746,13 +746,42 @@ export const layer = Layer.effect( } return yield* Effect.gen(function* () { - const startedAt = yield* localNow; - const startedAtIso = iso(startedAt); - // The in-memory snapshot may be stale: re-read before touching run // state. The task may have been deleted, paused, or postponed since // the poll loaded it — none of those may fire. - const active = yield* findTask(task.id); + let active = yield* findTask(task.id); + // Recovery cancels the old run before its continuation is dispatched. + // In that gap, queue mode would start a new run and suppress the continuation. + while (active !== null && active.threadId !== null) { + if ( + trigger === "manual" || + !active.enabled || + (webhook !== undefined && + (active.createdAt !== task.createdAt || active.schedule.type !== "webhook")) + ) + break; + const pending = yield* sql` + SELECT 1 FROM orchestration_v2_effect_outbox + WHERE thread_id = ${active.threadId} + AND effect_type = 'provider-runtime.continue' + AND status IN ('pending', 'running') + LIMIT 1 + `.pipe( + Effect.mapError((cause) => + taskError("Could not check schedule task restart continuation.", { + taskId: task.id, + cause, + }), + ), + ); + if (pending.length === 0) break; + if (trigger === "scheduled") return active; + // Keep the accepted delivery and its rendered body while recovery runs. + yield* Effect.sleep("5 seconds"); + active = yield* findTask(task.id); + } + const startedAt = yield* localNow; + const startedAtIso = iso(startedAt); if (active === null) { if (webhook !== undefined) { return yield* new WebhookDeliverySkipped({ diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index feb8461fa984a..a5685d2ab43b6 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -13,6 +13,8 @@ import * as Metric from "effect/Metric"; import * as Queue from "effect/Queue"; import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; @@ -142,6 +144,86 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () => ), ); +it.effect.each(["pending", "running"] as const)( + "holds a bound webhook while its restart continuation is %s", + (continuationStatus) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const sends = + yield* Queue.unbounded< + Parameters[0] + >(); + const gate = yield* Deferred.make(); + const dependencies = Layer.mergeAll( + NodePlatformCrypto.layer, + Scheduler.layer, + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Queue.offer(sends, input).pipe( + Effect.andThen(Deferred.await(gate)), + Effect.as({} as never), + ), + }), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + threadId: "thread:restart", + prompt: "Handle {{body.action}}: {{body}}.", + }), + ); + yield* sql`INSERT INTO orchestration_v2_effect_outbox ( + effect_id, command_id, thread_id, effect_type, payload_json, status, + available_at, created_at, updated_at + ) VALUES ( + 'effect:restart-continuation:cut', 'command:restart', 'thread:restart', + 'provider-runtime.continue', '{"type":"provider-runtime.continue","sourceRunId":"cut"}', + ${continuationStatus}, ${task.createdAt}, ${task.createdAt}, ${task.createdAt} + )`; + const subscribed = yield* Deferred.make(); + const completed = yield* service.subscribeList().pipe( + Stream.tap(() => Deferred.succeed(subscribed, undefined)), + Stream.filter((list) => list.tasks.some((task) => task.lastRunStatus === "succeeded")), + Stream.runHead, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + const bodyText = JSON.stringify({ action: "opened", number: 7 }); + const request = requestFor(task, { + relayDeliveryId: "held-after-restart", + body: new TextEncoder().encode(bodyText), + bodyText, + }); + const accepted = yield* service.triggerWebhook(request); + assert.equal(accepted._tag, "accepted"); + yield* TestClock.adjust("1 second"); + assert.equal(yield* Queue.size(sends), 0); + assert.equal((yield* service.list()).tasks[0]?.runCount, 0); + + yield* service.upsert( + yield* webhookTaskInput({ threadId: "thread:restart", prompt: "Changed {{body}}" }), + ); + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'succeeded'`; + yield* TestClock.adjust("5 seconds"); + const sent = yield* Queue.take(sends); + assert.equal(sent.threadId, "thread:restart"); + assert.equal(sent.mode, "queue"); + assert.equal(sent.scheduledTaskId, task.id); + assert.equal(sent.text, "Handle opened: " + bodyText + "."); + yield* Deferred.succeed(gate, undefined); + yield* Fiber.join(completed); + assert.equal((yield* service.list()).tasks[0]?.runCount, 1); + assert.equal((yield* service.list()).tasks[0]?.lastRunStatus, "succeeded"); + const duplicate = yield* service.triggerWebhook(request); + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate"); + assert.equal(yield* Queue.size(sends), 0); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => { const relayUrl = "https://relay.example.com/"; assert.equal( From 076584fef6bc8b65d75e269ea7148031d8c6b4fc Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:58 +0000 Subject: [PATCH 059/149] fix(server): MCP write receipts survive provider session rotation --- .../server/src/mcp/McpSessionRegistry.test.ts | 28 +++++++++++++++++++ apps/server/src/mcp/McpSessionRegistry.ts | 3 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 6b6207dc26601..709e4794bd369 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -59,6 +59,34 @@ it.effect("stores only a token hash, resolves the bearer token, and revokes by t }), ); +it.effect("preserves retry identities after restart while replacing the credential", () => + Effect.gen(function* () { + const request = { + threadId: ThreadId.make("thread-restarted"), + providerInstanceId: ProviderInstanceId.make("codex"), + }; + const before = yield* makeRegistry(() => 1_000); + const first = yield* before.issue(request); + const firstToken = first.config.authorizationHeader.replace(/^Bearer\s+/, ""); + const firstScope = yield* before.resolve(firstToken); + const after = yield* makeRegistry(() => 2_000); + expect(yield* after.resolve(firstToken)).toBeUndefined(); + const second = yield* after.issue(request); + const secondScope = yield* after.resolve( + second.config.authorizationHeader.replace(/^Bearer\s+/, ""), + ); + expect(second.config.authorizationHeader).not.toBe(first.config.authorizationHeader); + expect(secondScope?.thread.providerSessionId).not.toBe(firstScope?.thread.providerSessionId); + expect(secondScope?.requestNamespace).toBe(firstScope?.requestNamespace); + + const other = yield* after.issue({ ...request, threadId: ThreadId.make("thread-other") }); + const otherScope = yield* after.resolve( + other.config.authorizationHeader.replace(/^Bearer\s+/, ""), + ); + expect(otherScope?.requestNamespace).not.toBe(secondScope?.requestNamespace); + }), +); + it.effect("always grants pull-requests and gates browser and device access independently", () => Effect.gen(function* () { const registry = yield* makeRegistry(() => 1_000); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index f1e027f9eb85f..80e36dbcfcc0a 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -130,7 +130,8 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( const browserToolsAvailable = request.browserToolsAvailable ?? true; const scope: McpInvocationContext.McpThreadInvocationScope = { environmentId, - requestNamespace: providerSessionId, + // Retrying an accepted write after a restart must reuse its receipt. + requestNamespace: `thread:${encodeURIComponent(request.threadId)}:${encodeURIComponent(request.providerInstanceId)}`, thread: { threadId: ThreadId.make(request.threadId), providerSessionId, From 830b67c87774f826c00db5f27608ae5ce6594780 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:49:17 +0000 Subject: [PATCH 060/149] fix(server): restart continuation never answers for a waiting user A pending approval, input or secret request, or one recovery closed on the cut attempt, now blocks the automatic prompt, both when intent is recorded and when it is delivered. Tool calls, auth refreshes and message-capable questions do not block. A provider turn that already ended while the app run lagged is not resumed, and a recorded intent no longer wakes a snoozed or settled thread. Recovery also resumes a steering restart that never reached the provider with its replacement prompt, schedules a handoff's sole queued continuation in the new worktree, and rejects a cancelled handoff. The continuation keeps the source message's scheduled task. --- .../FoundationPersistence.test.ts | 2 + .../Orchestrator.control-reads.test.ts | 164 ++++++++ .../src/orchestration-v2/Orchestrator.ts | 46 ++- .../ProviderRuntimeRecoveryService.ts | 29 +- .../RestartBackgroundNote.test.ts | 21 + .../orchestration-v2/RestartBackgroundNote.ts | 37 +- .../RestartContinuation.test.ts | 375 ++++++++++++++++++ .../orchestration-v2/RestartContinuation.ts | 123 +++++- .../src/orchestration-v2/runtimeLayer.test.ts | 105 +++++ 9 files changed, 874 insertions(+), 28 deletions(-) diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 9a1e43a8b44b4..40fe37d78b5b7 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -2779,6 +2779,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { { id: sessionId, driver: "codex", providerInstanceId, status: "running" }, ], providerTurns: [{ providerThreadId, runAttemptId: attemptId, status: "running" }], + runtimeRequests: [], + turnItems: [], } as unknown as OrchestrationV2ThreadProjection; const recovery = yield* ProviderRuntimeRecovery.make.pipe( Effect.provide( diff --git a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts index 8e5fca79cf553..f0907479a4409 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts @@ -28,6 +28,9 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ThreadManagementService from "./ThreadManagementService.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { continueRestartedRun } from "./RestartContinuation.ts"; const instanceId = ProviderInstanceId.make("codex"); const modelSelection = { instanceId, model: "gpt-5.1-codex" }; @@ -49,6 +52,167 @@ const layerTest = Layer.mergeAll( ), ); +it.effect.each([ + "snoozed", + "settled", + "pending", + "expired", + "secret", + "resolved", + "completed", + "failed", + "interrupted", +] as const)("rechecks %s state when dispatching recorded restart intent", (state) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make(`thread:restart-${state}`); + const now = yield* DateTime.now; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create-restart-${state}`), + threadId, + projectId: ProjectId.make("project:control-dispatch"), + title: "Restart", + modelSelection, + runtimeMode: "full-access", + interactionMode: "plan", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`original-restart-${state}`), + threadId, + messageId: MessageId.make(`original-restart-${state}`), + text: "Make a plan.", + attachments: [], + dispatchMode: { type: "defer_start" }, + createdBy: "user", + creationSource: "web", + }); + const source = (yield* projections.getThreadRecords(threadId, ["runs"])).runs[0]!; + yield* projections.apply({ + id: EventId.make(`cut-restart-${state}`), + type: "run.updated", + threadId, + occurredAt: now, + payload: { ...source, status: "cancelled", completedAt: now }, + }); + const turnId = ProviderTurnId.make(`turn:restart-${state}`); + yield* projections.apply({ + id: EventId.make(`turn-restart-${state}`), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: turnId, + providerThreadId: source.providerThreadId!, + nodeId: source.rootNodeId!, + runAttemptId: source.activeAttemptId, + nativeTurnRef: null, + ordinal: 1, + status: "cancelled", + startedAt: now, + completedAt: now, + }, + }); + // The service reads eligible state before a user choice or provider state lands. + const before = yield* projections.getThreadProjection(threadId); + if (state === "snoozed" || state === "settled") { + yield* orchestrator.dispatch( + state === "snoozed" + ? { + type: "thread.snooze", + commandId: CommandId.make(`hold-restart-${state}`), + threadId, + snoozedUntil: "2099-01-01T00:00:00.000Z", + } + : { + type: "thread.settle", + commandId: CommandId.make(`hold-restart-${state}`), + threadId, + }, + ); + } else if (state === "completed" || state === "failed" || state === "interrupted") { + yield* projections.apply({ + id: EventId.make(`settled-turn-restart-${state}`), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { ...before.providerTurns[0]!, status: state }, + }); + } else if (state === "secret") { + yield* projections.apply({ + id: EventId.make(`secret-restart-${state}`), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: TurnItemId.make(`secret-restart-${state}`), + threadId, + runId: source.id, + nodeId: source.rootNodeId, + providerThreadId: source.providerThreadId, + providerTurnId: turnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "cancelled", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "secret_request", + label: "API key", + reason: "Connect to the provider.", + secretStatus: "cancelled", + }, + }); + } else { + yield* projections.apply({ + id: EventId.make(`request-restart-${state}`), + type: "runtime-request.updated", + threadId, + occurredAt: now, + payload: { + id: RuntimeRequestId.make(`request-restart-${state}`), + nodeId: source.rootNodeId!, + providerTurnId: turnId, + nativeRequestRef: null, + kind: "user_input", + status: state, + responseCapability: + state === "pending" + ? { type: "live", providerSessionId: ProviderSessionId.make("session:restart") } + : { type: "not_resumable", reason: "The server restarted." }, + createdAt: now, + resolvedAt: state === "pending" ? null : now, + }, + }); + } + yield* continueRestartedRun({ threadId, sourceRunId: source.id }).pipe( + Effect.provide( + Layer.merge( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => Effect.succeed(before), + dispatch: orchestrator.dispatch, + recoverDelegatedTask: () => Effect.void, + }), + ), + ), + ); + const after = yield* projections.getThreadProjection(threadId); + assert.lengthOf(after.runs, state === "resolved" ? 2 : 1); + assert.equal(after.thread.interactionMode, "plan"); + if (state === "snoozed") assert.isNotNull(after.thread.snoozedUntil); + if (state === "settled") assert.equal(after.thread.settledOverride, "settled"); + }).pipe(Effect.provide(layerTest)), +); + it.effect( "dispatches metadata, queue resume and request controls without hydrating unrelated history", () => diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 977b225f52566..cde3cfb1de10c 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -86,7 +86,8 @@ import { CommandPolicyV2, resolveMessageDispatchIntent } from "./CommandPolicy.t import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts"; import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts"; import { notificationTurnItem } from "@t3tools/provider-core/server/notification"; -import { isRestartNoteSource } from "./RestartBackgroundNote.ts"; +import { isRestartNoteSource, isWorktreeContinuationRun } from "./RestartBackgroundNote.ts"; +import { restartContinuationBlocked } from "./RestartContinuation.ts"; import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; @@ -4533,16 +4534,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (command.restartContinuationOfRunId !== undefined) { const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId); + const secretRequests = yield* projectionStore + .getThreadRecords(command.threadId, ["turnItems"], { + turnItemTypes: ["secret_request"], + }) + .pipe(mapDispatchError(command)); + const worktreeContinuation = + source !== undefined && + source.status === "queued" && + source.queueHeld !== true && + isWorktreeContinuationRun(source) && + projection.thread.worktreePath != null && + projection.runs.filter((run) => run.status === "queued").length === 1 && + !projection.runs.some(isBlockingRun); if ( !source || - source.status !== "cancelled" || + (source.status !== "cancelled" && !worktreeContinuation) || + (isWorktreeContinuationRun(source) && + source.status === "cancelled" && + !projection.providerTurns.some( + (turn) => turn.runAttemptId === source.activeAttemptId, + )) || isRestartNoteSource(source, projection.providerTurns) || - projection.thread.archivedAt !== null || - projection.thread.deletedAt !== null || + restartContinuationBlocked( + { ...projection, turnItems: secretRequests.turnItems }, + source, + ) || projection.thread.providerInstanceId !== source.providerInstanceId || // Held queued runs never started; they wait behind the continuation. projection.runs.some( - (run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source), + (run) => + run.id !== source.id && + run.status !== "queued" && + (worktreeContinuation ? run.ordinal > source.ordinal : runRanAfter(run, source)), ) || (yield* stopReachedRun(command, command.threadId, source.id)) ) { @@ -4559,6 +4583,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); return; } + if (worktreeContinuation) { + yield* dispatchQueuedRunCancel( + { + type: "queued-run.cancel", + commandId: command.commandId, + threadId: command.threadId, + runId: source.id, + }, + events, + ); + projection = yield* getProjectionWithPendingEvents(command.threadId, events); + } } if (projection.thread.settledOverride !== null) { diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 7a063bec065bc..8140d367d8eae 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -25,6 +25,7 @@ import { restartContinuationRun } from "./RestartContinuation.ts"; import { cancelledRosterTaskWork, cancelledTurnItemWork, + isWorktreeContinuationRun, mergeRestartCancelledBackgroundWork, } from "./RestartBackgroundNote.ts"; @@ -286,8 +287,20 @@ export const make = Effect.gen(function* () { }; // Queued runs have not started provider work. Preserve their execution // identities and order, but require explicit consent before draining them. + // A handoff's sole continuation already has that consent, and the old + // session was detached to run it in the newly bound worktree. + const queuedRuns = projection.runs.filter((run) => run.status === "queued"); + const handoffContinuation = + continueAfterRestart && + projection.thread.worktreePath != null && + queuedRuns.length === 1 && + queuedRuns[0]!.queueHeld !== true && + isWorktreeContinuationRun(queuedRuns[0]!) + ? queuedRuns[0] + : undefined; for (const run of projection.runs) { - if (run.status !== "queued" || run.queueHeld === true) continue; + if (run.status !== "queued" || run.queueHeld === true || run === handoffContinuation) + continue; events.push({ id: yield* allocateEventId(), type: "run.updated", @@ -650,7 +663,7 @@ export const make = Effect.gen(function* () { } const continuationRun = continueAfterRestart && trigger === "startup" - ? restartContinuationRun(projection) + ? (handoffContinuation ?? restartContinuationRun(projection)) : undefined; const effects: Array = continuationRun ? [ @@ -667,6 +680,18 @@ export const make = Effect.gen(function* () { ).length; let retiredEffects: number; if (events.length === 0) { + if (effects.length > 0) { + yield* eventSink.writeWithEffects({ commandId, events: [], effects }).pipe( + Effect.mapError( + (cause) => + new ProviderRuntimeRecoveryError({ + operation: "reconcile", + threadId: projection.thread.id, + cause, + }), + ), + ); + } const retiredEffectIds = yield* outbox .cancelUnsettled({ threadId: projection.thread.id, diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts index 2f22ed583a91d..40246f64e29c4 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts @@ -44,6 +44,27 @@ const turnFor = (source: OrchestrationV2Run) => ({ status: "completed" as const, }); +it("prompts a continuation whose steering restart never started a provider turn", () => { + const source = run(1, codexThread, { status: "cancelled" }); + const continuation = run(2, codexThread, { restartContinuationOfRunId: source.id }); + const attempt = { + id: source.activeAttemptId!, + runId: source.id, + reason: "steering_restart" as const, + }; + assert.isTrue(isRestartNoteContinuation(continuation, [source, continuation], [], [attempt])); + assert.isFalse( + isRestartNoteContinuation(continuation, [source, continuation], [turnFor(source)], [attempt]), + ); + const chained = run(3, codexThread, { restartContinuationOfRunId: continuation.id }); + assert.isTrue(isRestartNoteContinuation(chained, [source, continuation, chained], [], [attempt])); + const handoff = { + ...source, + userMessageId: MessageId.make("message:mcp:session:worktree-continuation:request"), + }; + assert.isTrue(isRestartNoteContinuation(continuation, [handoff, continuation], [], [])); +}); + it("keeps the note for the provider thread that lost the work across a provider switch", () => { const root = run(1, claudeThread, { restartCancelledBackgroundWork: lost }); const onCodex = run(2, codexThread); diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index 11b2cfe37c087..28db9481bdcca 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -10,6 +10,7 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; type Work = OrchestrationV2RestartCancelledBackgroundWork; type Attempt = Pick; +type PromptAttempt = Attempt & Partial>; const MAX_LABEL_LENGTH = 160; @@ -159,16 +160,43 @@ export function restartContinuationNote( return { work, settled: isRestartNoteSource(current, providerTurns) }; } +export const isWorktreeContinuationRun = (run: Pick) => + /^message:mcp:[^:]+:worktree-continuation:/.test(run.userMessageId); + +/** A handoff or replacement prompt that never reached the provider. */ +export function restartPromptSource( + source: OrchestrationV2Run, + runs: ReadonlyArray, + providerTurns: ReadonlyArray>, + attempts: ReadonlyArray, +): OrchestrationV2Run | undefined { + const visited = new Set(); + let current: OrchestrationV2Run | undefined = source; + while (current !== undefined && !visited.has(current.id)) { + visited.add(current.id); + if (providerTurns.some((turn) => turn.runAttemptId === current?.activeAttemptId)) return; + if ( + isWorktreeContinuationRun(current) || + attempts.some( + (attempt) => + attempt.id === current?.activeAttemptId && attempt.reason === "steering_restart", + ) + ) + return current; + current = runs.find((run) => run.id === current?.restartContinuationOfRunId); + } +} + /** - * A restart continuation prompted with the note rather than a native resume. + * A restart continuation prompted with the note or an undelivered steering restart. * A turn cut mid-way that lost background work is prompted too, so the note - * is delivered with it; only a continuation without a note resumes natively. + * is delivered with it; continuations with no new input resume natively. */ export function isRestartNoteContinuation( run: Pick, runs: ReadonlyArray, providerTurns: ReadonlyArray>, - attempts: ReadonlyArray, + attempts: ReadonlyArray, ): boolean { const source = run.restartContinuationOfRunId === undefined @@ -176,7 +204,8 @@ export function isRestartNoteContinuation( : runs.find((candidate) => candidate.id === run.restartContinuationOfRunId); return ( source !== undefined && - restartContinuationNote(source, runs, providerTurns, attempts).work.length > 0 + (restartContinuationNote(source, runs, providerTurns, attempts).work.length > 0 || + restartPromptSource(source, runs, providerTurns, attempts) !== undefined) ); } diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 3dcafae687171..1694326f6f389 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -1,6 +1,7 @@ import { assert, it } from "@effect/vitest"; import { MessageId, + NodeId, ProjectId, ProviderDriverKind, ProviderInstanceId, @@ -9,6 +10,8 @@ import { ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, + ScheduledTaskId, ThreadId, type OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; @@ -125,6 +128,69 @@ it("requires matching saved native state for an unfinished root run", () => { assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection)); }); +it.effect.each([ + ["user_input", "pending", "live", false], + ["user_input", "expired", "not_resumable", false], + ["user_input", "cancelled", "not_resumable", false], + ["command", "pending", "live", false], + ["command", "expired", "not_resumable", false], + ["command", "cancelled", "not_resumable", false], + ["user_input", "resolved", "live", true], + ["command", "cancelled", "live", true], + ["user_input", "pending", "message", true], + ["dynamic_tool_call", "pending", "live", true], + ["auth_refresh", "pending", "live", true], +] as const)( + "respects a %s request that is %s with %s response capability across restart", + ([kind, status, capability, shouldContinue]) => + Effect.gen(function* () { + const base = makeProjection(); + const projection = { + ...base, + runtimeRequests: [ + { + id: RuntimeRequestId.make("request:restart"), + nodeId: NodeId.make("node:restart-question"), + providerTurnId: base.providerTurns[0]!.id, + kind, + status, + responseCapability: + capability === "live" + ? { type: capability, providerSessionId: sessionId } + : capability === "not_resumable" + ? { + type: capability, + reason: "The server restarted before this runtime request was resolved.", + } + : { type: capability }, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + assert.equal(restartContinuationRun(projection) !== undefined, shouldContinue); + const dispatch = [] as Array; + yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe( + Effect.provide( + Layer.merge( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => + Effect.succeed({ + ...projection, + runs: [{ ...projection.runs[0]!, status: "cancelled" as const }], + }), + recoverDelegatedTask: () => Effect.void, + dispatch: (command) => { + dispatch.push(command); + return Effect.succeed({} as never); + }, + }), + ), + ), + ); + assert.lengthOf(dispatch, shouldContinue ? 1 : 0); + }), +); + it("continues a live turn whose session the adapter never marked running", () => { const projection = makeProjection(); // Codex, Claude, Cursor and ACP sessions stay "ready" for their whole life. @@ -157,6 +223,19 @@ it("recovers an admitted continuation after another crash before provider start" assert.equal(restartContinuationRun(starting)?.id, runId); }); +it("recovers an accepted steering restart before the replacement turn starts", () => { + const projection = makeProjection(); + const starting = { + ...projection, + runs: [{ ...projection.runs[0]!, status: "starting" as const }], + attempts: [{ id: attemptId, runId, reason: "steering_restart" }], + providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }], + providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection; + assert.equal(restartContinuationRun(starting)?.id, runId); +}); + it("does not continue settled root runs with restart-cancelled background work", () => { const projection = makeProjection(); const settled = { @@ -643,6 +722,90 @@ const cutMidTurn = (extra: Record = {}) => { } as unknown as OrchestrationV2ThreadProjection; }; +it.effect( + "delivers the replacement prompt when a steering restart never reached the provider", + () => + Effect.gen(function* () { + const projection = { + ...cutMidTurn(), + attempts: [{ id: attemptId, runId, reason: "steering_restart" }], + providerTurns: [], + messages: [ + { + id: MessageId.make("message:user"), + text: "Restart the review with the updated requirements.", + attachments: [], + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + assert.deepEqual(yield* continuationTexts(projection), [projection.messages[0]!.text]); + }), +); + +it.effect.each(["interval", "fixed_time", "webhook"] as const)( + "keeps %s task attribution through repeated restart continuations", + (trigger) => + Effect.gen(function* () { + const scheduledTaskId = ScheduledTaskId.make(`scheduled-task:${trigger}`); + const originalMessage = { + id: MessageId.make("message:user"), + scheduledTaskId, + text: trigger === "webhook" ? 'Handle {"action":"opened","number":7}.' : "Run the audit.", + attachments: [], + }; + const base = cutMidTurn(); + const messages = [originalMessage]; + const runs = [...base.runs]; + const projection = { + ...base, + messages, + runs, + } as unknown as OrchestrationV2ThreadProjection; + const commands: Parameters< + ThreadManagementService.ThreadManagementService["Service"]["dispatch"] + >[0][] = []; + const dependencies = Layer.merge( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => Effect.succeed(projection), + recoverDelegatedTask: () => Effect.void, + dispatch: (command) => { + commands.push(command); + return Effect.succeed({} as never); + }, + }), + ); + for (let ordinal = 1; ordinal <= 2; ordinal += 1) { + const source = projection.runs.at(-1)!; + yield* continueRestartedRun({ threadId, sourceRunId: source.id }).pipe( + Effect.provide(dependencies), + ); + const command = commands.at(-1)!; + assert.equal(command.type, "message.dispatch"); + if (command.type !== "message.dispatch") return; + assert.equal(command.scheduledTaskId, scheduledTaskId); + assert.equal(command.threadId, threadId); + assert.equal(command.restartContinuationOfRunId, source.id); + assert.equal(command.text, "Continue where you left off."); + messages.push({ + id: command.messageId, + text: command.text, + scheduledTaskId: command.scheduledTaskId, + attachments: [], + } as never); + runs.push({ + ...source, + id: RunId.make(`run:continuation:${ordinal}`), + ordinal: ordinal + 1, + userMessageId: command.messageId, + restartContinuationOfRunId: source.id, + }); + } + assert.lengthOf(commands, 2); + assert.equal(projection.messages[0]?.text, originalMessage.text); + }), +); + it.effect("stops continuing after three automatic restart continuations in a row", () => Effect.gen(function* () { const first = RunId.make("run:first-continuation"); @@ -696,6 +859,110 @@ it.effect("continues a cut run past queued follow-ups, which stay held", () => }), ); +it.effect.each([ + ["startup", true, false, false], + ["shutdown", true, false, false], + ["startup", false, false, false], + ["shutdown", false, false, false], + ["startup", true, true, false], + ["startup", true, false, true], +] as const)( + "preserves a queued worktree continuation through %s when enabled is %s, held is %s, extra queue is %s", + ([trigger, enabled, alreadyHeld, extraQueue]) => + Effect.gen(function* () { + const projection = { + ...makeProjection(), + thread: { ...makeProjection().thread, worktreePath: "/tmp/handoff-worktree" }, + runs: [ + { + ...queuedFollowUp, + queueHeld: alreadyHeld, + userMessageId: MessageId.make("message:mcp:credential:worktree-continuation:request"), + }, + ...(extraQueue ? [{ ...queuedFollowUp, id: RunId.make("run:other-queued") }] : []), + ], + providerThreads: [], + providerSessions: [], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection; + const events: Array< + Parameters[0]["events"][number] + > = []; + const effects: Array = []; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: enabled }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + writeWithEffects: (input) => { + effects.push(...input.effects); + return Effect.succeed([]); + }, + commitCommand: (input) => { + events.push(...input.events); + effects.push(...input.effects); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectOutbox.EffectOutboxV2)({ + cancelUnsettled: () => Effect.succeed([]), + signalCancellations: () => Effect.void, + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + yield* recovery.reconcile(trigger); + const held = + alreadyHeld || + events.some((event) => event.type === "run.updated" && event.payload.queueHeld === true); + assert.equal(held, !enabled || alreadyHeld || extraQueue); + assert.lengthOf( + effects, + trigger === "startup" && enabled && !alreadyHeld && !extraQueue ? 1 : 0, + ); + if (effects.length > 0) assert.equal(effects[0]!.request.type, "provider-runtime.continue"); + }), +); + +it.effect("delivers a queued handoff prompt once and respects cancellation before delivery", () => + Effect.gen(function* () { + const message = { + id: MessageId.make("message:mcp:credential:worktree-continuation:request"), + text: "Keep fixing the login bug in the worktree.", + attachments: [], + }; + const base = cutMidTurn(); + const projection = { + ...base, + thread: { ...base.thread, worktreePath: "/tmp/handoff-worktree" }, + runs: [ + { ...base.runs[0]!, id: RunId.make("run:before-handoff"), ordinal: 1 }, + { ...queuedFollowUp, id: runId, userMessageId: message.id, queueHeld: false }, + ], + messages: [message], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection; + assert.deepEqual(yield* continuationTexts(projection), [message.text]); + for (const status of ["queued", "cancelled"] as const) { + assert.deepEqual( + yield* continuationTexts({ + ...projection, + runs: projection.runs.map((run) => + run.id === runId ? { ...run, status, queueHeld: true } : run, + ), + }), + [], + ); + } + }), +); + it.effect("continues a resumed queued run that ran after an earlier continuation", () => Effect.gen(function* () { // The continuation (ordinal 3) ran ahead of the held queue, then the user @@ -841,3 +1108,111 @@ it.effect("prepares later threads' continuations when one thread fails", () => ); }), ); + +it.effect.each(["snoozed", "settled"] as const)( + "does not wake a %s thread after a restart", + (state) => + Effect.gen(function* () { + const live = makeProjection(); + const thread = { + ...live.thread, + ...(state === "snoozed" + ? { snoozedUntil: DateTime.makeUnsafe("2099-01-01T00:00:00.000Z") } + : { settledOverride: "settled" as const }), + }; + assert.isUndefined(restartContinuationRun({ ...live, thread })); + assert.deepEqual(yield* continuationTexts({ ...cutMidTurn(), thread }), []); + }), +); + +it.effect.each(["pending", "cancelled"] as const)( + "does not continue a run whose secret request is %s at shutdown", + (secretStatus) => + Effect.gen(function* () { + const completedAt = DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"); + const cut = cutMidTurn({ completedAt }); + const projection = { + ...cut, + turnItems: [{ type: "secret_request", runId, secretStatus, completedAt } as never], + }; + if (secretStatus === "pending") + assert.isUndefined( + restartContinuationRun({ ...makeProjection(), turnItems: projection.turnItems }), + ); + assert.deepEqual(yield* continuationTexts(projection), []); + }), +); + +it.effect("continues when an earlier attempt's question and secret were cancelled", () => + Effect.gen(function* () { + const completedAt = DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"); + const cut = cutMidTurn({ completedAt }); + const earlierTurn = { + ...cut.providerTurns[0]!, + id: ProviderTurnId.make("turn:earlier-question"), + runAttemptId: RunAttemptId.make("attempt:earlier-question"), + }; + const projection = { + ...cut, + providerTurns: [...cut.providerTurns, earlierTurn], + runtimeRequests: [ + { + kind: "user_input", + status: "expired", + providerTurnId: earlierTurn.id, + responseCapability: { type: "not_resumable", reason: "The server restarted." }, + } as never, + ], + turnItems: [ + { + type: "secret_request", + runId, + secretStatus: "cancelled", + completedAt: DateTime.subtract(completedAt, { seconds: 1 }), + } as never, + { + type: "secret_request", + runId: RunId.make("run:earlier-secret"), + secretStatus: "pending", + completedAt: null, + } as never, + ], + }; + assert.lengthOf(yield* continuationTexts(projection), 1); + }), +); + +it.effect.each(["completed", "failed", "interrupted"] as const)( + "does not continue when the provider turn %s before the app run settled", + (status) => + Effect.gen(function* () { + const cut = cutMidTurn(); + assert.deepEqual( + yield* continuationTexts({ + ...cut, + providerTurns: [{ ...cut.providerTurns[0]!, status }], + }), + [], + ); + }), +); + +it.effect("continues a cut goal turn after an earlier phase of the attempt completed", () => + Effect.gen(function* () { + const cut = cutMidTurn(); + assert.lengthOf( + yield* continuationTexts({ + ...cut, + providerTurns: [ + { ...cut.providerTurns[0]!, ordinal: 1, status: "completed" }, + { + ...cut.providerTurns[0]!, + id: ProviderTurnId.make("turn:goal-later"), + ordinal: 2, + }, + ], + }), + 1, + ); + }), +); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 077cd458d02c9..8524c0fe978fe 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -2,11 +2,13 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { CommandId, + latestProviderTurnForAttempt, MessageId, type OrchestrationV2Run, type RunId, type ThreadId, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts"; @@ -15,12 +17,63 @@ import { isNativeMaintenanceCommand } from "./Orchestrator.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import { isRestartNoteSource, + isWorktreeContinuationRun, restartCancelledBackgroundWorkNote, restartContinuationNote, + restartPromptSource, } from "./RestartBackgroundNote.ts"; const CONTINUE_PROMPT = "Continue where you left off."; +/** A recorded intent must not wake held threads, unanswered requests, or settled provider turns. */ +export function restartContinuationBlocked( + projection: Pick< + ProjectionRuntimeRecoveryState, + "thread" | "runtimeRequests" | "providerTurns" | "turnItems" + >, + source: OrchestrationV2Run, +): boolean { + if ( + projection.thread.archivedAt !== null || + projection.thread.deletedAt !== null || + projection.thread.settledOverride === "settled" || + projection.thread.snoozedUntil != null + ) + return true; + const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId); + return ( + (turn !== undefined && + (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")) || + projection.runtimeRequests.some( + (request) => + // Tool calls and auth refreshes do not wait on a person, and a + // message-capable question stays answerable after the restart. + request.kind !== "dynamic_tool_call" && + request.kind !== "auth_refresh" && + request.responseCapability.type !== "message" && + (request.status === "pending" || + // Recovery closes live callbacks; it does not answer their question. + ((request.status === "expired" || request.status === "cancelled") && + request.responseCapability.type === "not_resumable" && + source.activeAttemptId !== null && + projection.providerTurns.some( + (turn) => + turn.id === request.providerTurnId && turn.runAttemptId === source.activeAttemptId, + ))), + ) || + projection.turnItems.some( + (item) => + item.type === "secret_request" && + item.runId === source.id && + (item.secretStatus === "pending" || + (item.secretStatus === "cancelled" && + item.completedAt !== null && + source.completedAt !== null && + DateTime.Order(item.completedAt, source.completedAt) === 0)), + ) + ); +} + /** * Resume only an unfinished root turn. Leftover background work is cleaned up * separately and reported on the next user turn; it must not wake a settled run. @@ -28,10 +81,16 @@ const CONTINUE_PROMPT = "Continue where you left off."; export function restartContinuationRun( projection: Pick< ProjectionRuntimeRecoveryState, - "thread" | "runs" | "providerThreads" | "providerSessions" | "providerTurns" + | "thread" + | "runs" + | "attempts" + | "providerThreads" + | "providerSessions" + | "providerTurns" + | "runtimeRequests" + | "turnItems" >, ): OrchestrationV2Run | undefined { - if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) return; // Queued runs never started; recovery holds them behind the cut run. const run = projection.runs.reduce( (latest, candidate) => @@ -42,8 +101,12 @@ export function restartContinuationRun( ); if (!run) return; const preparedContinuation = - run.status === "starting" && run.restartContinuationOfRunId !== undefined; + run.status === "starting" && + (run.restartContinuationOfRunId !== undefined || + restartPromptSource(run, projection.runs, projection.providerTurns, projection.attempts) !== + undefined); if (run.status !== "running" && !preparedContinuation) return; + if (restartContinuationBlocked(projection, run)) return; const liveTurnRequired = !preparedContinuation; if (projection.thread.providerInstanceId !== run.providerInstanceId) return; const providerThread = projection.providerThreads.find( @@ -99,23 +162,35 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`); const projection = yield* threads.getThreadRecords( input.threadId, - ["messages", "runs", "providerTurns", "attempts"], - { messageIds: [messageId] }, + ["messages", "runs", "providerTurns", "attempts", "runtimeRequests", "turnItems"], + { messageIds: [messageId], turnItemTypes: ["secret_request"] }, ); if ( !resolveProjectSettings(enabled, projection.thread.projectId).settings .continueThreadsAfterServerUpdate ) return; - if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) return; - if (projection.messages.some((message) => message.id === messageId)) return; const source = projection.runs.find((run) => run.id === input.sourceRunId); + const worktreeContinuation = + source !== undefined && + source.status === "queued" && + source.queueHeld !== true && + isWorktreeContinuationRun(source) && + projection.thread.worktreePath != null && + projection.runs.filter((run) => run.status === "queued").length === 1 && + !projection.runs.some((run) => + ["preparing", "starting", "running", "waiting"].includes(run.status), + ); // Pending effects from older versions may target settled background work, // including waiting runs that reconciliation subsequently cancelled. if ( !source || - source.status !== "cancelled" || + (source.status !== "cancelled" && !worktreeContinuation) || + (isWorktreeContinuationRun(source) && + source.status === "cancelled" && + !projection.providerTurns.some((turn) => turn.runAttemptId === source.activeAttemptId)) || + restartContinuationBlocked(projection, source) || isRestartNoteSource(source, projection.providerTurns) ) return; @@ -123,7 +198,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta // prompt. Queued runs never started and stay held behind this one. if ( projection.runs.some( - (run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source), + (run) => + run.id !== source.id && + run.status !== "queued" && + (worktreeContinuation ? run.ordinal > source.ordinal : runRanAfter(run, source)), ) ) return; @@ -144,11 +222,17 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta }); return; } + const promptSource = restartPromptSource( + source, + projection.runs, + projection.providerTurns, + projection.attempts, + ); const sourceRecords = yield* threads.getThreadRecords( input.threadId, ["messages", "turnItems"], { - messageIds: [source.userMessageId], + messageIds: [source.userMessageId, ...(promptSource ? [promptSource.userMessageId] : [])], turnItemRunIds: [source.id], turnItemTypes: ["run_interrupt_request"], }, @@ -164,6 +248,12 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta (message) => message.id === source.userMessageId, ); if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return; + const replacementMessage = + promptSource === undefined + ? undefined + : sourceRecords.messages.find((message) => message.id === promptSource.userMessageId); + if (promptSource !== undefined && replacementMessage === undefined) return; + const prompt = replacementMessage?.text ?? CONTINUE_PROMPT; const note = restartContinuationNote( source, projection.runs, @@ -177,13 +267,12 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`), threadId: input.threadId, messageId, - text: - noteText === undefined - ? CONTINUE_PROMPT - : note.settled - ? noteText - : `${noteText}\n\n${CONTINUE_PROMPT}`, - attachments: [], + ...(sourceMessage?.scheduledTaskId === undefined + ? {} + : { scheduledTaskId: sourceMessage.scheduledTaskId }), + text: noteText === undefined ? prompt : note.settled ? noteText : `${noteText}\n\n${prompt}`, + attachments: replacementMessage?.attachments ?? [], + ...(replacementMessage?.context === undefined ? {} : { context: replacementMessage.context }), modelSelection: source.modelSelection, dispatchMode: { type: "start_immediately" }, createdBy: "agent", diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index b11184f446326..1fab8be056365 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -1,4 +1,5 @@ import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; +import { continueRestartedRun } from "./RestartContinuation.ts"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -3233,6 +3234,110 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect.each(["continue", "cancel", "stop"] as const)( + "recovers a queued worktree handoff after restart with %s before delivery", + (action) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make(`runtime-restart-handoff-${action}`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`handoff-create-${action}`), + threadId, + projectId: ProjectId.make("restart-project"), + title: "Handoff", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: "/tmp/runtime-restart-handoff", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`handoff-original-${action}`), + threadId, + messageId: MessageId.make(`handoff-original-${action}`), + text: "Original work", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`handoff-queue-${action}`), + threadId, + messageId: MessageId.make(`message:mcp:session:worktree-continuation:${action}`), + text: "Keep fixing the login bug in the worktree.", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + }); + const before = yield* orchestrator.getThreadProjection(threadId); + const original = before.runs[0]!; + const queued = before.runs[1]!; + const now = yield* DateTime.now; + yield* eventSink.commitCommand({ + commandId: CommandId.make(`command:runtime-reconcile:handoff-${action}`), + threadId, + commandType: "provider-runtime.reconcile", + acceptedAt: now, + events: [ + { + id: EventId.make(`handoff-reconcile-${action}`), + type: "run.updated", + threadId, + runId: original.id, + occurredAt: now, + payload: { ...original, status: "cancelled", completedAt: now }, + }, + ], + effects: [], + }); + if (action === "cancel") { + yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make(`handoff-cancel-${action}`), + threadId, + runId: queued.id, + }); + } else if (action === "stop") { + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make(`handoff-stop-${action}`), + threadId, + }); + } + const continuation = continueRestartedRun({ threadId, sourceRunId: queued.id }).pipe( + Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })), + ); + yield* continuation; + yield* continuation; + const after = yield* orchestrator.getThreadProjection(threadId); + assert.lengthOf(after.runs, action === "continue" ? 3 : 2); + if (action === "continue") { + assert.equal(after.runs[1]!.status, "cancelled"); + assert.equal(after.runs[2]!.status, "starting"); + assert.equal(after.runs[2]!.restartContinuationOfRunId, queued.id); + assert.equal( + after.messages.find((message) => message.id === after.runs[2]!.userMessageId)?.text, + "Keep fixing the login bug in the worktree.", + ); + } + yield* orchestrator.dispatch({ + type: "thread.archive", + commandId: CommandId.make(`handoff-archive-${action}`), + threadId, + }); + }), + ); + it.effect.each(["failed", "completed", "waiting", "cancelled"] as const)( "does not admit a restart continuation of a %s run with only lost background work", (status) => From dad17d39a47fd278a3bc5958b2a0d96a998942f2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:49:17 +0000 Subject: [PATCH 061/149] fix(server): restart recovery cancels orphan native subagents and runless child turns --- .../DelegatedCompletionDelivery.test.ts | 65 +++++++ .../ProjectionRecovery.test.ts | 182 +++++++++++++++++- .../src/orchestration-v2/ProjectionStore.ts | 56 +++++- .../ProviderRuntimeRecoveryService.test.ts | 34 ++-- .../ProviderRuntimeRecoveryService.ts | 74 ++++++- 5 files changed, 380 insertions(+), 31 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 5a41e5fd08e26..674778ae4d705 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1069,6 +1069,71 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); + it.effect("settles a delegated child once when its restart chain reaches the cap", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:restart-capped-parent"); + const projectId = ProjectId.make("project:restart-capped-parent"); + const runId = RunId.make("run:restart-capped-parent"); + const rootNodeId = NodeId.make("node:restart-capped-parent-root"); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make("node:restart-capped-parent-settled"), + deliveryState: "delivered", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "restart-capped-child", + completionWake: "always", + continuationPending: false, + now, + }); + let sourceRunId = child.childRunId; + for (let ordinal = 2; ordinal <= 4; ordinal += 1) { + const nextRunId = RunId.make(`run:restart-capped-child:${ordinal}`); + const event = runEvent({ + threadId: child.childThreadId, + runId: nextRunId, + ordinal, + status: "cancelled", + now, + }); + yield* eventSink.write({ + commandId: reconcileCommandId(`restart-capped-child:${ordinal}`), + events: [ + { ...event, payload: { ...event.payload, restartContinuationOfRunId: sourceRunId } }, + ], + }); + sourceRunId = nextRunId; + } + const continuation = continueRestartedRun({ + threadId: child.childThreadId, + sourceRunId, + }).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }))); + yield* continuation; + yield* continuation; + const parent = yield* orchestrator.getThreadProjection(threadId); + assert.equal(parent.subagents.find((task) => task.id === child.taskId)?.status, "cancelled"); + assert.equal( + parent.contextTransfers.filter( + (transfer) => transfer.sourceThreadId === child.childThreadId, + ).length, + 1, + ); + const stoppedChild = yield* orchestrator.getThreadProjection(child.childThreadId); + assert.equal(stoppedChild.runs.length, 4); + }), + ); + it.effect("settles a restart-cancelled child whose continuation declines to start", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index c7ea0530d3512..526522f99802f 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -12,6 +12,7 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, RunId, RuntimeRequestId, ThreadId, @@ -23,9 +24,13 @@ import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; +import * as EventSink from "./EventSink.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; import { restartContinuationRun } from "./RestartContinuation.ts"; const layerTest = Layer.mergeAll(ProjectionStore.layer, EffectOutbox.layer).pipe( @@ -160,6 +165,43 @@ it.effect("selects unfinished recovery work without reading settled thread histo }, }); const outboxOnly = yield* createThread("outbox-only"); + const orphanSubagent = yield* createThread("orphan-native-subagent"); + for (const origin of ["provider_native", "app_owned"] as const) { + const threadId = + origin === "provider_native" ? orphanSubagent : yield* createThread("app-owned-subagent"); + const runId = yield* createRun(threadId, "completed"); + const nodeId = NodeId.make(`node:recovery:orphan:${origin}`); + yield* projections.apply({ + id: EventId.make(`event:recovery:orphan:${origin}`), + type: "subagent.updated", + threadId, + runId, + nodeId, + occurredAt: now, + payload: { + id: nodeId, + threadId, + runId, + parentNodeId: NodeId.make(`node:recovery:orphan-root:${origin}`), + origin, + createdBy: "agent", + driver, + providerInstanceId, + providerThreadId: null, + childThreadId: + origin === "app_owned" ? ThreadId.make("thread:recovery:delegated-child") : null, + nativeTaskRef: null, + prompt: "unfinished native work", + title: null, + model: null, + status: "running", + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }); + } yield* outbox.enqueue([ { id: "effect:recovery:outbox-only", @@ -208,11 +250,14 @@ it.effect("selects unfinished recovery work without reading settled thread histo assert.deepEqual(yield* projections.getRecoveryThreadIds("queued-runs"), [queued]); assert.deepEqual( new Set(yield* projections.getRecoveryThreadIds("runtime")), - new Set([queued, archived, blocked, background, outboxOnly, requestOnly]), + new Set([queued, archived, blocked, background, outboxOnly, requestOnly, orphanSubagent]), ); assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [delivery]); assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []); const recoveryState = yield* projections.getRuntimeRecoveryProjection(queued); + const orphanState = yield* projections.getRuntimeRecoveryProjection(orphanSubagent); + assert.equal(orphanState.subagents.length, 1); + assert.equal(orphanState.runs[0]?.id, orphanState.subagents[0]?.runId); assert.deepEqual( recoveryState.runs.map((run) => run.id), [RunId.make(`run:${queued}:2`)], @@ -224,6 +269,141 @@ it.effect("selects unfinished recovery work without reading settled thread histo }).pipe(Effect.provide(layerTest)), ); +it.effect.each(["startup", "shutdown"] as const)( + "recovers native child turns and their runless streaming descendants on %s", + (trigger) => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = yield* createThread("native-child"); + const rootNodeId = NodeId.make("node:recovery:native-root"); + const textNodeId = NodeId.make("node:recovery:native-text"); + const providerThreadId = ProviderThreadId.make("provider-thread:recovery:native-child"); + const providerTurnId = ProviderTurnId.make("turn:recovery:native-child"); + yield* projections.apply({ + id: EventId.make("event:recovery:native-turn"), + type: "provider-turn.updated", + threadId, + nodeId: rootNodeId, + occurredAt: now, + payload: { + id: providerTurnId, + providerThreadId, + nodeId: rootNodeId, + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]); + for (const [id, kind] of [ + [rootNodeId, "root_turn"], + [textNodeId, "assistant_message"], + ] as const) { + yield* projections.apply({ + id: EventId.make(`event:${id}`), + type: "node.updated", + threadId, + nodeId: id, + occurredAt: now, + payload: { + id, + threadId, + runId: null, + parentNodeId: id === rootNodeId ? null : rootNodeId, + rootNodeId, + kind, + status: "running", + countsForRun: false, + providerThreadId, + providerTurnId, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }, + }); + } + const itemId = TurnItemId.make("item:recovery:native-text"); + yield* projections.apply({ + id: EventId.make("event:recovery:native-text-item"), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: itemId, + threadId, + runId: null, + nodeId: textNodeId, + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + type: "assistant_message", + status: "running", + title: null, + startedAt: now, + completedAt: null, + updatedAt: now, + messageId: MessageId.make("message:recovery:native-text"), + text: "unfinished native child output", + streaming: true, + }, + }); + const recovered = yield* projections.getRuntimeRecoveryProjection(threadId); + assert.deepEqual( + recovered.providerTurns.map((turn) => turn.id), + [providerTurnId], + ); + assert.deepEqual( + new Set(recovered.nodes.map((node) => node.id)), + new Set([rootNodeId, textNodeId]), + ); + assert.deepEqual( + recovered.turnItems.map((item) => item.id), + [itemId], + ); + yield* ProviderRuntimeRecovery.make.pipe( + Effect.flatMap((recovery) => recovery.reconcile(trigger)), + Effect.provide( + Layer.mergeAll( + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => + Effect.forEach(input.events, (event) => projections.apply(event), { + discard: true, + }).pipe( + Effect.orDie, + Effect.as({ committed: true, cancelledEffectCount: 0 } as never), + ), + }), + IdAllocator.layer, + ServerSettings.layerTest(), + ), + ), + ); + const after = yield* projections.getThreadProjection(threadId); + assert.deepEqual( + after.providerTurns.map((turn) => turn.status), + ["cancelled"], + ); + assert.deepEqual( + after.nodes.map((node) => node.status), + ["cancelled", "cancelled"], + ); + assert.equal(after.turnItems[0]?.status, "cancelled"); + assert.equal( + after.turnItems[0]?.type === "assistant_message" && after.turnItems[0].streaming, + false, + ); + assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []); + }).pipe(Effect.provide(layerTest)), +); + it.effect("recovers terminal subagent results until their cross-thread transfer exists", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index b88a828a9db8f..55469fc6404a1 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -611,6 +611,15 @@ function needsRecovery( ["command_execution", "dynamic_tool", "subagent"].includes(item.type) && ["pending", "running", "waiting"].includes(item.status) && !projection.runs.some((run) => run.id === item.runId && run.status === "rolled_back"), + ) || + projection.subagents.some( + (task) => + (task.origin !== "app_owned" || task.childThreadId === null) && + ["pending", "running", "waiting"].includes(task.status) && + !projection.runs.some((run) => run.id === task.runId && run.status === "rolled_back"), + ) || + projection.providerTurns.some( + (turn) => turn.runAttemptId === null && ["pending", "running"].includes(turn.status), ) ); } @@ -3694,6 +3703,17 @@ export const layer: Layer.Layer = UNION SELECT thread_id FROM orchestration_v2_effect_outbox WHERE status IN ('pending', 'running') + UNION + SELECT task.thread_id FROM orchestration_v2_projection_subagents AS task + WHERE (task.origin != 'app_owned' OR task.child_thread_id IS NULL) + AND task.status IN ('pending', 'running', 'waiting') + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs AS run + WHERE run.run_id = task.run_id AND run.status = 'rolled_back' + ) + UNION + SELECT thread_id FROM orchestration_v2_projection_provider_turns + WHERE run_attempt_id IS NULL AND status IN ('pending', 'running') `; } })(); @@ -4050,6 +4070,12 @@ export const layer: Layer.Layer = AND item.status IN ('pending', 'running', 'waiting') AND item.run_id IS NOT NULL ) + OR run.run_id IN ( + SELECT run_id FROM orchestration_v2_projection_subagents + WHERE thread_id = ${threadId} + AND status IN ('pending', 'running', 'waiting') + AND (origin != 'app_owned' OR child_thread_id IS NULL) + ) ) ORDER BY run.ordinal ASC `, @@ -4069,7 +4095,7 @@ export const layer: Layer.Layer = WHERE node.thread_id = ${threadId} AND node.status IN ('pending', 'starting', 'running', 'waiting') AND ( - (node.run_id IS NULL AND node.kind = 'root_turn') + node.run_id IS NULL OR node.run_id IN ( SELECT run_id FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} @@ -4087,6 +4113,12 @@ export const layer: Layer.Layer = WHERE item.thread_id = ${threadId} AND item.type = 'subagent' AND item.status IN ('pending', 'running', 'waiting') ) + OR node.node_id IN ( + SELECT subagent_id FROM orchestration_v2_projection_subagents + WHERE thread_id = ${threadId} + AND status IN ('pending', 'running', 'waiting') + AND (origin != 'app_owned' OR child_thread_id IS NULL) + ) ) ORDER BY COALESCE(node.started_at, ''), node.node_id ASC `, @@ -4106,6 +4138,8 @@ export const layer: Layer.Layer = WHERE item.thread_id = ${threadId} AND item.type = 'subagent' AND item.status IN ('pending', 'running', 'waiting') ) + OR subagent.origin != 'app_owned' + OR subagent.child_thread_id IS NULL ) ORDER BY COALESCE(subagent.started_at, ''), subagent.subagent_id ASC `, @@ -4170,14 +4204,17 @@ export const layer: Layer.Layer = FROM orchestration_v2_projection_provider_turns AS provider_turn WHERE provider_turn.thread_id = ${threadId} AND provider_turn.status IN ('pending', 'starting', 'running', 'waiting') - AND provider_turn.run_attempt_id IN ( - SELECT attempt_id FROM orchestration_v2_projection_run_attempts - WHERE thread_id = ${threadId} - AND run_id IN ( - SELECT run_id FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - ) + AND ( + provider_turn.run_attempt_id IS NULL + OR provider_turn.run_attempt_id IN ( + SELECT attempt_id FROM orchestration_v2_projection_run_attempts + WHERE thread_id = ${threadId} + AND run_id IN ( + SELECT run_id FROM orchestration_v2_projection_runs + WHERE thread_id = ${threadId} + AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + ) + ) ) ORDER BY provider_turn.provider_thread_id ASC, provider_turn.ordinal ASC `, @@ -4213,7 +4250,6 @@ export const layer: Layer.Layer = SELECT node_id FROM orchestration_v2_projection_nodes WHERE thread_id = ${threadId} AND run_id IS NULL - AND kind = 'root_turn' AND status IN ('pending', 'running', 'waiting') ) ) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index da764d41d7301..7cd4cf805bdd3 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -1189,9 +1189,9 @@ it.effect( }, ); -it.effect( - "terminalizes the linked subagent and node for a stale subagent item on a settled run", - () => { +it.effect.each([true, false])( + "terminalizes a native subagent on a settled run with a turn item: %s", + (hasTurnItem) => { const threadId = ThreadId.make("thread_recovery_subagent"); const settledRunId = RunId.make("run_recovery_subagent_settled"); const providerThreadId = ProviderThreadId.make("provider_thread_recovery_subagent"); @@ -1243,16 +1243,20 @@ it.effect( ], messages: [], turnItems: [ - { - id: staleItemId, - runId: settledRunId, - nodeId: staleSubagentNodeId, - providerThreadId, - type: "subagent", - status: "running", - subagentId: staleSubagentNodeId, - providerInstanceId: claudeInstanceId, - }, + ...(hasTurnItem + ? [ + { + id: staleItemId, + runId: settledRunId, + nodeId: staleSubagentNodeId, + providerThreadId, + type: "subagent", + status: "running", + subagentId: staleSubagentNodeId, + providerInstanceId: claudeInstanceId, + }, + ] + : []), { id: doneItemId, runId: settledRunId, @@ -1285,6 +1289,8 @@ it.effect( }), Layer.mock(EffectOutbox.EffectOutboxV2)({ listByCommandId: () => Effect.succeed([]), + cancelUnsettled: () => Effect.succeed([]), + signalCancellations: () => Effect.void, reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), }), ), @@ -1299,7 +1305,7 @@ it.effect( const turnItemCancels = events.filter( (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", ); - assert.equal(turnItemCancels.length, 1); + assert.equal(turnItemCancels.length, hasTurnItem ? 1 : 0); // The linked subagent entity is terminalized alongside its turn item. const subagentCancels = events.filter((event) => event.type === "subagent.updated"); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 8140d367d8eae..cc15e893bf0f7 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -541,17 +541,64 @@ export const make = Effect.gen(function* () { }); } } - // A provider-native subagent thread has no runs: its work is a runless - // root turn, plus items under it (Claude's live progress item), that - // only the dead provider process could settle. Left running, the child - // would show as working forever. + // A crash can persist the task before its turn item. It still died with + // the provider, while an app-owned child settles through its own thread. + const cancelledSubagentIds = new Set( + events.flatMap((event) => (event.type === "subagent.updated" ? [event.payload.id] : [])), + ); + for (const subagent of projection.subagents ?? []) { + if ( + isAppOwnedDelegation(subagent) || + !isNonterminalSubagentStatus(subagent.status) || + cancelledSubagentIds.has(subagent.id) || + projection.runs.some((run) => run.id === subagent.runId && run.status === "rolled_back") + ) + continue; + recordCancelledBackgroundWork( + projection.runs.find((run) => run.id === subagent.runId)?.providerThreadId ?? + subagent.providerThreadId, + { kind: "subagent", label: subagent.title?.slice(0, 160) ?? "subagent", id: subagent.id }, + ); + events.push({ + id: yield* allocateEventId(), + type: "subagent.updated", + threadId: projection.thread.id, + ...(subagent.runId === null ? {} : { runId: subagent.runId }), + nodeId: subagent.id, + driver: subagent.driver, + providerInstanceId: subagent.providerInstanceId, + occurredAt: now, + payload: { ...subagent, status: "cancelled", completedAt: now, updatedAt: now }, + }); + const node = projection.nodes.find((candidate) => candidate.id === subagent.id); + if ( + node !== undefined && + isNonterminalNodeStatus(node.status) && + !cancelledStaleNodeIds.has(node.id) + ) { + cancelledStaleNodeIds.add(node.id); + events.push({ + id: yield* allocateEventId(), + type: "node.updated", + threadId: projection.thread.id, + ...(node.runId === null ? {} : { runId: node.runId }), + nodeId: node.id, + providerInstanceId: subagent.providerInstanceId, + occurredAt: now, + payload: { ...node, status: "cancelled", completedAt: now }, + }); + } + } + // A native child has no app runs. Its nodes and streaming items must + // settle too, or the dead provider leaves it looking busy forever. const cancelledStaleItemIds = new Set( events.flatMap((event) => (event.type === "turn-item.updated" ? [event.payload.id] : [])), ); for (const node of projection.nodes) { if ( - node.kind !== "root_turn" || node.runId !== null || + messageRequestNodeIds.has(node.id) || + delegatedTaskNodeIds.has(node.id) || !isNonterminalNodeStatus(node.status) || cancelledStaleNodeIds.has(node.id) ) { @@ -589,13 +636,28 @@ export const make = Effect.gen(function* () { status: "cancelled", completedAt: now, updatedAt: now, - ...(item.type === "reasoning" || item.type === "assistant_message" + ...(item.type === "reasoning" || + item.type === "assistant_message" || + item.type === "proposed_plan" ? { streaming: false } : {}), }, }); } } + for (const turn of projection.providerTurns ?? []) { + if (turn.runAttemptId !== null || (turn.status !== "pending" && turn.status !== "running")) + continue; + events.push({ + id: yield* allocateEventId(), + type: "provider-turn.updated", + threadId: projection.thread.id, + nodeId: turn.nodeId, + providerInstanceId: projection.thread.providerInstanceId, + occurredAt: now, + payload: { ...turn, status: "cancelled", completedAt: now }, + }); + } // All provider processes are gone on startup/shutdown: clear any // persisted Waiting roster (including idle threads from settled roots) // and idle active threads without resurrecting active status. From ab46dcddcc6fa73f9e662729e6393e3c9ead9248 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:49:17 +0000 Subject: [PATCH 062/149] fix(server): MCP environment tools expose thread continuation, and an unreadable settings file keeps it off --- .../mcp/toolkits/environment/handlers.test.ts | 49 ++++++++++--------- .../src/mcp/toolkits/environment/handlers.ts | 2 + .../src/mcp/toolkits/environment/tools.ts | 2 + apps/server/src/serverSettings.test.ts | 5 +- apps/server/src/serverSettings.ts | 8 ++- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/apps/server/src/mcp/toolkits/environment/handlers.test.ts b/apps/server/src/mcp/toolkits/environment/handlers.test.ts index bf32658886a00..2dd4931253e6e 100644 --- a/apps/server/src/mcp/toolkits/environment/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/environment/handlers.test.ts @@ -52,31 +52,36 @@ const layerEnvironment = Layer.mock(Environment.ServerEnvironment)({ it.effect("updates preferences for a thread caller through the /mcp registration", () => Effect.gen(function* () { const server = yield* McpServer.McpServer; - const result = yield* server - .callTool({ - name: "t3_environment_preferences_update", - arguments: { newWorktreesStartFromOrigin: true }, - }) - .pipe( - Effect.provideService(McpInvocationContext.McpInvocationContext, caller), - Effect.provideService( - McpSchema.McpServerClient, - McpSchema.McpServerClient.of({ - clientId: 1, - protocolVersion: "2025-06-18", - clientCapabilities: {}, - clientInfo: { name: "preferences", version: "1" }, - initializePayload: { + const expected = { newWorktreesStartFromOrigin: true, continueThreadsAfterServerUpdate: false }; + for (const name of ["t3_environment_preferences_update", "t3_environment_read"] as const) { + const result = yield* server + .callTool({ + name, + arguments: name === "t3_environment_preferences_update" ? expected : {}, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, caller), + Effect.provideService( + McpSchema.McpServerClient, + McpSchema.McpServerClient.of({ + clientId: 1, protocolVersion: "2025-06-18", - capabilities: {}, + clientCapabilities: {}, clientInfo: { name: "preferences", version: "1" }, - }, - getClient: Effect.die("unused"), - }), - ), + initializePayload: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "preferences", version: "1" }, + }, + getClient: Effect.die("unused"), + }), + ), + ); + expect(result.isError).toBe(false); + expect(result.structuredContent).toMatchObject( + name === "t3_environment_read" ? { preferences: expected } : expected, ); - expect(result.isError).toBe(false); - expect(result.structuredContent).toMatchObject({ newWorktreesStartFromOrigin: true }); + } }).pipe( Effect.provide( McpHttpServer.layerEnvironmentToolkit.pipe( diff --git a/apps/server/src/mcp/toolkits/environment/handlers.ts b/apps/server/src/mcp/toolkits/environment/handlers.ts index dc51226bcf227..cf466145c5ddf 100644 --- a/apps/server/src/mcp/toolkits/environment/handlers.ts +++ b/apps/server/src/mcp/toolkits/environment/handlers.ts @@ -14,6 +14,7 @@ export function preferences(settings: ServerSettings) { newWorktreesStartFromOrigin, enableProviderUpdateChecks, automaticUpdates, + continueThreadsAfterServerUpdate, backgroundActivity, sourceControlWritingStyle, } = settings; @@ -23,6 +24,7 @@ export function preferences(settings: ServerSettings) { newWorktreesStartFromOrigin, enableProviderUpdateChecks, automaticUpdates, + continueThreadsAfterServerUpdate, backgroundActivity: { profile: backgroundActivity.profile }, sourceControlWritingStyle: { ...sourceControlWritingStyle, diff --git a/apps/server/src/mcp/toolkits/environment/tools.ts b/apps/server/src/mcp/toolkits/environment/tools.ts index 88e103ce0debe..545124d4c5a39 100644 --- a/apps/server/src/mcp/toolkits/environment/tools.ts +++ b/apps/server/src/mcp/toolkits/environment/tools.ts @@ -19,6 +19,7 @@ const PreferenceFields = { newWorktreesStartFromOrigin: ServerSettings.fields.newWorktreesStartFromOrigin, enableProviderUpdateChecks: ServerSettings.fields.enableProviderUpdateChecks, automaticUpdates: ServerSettings.fields.automaticUpdates, + continueThreadsAfterServerUpdate: ServerSettings.fields.continueThreadsAfterServerUpdate, backgroundActivity: Schema.Struct({ profile: BackgroundActivityProfileSelection }), sourceControlWritingStyle: Schema.Struct({ mode: Schema.String, @@ -61,6 +62,7 @@ const EnvironmentPreferencesTool = Tool.make("t3_environment_preferences_update" newWorktreesStartFromOrigin: ServerSettingsPatch.fields.newWorktreesStartFromOrigin, enableProviderUpdateChecks: ServerSettingsPatch.fields.enableProviderUpdateChecks, automaticUpdates: ServerSettingsPatch.fields.automaticUpdates, + continueThreadsAfterServerUpdate: ServerSettingsPatch.fields.continueThreadsAfterServerUpdate, backgroundActivity: Schema.optionalKey(Schema.Struct({ profile: BackgroundActivityProfile })), sourceControlWritingStyle: ServerSettingsPatch.fields.sourceControlWritingStyle, }), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index b32457f71305f..71361bfdec1eb 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1035,7 +1035,8 @@ it.layer(NodeServices.layer)("server settings", (it) => { const serverConfig = yield* ServerConfig.ServerConfig; const fileSystem = yield* FileSystem.FileSystem; const serverSettings = yield* ServerSettingsModule.ServerSettingsService; - const raw = '{"addProjectBaseDirectory":42,"providers":{"codex":{"binaryPath":"/x"}}}'; + const raw = + '{"addProjectBaseDirectory":42,"continueThreadsAfterServerUpdate":"false","providers":{"codex":{"binaryPath":"/x"}}}'; yield* fileSystem.writeFileString(serverConfig.settingsPath, raw); const settings = yield* serverSettings.getSettings; @@ -1047,6 +1048,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { }); // An unreadable file may hold an opt-out, so background updates stay off. assert.equal(settings.automaticUpdates, false); + assert.isFalse(settings.continueThreadsAfterServerUpdate); assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw); }).pipe(Effect.provide(layerServerSettings())), ); @@ -1899,6 +1901,7 @@ it.layer(NodeServices.layer)("server settings", (it) => { const settings = yield* serverSettings.getSettings; assert.isFalse(settings.projectSettingsFolded); + assert.isFalse(settings.continueThreadsAfterServerUpdate); assert.deepEqual(settings.projectSettingsOverrides, {}); // The user's file is still there to repair; nothing was written over it. assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), broken); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index da3f0a53b3751..0b0840738d275 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -786,8 +786,12 @@ const make = Effect.gen(function* () { if (decoded._tag === "Failure" || legacySettings._tag === "Failure") { const failure = decoded._tag === "Failure" ? decoded : legacySettings; settingsFileTrusted = false; - // The file may hold an opt-out we can no longer read; never install unattended on a guess. - settings = { ...DEFAULT_SERVER_SETTINGS, automaticUpdates: false }; + // The file may hold an opt-out we can no longer read; never install or resume on a guess. + settings = { + ...DEFAULT_SERVER_SETTINGS, + automaticUpdates: false, + continueThreadsAfterServerUpdate: false, + }; if (failure._tag === "Failure") { yield* Effect.logWarning("failed to parse settings.json, using defaults", { path: settingsPath, From ba5ff50b9f8ce78e842388e5c0e048a08286b2b9 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:57:16 +0000 Subject: [PATCH 063/149] fix(server): restart continuation resumes a thread whose snooze elapsed --- .../src/orchestration-v2/Orchestrator.ts | 1 + .../ProjectionRecovery.test.ts | 5 +- .../ProviderRuntimeRecoveryService.ts | 4 +- .../RestartContinuation.test.ts | 66 ++++++++++++------- .../orchestration-v2/RestartContinuation.ts | 12 +++- 5 files changed, 60 insertions(+), 28 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index c950f9d7ef713..cf88d9b3e1a17 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -4585,6 +4585,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio restartContinuationBlocked( { ...projection, turnItems: secretRequests.turnItems }, source, + yield* DateTime.now, ) || projection.thread.providerInstanceId !== source.providerInstanceId || // Held queued runs never started; they wait behind the continuation. diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index c2ce589a457da..158d82740cafb 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -682,7 +682,10 @@ it.effect("includes shared sessions and provider-owned background rosters in rec preparedState.providerSessions.map((session) => session.id), [preparedSessionId], ); - assert.equal(restartContinuationRun(preparedState)?.id, preparedRunId); + assert.equal( + restartContinuationRun(preparedState, DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"))?.id, + preparedRunId, + ); assert.deepEqual(yield* projections.getUnreadableThreadIds(), []); const sql = yield* SqlClient.SqlClient; yield* sql` diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index e1219ee28b0bb..0a8389878c1b5 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -646,7 +646,7 @@ export const make = Effect.gen(function* () { } const continuationRun = continueAfterRestart && trigger === "startup" - ? (handoffContinuation ?? restartContinuationRun(projection)) + ? (handoffContinuation ?? restartContinuationRun(projection, now)) : undefined; const effects: Array = continuationRun ? [ @@ -794,7 +794,7 @@ export const make = Effect.gen(function* () { .continueThreadsAfterServerUpdate ) return; - const run = restartContinuationRun(projection); + const run = restartContinuationRun(projection, yield* DateTime.now); if (!run) return; const commandId = CommandId.make(`command:restart-prepare:${run.id}`); yield* eventSink.writeWithEffects({ diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index be24e36d63e21..1abf4fe869cf6 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -36,6 +36,7 @@ const driver = ProviderDriverKind.make("codex"); const providerThreadId = ProviderThreadId.make("provider-thread:restart"); const sessionId = ProviderSessionId.make("session:restart"); const attemptId = RunAttemptId.make("attempt:restart"); +const testNow = DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"); // "No project" threads belong to the environment's Scratch project. const scratchProjectId = ProjectId.make("project:scratch"); @@ -93,7 +94,7 @@ function makeProjection() { it("requires matching saved native state for an unfinished root run", () => { const projection = makeProjection(); - assert.equal(restartContinuationRun(projection)?.id, runId); + assert.equal(restartContinuationRun(projection, testNow)?.id, runId); for (const invalid of [ { ...projection, thread: { ...projection.thread, archivedAt: {} } }, { ...projection, thread: { ...projection.thread, deletedAt: {} } }, @@ -126,7 +127,7 @@ it("requires matching saved native state for an unfinished root run", () => { "interrupted", ].map((status) => ({ ...projection, runs: [{ ...projection.runs[0]!, status }] })), ]) - assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection)); + assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection, testNow)); }); it.effect.each([ @@ -167,7 +168,7 @@ it.effect.each([ }, ], } as unknown as OrchestrationV2ThreadProjection; - assert.equal(restartContinuationRun(projection) !== undefined, shouldContinue); + assert.equal(restartContinuationRun(projection, testNow) !== undefined, shouldContinue); const dispatch = [] as Array; yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe( Effect.provide( @@ -201,9 +202,9 @@ it("continues a live turn whose session the adapter never marked running", () => providerSessions: [{ ...projection.providerSessions[0]!, status }], }) as OrchestrationV2ThreadProjection; for (const status of ["starting", "ready", "running", "waiting"]) - assert.equal(restartContinuationRun(withSessionStatus(status))?.id, runId, status); + assert.equal(restartContinuationRun(withSessionStatus(status), testNow)?.id, runId, status); for (const status of ["stopped", "error"]) - assert.isUndefined(restartContinuationRun(withSessionStatus(status)), status); + assert.isUndefined(restartContinuationRun(withSessionStatus(status), testNow), status); }); it("recovers an admitted continuation after another crash before provider start", () => { @@ -221,7 +222,7 @@ it("recovers an admitted continuation after another crash before provider start" providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }], providerTurns: [], }; - assert.equal(restartContinuationRun(starting)?.id, runId); + assert.equal(restartContinuationRun(starting, testNow)?.id, runId); }); it("recovers an accepted steering restart before the replacement turn starts", () => { @@ -234,7 +235,7 @@ it("recovers an accepted steering restart before the replacement turn starts", ( providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }], providerTurns: [], } as unknown as OrchestrationV2ThreadProjection; - assert.equal(restartContinuationRun(starting)?.id, runId); + assert.equal(restartContinuationRun(starting, testNow)?.id, runId); }); it("does not continue settled root runs with restart-cancelled background work", () => { @@ -255,11 +256,14 @@ it("does not continue settled root runs with restart-cancelled background work", for (const projectId of [scratchProjectId, projection.thread.projectId]) for (const status of ["completed", "waiting"] as const) assert.isUndefined( - restartContinuationRun({ - ...settled, - thread: { ...settled.thread, projectId }, - runs: [{ ...settled.runs[0]!, status }], - }), + restartContinuationRun( + { + ...settled, + thread: { ...settled.thread, projectId }, + runs: [{ ...settled.runs[0]!, status }], + }, + testNow, + ), ); }); @@ -851,10 +855,13 @@ it.effect("continues a cut run past queued follow-ups, which stay held", () => Effect.gen(function* () { const live = makeProjection(); assert.equal( - restartContinuationRun({ - ...live, - runs: [...live.runs, queuedFollowUp], - } as unknown as OrchestrationV2ThreadProjection)?.id, + restartContinuationRun( + { + ...live, + runs: [...live.runs, queuedFollowUp], + } as unknown as OrchestrationV2ThreadProjection, + testNow, + )?.id, runId, ); const cut = cutMidTurn(); @@ -984,10 +991,13 @@ it.effect("continues a resumed queued run that ran after an earlier continuation }; const live = makeProjection(); assert.equal( - restartContinuationRun({ - ...live, - runs: [...live.runs, finishedContinuation], - } as unknown as OrchestrationV2ThreadProjection)?.id, + restartContinuationRun( + { + ...live, + runs: [...live.runs, finishedContinuation], + } as unknown as OrchestrationV2ThreadProjection, + testNow, + )?.id, runId, ); const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") }); @@ -1139,11 +1149,23 @@ it.effect.each(["snoozed", "settled"] as const)( ? { snoozedUntil: DateTime.makeUnsafe("2099-01-01T00:00:00.000Z") } : { settledOverride: "settled" as const }), }; - assert.isUndefined(restartContinuationRun({ ...live, thread })); + assert.isUndefined(restartContinuationRun({ ...live, thread }, testNow)); assert.deepEqual(yield* continuationTexts({ ...cutMidTurn(), thread }), []); }), ); +it.effect("continues a thread whose snooze already elapsed", () => + Effect.gen(function* () { + const live = makeProjection(); + const thread = { + ...live.thread, + snoozedUntil: DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"), + }; + assert.equal(restartContinuationRun({ ...live, thread }, testNow)?.id, runId); + assert.lengthOf(yield* continuationTexts({ ...cutMidTurn(), thread }), 1); + }), +); + it.effect.each(["pending", "cancelled"] as const)( "does not continue a run whose secret request is %s at shutdown", (secretStatus) => @@ -1156,7 +1178,7 @@ it.effect.each(["pending", "cancelled"] as const)( }; if (secretStatus === "pending") assert.isUndefined( - restartContinuationRun({ ...makeProjection(), turnItems: projection.turnItems }), + restartContinuationRun({ ...makeProjection(), turnItems: projection.turnItems }, testNow), ); assert.deepEqual(yield* continuationTexts(projection), []); }), diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index d3a8d3fcc7af0..bce2dd9ab4edc 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -32,12 +32,17 @@ export function restartContinuationBlocked( "thread" | "runtimeRequests" | "providerTurns" | "turnItems" >, source: OrchestrationV2Run, + now: DateTime.Utc, ): boolean { + // An elapsed snooze no longer holds the thread. The early wakes the settlement + // service counts (a run that completed or failed after the snooze) cannot + // apply here: the source is the latest run and it was cut, not finished. if ( projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.thread.settledOverride === "settled" || - projection.thread.snoozedUntil != null + (projection.thread.snoozedUntil != null && + DateTime.isGreaterThan(projection.thread.snoozedUntil, now)) ) return true; const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId); @@ -90,6 +95,7 @@ export function restartContinuationRun( | "runtimeRequests" | "turnItems" >, + now: DateTime.Utc, ): OrchestrationV2Run | undefined { // Queued runs never started; recovery holds them behind the cut run. const run = projection.runs.reduce( @@ -106,7 +112,7 @@ export function restartContinuationRun( restartPromptSource(run, projection.runs, projection.providerTurns, projection.attempts) !== undefined); if (run.status !== "running" && !preparedContinuation) return; - if (restartContinuationBlocked(projection, run)) return; + if (restartContinuationBlocked(projection, run, now)) return; const liveTurnRequired = !preparedContinuation; if (projection.thread.providerInstanceId !== run.providerInstanceId) return; const providerThread = projection.providerThreads.find( @@ -190,7 +196,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta (isWorktreeContinuationRun(source) && source.status === "cancelled" && !projection.providerTurns.some((turn) => turn.runAttemptId === source.activeAttemptId)) || - restartContinuationBlocked(projection, source) || + restartContinuationBlocked(projection, source, yield* DateTime.now) || isRestartNoteSource(source, projection.providerTurns) ) return; From bd9e711ae89467faa98315fbec1dc5c675bda12f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:58:22 +0000 Subject: [PATCH 064/149] fix(claude): a roster update keeps a consumed rollback cursor cleared --- .../Adapters/ClaudeAdapterV2.test.ts | 15 +++++++++++++-- .../orchestration-v2/Adapters/ClaudeAdapterV2.ts | 8 ++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index a4be048c8af5b..e59b69b5d0e3f 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2605,11 +2605,22 @@ describe("ClaudeAdapterV2 background wake turns", () => { session_id: WAKE_NATIVE_SESSION, }), ); + // A later roster frame must not bring the consumed cursor back. + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [], + uuid: "post-rollback-roster", + session_id: WAKE_NATIVE_SESSION, + }), + ); yield* Queue.shutdown(harness.sdkMessages); yield* Queue.take(harness.terminalReceipts); const updates = harness.events.filter((event) => event.type === "provider_thread.updated"); - assert.lengthOf(updates, 2); - assert.isNull(updates[0]?.providerThread.nativeConversationHeadRef); + assert.lengthOf(updates, 3); + for (const update of updates) + assert.isNull(update.providerThread.nativeConversationHeadRef); const saved = updates.at(-1); assert.equal(saved?.type, "provider_thread.updated"); if (saved?.type !== "provider_thread.updated") return; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6cf7aacbc3d4e..2f93d53561b3a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -5933,9 +5933,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return false; } + // A rollback cursor this turn consumed must not come back with the roster. const baseThread = - input.activeContext?.input.providerThread ?? - (yield* Ref.get(lastProviderThreadByNativeThread)).get(input.nativeThreadId); + input.activeContext === null + ? (yield* Ref.get(lastProviderThreadByNativeThread)).get(input.nativeThreadId) + : input.activeContext.nativeConversationHeadConsumed + ? { ...input.activeContext.input.providerThread, nativeConversationHeadRef: null } + : input.activeContext.input.providerThread; if (baseThread === undefined) { return true; } From e6e6c7aeaf680a510704e7746ff993fa9db7c4bd Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:00:59 +0000 Subject: [PATCH 065/149] fix(opencode): a surviving-server history replay no longer rolls back live parts --- .../src/server/adapter.test.ts | 85 +++++++++++-------- .../provider-opencode/src/server/adapter.ts | 8 ++ 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index 305a39b709de2..70daeac9c2223 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -248,6 +248,13 @@ describe("OpenCodeAdapterV2", () => { (status) => Effect.gen(function* () { const nativeEvents = asyncEventStream(); + const livePart = { + type: "text", + id: "part-live", + sessionID: "root", + messageID: "assistant-gap", + text: "live", + }; let prompts = 0; const harness = yield* makeOpenCodeRuntimeHarness(`restart-${status}`, "root", { event: { subscribe: async () => ({ stream: nativeEvents.stream }) }, @@ -255,37 +262,45 @@ describe("OpenCodeAdapterV2", () => { create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), get: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), status: async () => ({ data: { root: { type: status } } }), - messages: async () => ({ - data: [ - { - info: { - id: "original-prompt", - role: "user", - sessionID: "root", - time: { created: 1 }, - }, - parts: [], - }, - { - info: { - id: "assistant-gap", - role: "assistant", - sessionID: "root", - parentID: "original-prompt", - time: { created: 2 }, + messages: async () => { + // A live update lands while history loads; the older snapshot must not undo it. + await nativeEvents.push({ + type: "message.part.updated", + properties: { sessionID: "root", part: { ...livePart, text: "live, then more" } }, + }); + return { + data: [ + { + info: { + id: "original-prompt", + role: "user", + sessionID: "root", + time: { created: 1 }, + }, + parts: [], }, - parts: [ - { - type: "text", - id: "part-gap", + { + info: { + id: "assistant-gap", + role: "assistant", sessionID: "root", - messageID: "assistant-gap", - text: "work during restart", + parentID: "original-prompt", + time: { created: 2 }, }, - ], - }, - ], - }), + parts: [ + { + type: "text", + id: "part-gap", + sessionID: "root", + messageID: "assistant-gap", + text: "work during restart", + }, + livePart, + ], + }, + ], + }; + }, promptAsync: async () => { prompts++; return { data: true }; @@ -314,14 +329,14 @@ describe("OpenCodeAdapterV2", () => { ); const collected = yield* Fiber.join(terminal); assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); - assert.include( - collected.flatMap((event) => - event.type === "turn_item.updated" && event.turnItem.type === "assistant_message" - ? [event.turnItem.text] - : [], - ), - "work during restart", + const texts = collected.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "assistant_message" + ? [event.turnItem.text] + : [], ); + assert.include(texts, "work during restart"); + assert.include(texts, "live, then more"); + assert.notInclude(texts, "live"); } }).pipe( Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index b2c543a527776..02736580997c8 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -3273,6 +3273,14 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function properties: { sessionID: sessionId, info: entry.info }, }); for (const part of entry.parts) { + // Live events since this turn became active already + // hold a newer copy; the snapshot would roll it back. + if ( + part.type === "tool" + ? turn.toolNamesByCallId.has(part.callID) + : turn.parts.has(part.id) + ) + continue; yield* handlePartUpdated({ id: part.id, type: "message.part.updated", From 6d9bb9bc7b9f2d7d362470c37d3958c01157a9c5 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:04:38 +0000 Subject: [PATCH 066/149] test(server): pin the clock for the elapsed snooze restart case --- apps/server/src/orchestration-v2/RestartContinuation.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 1abf4fe869cf6..a01f78b8c238a 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -19,6 +19,7 @@ import { import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerSettings from "../serverSettings.ts"; import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; @@ -1162,6 +1163,7 @@ it.effect("continues a thread whose snooze already elapsed", () => snoozedUntil: DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"), }; assert.equal(restartContinuationRun({ ...live, thread }, testNow)?.id, runId); + yield* TestClock.setTime(DateTime.toEpochMillis(testNow)); assert.lengthOf(yield* continuationTexts({ ...cutMidTurn(), thread }), 1); }), ); From 792dded6a9de545d099f77af0c1b3043f3ef4300 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:42:41 +0000 Subject: [PATCH 067/149] fix(opencode): following a surviving run never re-prompts a finished turn or drops its usage A run that finishes while the status probe is in flight no longer gets a fresh continuation prompt, and assistant messages that arrive before the history replay stay unclassified until the replay names the followed prompt, so their token usage still counts. --- packages/provider-opencode/src/server/adapter.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 02736580997c8..7ab9c55b79344 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -234,6 +234,9 @@ interface OpenCodeTurnTokenUsageAccumulator { reasoningTokens: number; hasSubagents: boolean; complete: boolean; + // While a restart follows a surviving run, its prompt is not known yet, so + // no assistant message can be ruled out. + followingRestart: boolean; } function makeOpenCodeTurnTokenUsageAccumulator(): OpenCodeTurnTokenUsageAccumulator { @@ -242,6 +245,7 @@ function makeOpenCodeTurnTokenUsageAccumulator(): OpenCodeTurnTokenUsageAccumula promptMessageIds: new Set(), assistantOwnershipByMessageId: new Map(), unresolvedStepsByMessageId: new Map(), + followingRestart: false, inputTokens: 0, cachedInputTokens: 0, cacheCreationTokens: 0, @@ -2423,7 +2427,9 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function ? "unknown" : usage.promptMessageIds.has(message.parentID) ? "owned" - : "other"; + : usage.followingRestart + ? "unknown" + : "other"; usage.assistantOwnershipByMessageId.set(message.id, ownership); if (ownership !== "unknown") { if (ownership === "owned") { @@ -3232,6 +3238,7 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function }); yield* updateProviderSession("running", null); if (turnInput.restartContinuationOfRunId !== undefined) { + turn.usage.followingRestart = true; const statuses = unwrapData( "session.status", yield* sdkCall("session.status", { sessionID: sessionId }, () => @@ -3244,7 +3251,11 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function ), ), ); - if (statuses[sessionId] !== undefined && statuses[sessionId]?.type !== "idle") { + // Live events may have finished the surviving run meanwhile. + if (turn.finalized || state.activeTurn !== turn) return; + if (statuses[sessionId] === undefined || statuses[sessionId]?.type === "idle") { + turn.usage.followingRestart = false; + } else { // A server that outlived T3 still owns this turn; follow it // instead of adding another prompt to its running execution. turn.admissionMessageId = null; @@ -3292,6 +3303,7 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function }); } } + turn.usage.followingRestart = false; if (admissionAction === "reconcile-idle") { yield* reconcilePromptAdmission(state, turn); } From 06a61c66358d22e2d504039ca8d4d57900cb031e Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:28:36 +0000 Subject: [PATCH 068/149] fix(opencode): a remote turn that finished while T3 was down is not prompted again After a restart, an idle OpenCode session whose newest prompt T3 never settled and whose last reply finished cleanly already completed the cut run. T3 now replays that reply and completes the run instead of sending another continue prompt. --- .../provider-opencode/src/server/adapter.ts | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 7ab9c55b79344..1b3316d9f4793 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -3251,9 +3251,34 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function ), ), ); + const readHistory = sdkCall("session.messages", { sessionID: sessionId }, () => + client.session.messages({ sessionID: sessionId }), + ).pipe( + Effect.map((response) => unwrapData("session.messages", response)), + Effect.tapError((cause) => + finalizeTurn(state, turn, "failed", { + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ); + const idle = + statuses[sessionId] === undefined || statuses[sessionId]?.type === "idle"; + const idleHistory = idle ? yield* readHistory : undefined; // Live events may have finished the surviving run meanwhile. if (turn.finalized || state.activeTurn !== turn) return; - if (statuses[sessionId] === undefined || statuses[sessionId]?.type === "idle") { + // An idle session whose newest prompt T3 never settled, answered by + // a clean final reply, finished the cut run while T3 was down. + const lastPrompt = idleHistory?.findLast((entry) => entry.info.role === "user"); + const lastInfo = idleHistory?.at(-1)?.info; + const finishedWhileDown = + lastPrompt !== undefined && + lastPrompt.info.id !== + turnInput.providerThread.nativeConversationHeadRef?.nativeId && + lastInfo?.role === "assistant" && + lastInfo.time.completed !== undefined && + lastInfo.error === undefined && + lastInfo.finish !== "tool-calls"; + if (idle && !finishedWhileDown) { turn.usage.followingRestart = false; } else { // A server that outlived T3 still owns this turn; follow it @@ -3263,18 +3288,7 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function const admissionAction = advanceOpenCodePromptAdmission(turn, "accepted"); yield* Deferred.succeed(admissionSettled, undefined); turn.admissionAbortController = null; - const history = unwrapData( - "session.messages", - yield* sdkCall("session.messages", { sessionID: sessionId }, () => - client.session.messages({ sessionID: sessionId }), - ).pipe( - Effect.tapError((cause) => - finalizeTurn(state, turn, "failed", { - failure: makeProviderFailure({ cause, class: "provider_error" }), - }), - ), - ), - ); + const history = idleHistory ?? (yield* readHistory); const userIndex = history.findLastIndex((entry) => entry.info.role === "user"); for (const entry of userIndex < 0 ? [] : history.slice(userIndex)) { if (turn.finalized) break; @@ -3304,7 +3318,9 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function } } turn.usage.followingRestart = false; - if (admissionAction === "reconcile-idle") { + if (idle) { + if (!turn.finalized) yield* finalizeTurn(state, turn, "completed"); + } else if (admissionAction === "reconcile-idle") { yield* reconcilePromptAdmission(state, turn); } return; From 8f13ec7106d8b0f06b5760ba2a97a331fc44b59c Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:45:11 +0000 Subject: [PATCH 069/149] fix(server): background provider installs only update installs they own Background installs now require proven ownership of the executable, run with the instance's own environment, pin the checked version instead of @latest, install a shared installation once per pass and refresh its sibling instances. Mise data directories stay manual-only and OpenCode refreshes its advisory after an update. --- .../providerMaintenanceRunner.test.ts | 409 ++++++++++++++ .../src/provider/providerMaintenanceRunner.ts | 157 ++++-- .../server/src/updates/ProviderAutoUpdater.ts | 26 +- .../provider-core/src/server/maintenance.ts | 2 + .../src/server/maintenanceResolver.test.ts | 509 ++++++++++++++++-- .../src/server/maintenanceResolver.ts | 130 +++-- .../src/server/managedProvider.ts | 36 +- .../provider-grok/src/server/driver.test.ts | 38 ++ packages/provider-grok/src/server/driver.ts | 47 +- .../src/server/maintenance.test.ts | 1 + .../provider-muse/src/server/maintenance.ts | 1 + .../src/server/driver.test.ts | 62 ++- .../provider-opencode/src/server/driver.ts | 2 +- 13 files changed, 1274 insertions(+), 146 deletions(-) diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index fb0d6cb21a4c5..833d814af7e4d 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -1,5 +1,6 @@ import { describe, it, assert } from "@effect/vitest"; import { + DEFAULT_SERVER_SETTINGS, ProviderDriverKind, ProviderInstanceId, type ServerProvider, @@ -24,6 +25,10 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; import * as ProviderRegistry from "./ProviderRegistry.ts"; +import * as ProviderAutoUpdater from "../updates/ProviderAutoUpdater.ts"; +import * as UpdateWindow from "../updates/UpdateWindow.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as ServerConfig from "../config.ts"; import * as ModelManifest from "./ModelManifest.ts"; import * as ProviderMaintenanceRunner from "./providerMaintenanceRunner.ts"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; @@ -1084,3 +1089,407 @@ it.effect("refuses incompatible latest versions and unapproved or unpinnable tar ), ); }); + +it.effect.each([ + { exitCode: 0, targetVersion: "2.0.0" }, + { exitCode: 1, targetVersion: "2.0.0" }, + { exitCode: 0, targetVersion: "2.0.0-beta.1" }, +])( + "background installs pin $targetVersion and refresh shared instances after exit $exitCode", + ({ exitCode, targetVersion }) => { + const calls: string[] = []; + const refreshed: ProviderInstanceId[] = []; + let installedVersion = "1.0.0"; + const sibling = ProviderInstanceId.make("codex-work"); + const unrelated = ProviderInstanceId.make("codex-other"); + return Effect.gen(function* () { + const { registry, providersRef } = yield* makeRegistry([ + baseProvider, + { ...baseProvider, instanceId: sibling }, + { ...baseProvider, instanceId: unrelated }, + ]); + const capabilities = makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "--prefix", "/owned", "@openai/codex@latest"], + updateLockKey: "npm-global:/owned", + installationKey: "codex:/owned", + env: { PATH: "/instance/node/bin", npm_config_registry: "https://private.invalid/" }, + }); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (id) => + Effect.succeed( + id === unrelated + ? { + ...capabilities, + update: { ...capabilities.update!, installationKey: "codex:/other" }, + } + : capabilities, + ), + refreshInstance: (id) => + Effect.sync(() => refreshed.push(id)).pipe( + Effect.andThen( + Ref.updateAndGet(providersRef, (providers) => + providers.map((entry) => + entry.instanceId === id ? { ...entry, version: installedVersion } : entry, + ), + ), + ), + ), + }); + const result = yield* runner.updateProvider({ + provider: CODEX_DRIVER, + ...(exitCode === 0 ? { targetVersion } : {}), + quiet: true, + background: true, + }); + assert.deepStrictEqual(calls, [`install -g --prefix /owned @openai/codex@${targetVersion}`]); + assert.deepStrictEqual(refreshed, [CODEX_INSTANCE_ID, CODEX_INSTANCE_ID, sibling]); + assert.strictEqual( + result.providers[0]?.updateState?.status, + exitCode === 0 ? "succeeded" : "failed", + ); + assert.strictEqual(result.providers[1]?.version, targetVersion); + assert.strictEqual(result.providers[2]?.version, "0.0.0"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient(exitCode === 0 ? "9.0.0" : "2.0.0"), + layerMockSpawner((_command, args, options) => { + calls.push(args.join(" ")); + assert.strictEqual(options.env?.npm_config_registry, "https://private.invalid/"); + // The configured registry's latest is older than the checked public release. + installedVersion = args.includes(`@openai/codex@${targetVersion}`) + ? targetVersion + : "0.9.0"; + return { stdout: "updated", code: exitCode }; + }), + ), + ), + ); + }, +); + +it.effect.each(["2.0.0", "3.0.0", "v3.0.0", "unknown"])( + "background installs never downgrade a freshly probed %s", + (version) => + Effect.gen(function* () { + const { registry, providersRef } = yield* makeRegistry({ ...baseProvider, version: "1.0.0" }); + const capabilities = makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "@openai/codex@latest"], + updateLockKey: "npm-global:/owned", + installationKey: "codex:/owned", + }); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: () => Effect.succeed(capabilities), + refreshInstance: () => + Ref.updateAndGet(providersRef, (providers) => + providers.map((entry) => ({ ...entry, version })), + ), + }); + const result = yield* runner.updateProvider({ + provider: CODEX_DRIVER, + background: true, + }); + assert.strictEqual(result.providers[0]?.updateState?.status, "unchanged"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("2.0.0"), + layerMockSpawner(() => { + throw new Error("must not install or downgrade"); + }), + ), + ), + ), +); + +it.effect.each([false, true])( + "background installs reject unproven or replaced ownership (%s)", + (changed) => { + const calls: string[] = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry({ ...baseProvider, version: "1.0.0" }); + const capabilities = makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "/opt/pinned/codex", + updateArgs: ["update"], + updateLockKey: "codex-native", + ...(changed ? { installationKey: "old" } : {}), + }); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (_id, _provider, options) => + Effect.succeed( + changed && options?.fresh + ? { + ...capabilities, + update: { ...capabilities.update!, installationKey: "replacement" }, + } + : capabilities, + ), + }); + const result = yield* runner.updateProvider({ + provider: CODEX_DRIVER, + background: true, + }); + assert.strictEqual(result.providers[0]?.updateState?.status, "failed"); + assert.deepStrictEqual(calls, []); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("2.0.0"), + layerMockSpawner((command) => { + calls.push(command); + return {}; + }), + ), + ), + ); + }, +); + +it.effect("Windows updater lookup uses the same instance PATH as the spawned process", () => { + const captured: string[] = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry(); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: () => + Effect.succeed( + makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "@openai/codex@latest"], + updateLockKey: "npm-global:instance", + env: { PATH: "C:\\instance\\bin", PATHEXT: ".CMD" }, + }), + ), + }); + yield* runner.updateProvider(CODEX_DRIVER); + assert.strictEqual(captured.length, 1); + assert.include(captured[0]!, "instance"); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "win32"), + Layer.succeed(HostProcess.Environment, { PATH: "C:\\host\\bin" }), + Layer.succeed( + SpawnExecutableResolution, + (_command, _platform, env) => `${env.PATH}\\npm.cmd`, + ), + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner((command, _args, options) => { + captured.push(command); + assert.strictEqual(options.env?.PATH, "C:\\instance\\bin"); + return {}; + }), + ), + ), + ); +}); + +it.effect( + "background passes share retries per installation, pin targets and skip custom binaries", + () => + Effect.gen(function* () { + const instances = ["a", "b", "separate", "pinned"].map((id) => ProviderInstanceId.make(id)); + const { registry, providersRef } = yield* makeRegistry( + instances.map((id) => ({ + ...baseProvider, + instanceId: id, + versionAdvisory: { + status: "behind_latest", + currentVersion: "1.0.0", + latestVersion: id === instances[1] ? "3.0.0" : "2.0.0", + updateCommand: "npm install", + canUpdate: true, + canInstallVersion: true, + checkedAt: null, + message: null, + }, + })), + ); + const calls: Array<{ + instanceId: ProviderInstanceId | undefined; + targetVersion: string | undefined; + }> = []; + const finished = yield* Deferred.make(); + const secondPass = yield* Deferred.make(); + const thirdPass = yield* Deferred.make(); + const retried = yield* Deferred.make(); + let windowCalls = 0; + const capabilitiesFor = (id: ProviderInstanceId) => + makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "@openai/codex@latest"], + updateLockKey: "npm-global:/owned", + ...(id === instances[3] + ? {} + : { installationKey: id === instances[2] ? "separate" : "shared" }), + }); + yield* Layer.build( + ProviderAutoUpdater.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ServerConfig.ServerConfig, { + devUrl: undefined, + } as ServerConfig.ServerConfig["Service"]), + Layer.succeed(ProviderRegistry.ProviderRegistry, { + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (id) => + Effect.succeed(capabilitiesFor(id)), + }), + Layer.succeed(ProviderMaintenanceRunner.ProviderMaintenanceRunner, { + updateProvider: (target) => + Effect.gen(function* () { + assert.notStrictEqual(typeof target, "string"); + if (typeof target === "string") throw new Error("instance target expected"); + calls.push({ + instanceId: target.instanceId, + targetVersion: target.targetVersion, + }); + assert.strictEqual(target.background, true); + assert.strictEqual(target.quiet, true); + return yield* new ServerProviderUpdateError({ + provider: CODEX_DRIVER, + reason: "isolated installer failure", + }); + }), + }), + Layer.succeed(UpdateWindow.UpdateWindow, { + runIfOpen: (install) => + install.pipe( + Effect.tap(() => { + windowCalls += 1; + return windowCalls === 4 + ? Deferred.succeed(finished, undefined) + : windowCalls === 8 + ? Deferred.succeed(secondPass, undefined) + : windowCalls === 12 + ? Deferred.succeed(thirdPass, undefined) + : windowCalls === 156 + ? Deferred.succeed(retried, undefined) + : Effect.void; + }), + Effect.asSome, + ), + }), + Layer.succeed(ServerSettings.ServerSettingsService, { + start: Effect.void, + ready: Effect.void, + getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS), + updateSettings: () => Effect.die("unused"), + updateProviderInstance: () => Effect.die("unused"), + withSettingsSnapshot: (use) => use(DEFAULT_SERVER_SETTINGS), + streamChanges: Stream.empty, + subscribeChanges: Effect.succeed(Stream.empty), + }), + ), + ), + ), + ); + yield* TestClock.adjust("2 minutes"); + yield* Deferred.await(finished); + assert.deepStrictEqual(calls, [ + { instanceId: instances[0], targetVersion: "2.0.0" }, + { instanceId: instances[2], targetVersion: "2.0.0" }, + ]); + yield* Ref.update(providersRef, (providers) => + providers.map((entry) => ({ + ...entry, + versionAdvisory: { ...entry.versionAdvisory!, latestVersion: "2.0.0" }, + })), + ); + yield* TestClock.adjust("10 minutes"); + yield* Deferred.await(secondPass); + assert.strictEqual(calls.length, 2); + yield* Ref.update(providersRef, (providers) => + providers.map((entry) => ({ + ...entry, + versionAdvisory: { ...entry.versionAdvisory!, latestVersion: "3.0.0" }, + })), + ); + yield* TestClock.adjust("10 minutes"); + yield* Deferred.await(thirdPass); + assert.deepStrictEqual(calls.slice(2), [ + { instanceId: instances[0], targetVersion: "3.0.0" }, + { instanceId: instances[2], targetVersion: "3.0.0" }, + ]); + yield* TestClock.adjust("6 hours"); + // At 6h after the first pass, the newer release has had only 5h40m; at 6h after its attempt it retries. + yield* Deferred.await(retried); + assert.strictEqual(calls.length, 6); + }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), +); + +it.effect( + "background native updates recheck the owning channel instead of a stale advisory target", + () => { + const calls: string[] = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry({ ...baseProvider, version: "1.0.0" }); + const capabilities = makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "brew", + updateArgs: ["upgrade", "codex"], + updateLockKey: "homebrew", + installationKey: "codex:/brew", + latestVersion: "3.0.0", + }); + const runner = yield* makeTestRunner( + { + ...registry, + getProviderMaintenanceCapabilitiesForInstance: () => Effect.succeed(capabilities), + }, + { + version: 1, + currentModels: {}, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + recommendedVersion: "2.0.0", + ranges: [ + { range: "=2.0.0", status: "supported" }, + { range: "=3.0.0", status: "broken" }, + ], + }, + ], + }, + ); + const result = yield* runner.updateProvider({ + provider: CODEX_DRIVER, + targetVersion: "2.0.0", + background: true, + }); + assert.strictEqual(result.providers[0]?.updateState?.status, "failed"); + assert.deepStrictEqual(calls, []); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("2.0.0"), + layerMockSpawner((command) => { + calls.push(command); + return {}; + }), + ), + ), + ); + }, +); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 6dca9f2603095..3f55abf719119 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -8,6 +8,7 @@ import { type ServerProviderUpdateState, } from "@t3tools/contracts"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; +import { compareSemverVersions, parseSemver } from "@t3tools/shared/semver"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as Data from "effect/Data"; @@ -68,6 +69,8 @@ export interface ProviderMaintenanceRunnerShape { readonly targetVersion?: string | undefined; /** Skips installer output progress, for background installs nobody watches. */ readonly quiet?: boolean | undefined; + /** Requires proven ownership and pins the checked release when supported. */ + readonly background?: boolean | undefined; }, ) => Effect.Effect; } @@ -104,7 +107,10 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR // which a bare ChildProcess.spawn cannot launch (spawn npm ENOENT); // resolveSpawnCommand finds the real `.cmd` and routes it through the // shell. On Linux/macOS (incl. the WSL backend) this is a no-op. - const resolved = yield* resolveSpawnCommand(input.command, input.args); + const resolved = yield* resolveSpawnCommand(input.command, input.args, { + ...(input.env ? { env: input.env } : {}), + extendEnv: true, + }); const child = yield* input.spawner .spawn( ChildProcess.make(resolved.command, resolved.args, { @@ -318,33 +324,60 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { }), }); + const resolveInstallationInstances = ( + provider: ProviderDriverKind, + instanceId: ProviderInstanceId, + installationKey: string | undefined, + ) => + providerRegistry.getProviders.pipe( + Effect.flatMap((providers) => + Effect.forEach(providers, (candidate) => + candidate.instanceId === instanceId + ? Effect.succeed(candidate.instanceId) + : candidate.driver !== provider || !installationKey + ? Effect.succeed(null) + : providerRegistry + .getProviderMaintenanceCapabilitiesForInstance( + candidate.instanceId, + candidate.driver, + ) + .pipe( + Effect.map((capabilities) => + capabilities.update?.installationKey === installationKey + ? candidate.instanceId + : null, + ), + ), + ), + ), + Effect.map((ids) => ids.filter((id) => id !== null)), + ); + + const refreshInstallationInstances = ( + provider: ProviderDriverKind, + instanceIds: ReadonlyArray, + alreadyFresh?: ProviderInstanceId, + ) => + Effect.forEach( + instanceIds, + (id) => + (id === alreadyFresh + ? Effect.void + : providerRegistry.getProviderMaintenanceCapabilitiesForInstance(id, provider, { + fresh: true, + }) + ).pipe(Effect.andThen(providerRegistry.refreshInstance(id))), + { discard: true }, + ); + const verifyRefreshedProvider = ( provider: ProviderDriverKind, maintenanceCapabilities: ProviderMaintenanceCapabilities, instanceId: ProviderInstanceId, + instanceIds: ReadonlyArray, ): Effect.Effect => - providerRegistry.getProviders.pipe( - Effect.map((providers) => { - const instanceIds: Array = []; - for (const candidate of providers) { - if (candidate.driver === provider && candidate.instanceId === instanceId) { - instanceIds.push(candidate.instanceId); - } - } - return instanceIds; - }), - Effect.flatMap((instanceIds) => - instanceIds.length === 0 - ? providerRegistry.refreshInstance(instanceId) - : Effect.forEach( - instanceIds, - (instanceId) => providerRegistry.refreshInstance(instanceId), - { - concurrency: "unbounded", - discard: true, - }, - ).pipe(Effect.andThen(providerRegistry.getProviders)), - ), + refreshInstallationInstances(provider, instanceIds, instanceId).pipe( + Effect.andThen(providerRegistry.getProviders), Effect.flatMap((providers) => { const refreshedProviders = providers.filter( (candidate) => candidate.driver === provider && candidate.instanceId === instanceId, @@ -398,6 +431,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { : (target.instanceId ?? defaultInstanceIdForDriver(provider)); const targetVersion = typeof target === "string" ? undefined : target.targetVersion; const quiet = typeof target !== "string" && target.quiet === true; + const background = typeof target !== "string" && target.background === true; const targetKey = `instance:${instanceId}`; const capabilities = yield* providerRegistry.getProviderMaintenanceCapabilitiesForInstance( instanceId, @@ -455,7 +489,13 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { provider, { fresh: true }, ); - if (!fresh.update || fresh.update.lockKey !== update.lockKey) { + if ( + !fresh.update || + fresh.update.lockKey !== update.lockKey || + (background && + (!fresh.update.installationKey || + fresh.update.installationKey !== update.installationKey)) + ) { return yield* finish( makeUpdateState({ status: "failed", @@ -467,8 +507,14 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { } const manifest = yield* manifestService.current; + const checkedTarget = targetVersion + ? makeTargetedProviderUpdateAction(fresh, targetVersion) + : null; + if (background && !checkedTarget && fresh.packageName) { + yield* latestVersions.invalidate(fresh.packageName); + } const candidateVersion = - targetVersion ?? + (!background || checkedTarget !== null ? targetVersion : undefined) ?? (yield* resolveLatestProviderVersion(fresh).pipe( Effect.provideService(HttpClient.HttpClient, httpClient), Effect.provideService( @@ -483,12 +529,48 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { provider, candidateVersion, ); - const command = - targetVersion !== undefined + if (background) { + // Re-probe under the install lock: a manual/external update may have won the race. + const current = (yield* providerRegistry.refreshInstance(instanceId)).find( + (candidate) => candidate.instanceId === instanceId, + ); + if ( + !current?.enabled || + !current.installed || + !current.version || + !candidateVersion || + !parseSemver(current.version) || + !parseSemver(candidateVersion) || + (fresh.compareVersions ?? compareSemverVersions)( + current.version, + candidateVersion, + ) >= 0 + ) { + return yield* finish( + makeUpdateState({ + status: "unchanged", + startedAt, + finishedAt: yield* nowIso, + message: "No newer provider version is available.", + }), + ); + } + } + const pinnedCommand = candidateVersion + ? makeTargetedProviderUpdateAction(fresh, candidateVersion) + : null; + const command = background + ? (pinnedCommand ?? + (fresh.packageName === null || + fresh.latestVersion !== undefined || + fresh.update.lockKey.endsWith("-native") + ? fresh.update + : null)) + : targetVersion !== undefined ? makeTargetedProviderUpdateAction(fresh, targetVersion) : fresh.update; const rejected = - targetVersion !== undefined + !background && targetVersion !== undefined ? !command || advisory?.recommendedVersion !== targetVersion || advisory.status !== "supported" @@ -506,6 +588,11 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { }), ); } + const affectedInstances = yield* resolveInstallationInstances( + provider, + instanceId, + fresh.update.installationKey, + ); yield* setRunningMessage(`Running ${describeCommand(command)}`); const result = yield* runMaintenanceCommand( command, @@ -513,6 +600,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { ); const finishedAt = yield* nowIso; if (result.timedOut || result.exitCode !== 0) { + yield* refreshInstallationInstances(provider, affectedInstances); return yield* finish( makeUpdateState({ status: "failed", @@ -535,6 +623,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { provider, verified, instanceId, + affectedInstances, ); // "Succeeded" needs the provider to still be installed: an // installer that exits 0 and leaves the binary missing is not a @@ -545,8 +634,16 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { verifiedProviders.some( (verifiedProvider) => !isStillInstalled(verifiedProvider) || - (targetVersion !== undefined && - verifiedProvider.version?.replace(/^v/, "") !== targetVersion), + (background && + (!verifiedProvider.version || + !candidateVersion || + (verified.compareVersions ?? compareSemverVersions)( + verifiedProvider.version, + candidateVersion, + ) < 0)) || + (((!background && targetVersion !== undefined) || + (background && pinnedCommand !== null)) && + verifiedProvider.version?.replace(/^v/, "") !== candidateVersion), ); const stillOutdated = targetVersion === undefined && diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index 7d04e6727611e..952d4e33e687e 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -47,12 +47,13 @@ export const layer = Layer.effectDiscard( const runner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; - // Keyed by instance and target version, so a newer release retries at once. + // Instances sharing an install share its retry budget; a new release retries at once. const attempts = new Map(); const pass = Effect.gen(function* () { const settings = yield* serverSettings.getSettings; if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; + const installations = new Set(); for (const { instanceId } of yield* providers.getProviders) { const install = Effect.gen(function* () { const settings = yield* serverSettings.getSettings; @@ -61,20 +62,37 @@ export const layer = Layer.effectDiscard( (candidate) => candidate.instanceId === instanceId, ); if (!current || !isAutoUpdatable(current)) return; + const capabilities = yield* providers.getProviderMaintenanceCapabilitiesForInstance( + instanceId, + current.driver, + ); + const installationKey = capabilities.update?.installationKey; + if (!installationKey || installations.has(installationKey)) return; const targetVersion = current.versionAdvisory?.latestVersion; - const attemptKey = `${instanceId}@${targetVersion}`; - const nowMs = yield* Clock.currentTimeMillis; + if (!targetVersion) return; + const attemptKey = `${installationKey}@${targetVersion}`; + const nowMs = Number(yield* Clock.monotonicTimeNanos) / 1_000_000; + for (const [key, attemptedAt] of attempts) { + if (nowMs - attemptedAt >= Duration.toMillis(RETRY_AFTER)) attempts.delete(key); + } const attemptedAt = attempts.get(attemptKey); if (attemptedAt !== undefined && nowMs - attemptedAt < Duration.toMillis(RETRY_AFTER)) return; attempts.set(attemptKey, nowMs); + installations.add(installationKey); yield* Effect.logInfo("Updating provider in the background", { instanceId, fromVersion: current.version, targetVersion, }); yield* runner - .updateProvider({ provider: current.driver, instanceId, quiet: true }) + .updateProvider({ + provider: current.driver, + instanceId, + targetVersion, + quiet: true, + background: true, + }) .pipe( Effect.catchCause((cause) => Effect.logWarning("Background provider update failed", { instanceId, cause }), diff --git a/packages/provider-core/src/server/maintenance.ts b/packages/provider-core/src/server/maintenance.ts index 62f9a2ab63a61..06846b18bc40f 100644 --- a/packages/provider-core/src/server/maintenance.ts +++ b/packages/provider-core/src/server/maintenance.ts @@ -20,6 +20,8 @@ export interface ProviderMaintenanceCommandAction { readonly executable: string; readonly args: ReadonlyArray; readonly lockKey: string; + /** Stable identity of an install whose updater ownership is proven. Absent for manual fallbacks. */ + readonly installationKey?: string | undefined; /** * Extra environment for the spawned updater, on top of the server's own. * A native updater finds its install through the same variables the diff --git a/packages/provider-core/src/server/maintenanceResolver.test.ts b/packages/provider-core/src/server/maintenanceResolver.test.ts index b0b1be74ee887..3db55d621ad5b 100644 --- a/packages/provider-core/src/server/maintenanceResolver.test.ts +++ b/packages/provider-core/src/server/maintenanceResolver.test.ts @@ -237,11 +237,15 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { ), ); - it.effect.skipIf(!symlinksSupported)( - "pins npm updates to the global prefix that owns the package", - () => + it.effect.each(["", ".nvm/versions/node/24", "Cellar/node/24"])( + "pins npm updates to the owning global prefix even under %s", + (prefix) => Effect.gen(function* () { - const tempDir = yield* makeTempDir("t3-npm-capabilities"); + const root = yield* makeTempDir("t3-npm-capabilities"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const tempDir = NodePath.join(root, prefix); const link = linkIntoPackage(tempDir, "package-tool", [ "lib", "node_modules", @@ -257,7 +261,8 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { }, ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)); - expect(capabilities).toEqual({ + expect(capabilities.update?.installationKey).toBeDefined(); + expect(capabilities).toMatchObject({ provider: driver("packageTool"), packageName: "@example/package-tool", update: { @@ -274,7 +279,8 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { lockKey: `npm-global:${normalizeCommandPath(tempDir)}`, }, }); - }), + }).pipe(Effect.scoped), + { skip: windowsHost || !symlinksSupported }, ); it("derives the npm prefix only from the global lib/node_modules layout", () => { @@ -458,7 +464,7 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), ); - expect(capabilities).toEqual({ + expect(capabilities).toMatchObject({ provider: driver("nativePackageTool"), packageName: "@example/native-package-tool", update: { @@ -589,7 +595,7 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { env: { PATH: "" }, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)); - expect(capabilities.update?.env).toEqual({ TOOL_HOME: tempDir }); + expect(capabilities.update?.env).toEqual({ PATH: "", TOOL_HOME: tempDir }); }), ); @@ -679,7 +685,7 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { [brewPath, "--prefix"], [brewPath, "info", "--json=v2", fixture.name], ]); - expect(capabilities).toEqual({ + expect(capabilities).toMatchObject({ provider: driver("packageTool"), packageName: "@example/package-tool", latestVersion: "0.148.0", @@ -879,6 +885,404 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { }), ); + it.effect.each(["npm", "pnpm", "bun", "yarn", "volta", "vp", "brew", "native"] as const)( + "executes the owning %s updater with the instance environment", + (method) => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-execution"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const tools = NodePath.join(root, "tools"); + const env = { + PATH: tools, + BUN_INSTALL: NodePath.join(root, "custom-bun"), + npm_config_prefix: NodePath.join(root, "other-prefix"), + npm_config_registry: "https://private.invalid/", + CODEX_HOME: NodePath.join(root, "instance-home"), + }; + const executable = NodePath.join(tools, method === "native" ? "package-tool" : method); + writeExecutable(executable); + NodeFS.writeFileSync( + executable, + `#!/bin/sh\nprintf '%s\n' "$BUN_INSTALL" "$npm_config_prefix" "$CODEX_HOME" "$npm_config_registry" "$@"\n`, + ); + const nativeRoot = NodePath.join(root, ".local", "bin"); + const resolved = + method === "bun" + ? NodePath.join(env.BUN_INSTALL, "bin", "package-tool") + : method === "vp" + ? NodePath.join(root, ".vite-plus", "bin", "package-tool") + : method === "volta" + ? NodePath.join(root, ".volta", "bin", "package-tool") + : method === "native" + ? NodePath.join(nativeRoot, "package-tool") + : NodePath.join(root, "bin", "package-tool"); + const real = + method === "npm" + ? NodePath.join( + root, + "lib", + "node_modules", + "@example", + "package-tool", + "bin", + "tool.js", + ) + : method === "pnpm" + ? NodePath.join( + root, + ".local", + "share", + "pnpm", + "global", + "node_modules", + "@example", + "package-tool", + "bin", + "tool.js", + ) + : method === "yarn" + ? NodePath.join( + root, + ".config", + "yarn", + "global", + "node_modules", + "@example", + "package-tool", + "bin", + "tool.js", + ) + : method === "volta" + ? NodePath.join(root, ".volta", "bin", "volta-shim") + : method === "brew" + ? NodePath.join(root, "Cellar", "package-tool", "1.0.0", "bin", "tool") + : resolved; + writeExecutable(real); + if (method === "native") NodeFS.copyFileSync(executable, resolved); + if (method === "volta") + NodeFS.mkdirSync( + NodePath.join(root, ".volta", "tools", "image", "packages", "@example", "package-tool"), + { recursive: true }, + ); + const resolver = { + provider: driver("packageTool"), + npmPackageName: "@example/package-tool", + nativeUpdate: { + args: ["update"], + isCommandPath: isNativeTestCommandPath("/.local/bin/package-tool"), + env: { CODEX_HOME: "shared-home" }, + }, + }; + const capabilities = yield* resolvePackageManagedProviderMaintenance(resolver, { + binaryPath: resolved, + resolvedCommandPath: resolved, + realCommandPath: real, + env, + platform: "linux", + }).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + stdoutSpawner((_command, args) => + args[0] === "--prefix" + ? root + : JSON.stringify({ formulae: [{ versions: { stable: "2.0.0" } }] }), + ), + ), + ); + const action = capabilities.update!; + expect(action).not.toBeNull(); + const expectedEnv = method === "native" ? { ...env, CODEX_HOME: "shared-home" } : env; + expect(action.env).toEqual(expectedEnv); + expect(action.installationKey).toBeDefined(); + const result = NodeChildProcess.spawnSync(action.executable, action.args, { + env: { ...process.env, ...action.env }, + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + env.BUN_INSTALL, + env.npm_config_prefix, + expectedEnv.CODEX_HOME, + env.npm_config_registry, + ...action.args, + ]); + if (method === "npm") expect(action.args).toContain(root); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.each([ + "installs/npm-codex/1.0.0/lib/node_modules/@example/package-tool/bin/tool.js", + "installs/package-tool/1.0.0/bin/tool", + "shims/tool", + ])( + "keeps custom MISE_DATA_DIR %s manual-only before choosing an updater", + (suffix) => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-mise"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const binary = NodePath.join(root, "custom-data", suffix); + writeExecutable(binary); + const capabilities = yield* resolveProviderMaintenanceCapabilitiesEffect( + makePackageManagedProviderMaintenanceResolver({ + provider: driver("packageTool"), + npmPackageName: "@example/package-tool", + nativeUpdate: { args: ["update"] }, + }), + { + binaryPath: binary, + env: { PATH: "", MISE_DATA_DIR: NodePath.join(root, "custom-data") }, + }, + ); + expect(capabilities.update).toBeNull(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.skipIf(windowsHost)( + "allows npm globals in custom mise Node installs but keeps wrappers manual", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-mise-node"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const binary = NodePath.join( + root, + "data", + "installs", + "node", + "24", + "lib", + "node_modules", + "@example", + "package-tool", + "bin", + "tool.js", + ); + writeExecutable(binary); + const env = { PATH: "", MISE_DATA_DIR: NodePath.join(root, "data") }; + const capabilities = yield* resolveProviderMaintenanceCapabilitiesEffect( + packageToolUpdate, + { + binaryPath: binary, + env, + }, + ); + expect(capabilities.update?.executable).toBe("npm"); + NodeFS.writeFileSync(binary, "#!/bin/sh\nexec mise x package-tool -- package-tool\n"); + expect( + (yield* resolveProviderMaintenanceCapabilitiesEffect(packageToolUpdate, { + binaryPath: binary, + env, + })).update, + ).toBeNull(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.skipIf(windowsHost)( + "distinguishes automatic native installs from custom pinned and SDK binaries", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-ownership"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + for (const suffix of [ + "opt/pinned/tool", + "node_modules/@example/sdk/bin/tool", + ".local/bin/native-package-tool", + ]) { + const binary = NodePath.join(root, suffix); + writeExecutable(binary); + const capabilities = yield* resolveProviderMaintenanceCapabilitiesEffect( + nativePackageToolUpdate, + { binaryPath: binary, env: { PATH: "" } }, + ); + if (suffix.startsWith(".local")) + expect(capabilities.update?.installationKey).toBeDefined(); + else expect(capabilities.update?.installationKey).toBeUndefined(); + if (suffix.startsWith("opt")) expect(capabilities.update?.args).toEqual(["update"]); + if (suffix.startsWith("node_modules")) expect(capabilities.update).toBeNull(); + } + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.skipIf(windowsHost)( + "native aliases share stable ownership but configured release paths stay pinned", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-native-alias"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const release = NodePath.join( + root, + ".local", + "share", + "claude", + "versions", + "1.0.0", + "claude", + ); + writeExecutable(release); + const alias = NodePath.join(root, ".local", "bin", "claude"); + NodeFS.mkdirSync(NodePath.dirname(alias), { recursive: true }); + NodeFS.symlinkSync(release, alias); + const resolver = makePackageManagedProviderMaintenanceResolver({ + provider: driver("claudeAgent"), + npmPackageName: "@anthropic-ai/claude-code", + nativeUpdate: { + args: ["update"], + isCommandPath: (value) => + value.includes("/.local/share/claude/") || value.endsWith("/.local/bin/claude"), + }, + }); + const resolve = (binaryPath: string) => + resolveProviderMaintenanceCapabilitiesEffect(resolver, { binaryPath, env: { PATH: "" } }); + const active = yield* resolve(alias); + const pinned = yield* resolve(release); + expect(active.update?.installationKey).toBeDefined(); + expect(pinned.update?.installationKey).toBeUndefined(); + const newer = NodePath.join( + root, + ".local", + "share", + "claude", + "versions", + "2.0.0", + "claude", + ); + writeExecutable(newer); + NodeFS.unlinkSync(alias); + NodeFS.symlinkSync(newer, alias); + expect((yield* resolve(alias)).update?.installationKey).toBe( + active.update?.installationKey, + ); + const foreign = NodePath.join(root, "opt", "pinned", "claude"); + writeExecutable(foreign); + NodeFS.unlinkSync(alias); + NodeFS.symlinkSync(foreign, alias); + expect((yield* resolve(alias)).update?.installationKey).toBeUndefined(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.skipIf(windowsHost)( + "standalone Codex only owns updates for its installation home", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-codex-home"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const binary = NodePath.join(root, "packages", "standalone", "current", "bin", "codex"); + writeExecutable(binary); + const resolve = (home: string) => + resolveProviderMaintenanceCapabilitiesEffect( + makePackageManagedProviderMaintenanceResolver({ + provider: driver("codex"), + npmPackageName: "@openai/codex", + nativeUpdate: { + args: ["update"], + env: { CODEX_HOME: home }, + isCommandPath: (value) => value.includes("/packages/standalone/"), + }, + }), + { binaryPath: binary, env: { PATH: "instance-path", CODEX_HOME: "shadow-home" } }, + ); + const owned = yield* resolve(root); + expect(owned.update?.installationKey).toBeDefined(); + expect(owned.update?.env).toEqual({ PATH: "instance-path", CODEX_HOME: root }); + expect( + (yield* resolve(NodePath.join(root, "other"))).update?.installationKey, + ).toBeUndefined(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.skipIf(windowsHost)( + "Homebrew active aliases keep identity across keg versions and explicit kegs stay pinned", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-brew-identity"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const brew = NodePath.join(root, "bin", "brew"); + writeExecutable(brew); + const resolve = (version: string, pinned: boolean) => { + const real = NodePath.join(root, "Cellar", "package-tool", version, "bin", "tool"); + writeExecutable(real); + return resolvePackageManagedProviderMaintenance( + { + provider: driver("packageTool"), + npmPackageName: "@example/package-tool", + nativeUpdate: null, + }, + { + binaryPath: pinned ? real : "tool", + resolvedCommandPath: NodePath.join(root, "bin", "tool"), + realCommandPath: real, + env: { PATH: NodePath.dirname(brew) }, + platform: "linux", + }, + ).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + stdoutSpawner((_command, args) => (args[0] === "--prefix" ? root : "{}")), + ), + ); + }; + const old = yield* resolve("1.0.0", false); + const next = yield* resolve("2.0.0", false); + expect(old.update?.installationKey).toBeDefined(); + expect(next.update?.installationKey).toBe(old.update?.installationKey); + expect((yield* resolve("1.0.0", true)).update?.installationKey).toBeUndefined(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + + it.effect.skipIf(!symlinksSupported || windowsHost)( + "keeps aliased custom mise data directories manual-only", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("pf-09-mise-alias"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const data = NodePath.join(root, "physical"); + const binary = NodePath.join( + data, + "installs", + "npm-package-tool", + "1.0.0", + "lib", + "node_modules", + "@example", + "package-tool", + "bin", + "tool.js", + ); + writeExecutable(binary); + const logical = NodePath.join(root, "logical"); + NodeFS.symlinkSync(data, logical); + const alias = NodePath.join(root, "tool"); + NodeFS.symlinkSync(binary, alias); + const result = yield* resolveProviderMaintenanceCapabilitiesEffect(packageToolUpdate, { + binaryPath: alias, + env: { PATH: "", MISE_DATA_DIR: logical }, + }); + expect(result.update).toBeNull(); + }).pipe(Effect.scoped), + ); + it.effect("caches resolution until a fresh read is requested", () => Effect.gen(function* () { let resolutions = 0; @@ -953,44 +1357,49 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { ); }); -it("pins only owned package-manager installs and preserves their execution context", () => { - const capabilities = makeProviderMaintenanceCapabilities({ - provider: driver("codex"), - packageName: "@openai/codex", - updateExecutable: "npm", - updateLockKey: "npm-global:/opt/node", - updateArgs: [ - "install", - "-g", - "--prefix", - "/opt/node", - "--allow-scripts=@openai/codex", - "@openai/codex@latest", - ], - env: { PATH: "/opt/node/bin" }, - }); - const pinned = makeTargetedProviderUpdateAction(capabilities, "2.0.0"); - expect(pinned).toMatchObject({ - executable: "npm", - lockKey: capabilities.update?.lockKey, - env: capabilities.update?.env, - args: [ - "install", - "-g", - "--prefix", - "/opt/node", - "--allow-scripts=@openai/codex", - "@openai/codex@2.0.0", - ], - }); - expect(pinned?.command).toContain("@openai/codex@2.0.0"); - for (const lockKey of ["codex-native", "homebrew", "manual"]) { - expect( - makeTargetedProviderUpdateAction( - { ...capabilities, update: { ...capabilities.update!, lockKey } }, - "2.0.0", - ), - ).toBeNull(); - } - expect(makeTargetedProviderUpdateAction(capabilities, "2.0.0; rm -rf /")).toBeNull(); -}); +it.each(["2.0.0", "2.0.0-beta.1"])( + "pins only owned package-manager installs to %s and preserves their execution context", + (version) => { + const capabilities = makeProviderMaintenanceCapabilities({ + provider: driver("codex"), + packageName: "@openai/codex", + updateExecutable: "npm", + updateLockKey: "npm-global:/opt/node", + updateArgs: [ + "install", + "-g", + "--prefix", + "/opt/node", + "--allow-scripts=@openai/codex", + "@openai/codex@latest", + ], + env: { PATH: "/opt/node/bin" }, + }); + const pinned = makeTargetedProviderUpdateAction(capabilities, version); + expect(pinned).toMatchObject({ + executable: "npm", + lockKey: capabilities.update?.lockKey, + env: capabilities.update?.env, + args: [ + "install", + "-g", + "--prefix", + "/opt/node", + "--allow-scripts=@openai/codex", + `@openai/codex@${version}`, + ], + }); + expect(pinned?.command).toContain(`@openai/codex@${version}`); + for (const lockKey of ["codex-native", "homebrew", "manual"]) { + expect( + makeTargetedProviderUpdateAction( + { ...capabilities, update: { ...capabilities.update!, lockKey } }, + "2.0.0", + ), + ).toBeNull(); + } + for (const invalid of ["2.0.0; rm -rf /", "2.0.0-beta; echo injected", "2.0.0-beta..1"]) { + expect(makeTargetedProviderUpdateAction(capabilities, invalid)).toBeNull(); + } + }, +); diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index 48b01d495ca50..c91c398a47418 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -16,7 +16,6 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import { causeErrorTag } from "@t3tools/shared/observability"; import { resolveCommandPath } from "@t3tools/shared/shell"; import * as Cache from "effect/Cache"; -import * as Config from "effect/Config"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -46,25 +45,6 @@ const PROVIDER_UPDATE_ACTION_TOAST_MESSAGE = "Install the update now or review p */ const MAINTENANCE_CAPABILITIES_CACHE_TTL = Duration.hours(1); -const compactEnv = (input: Record>): NodeJS.ProcessEnv => - Object.fromEntries( - Object.entries(input).flatMap(([key, value]) => - Option.match(value, { - onNone: () => [], - onSome: (resolved) => [[key, resolved]], - }), - ), - ); - -const CommandLookupEnvConfig = Config.all({ - PATH: Config.String("PATH").pipe(Config.option), - Path: Config.String("Path").pipe(Config.option), - path: Config.String("path").pipe(Config.option), - PATHEXT: Config.String("PATHEXT").pipe(Config.option), -}).pipe(Config.map(compactEnv)); - -const readCommandLookupEnv = CommandLookupEnvConfig.pipe(Effect.orElseSucceed(() => ({}))); - /** Where the provider executable was found; every path is absolute. */ export interface ProviderMaintenanceResolutionContext { readonly binaryPath: string; @@ -135,6 +115,7 @@ export function makeProviderMaintenanceCapabilities(input: { readonly updateExecutable: string | null; readonly updateArgs: ReadonlyArray; readonly updateLockKey: string | null; + readonly installationKey?: string | undefined; /** Shown to the user instead of ` `; use for a bare tool name like `brew`. */ readonly updateCommand?: string; readonly platform?: NodeJS.Platform; @@ -155,6 +136,7 @@ export function makeProviderMaintenanceCapabilities(input: { executable: input.updateExecutable, args: input.updateArgs, lockKey: input.updateLockKey, + ...(input.installationKey ? { installationKey: input.installationKey } : {}), ...(input.env ? { env: input.env } : {}), }; return { @@ -170,7 +152,7 @@ export function makeTargetedProviderUpdateAction( capabilities: ProviderMaintenanceCapabilities, version: string, ): ProviderMaintenanceCommandAction | null { - if (!/^\d+\.\d+\.\d+$/.test(version)) return null; + if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/.test(version)) return null; const update = capabilities.update; const packageName = capabilities.packageName; if (!update || !packageName) return null; @@ -227,9 +209,17 @@ function isYarnGlobalCommandPath(commandPath: string): boolean { } /** Version-manager installs are pinned in its config, so updating them means editing that. */ -function isMiseCommandPath(commandPath: string): boolean { +function isMiseCommandPath(commandPath: string, env: NodeJS.ProcessEnv): boolean { const normalized = normalizeCommandPath(commandPath); - return normalized.includes("/mise/installs/") || normalized.includes("/mise/shims/"); + const dataDir = env.MISE_DATA_DIR?.trim().replace(/[\\/]+$/, ""); + const roots = ["/mise", ...(dataDir ? [normalizeCommandPath(dataDir)] : [])]; + return roots.some((root) => { + if (normalized.includes(`${root}/shims/`)) return true; + const installs = `${root}/installs/`; + const tool = normalized.slice(normalized.indexOf(installs) + installs.length).split("/")[0]; + // A global package under mise's Node install belongs to npm, not mise. + return normalized.includes(installs) && tool !== "node"; + }); } function isPnpmGlobalCommandPath(commandPath: string): boolean { @@ -386,6 +376,34 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( const commandPaths = [context.resolvedCommandPath, context.realCommandPath]; const packageName = definition.npmPackageName; + const fileSystem = yield* FileSystem.FileSystem; + const miseDataDir = context.env.MISE_DATA_DIR; + const realMiseDataDir = miseDataDir + ? yield* fileSystem.realPath(miseDataDir).pipe(Effect.orElseSucceed(() => miseDataDir)) + : undefined; + if ( + commandPaths.some((commandPath) => + isMiseCommandPath(commandPath, { ...context.env, MISE_DATA_DIR: realMiseDataDir }), + ) || + (yield* isMiseWrapperScript(context)) + ) { + return manual; + } + const identity = (value: string) => + context.platform === "win32" ? normalizeCommandPath(value) : value.replaceAll("\\", "/"); + // A configured version directory stays pinned even if an installer owns the active alias. + const pinned = + /[\\/](?:\.local[\\/]share[\\/]claude[\\/]versions|packages[\\/]standalone[\\/]releases|Cellar|Caskroom|\.pnpm)[\\/]/i.test( + context.binaryPath, + ); + const keyFor = (root: string) => (pinned ? undefined : `${packageName}:${identity(root)}`); + const installationKey = keyFor(context.realCommandPath); + const rootAt = (marker: string) => { + const owned = commandPaths.find((value) => normalizeCommandPath(value).includes(marker)); + if (!owned) return context.realCommandPath; + const slash = owned.replaceAll("\\", "/"); + return slash.slice(0, normalizeCommandPath(slash).indexOf(marker) + marker.length - 1); + }; const nativeUpdate = definition.nativeUpdate; const native = nativeUpdate ? makeProviderMaintenanceCapabilities({ @@ -395,11 +413,28 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( updateArgs: nativeUpdate.args, updateLockKey: `${definition.provider}-native`, platform: context.platform, - ...(nativeUpdate.env ? { env: nativeUpdate.env } : {}), + env: { ...context.env, ...nativeUpdate.env }, }) : manual; - if (nativeUpdate?.isCommandPath && commandPaths.some(nativeUpdate.isCommandPath)) { - return native; + if (nativeUpdate?.isCommandPath && nativeUpdate.isCommandPath(context.realCommandPath)) { + const nativeRoot = context.realCommandPath + .replaceAll("\\", "/") + .replace(/(\/packages\/standalone)\/.*$/i, "$1") + .replace(/(\/\.local\/share\/claude)\/.*$/i, "$1"); + const codexHome = nativeUpdate.env?.CODEX_HOME; + const ownsHome = + definition.provider !== "codex" || + !codexHome || + identity(nativeRoot) === identity(`${codexHome.replace(/[\\/]+$/, "")}/packages/standalone`); + return { + ...native, + update: native.update + ? { + ...native.update, + ...(ownsHome && keyFor(nativeRoot) ? { installationKey: keyFor(nativeRoot) } : {}), + } + : null, + }; } // A `node_modules` path not proven below belongs to another package or a // project, so the provider's own updater could act on the wrong install. @@ -411,24 +446,44 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( if (commandPaths.some(isVitePlusGlobalCommandPath)) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, + installationKey: keyFor(rootAt("/.vite-plus/")), packageName, updateExecutable: "vp", updateArgs: ["i", "-g", packageName], updateLockKey: "vite-plus-global", }); } - if (commandPaths.some(isBunGlobalCommandPath)) { + const bunBin = context.env.BUN_INSTALL?.trim().replace(/[\\/]+$/, ""); + if ( + commandPaths.some(isBunGlobalCommandPath) || + (bunBin && + commandPaths.some((value) => + normalizeCommandPath(value).startsWith(`${normalizeCommandPath(bunBin)}/bin/`), + )) + ) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, + installationKey: keyFor(bunBin || rootAt("/.bun/")), packageName, updateExecutable: "bun", updateArgs: ["i", "-g", `${packageName}@latest`], updateLockKey: "bun-global", }); } - if (commandPaths.some(isPnpmGlobalCommandPath)) { + const pnpmHome = context.env.PNPM_HOME?.trim().replace(/[\\/]+$/, ""); + if ( + commandPaths.some(isPnpmGlobalCommandPath) || + (pnpmHome && commandPaths.some((value) => identity(value).startsWith(`${identity(pnpmHome)}/`))) + ) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, + installationKey: keyFor(context.realCommandPath.replaceAll("\\", "/").split("/global/")[0]!), packageName, updateExecutable: "pnpm", updateArgs: ["add", "-g", `${packageName}@latest`], @@ -439,6 +494,9 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( if (commandPaths.some(isYarnGlobalCommandPath)) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, + installationKey, packageName, updateExecutable: "yarn", updateArgs: ["global", "add", `${packageName}@latest`], @@ -448,6 +506,9 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( if (yield* isVoltaPackageInstall(context, packageName)) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, + installationKey: keyFor(context.resolvedCommandPath.replaceAll("\\", "/").split("/bin/")[0]!), packageName, updateExecutable: "volta", updateArgs: ["install", `${packageName}@latest`], @@ -467,6 +528,8 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( // Older npm warns about the unknown config and continues. return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, packageName, updateExecutable: "npm", updateArgs: [ @@ -478,20 +541,16 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( `${packageName}@latest`, ], updateLockKey: `npm-global:${normalizeCommandPath(npmPrefix)}`, + installationKey: `${packageName}:${identity(npmPrefix)}`, }); } - if (commandPaths.some(isMiseCommandPath) || (yield* isMiseWrapperScript(context))) { - return manual; - } - const homebrew = homebrewOwnershipFromCommandPath(context.realCommandPath); if (homebrew) { // Mise shims resolve to the version manager, not the provider. if (homebrew.kind === "formula" && homebrew.name.toLowerCase() === "mise") { return manual; } - const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; // The keg's own brew works even when a GUI-launched server has no // Homebrew on PATH; the prefix check below still applies to it. @@ -523,10 +582,13 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( const info = yield* runHomebrew(brewPath, ["info", "--json=v2", homebrew.name], context.env); return makeProviderMaintenanceCapabilities({ provider: definition.provider, + env: context.env, + platform: context.platform, packageName, updateExecutable: brewPath, updateArgs: args, updateLockKey: "homebrew", + installationKey: keyFor(`${homebrew.prefix}/${homebrew.kind}/${homebrew.name}`), updateCommand: ["brew", ...args].join(" "), latestVersion: info ? parseHomebrewLatestVersion(info, homebrew) : null, }); @@ -644,7 +706,7 @@ export const resolveProviderMaintenanceCapabilitiesEffect = Effect.fn( return yield* resolver.resolve(null); } - const env = options?.env ?? (yield* readCommandLookupEnv); + const env = options?.env ?? (yield* HostProcess.Environment); // resolveCommandPath checks explicit paths for existence too, so a missing // binary always lands in the no-context branch and never gets an update // command it cannot run. diff --git a/packages/provider-core/src/server/managedProvider.ts b/packages/provider-core/src/server/managedProvider.ts index a11734a1661b4..effd399a9ab39 100644 --- a/packages/provider-core/src/server/managedProvider.ts +++ b/packages/provider-core/src/server/managedProvider.ts @@ -51,7 +51,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( readonly publishSnapshot: (snapshot: ServerProvider) => Effect.Effect; }) => Effect.Effect; readonly refreshInterval?: Duration.Input; - readonly refreshOnInterval?: boolean; + readonly refreshOnInterval?: boolean | "enrich"; readonly checkProviderOnSettingsChange?: (previous: Settings, next: Settings) => boolean; }): Effect.fn.Return< ManagedServerProvider, @@ -224,6 +224,16 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( return yield* applySnapshot(nextSettings, { forceRefresh: true }); }); + const refreshEnrichment = refreshSemaphore.withPermits(1)( + Effect.gen(function* () { + const settings = yield* input.getSettings; + const state = yield* Ref.get(snapshotStateRef); + const generation = state.enrichmentGeneration + 1; + yield* Ref.set(snapshotStateRef, { ...state, enrichmentGeneration: generation }); + yield* restartSnapshotEnrichment(settings, state.snapshot, generation); + }), + ); + const hasProviderStatusDemand = Effect.gen(function* () { const state = yield* Ref.get(snapshotStateRef); const instanceId = state.snapshot.instanceId; @@ -234,7 +244,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( return genericDemand || instanceDemand; }); - const getRefreshInterval = + const configuredRefreshInterval = input.refreshInterval !== undefined ? Effect.succeed(input.refreshInterval) : host.settings.get.pipe( @@ -245,6 +255,15 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( Effect.orElseSucceed(() => DEFAULT_PROVIDER_HEALTH_REFRESH_INTERVAL), ); + const getRefreshInterval = configuredRefreshInterval.pipe( + Effect.map((interval) => + input.refreshOnInterval === "enrich" && + Duration.toMillis(Duration.fromInputUnsafe(interval)) <= 0 + ? Duration.hours(1) + : interval, + ), + ); + const refreshIntervalChanges = yield* Queue.sliding(1); if (input.refreshInterval === undefined) { const serverSettingsChanges = yield* host.settings.subscribe; @@ -282,7 +301,18 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( Duration.toMillis(Duration.fromInputUnsafe(refreshInterval)) > 0 ? hasProviderStatusDemand.pipe( Effect.flatMap((shouldRefresh) => - shouldRefresh ? refreshSnapshot().pipe(Effect.asVoid) : Effect.void, + input.refreshOnInterval === "enrich" + ? host.settings.get.pipe( + Effect.flatMap((settings) => + shouldRefresh || + (settings.automaticUpdates && settings.enableProviderUpdateChecks) + ? refreshEnrichment + : Effect.void, + ), + ) + : shouldRefresh + ? refreshSnapshot().pipe(Effect.asVoid) + : Effect.void, ), ) : Effect.void, diff --git a/packages/provider-grok/src/server/driver.test.ts b/packages/provider-grok/src/server/driver.test.ts index b3e96bad0d040..42f2ae4841f24 100644 --- a/packages/provider-grok/src/server/driver.test.ts +++ b/packages/provider-grok/src/server/driver.test.ts @@ -72,12 +72,50 @@ it.layer(layerTest)("GrokDriver", (it) => { }); // `grok update` installs under GROK_HOME, so it must target this instance's home. expect(capabilities.update?.env?.GROK_HOME).toBe(grokHome); + expect(capabilities.update?.installationKey).toBeDefined(); }).pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawner), Effect.scoped, ), ); + it.effect.skipIf(windowsHost)( + "keeps custom Grok binaries manual and pins npm-owned installs", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "pf-09-grok-" }); + const binaries = [ + path.join(root, "pinned", "grok"), + path.join(root, "lib", "node_modules", "@xai-official", "grok", "bin", "grok"), + ]; + for (const binaryPath of binaries) { + yield* fs.makeDirectory(path.dirname(binaryPath), { recursive: true }); + yield* fs.writeFileString(binaryPath, "#!/bin/sh\n"); + yield* fs.chmod(binaryPath, 0o755); + const instance = yield* GrokDriver.create({ + instanceId: ProviderInstanceId.make("grok-owned"), + displayName: undefined, + enabled: false, + environment: [{ name: "GROK_HOME", value: root, sensitive: false }], + config: { ...GrokDriver.defaultConfig(), binaryPath }, + }); + const capabilities = yield* instance.snapshot.resolveMaintenance(); + if (binaryPath === binaries[0]) + expect(capabilities.update?.installationKey).toBeUndefined(); + else { + expect(capabilities.update?.executable).toBe("npm"); + expect(capabilities.update?.installationKey).toBeDefined(); + expect(capabilities.update?.args).toContain(root); + } + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawner), + Effect.scoped, + ), + ); + it.effect("stays manual-only when the configured executable does not exist", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/provider-grok/src/server/driver.ts b/packages/provider-grok/src/server/driver.ts index fe5b8705471aa..588b57af5f54c 100644 --- a/packages/provider-grok/src/server/driver.ts +++ b/packages/provider-grok/src/server/driver.ts @@ -32,8 +32,8 @@ import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/ import { discoverGrokSkills } from "./skills.ts"; import { makeCachedProviderMaintenanceResolution, - makeManualOnlyProviderMaintenanceCapabilities, - makeProviderMaintenanceCapabilities, + resolvePackageManagedProviderMaintenance, + normalizeCommandPath, type ProviderMaintenanceCapabilitiesResolver, resolveProviderMaintenanceCapabilitiesEffect, } from "@t3tools/provider-core/server/maintenanceResolver"; @@ -48,28 +48,29 @@ const DRIVER_KIND = ProviderDriverKind.make("grok"); // npm's `latest` tracks Grok's stable channel, the one `grok update` installs // by default, so the registry stays the source for "latest". const GROK_NPM_PACKAGE = "@xai-official/grok"; -// `grok update` finds the installer that owns the binary itself, so the -// resolved executable is its own updater. It installs under `GROK_HOME`, so it -// runs with the instance's environment. No executable means nothing to update, -// not "whatever is on PATH". +// Native updates target GROK_HOME; package-manager installs stay with their owner. const UPDATE: ProviderMaintenanceCapabilitiesResolver = { - resolve: (context) => - Effect.succeed( - context - ? makeProviderMaintenanceCapabilities({ - provider: DRIVER_KIND, - packageName: GROK_NPM_PACKAGE, - updateExecutable: context.resolvedCommandPath, - updateArgs: ["update"], - updateLockKey: "grok", - platform: context.platform, - env: context.env, - }) - : makeManualOnlyProviderMaintenanceCapabilities({ - provider: DRIVER_KIND, - packageName: GROK_NPM_PACKAGE, - }), - ), + resolve: (context) => { + const home = ( + context?.env.GROK_HOME || (context?.env.HOME ? `${context.env.HOME}/.grok` : undefined) + ) + ?.trim() + .replace(/[\\/]+$/, ""); + return resolvePackageManagedProviderMaintenance( + { + provider: DRIVER_KIND, + npmPackageName: GROK_NPM_PACKAGE, + nativeUpdate: { + args: ["update"], + isCommandPath: (value) => + Boolean(home) && + normalizeCommandPath(value).replace(/\.exe$/, "") === + normalizeCommandPath(`${home}/bin/grok`), + }, + }, + context, + ); + }, }; export type GrokDriverEnv = diff --git a/packages/provider-muse/src/server/maintenance.test.ts b/packages/provider-muse/src/server/maintenance.test.ts index 869aa6dca2fec..fa95b0ed13997 100644 --- a/packages/provider-muse/src/server/maintenance.test.ts +++ b/packages/provider-muse/src/server/maintenance.test.ts @@ -212,6 +212,7 @@ it.layer(NodeServices.layer)("Muse maintenance", (it) => { executable: binaryPath, args: ["--version"], lockKey: `muse:${binaryPath}`, + installationKey: `muse:${binaryPath}`, env: { MUSE_CHANNEL: "muse-canary", MUSE_NO_AUTO_UPDATE: "0", diff --git a/packages/provider-muse/src/server/maintenance.ts b/packages/provider-muse/src/server/maintenance.ts index 776e9b0f3c239..a415f2fc73e01 100644 --- a/packages/provider-muse/src/server/maintenance.ts +++ b/packages/provider-muse/src/server/maintenance.ts @@ -78,6 +78,7 @@ export const museMaintenance: ProviderMaintenanceCapabilitiesResolver = { "--version", ].join(" "), updateLockKey: `muse:${context.realCommandPath}`, + installationKey: `muse:${context.realCommandPath}`, platform: context.platform, env: { ...context.env, ...updateEnv }, }), diff --git a/packages/provider-opencode/src/server/driver.test.ts b/packages/provider-opencode/src/server/driver.test.ts index 64b18e0db88ba..f45855207a989 100644 --- a/packages/provider-opencode/src/server/driver.test.ts +++ b/packages/provider-opencode/src/server/driver.test.ts @@ -5,14 +5,17 @@ import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; -import { ProviderInstanceId } from "@t3tools/contracts"; +import { DEFAULT_SERVER_SETTINGS, ProviderInstanceId } from "@t3tools/contracts"; import type { OpenCodeSettings } from "../settings.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; +import * as Duration from "effect/Duration"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; @@ -225,6 +228,63 @@ const layerUpdate = Layer.mergeAll( ).pipe(Layer.provideMerge(NodeServices.layer)); it.layer(layerUpdate)("OpenCodeDriver updates", (it) => { + it.effect.each([Duration.minutes(1), Duration.zero])( + "refreshes OpenCode release advisories without foreground demand at health interval %s", + (interval) => + Effect.gen(function* () { + yield* (yield* ProviderLatestVersions.ProviderLatestVersions).invalidate("@opencode/cli"); + versionOutput.current = "opencode v2.0.18\n"; + serverStarts.length = 0; + const root = NodeFS.realpathSync( + NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "pf-09-opencode-")), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const binaryPath = npmGlobalInstall(root, ["@opencode", "cli"], "opencode.exe"); + let latest = "2.0.18"; + const http = HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + request.url.startsWith("https://registry.npmjs.org/") + ? Response.json({ version: latest }) + : new Response("", { status: 404 }), + ), + ), + ); + const instance = yield* create({ binaryPath }, http).pipe( + Effect.provide( + TestProviderHost.layer({ + runBackgroundWork: false, + settings: { ...DEFAULT_SERVER_SETTINGS, providerHealthRefreshInterval: interval }, + }).pipe(Layer.provide(NodeServices.layer)), + ), + ); + const boot = yield* instance.snapshot.streamChanges.pipe( + Stream.filter((snapshot) => snapshot.versionAdvisory?.latestVersion === latest), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Fiber.join(boot); + const startsAtBoot = serverStarts.length; + const probesAtBoot = versionProbes.length; + latest = "2.0.19"; + const updated = yield* instance.snapshot.streamChanges.pipe( + Stream.filter((snapshot) => snapshot.versionAdvisory?.latestVersion === latest), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* TestClock.adjust("1 hour"); + const snapshots = yield* Fiber.join(updated); + assert.strictEqual(snapshots[0]?.versionAdvisory?.status, "behind_latest"); + assert.strictEqual(serverStarts.length, startsAtBoot); + assert.strictEqual(versionProbes.length, probesAtBoot); + }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + ); + it.effect("never runs the binary for a disabled instance's update check", () => Effect.gen(function* () { versionProbes.length = 0; diff --git a/packages/provider-opencode/src/server/driver.ts b/packages/provider-opencode/src/server/driver.ts index cf1bffebf05bc..46ab9ffe587a8 100644 --- a/packages/provider-opencode/src/server/driver.ts +++ b/packages/provider-opencode/src/server/driver.ts @@ -454,7 +454,7 @@ export const OpenCodeDriver: ProviderDriver< streamSettings: snapshotSettings.streamSettings, haveSettingsChanged: haveProviderSnapshotSettingsChanged, checkProviderOnSettingsChange: () => false, - refreshOnInterval: false, + refreshOnInterval: "enrich", initialSnapshot: (settings) => makePendingOpenCodeProvider(settings.provider).pipe(Effect.map(stampIdentity)), checkProvider, From 4ae068e30bb75b1b2d3d9bf536624bc811d6c345 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:45:11 +0000 Subject: [PATCH 070/149] fix(server): provider installs wait for turns on every T3 server on the host A user-wide reader/writer gate in the user cache coordinates independent T3 processes. Installs take the write lease; turns, compaction, session operations and text generation take read leases that last until their background work drains. Crashed holders are reclaimed by PID identity. --- .../ProviderSessionManager.test.ts | 110 ++++++++ .../ProviderSessionManager.ts | 249 ++++++++++++------ .../provider/ProviderMaintenanceHostLock.ts | 147 +++++++++++ .../providerMaintenanceCommandCoordinator.ts | 4 +- .../providerMaintenanceRunner.test.ts | 199 +++++++++++++- .../src/textGeneration/TextGeneration.test.ts | 58 ++++ .../src/textGeneration/TextGeneration.ts | 18 +- 7 files changed, 692 insertions(+), 93 deletions(-) create mode 100644 apps/server/src/provider/ProviderMaintenanceHostLock.ts diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index ceb230b008940..c4a22376ba919 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -50,8 +50,14 @@ import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; +import * as HostLock from "../provider/ProviderMaintenanceHostLock.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +const layerHostLock = Layer.succeed(HostLock.ProviderMaintenanceHostLock, { + acquireRead: Effect.succeed(Effect.void), + withRead: (effect) => effect, + withWrite: (effect) => effect, +}); const layerTestDatabase = SqlitePersistence.layerMemory; const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( Layer.provide(layerTestDatabase), @@ -470,6 +476,7 @@ function layerTest(input: { readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; + readonly hostLock?: ReturnType; readonly serverSettingsLayer?: ReturnType; readonly projectServiceLayer?: Layer.Layer; }) { @@ -531,6 +538,9 @@ function layerTest(input: { Layer.provide( Layer.mergeAll( layerRegistry.pipe(Layer.provideMerge(McpProviderSessions.layer)), + input.hostLock === undefined + ? layerHostLock + : Layer.succeed(HostLock.ProviderMaintenanceHostLock, input.hostLock), layerConfiguredEventSink, IdAllocator.layer, layerProviderEventIngestorTest, @@ -4723,3 +4733,103 @@ it.effect( assert.isFalse(denied?.capabilities?.has("device")); }), ); + +it.effect("ProviderSessionManagerV2 holds the host lease through terminal background work", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const pending = yield* Ref.make(true); + const held = yield* Ref.make(0); + const hostLock = { + withRead: (effect: Effect.Effect) => effect, + withWrite: (effect: Effect.Effect) => effect, + acquireRead: Ref.update(held, (value) => value + 1).pipe( + Effect.as(Ref.update(held, (value) => value - 1)), + ), + }; + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-host-maintenance-background"); + const providerSessionId = idAllocator.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: "native-host-maintenance", + }); + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + assert.equal(yield* Ref.get(held), 1); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + const terminal = { + type: "turn.terminal" as const, + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-host-turn", + }), + runOrdinal: 1, + status: "completed" as const, + failure: null, + threadDisposition: "reusable" as const, + }; + for (const background of [true, false]) { + yield* Ref.set(pending, background); + const subscription = yield* runtime.subscribeEvents!; + const observed = yield* subscription.events.pipe( + Stream.take(1), + Stream.runDrain, + Effect.forkChild, + ); + yield* Queue.offer(queue, terminal); + yield* Fiber.join(observed); + assert.equal(yield* Ref.get(held), background ? 1 : 0); + } + yield* manager.close(providerSessionId); + assert.equal(yield* Ref.get(held), 0); + }).pipe( + Effect.provide( + layerTest({ + state, + hostLock, + hasPendingBackgroundWork: Ref.get(pending), + idleTimeoutMs: 60_000, + }), + ), + ); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 99bcc097d244d..db0c3f4ca334a 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -50,6 +50,7 @@ import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as HostLock from "../provider/ProviderMaintenanceHostLock.ts"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; @@ -347,6 +348,33 @@ export const layerWithOptions = ( ProviderSessionManagerV2, Effect.gen(function* () { const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; + const hostLock = yield* HostLock.ProviderMaintenanceHostLock; + const hostLeases = new Map< + ProviderAdapter.ProviderAdapterV2SessionRuntime, + Effect.Effect + >(); + const hostLeaseLocks = yield* KeyedLock.make(); + const releaseHostLease = (runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime) => + Effect.suspend(() => { + const release = hostLeases.get(runtime); + hostLeases.delete(runtime); + return release ?? Effect.void; + }); + const releaseIdleHostLease = (providerSessionId: ProviderSessionId) => + hostLeaseLocks.withLock( + providerSessionId, + Effect.gen(function* () { + const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); + if (!entry || entry.busyTurns.size > 0 || !hostLeases.has(entry.runtime)) return; + const pending = + entry.runtime.hasPendingBackgroundWork === undefined + ? false + : yield* entry.runtime.hasPendingBackgroundWork.pipe( + Effect.catchCause(() => Effect.succeed(true)), + ); + if (!pending) yield* releaseHostLease(entry.runtime); + }), + ); const fileSystem = yield* FileSystem.FileSystem; const mcpSessionRegistry = yield* McpSessionRegistry.McpSessionRegistry; const mcpSessions = yield* McpProviderSessions.McpProviderSessions; @@ -1400,28 +1428,44 @@ export const layerWithOptions = ( }); const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) => - withActivityError( + hostLeaseLocks.withLock( providerSessionId, - Effect.gen(function* () { - const key = sessionKey(providerSessionId); - const now = yield* Clock.currentTimeMillis; - const idleFiber = yield* Ref.modify(sessions, (current) => { - const entry = current.get(key); - if (entry === undefined) { - return [null, current] as const; + withActivityError( + providerSessionId, + Effect.gen(function* () { + const key = sessionKey(providerSessionId); + const entry = (yield* Ref.get(sessions)).get(key); + if (entry && !hostLeases.has(entry.runtime)) { + yield* Effect.uninterruptible( + Effect.gen(function* () { + const release = yield* hostLock.acquireRead; + if ((yield* Ref.get(sessions)).get(key)?.runtime === entry.runtime) { + hostLeases.set(entry.runtime, release); + } else { + yield* release; + } + }), + ); } - const updated = new Map(current); - updated.set(key, { - ...entry, - busyTurns: new Set(entry.busyTurns).add(turnKey), - idleFiber: null, - lastActivityAtMs: now, - pinnedSinceMs: null, + const now = yield* Clock.currentTimeMillis; + const idleFiber = yield* Ref.modify(sessions, (current) => { + const entry = current.get(key); + if (entry === undefined) { + return [null, current] as const; + } + const updated = new Map(current); + updated.set(key, { + ...entry, + busyTurns: new Set(entry.busyTurns).add(turnKey), + idleFiber: null, + lastActivityAtMs: now, + pinnedSinceMs: null, + }); + return [entry.idleFiber, updated] as const; }); - return [entry.idleFiber, updated] as const; - }); - yield* cancelIdleFiber(idleFiber); - }), + yield* cancelIdleFiber(idleFiber); + }), + ), ); // Clearing a turn that is not marked busy (one whose failed start already @@ -1454,6 +1498,7 @@ export const layerWithOptions = ( }); yield* scheduleIdleReleaseInternal(providerSessionId); yield* scheduleThreadUnload(providerSessionId, providerThreadId); + yield* releaseIdleHostLease(providerSessionId); }), ); @@ -1699,6 +1744,46 @@ export const layerWithOptions = ( modelFamily: normalizeModelMetricLabel(model), }, }); + const startTurn = (input: Parameters[0], compact = false) => + observeActivity( + providerSessionId, + ensureThreadAttached({ + providerSessionId, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + }), + ).pipe( + Effect.andThen( + holdThreadLoaded({ + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + }), + ), + Effect.andThen( + // Failed starts may not emit turn.terminal; clear only their own busy key. + Effect.acquireUseRelease( + observeActivity( + providerSessionId, + markBusy( + providerSessionId, + busyTurnKey(input.providerThread.id, input.runOrdinal), + ), + ), + () => + (compact ? runtime.compactThread!(input) : runtime.startTurn(input)).pipe( + turnMetrics("send", input.modelSelection.model), + ), + (_, exit) => + Exit.isFailure(exit) + ? observeActivity( + providerSessionId, + markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), + ) + : Effect.void, + ), + ), + ); return { ...runtime, subscribeEvents, @@ -1714,7 +1799,7 @@ export const layerWithOptions = ( providerInstanceId: runtime.instanceId, }), ).pipe( - Effect.andThen(runtime.ensureThread(input)), + Effect.andThen(hostLock.withRead(runtime.ensureThread(input))), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, @@ -1730,7 +1815,7 @@ export const layerWithOptions = ( resumeThread: (input) => { const threadId = input.threadId ?? input.providerThread.appThreadId; if (threadId === null || threadId === undefined) { - return runtime.resumeThread(input); + return hostLock.withRead(runtime.resumeThread(input)); } const providerThreadKey = providerThreadLoadKey({ providerThread: input.providerThread, @@ -1758,7 +1843,9 @@ export const layerWithOptions = ( isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }), ), Effect.flatMap((loaded) => - loaded ? Effect.succeed(input.providerThread) : runtime.resumeThread(input), + loaded + ? Effect.succeed(input.providerThread) + : hostLock.withRead(runtime.resumeThread(input)), ), Effect.tap((providerThread) => markProviderThreadLoaded({ @@ -1786,7 +1873,7 @@ export const layerWithOptions = ( providerInstanceId: runtime.instanceId, }), ).pipe( - Effect.andThen(runtime.forkThread(input)), + Effect.andThen(hostLock.withRead(runtime.forkThread(input))), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, @@ -1803,61 +1890,38 @@ export const layerWithOptions = ( }), ), ), - startTurn: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, + readThreadSnapshot: (input) => hostLock.withRead(runtime.readThreadSnapshot(input)), + rollbackThread: (input) => hostLock.withRead(runtime.rollbackThread(input)), + ...(runtime.injectHistory === undefined + ? {} + : { + injectHistory: (input: Parameters>[0]) => + hostLock.withRead(runtime.injectHistory!(input)), + }), + startTurn: (input) => startTurn(input), + ...(runtime.compactThread === undefined + ? {} + : { + compactThread: (input: Parameters[0]) => + startTurn(input, true), }), - ).pipe( - // A start that fails or is stopped may never emit turn.terminal, - // so it clears its own turn or the session never goes idle. If - // the adapter emits the terminal anyway, clearing the same turn - // again changes nothing, so another thread's turn on a shared - // session stays busy either way. - Effect.andThen( - holdThreadLoaded({ - providerSessionId, - threadId: input.threadId, - providerThread: input.providerThread, - }), - ), - Effect.andThen( - Effect.acquireUseRelease( - observeActivity( - providerSessionId, - markBusy( - providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), - ), - ), - () => - runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), - (_, exit) => - Exit.isFailure(exit) - ? observeActivity( - providerSessionId, - markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), - ) - : Effect.void, - ), - ), - ), steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( - Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))), + Effect.andThen( + hostLock.withRead(runtime.steerTurn(input)).pipe(turnMetrics("steer")), + ), ), interruptTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( - Effect.andThen(runtime.interruptTurn(input).pipe(turnMetrics("interrupt"))), + Effect.andThen( + hostLock.withRead(runtime.interruptTurn(input)).pipe(turnMetrics("interrupt")), + ), ), respondToRuntimeRequest: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen( - runtime - .respondToRuntimeRequest(input) + hostLock + .withRead(runtime.respondToRuntimeRequest(input)) .pipe(turnMetrics("runtime-request-response")), ), ), @@ -1914,6 +1978,7 @@ export const layerWithOptions = ( ) : touchActivity(entry.runtime.providerSessionId, { providerActivity: true }), ).pipe( + Effect.andThen(releaseIdleHostLease(entry.runtime.providerSessionId)), Effect.andThen( event.type === "provider_session.updated" ? persistProviderSessionUpdate(entry, event) @@ -2113,28 +2178,37 @@ export const layerWithOptions = ( } }); const sessionScope = yield* Scope.fork(sessionScopes); + let leaseRuntime: ProviderAdapter.ProviderAdapterV2SessionRuntime | undefined; + yield* Scope.addFinalizer( + sessionScope, + Effect.suspend(() => + leaseRuntime === undefined ? Effect.void : releaseHostLease(leaseRuntime), + ), + ); // The previous session's agent scope must not explain this // session's failures, even when this provider runs without one. yield* agentScope.clear(input.threadId); - const runtime = yield* adapter - .openSession({ - threadId: input.threadId, - providerSessionId: input.providerSessionId, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - ...(input.resumeFromSession === undefined - ? {} - : { resumeFromSession: input.resumeFromSession }), - ...(input.initialNativeThreadId === undefined - ? {} - : { initialNativeThreadId: input.initialNativeThreadId }), - ...(input.initialProviderItemIdentityVersion === undefined - ? {} - : { - initialProviderItemIdentityVersion: - input.initialProviderItemIdentityVersion, - }), - }) + const runtime = yield* hostLock + .withRead( + adapter.openSession({ + threadId: input.threadId, + providerSessionId: input.providerSessionId, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + ...(input.resumeFromSession === undefined + ? {} + : { resumeFromSession: input.resumeFromSession }), + ...(input.initialNativeThreadId === undefined + ? {} + : { initialNativeThreadId: input.initialNativeThreadId }), + ...(input.initialProviderItemIdentityVersion === undefined + ? {} + : { + initialProviderItemIdentityVersion: + input.initialProviderItemIdentityVersion, + }), + }), + ) .pipe( Effect.provideService(Scope.Scope, sessionScope), // Any failure, including a Stop that interrupts a slow @@ -2174,6 +2248,7 @@ export const layerWithOptions = ( attributes: { provider: adapter.driver, operation: "open" }, }), ); + leaseRuntime = runtime; const eventSubscribers = yield* Ref.make< ReadonlyMap> >(new Map()); diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts new file mode 100644 index 0000000000000..e594389ef3388 --- /dev/null +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -0,0 +1,147 @@ +// @effect-diagnostics nodeBuiltinImport:off - Host leases coordinate independent Node servers. +// oxlint-disable t3code/no-global-process-runtime -- Leases belong to the OS user and process, not an environment. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { lock } from "proper-lockfile"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Semaphore from "effect/Semaphore"; + +// Linux exposes a process start identity, so recycled PIDs cannot pin a dead reader. +const processIdentity = (pid: number) => { + if (NodeOS.platform() !== "linux") return null; + const fields = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").at(-1)!.split(" "); + return fields[0] === "Z" ? "" : fields[19]!; +}; + +/** + * One user-wide reader/writer gate, deliberately covering every CLI: package + * managers can replace several providers in one installation. Readers overlap; + * an installer closes admission before waiting for active turns to drain. + */ +export const make = (directory: string) => { + const readers = NodePath.join(directory, "readers"); + const prepare = Effect.sync(() => NodeFS.mkdirSync(readers, { recursive: true })); + const acquireGate = Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* prepare; + const compromised = yield* Deferred.make(); + const services = yield* Effect.context(); + while (true) { + const result = yield* Effect.tryPromise(() => + lock(NodePath.join(directory, "gate"), { + realpath: false, + stale: 120_000, + update: 10_000, + onCompromised: (error) => + Effect.runSyncWith(services)(Deferred.die(compromised, error)), + }), + ).pipe(Effect.result); + if (result._tag === "Success") return { release: result.success, compromised }; + const cause = result.failure.cause; + if (!(cause instanceof Error) || !("code" in cause) || cause.code !== "ELOCKED") { + return yield* Effect.die(cause); + } + yield* Effect.interruptible(restore(Effect.sleep("100 millis"))); + } + }), + ); + const withGate = (effect: Effect.Effect) => + Effect.acquireUseRelease( + acquireGate, + ({ compromised }) => Effect.raceFirst(effect, Deferred.await(compromised)), + ({ release }) => Effect.promise(() => release()), + ); + + const readPermit = Semaphore.makeUnsafe(1); + let readCount = 0; + let readFile: string | undefined; + // A provider can launch helper CLIs while its turn holds a read lease. + // Joining that lease must not wait behind an installer draining this turn. + const acquireRead = Effect.uninterruptible( + readPermit.withPermits(1)( + Effect.gen(function* () { + if (readCount === 0) { + readFile = yield* withGate( + Effect.sync(() => { + const file = NodePath.join(readers, `${process.pid}-${NodeCrypto.randomUUID()}`); + NodeFS.writeFileSync(file, processIdentity(process.pid) ?? "", { flag: "wx" }); + return file; + }), + ); + } + readCount++; + let released = false; + return readPermit.withPermits(1)( + Effect.sync(() => { + if (released) return; + released = true; + if (--readCount === 0 && readFile !== undefined) { + NodeFS.rmSync(readFile, { force: true }); + readFile = undefined; + } + }), + ); + }), + ), + ); + const withRead = (effect: Effect.Effect) => + Effect.acquireUseRelease( + acquireRead, + () => effect, + (release) => release, + ); + const drained = Effect.sync(() => { + let busy = false; + for (const file of NodeFS.readdirSync(readers)) { + const pid = Number(file.split("-")[0]); + try { + process.kill(pid, 0); + const identity = processIdentity(pid); + if ( + identity !== null && + identity !== NodeFS.readFileSync(NodePath.join(readers, file), "utf8") + ) { + NodeFS.rmSync(NodePath.join(readers, file), { force: true }); + } else { + busy = true; + } + } catch (error) { + if ( + !(error instanceof Error) || + !("code" in error) || + (error.code !== "ESRCH" && error.code !== "ENOENT") + ) { + throw error; + } + NodeFS.rmSync(NodePath.join(readers, file), { force: true }); + } + } + return !busy; + }); + const withWrite = (effect: Effect.Effect) => + withGate( + Effect.gen(function* () { + while (!(yield* drained)) yield* Effect.sleep("100 millis"); + return yield* effect; + }), + ); + return { acquireRead, withRead, withWrite }; +}; + +export class ProviderMaintenanceHostLock extends Context.Reference>( + "t3/provider/ProviderMaintenanceHostLock", + { + defaultValue: () => + make( + NodePath.join( + process.env.XDG_CACHE_HOME || NodePath.join(NodeOS.homedir(), ".cache"), + "t3", + "provider-maintenance", + ), + ), + }, +) {} diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index 28cac15d4b23b..95c6ad5d9027a 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -3,6 +3,7 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Ref from "effect/Ref"; import * as Semaphore from "effect/Semaphore"; +import * as HostLock from "./ProviderMaintenanceHostLock.ts"; /** One permit per process: background installs hold it, provider turn starts wait for it. */ export class ProviderMaintenanceAdmission extends Context.Reference<{ @@ -26,6 +27,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( )(function* (input: { readonly makeAlreadyRunningError: (targetKey: string) => E }) { const runningTargetsRef = yield* Ref.make>(new Set()); const locks = yield* KeyedLock.make(); + const hostLock = yield* HostLock.ProviderMaintenanceHostLock; const acquireTarget = Effect.fn("acquireTarget")(function* (targetKey: string) { return yield* Ref.modify(runningTargetsRef, (runningTargets) => { @@ -59,7 +61,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( } return yield* (onQueued ?? Effect.void).pipe( - Effect.andThen(locks.withLock(lockKey, run)), + Effect.andThen(locks.withLock(lockKey, hostLock.withWrite(run))), Effect.onInterrupt(() => onInterrupted ?? Effect.void), Effect.ensuring(releaseTarget(targetKey)), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 833d814af7e4d..975c5fc11a4e2 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -1,4 +1,12 @@ +// @effect-diagnostics nodeBuiltinImport:off - Separate processes exercise host-wide maintenance leases. import { describe, it, assert } from "@effect/vitest"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as FileSystem from "effect/FileSystem"; +import * as HostLock from "./ProviderMaintenanceHostLock.ts"; +import { makeProviderMaintenanceCommandCoordinator } from "./providerMaintenanceCommandCoordinator.ts"; import { DEFAULT_SERVER_SETTINGS, ProviderDriverKind, @@ -50,7 +58,14 @@ const encoder = new TextEncoder(); // `{ command, args }` assertions below hold deterministically on any host // (including Windows). Windows-specific resolution is covered by the dedicated // win32 case at the end of this suite. -const layerNonWindowsPlatform = Layer.succeed(HostProcess.Platform, "linux"); +const layerNonWindowsPlatform = Layer.merge( + Layer.succeed(HostProcess.Platform, "linux"), + Layer.succeed(HostLock.ProviderMaintenanceHostLock, { + acquireRead: Effect.succeed(Effect.void), + withRead: (effect) => effect, + withWrite: (effect) => effect, + }), +); function lifecycleFor(provider: ProviderDriverKind): ProviderMaintenanceCapabilities { if (provider === NATIVE_CLI_DRIVER) { @@ -1493,3 +1508,185 @@ it.effect( ); }, ); + +describe("provider maintenance host leases", () => { + const coordinatedWrite = (directory: string, run: Effect.Effect) => + Effect.gen(function* () { + const coordinator = yield* makeProviderMaintenanceCommandCoordinator({ + makeAlreadyRunningError: () => new Error("already running"), + }); + return yield* coordinator.withCommandLock({ targetKey: "provider", lockKey: "install", run }); + }).pipe(Effect.provideService(HostLock.ProviderMaintenanceHostLock, HostLock.make(directory))); + const childLease = (directory: string, mode: "withRead" | "withWrite") => + Effect.acquireRelease( + Effect.sync(() => { + const child = NodeChildProcess.spawn( + process.execPath, + [ + "--input-type=module", + "-e", + ` + import * as Effect from "effect/Effect"; + import { make } from ${JSON.stringify(new URL("./ProviderMaintenanceHostLock.ts", import.meta.url).href)}; + await Effect.runPromise(make(process.argv[1])[process.argv[2]](Effect.promise(() => + new Promise(resolve => { process.stdout.write("held\\n"); process.stdin.once("data", resolve); }) + ))); + `, + directory, + mode, + ], + { cwd: NodePath.resolve(import.meta.dirname, "../.."), stdio: "pipe" }, + ); + const held = new Promise((resolve, reject) => { + child.stdout.once("data", () => resolve()); + child.once("error", reject); + child.once("exit", (code) => code && reject(new Error(`holder exited ${code}`))); + }); + const exited = new Promise((resolve) => child.once("exit", () => resolve())); + return { child, held, exited }; + }), + ({ child, exited }) => + Effect.promise(async () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await exited; + }), + ); + + const waitForGate = (directory: string) => + Effect.callback((resume) => { + const gate = NodePath.join(directory, "gate.lock"); + const watcher = NodeFS.watch(directory, () => { + if (NodeFS.existsSync(gate)) resume(Effect.void); + }); + watcher.once("error", (error) => resume(Effect.die(error))); + if (NodeFS.existsSync(gate)) resume(Effect.void); + return Effect.sync(() => watcher.close()); + }); + + it.live("holds another server's install through a turn and admits nested CLI calls", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const holder = yield* childLease(directory, "withRead"); + yield* Effect.promise(() => holder.held); + const local = HostLock.make(directory); + const release = yield* local.acquireRead; + const entered = yield* Deferred.make(); + const gate = Effect.raceFirst( + waitForGate(directory), + Deferred.await(entered).pipe(Effect.andThen(Effect.die("install bypassed active turn"))), + ); + const install = yield* coordinatedWrite( + directory, + Deferred.succeed(entered, undefined), + ).pipe(Effect.forkScoped); + yield* gate; + assert.isFalse(yield* Deferred.isDone(entered)); + yield* local.withRead(Effect.void); + yield* release; + assert.isFalse(yield* Deferred.isDone(entered)); + holder.child.stdin.end("release"); + yield* Effect.promise(() => holder.exited); + yield* Fiber.join(install); + assert.isTrue(yield* Deferred.isDone(entered)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("serializes installs across independent server processes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const holder = yield* childLease(directory, "withWrite"); + yield* Effect.promise(() => holder.held); + const entered = yield* Deferred.make(); + const install = yield* coordinatedWrite( + directory, + Deferred.succeed(entered, undefined), + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isFalse(yield* Deferred.isDone(entered)); + holder.child.stdin.end("release"); + yield* Effect.promise(() => holder.exited); + yield* Fiber.join(install); + assert.isTrue(yield* Deferred.isDone(entered)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("releases active and waiting host leases on interruption", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const local = HostLock.make(directory); + const entered = yield* Deferred.make(); + const turn = yield* local + .withRead(Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never))) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const gate = waitForGate(directory); + const install = yield* local.withWrite(Effect.never).pipe(Effect.forkScoped); + yield* gate; + yield* Fiber.interrupt(install); + yield* Fiber.interrupt(turn); + assert.equal(NodeFS.readdirSync(NodePath.join(directory, "readers")).length, 0); + assert.equal(yield* local.withWrite(Effect.succeed("installed")), "installed"); + const writer = yield* childLease(directory, "withWrite"); + yield* Effect.promise(() => writer.held); + const queued = yield* local.withRead(Effect.never).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Fiber.interrupt(queued); + writer.child.stdin.end("release"); + yield* Effect.promise(() => writer.exited); + assert.equal(yield* local.withWrite(Effect.succeed("installed")), "installed"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("reclaims a dead reader whose PID has been reused", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) !== "linux") return; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const readers = NodePath.join(directory, "readers"); + NodeFS.mkdirSync(readers); + NodeFS.writeFileSync( + NodePath.join(readers, `${process.pid}-old-reader`), + "previous-process-start", + ); + assert.equal( + yield* HostLock.make(directory).withWrite(Effect.succeed("installed")), + "installed", + ); + assert.deepEqual(NodeFS.readdirSync(readers), []); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("reclaims a crashed server's turn and install leases", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + for (const mode of ["withRead", "withWrite"] as const) { + const holder = yield* childLease(directory, mode); + yield* Effect.promise(() => holder.held); + holder.child.kill("SIGKILL"); + yield* Effect.promise(() => holder.exited); + if (mode === "withWrite") { + // Advance the dead heartbeat, without sleeping through its stale interval. + NodeFS.utimesSync(NodePath.join(directory, "gate.lock"), 0, 0); + } + assert.equal( + yield* HostLock.make(directory).withWrite(Effect.succeed("installed")), + "installed", + ); + } + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/textGeneration/TextGeneration.test.ts b/apps/server/src/textGeneration/TextGeneration.test.ts index 61baaebe4dc20..49f445526d688 100644 --- a/apps/server/src/textGeneration/TextGeneration.test.ts +++ b/apps/server/src/textGeneration/TextGeneration.test.ts @@ -1,5 +1,9 @@ import { it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as Semaphore from "effect/Semaphore"; +import * as HostLock from "../provider/ProviderMaintenanceHostLock.ts"; import * as PubSub from "effect/PubSub"; import * as Result from "effect/Result"; import * as Stream from "effect/Stream"; @@ -177,3 +181,57 @@ describe("TextGeneration.make", () => { }), ); }); + +it.effect("holds the host lease until the text generation provider completes", () => + Effect.gen(function* () { + const permit = yield* Semaphore.make(1); + const entered = yield* Deferred.make(); + const complete = yield* Deferred.make(); + const installed = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("codex"); + const instance = makeStubInstance( + instanceId, + makeStubTextGeneration({ + generateBranchName: () => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(complete)), + Effect.as({ branch: "generated" }), + ), + }), + ); + const host = { + acquireRead: Effect.succeed(Effect.void), + withRead: permit.withPermits(1), + withWrite: permit.withPermits(1), + }; + const generation = yield* TextGeneration.make.pipe( + Effect.provideService( + ProviderInstanceRegistry.ProviderInstanceRegistry, + makeStubRegistry([instance]), + ), + Effect.provideService(HostLock.ProviderMaintenanceHostLock, host), + Effect.provide( + Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ + resolveLink: () => Effect.die("unused"), + }), + ), + ); + const request = yield* generation + .generateBranchName({ + cwd: process.cwd(), + message: "branch", + modelSelection: createModelSelection(instanceId, "gpt-5"), + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const install = yield* host + .withWrite(Deferred.succeed(installed, undefined)) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(yield* Deferred.isDone(installed)).toBe(false); + yield* Deferred.succeed(complete, undefined); + expect(yield* Fiber.join(request)).toEqual({ branch: "generated" }); + yield* Fiber.join(install); + expect(yield* Deferred.isDone(installed)).toBe(true); + }), +); diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index ce99d1812ceed..7ab79ed3fa0df 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -10,6 +10,7 @@ import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenance import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; import * as ThreadTitleLinks from "./ThreadTitleLinks.ts"; +import * as HostLock from "../provider/ProviderMaintenanceHostLock.ts"; export type { BranchNameGenerationInput, @@ -58,6 +59,7 @@ export const make = Effect.gen(function* () { const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; const sourceControl = yield* SourceControlProviderRegistry.SourceControlProviderRegistry; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; + const hostLock = yield* HostLock.ProviderMaintenanceHostLock; // Waits out a background install so a provider CLI is never launched mid-replacement. const instanceFor = (operation: TextGenerationOp, instanceId: ProviderInstanceId) => admission @@ -66,15 +68,21 @@ export const make = Effect.gen(function* () { return TextGeneration.of({ generateCommitMessage: (input) => instanceFor("generateCommitMessage", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => textGeneration.generateCommitMessage(input)), + Effect.flatMap((textGeneration) => + hostLock.withRead(textGeneration.generateCommitMessage(input)), + ), ), generatePrContent: (input) => instanceFor("generatePrContent", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => textGeneration.generatePrContent(input)), + Effect.flatMap((textGeneration) => + hostLock.withRead(textGeneration.generatePrContent(input)), + ), ), generateBranchName: (input) => instanceFor("generateBranchName", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => textGeneration.generateBranchName(input)), + Effect.flatMap((textGeneration) => + hostLock.withRead(textGeneration.generateBranchName(input)), + ), ), generateThreadTitle: (input) => instanceFor("generateThreadTitle", input.modelSelection.instanceId).pipe( @@ -88,7 +96,9 @@ export const make = Effect.gen(function* () { sourceControl, ), )); - return yield* textGeneration.generateThreadTitle({ ...input, linkedContext }); + return yield* hostLock.withRead( + textGeneration.generateThreadTitle({ ...input, linkedContext }), + ); }), ), ), From ee29b4290d9c4b14664ba97c1ccb7ba2761135ff Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:05:42 +0000 Subject: [PATCH 071/149] fix(server): every provider CLI launch waits for a running install Probes, auth, ACP spawns, OpenCode commands, managed refreshes and text generation now hold a shared launch permit that installs take exclusively, with reentry for an install's own verification. Launches still run side by side, so a probe or a title never delays a turn. Installs drain host leases before closing admission, and background installs run after the update window check releases admission, so a session holding a lease can always finish launching its CLI. The update window stops at the first blocker, rechecks recent interaction before installing, and leaves frequent scheduled tasks out of its query unless they run. Idle foreground mobile clients no longer count as interaction. --- apps/mobile/src/App.tsx | 3 +- apps/mobile/src/components/ComposerEditor.tsx | 2 + .../connection/background-activity-scopes.ts | 16 ++ .../connection/background-activity.test.ts | 35 +++ .../src/connection/background-activity.ts | 5 +- .../src/background/BackgroundPolicy.test.ts | 30 +++ .../CheckpointRollbackService.ts | 2 +- .../ProviderTurnStartService.ts | 2 +- .../src/provider/AcpRegistryProbe.test.ts | 34 +++ .../provider/AntigravityInstallation.test.ts | 28 +++ .../src/provider/AntigravityInstallation.ts | 26 ++- .../provider/ClaudeCapabilitiesProbe.test.ts | 35 +++ apps/server/src/provider/ClaudeProvider.ts | 5 + .../src/provider/CodexInstallation.test.ts | 27 +++ apps/server/src/provider/CodexInstallation.ts | 77 ++++--- .../server/src/provider/CodexProvider.test.ts | 34 +++ apps/server/src/provider/CodexProvider.ts | 107 ++++----- .../providerMaintenanceCommandCoordinator.ts | 15 +- .../providerMaintenanceRunner.test.ts | 208 ++++++++++++++++++ .../ScheduledTaskService.test.ts | 31 +++ .../scheduledTasks/ScheduledTaskService.ts | 8 +- .../src/textGeneration/TextGeneration.test.ts | 49 +++++ .../src/textGeneration/TextGeneration.ts | 26 +-- .../server/src/updates/ProviderAutoUpdater.ts | 29 ++- apps/server/src/updates/UpdateWindow.ts | 52 +++-- .../AcpRegistryRuntimeCoordinator.test.ts | 23 ++ .../server/AcpRegistryRuntimeCoordinator.ts | 7 +- .../src/server/auth.test.ts | 28 +++ .../provider-acp-registry/src/server/auth.ts | 9 +- .../provider-acp-registry/src/server/probe.ts | 9 + .../src/server/AcpSessionRuntime.ts | 5 + .../provider-core/src/server/maintenance.ts | 75 +++++++ .../src/server/managedProvider.test.ts | 45 ++++ .../src/server/managedProvider.ts | 8 +- .../src/server/snapshotProbe.test.ts | 150 +++++++++++++ .../provider-core/src/server/snapshotProbe.ts | 7 +- .../server/OpenCodeRuntime.inventory.test.ts | 24 ++ .../src/server/OpenCodeRuntime.ts | 4 + 38 files changed, 1121 insertions(+), 159 deletions(-) diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index 9be57907b358d..a911c1d0ae9d1 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -1,3 +1,4 @@ +import { recordMobileInteraction } from "./connection/background-activity-scopes"; import { PermissionUpdateNotice } from "./components/PermissionUpdateNotice"; import * as Linking from "expo-linking"; import * as SplashScreen from "expo-splash-screen"; @@ -81,7 +82,7 @@ function AppContent() { - + diff --git a/apps/mobile/src/components/ComposerEditor.tsx b/apps/mobile/src/components/ComposerEditor.tsx index 1eff2e8995008..ba63319d44f09 100644 --- a/apps/mobile/src/components/ComposerEditor.tsx +++ b/apps/mobile/src/components/ComposerEditor.tsx @@ -1,3 +1,4 @@ +import { recordMobileInteraction } from "../connection/background-activity-scopes"; import { ComposerContextId } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import { AsyncResult } from "effect/reactivity"; @@ -68,6 +69,7 @@ export function ComposerEditor({ text, draftKey ? getComposerDraftSnapshot(draftKey) : draft, ); + recordMobileInteraction(); props.onChangeText(text); if (draftKey) { setComposerDraftContext(draftKey, restored.context); diff --git a/apps/mobile/src/connection/background-activity-scopes.ts b/apps/mobile/src/connection/background-activity-scopes.ts index 2e2a343ff0d88..bd60d7ce669c1 100644 --- a/apps/mobile/src/connection/background-activity-scopes.ts +++ b/apps/mobile/src/connection/background-activity-scopes.ts @@ -2,6 +2,22 @@ import type { EnvironmentRpcSubscriptionObservation } from "@t3tools/client-runt import { type BackgroundScope, type EnvironmentId, WS_METHODS } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +const INTERACTION_WINDOW_MS = 30_000; +let lastInteractionAt = Number.NEGATIVE_INFINITY; +let lastInteractionReportAt = Number.NEGATIVE_INFINITY; + +export function recordMobileInteraction(): void { + lastInteractionAt = performance.now(); + if (lastInteractionAt - lastInteractionReportAt >= 1_000) { + lastInteractionReportAt = lastInteractionAt; + notify(); + } +} + +export function mobileRecentlyInteracted(now = performance.now()): boolean { + return now >= lastInteractionAt && now - lastInteractionAt < INTERACTION_WINDOW_MS; +} + interface RetainedScope { readonly environmentId: EnvironmentId; readonly scope: BackgroundScope; diff --git a/apps/mobile/src/connection/background-activity.test.ts b/apps/mobile/src/connection/background-activity.test.ts index 170ba4efa9c90..aa790e5a1e547 100644 --- a/apps/mobile/src/connection/background-activity.test.ts +++ b/apps/mobile/src/connection/background-activity.test.ts @@ -1,9 +1,12 @@ import { EnvironmentId, WS_METHODS } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import { vi } from "vite-plus/test"; import { onRetainedMobileBackgroundScopesChange, + recordMobileInteraction, + mobileRecentlyInteracted, observeMobileBackgroundActivitySubscription, retainedMobileBackgroundScopes, } from "./background-activity-scopes"; @@ -86,3 +89,35 @@ describe("mobile background activity", () => { }), ); }); + +it("expires real mobile interaction independently of foreground heartbeats", () => { + recordMobileInteraction(); + const now = performance.now(); + expect(mobileRecentlyInteracted(now)).toBe(true); + for (let elapsed = 25_000; elapsed <= 15 * 60_000; elapsed += 25_000) { + expect(mobileRecentlyInteracted(now + elapsed)).toBe(elapsed < 30_000); + } + recordMobileInteraction(); + expect(mobileRecentlyInteracted()).toBe(true); +}); + +it("reports continuous interaction without starving the reporter debounce", () => { + const start = performance.now() + 10_000; + const clock = vi.spyOn(performance, "now"); + let reports = 0; + const unsubscribe = onRetainedMobileBackgroundScopesChange(() => { + reports++; + }); + try { + for (let elapsed = 0; elapsed < 10_000; elapsed += 100) { + clock.mockReturnValue(start + elapsed); + recordMobileInteraction(); + } + expect(reports).toBe(10); + expect(mobileRecentlyInteracted(start + 10_000)).toBe(true); + expect(mobileRecentlyInteracted(start + 40_000)).toBe(false); + } finally { + unsubscribe(); + clock.mockRestore(); + } +}); diff --git a/apps/mobile/src/connection/background-activity.ts b/apps/mobile/src/connection/background-activity.ts index 80e185b9e05d3..64473c83b627a 100644 --- a/apps/mobile/src/connection/background-activity.ts +++ b/apps/mobile/src/connection/background-activity.ts @@ -19,6 +19,8 @@ import { AppState } from "react-native"; import * as MobileStorage from "../persistence/mobile-storage"; import { observeMobileBackgroundActivitySubscription, + mobileRecentlyInteracted, + recordMobileInteraction, onRetainedMobileBackgroundScopesChange, retainedMobileBackgroundScopes, } from "./background-activity-scopes"; @@ -71,7 +73,7 @@ export const layerReporter = Layer.effectDiscard( clientKind: "mobile", visible: active, focused: active, - recentlyInteracted: active, + recentlyInteracted: active && mobileRecentlyInteracted(), appState: normalizeAppState(appState), scopes: [ ...BASELINE_SCOPES, @@ -90,6 +92,7 @@ export const layerReporter = Layer.effectDiscard( Effect.sync(() => { const removeScopeListener = onRetainedMobileBackgroundScopesChange(requestReport); const subscription = AppState.addEventListener("change", (nextState) => { + if (nextState === "active" && appState !== "active") recordMobileInteraction(); appState = nextState; requestReport(); }); diff --git a/apps/server/src/background/BackgroundPolicy.test.ts b/apps/server/src/background/BackgroundPolicy.test.ts index 078e3441d8abb..71291e0ccc1cc 100644 --- a/apps/server/src/background/BackgroundPolicy.test.ts +++ b/apps/server/src/background/BackgroundPolicy.test.ts @@ -1,3 +1,5 @@ +import * as Duration from "effect/Duration"; +import * as TestClock from "effect/testing/TestClock"; import { assert, describe, it } from "@effect/vitest"; import { AuthSessionId, @@ -384,3 +386,31 @@ describe("BackgroundPolicy", () => { ), ); }); + +it.effect("an idle foreground mobile lease leaves the fifteen minute quiet period intact", () => + Effect.gen(function* () { + const policy = yield* BackgroundPolicy.BackgroundPolicy; + for (let report = 0; report < 36; report++) { + yield* policy.reportClientActivity( + AuthSessionId.make("mobile-session"), + RpcClientId.make(5), + makeReport({ clientKind: "mobile", appState: "active", recentlyInteracted: false }), + ); + yield* TestClock.adjust("25 seconds"); + } + assert.isTrue( + Duration.isGreaterThanOrEqualTo( + yield* policy.sinceLastClientInteraction, + Duration.minutes(15), + ), + ); + yield* policy.reportClientActivity( + AuthSessionId.make("mobile-session"), + RpcClientId.make(5), + makeReport({ clientKind: "mobile", recentlyInteracted: true }), + ); + assert.isTrue( + Duration.isLessThan(yield* policy.sinceLastClientInteraction, Duration.seconds(1)), + ); + }).pipe(Effect.provide(layerFor(nominalHostPower))), +); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 8efe1b732d20a..316951c317f23 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -186,7 +186,7 @@ export const layer: Layer.Layer< (candidate) => candidate.id === providerThread.providerSessionId, ); // Opening a session can launch the provider CLI; wait out a background install. - yield* admission.withPermit(Effect.void); + yield* admission.withLaunchPermit(Effect.void); const session = yield* sessions.open({ threadId: input.threadId, providerSessionId: providerThread.providerSessionId, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3fd3755d0426f..989c4260cf2fe 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -226,7 +226,7 @@ export const layer: Layer.Layer< // The run is durably starting before this effect is dispatched. Wait for // an admitted install, then reread: active-work blocks later installs, // without holding the permit through provider commands such as /compact. - yield* admission.withPermit(Effect.void); + yield* admission.withLaunchPermit(Effect.void); const projection = yield* projectionStore.getTurnStartContext(input.threadId, runId); const run = projection.runs.find((candidate) => candidate.id === runId); if (run === undefined) { diff --git a/apps/server/src/provider/AcpRegistryProbe.test.ts b/apps/server/src/provider/AcpRegistryProbe.test.ts index 7b69ca58baaaf..cb010b121e674 100644 --- a/apps/server/src/provider/AcpRegistryProbe.test.ts +++ b/apps/server/src/provider/AcpRegistryProbe.test.ts @@ -1,3 +1,5 @@ +import * as Layer from "effect/Layer"; +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; // @effect-diagnostics nodeBuiltinImport:off import * as NodePath from "node:path"; import * as NodeURL from "node:url"; @@ -604,3 +606,35 @@ describe("ACP Registry probe", () => { ).toBe("probe_failed"); }); }); + +it.effect( + "ACP discovery waits before resolving an installable CLI during provider maintenance", + () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + // Resolving an agent can install it. None of its dependencies may be read yet. + const probe = yield* probeAcpRegistryConfiguration({ + instanceId, + settings: decodeSettings({ agentId: "probe-agent" }), + cwd: "/workspace", + environment: {}, + }).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(probe.pollUnsafe() === undefined).toBe(true); + yield* Fiber.interrupt(probe); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + }).pipe( + Effect.provide( + Layer.mergeAll(NodeServices.layer, Layer.mock(AcpRegistrySupport.AcpRegistryCatalog)({})), + ), + ), +); diff --git a/apps/server/src/provider/AntigravityInstallation.test.ts b/apps/server/src/provider/AntigravityInstallation.test.ts index e99c2f684954a..15d71e68180ee 100644 --- a/apps/server/src/provider/AntigravityInstallation.test.ts +++ b/apps/server/src/provider/AntigravityInstallation.test.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -975,3 +976,30 @@ it.layer(NodeServices.layer)("Antigravity installation", (it) => { expect(resolveAntigravityReleaseAsset("freebsd", "x64")).toBeNull(); }); }); + +it.effect( + "Antigravity first installs wait for admission before changing state or downloading", + () => + Effect.gen(function* () { + const h = yield* makeHarness(); + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const install = yield* h.installation.start.pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(install.pollUnsafe() === undefined).toBe(true); + expect(h.requests.length).toBe(0); + expect((yield* h.installation.state).phase).toBe("idle"); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + yield* Fiber.join(install); + expect((yield* terminalState(h.installation)).phase).toBe("succeeded"); + expect(h.requests.length).toBe(1); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/AntigravityInstallation.ts b/apps/server/src/provider/AntigravityInstallation.ts index c57200f4afdad..09277fad47671 100644 --- a/apps/server/src/provider/AntigravityInstallation.ts +++ b/apps/server/src/provider/AntigravityInstallation.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; // @effect-diagnostics nodeBuiltinImport:off - Effect has no incremental digest or free-space query. import * as EffectNodeStream from "@effect/platform-node/NodeStream"; import { ProviderDriverKind, type ProviderInstallState } from "@t3tools/contracts"; @@ -146,6 +147,7 @@ function isRunning(state: ProviderInstallState) { export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.make")(function* ( options: AntigravityInstallationOptions, ) { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; @@ -753,7 +755,9 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); }); - const remove = Effect.fn("AntigravityInstallation.remove")( + const remove: AntigravityInstallationService["remove"] = Effect.fn( + "AntigravityInstallation.remove", + )( function* (protectedBinaryPaths: ReadonlyArray = []) { yield* gate.withPermit( Effect.gen(function* () { @@ -837,10 +841,26 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma managedDirectory, resolve, acquire, - start, + start: SubscriptionRef.get(state).pipe( + Effect.flatMap((current) => + isRunning(current) ? Effect.succeed(current) : admission.withPermit(start), + ), + ), cancel, state: SubscriptionRef.get(state), changes: SubscriptionRef.changes(state), - remove, + remove: (protectedBinaryPaths) => + SubscriptionRef.get(state).pipe( + Effect.flatMap((current) => + isRunning(current) + ? Effect.fail( + installationError( + "remove", + "Stop provider installation before removing its managed runtime.", + ), + ) + : admission.withPermit(remove(protectedBinaryPaths)), + ), + ), }); }); diff --git a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts index d617eb999fa61..1ebbae894caa6 100644 --- a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts +++ b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; // @effect-diagnostics nodeBuiltinImport:off - cleanup uses Node's retrying rm, which the FileSystem service does not expose. import * as ClaudeSdk from "@anthropic-ai/claude-agent-sdk"; import { vi } from "vite-plus/test"; @@ -374,3 +375,37 @@ it.effect("asks for usage without the local transcript scan", () => assert.deepEqual(usageOptions, { skipBehaviors: true }); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + +it.effect("keeps Claude SDK probes behind installation admission", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const query = vi.spyOn(ClaudeSdk, "query").mockImplementation( + () => + ({ + initializationResult: async () => ({ commands: [] }), + }) as unknown as ReturnType, + ); + yield* Effect.addFinalizer(() => Effect.sync(() => query.mockRestore())); + const probe = yield* probeClaudeCapabilities( + decodeClaudeSettings({ binaryPath: "claude" }), + undefined, + undefined, + false, + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.strictEqual(query.mock.calls.length, 0); + assert.isUndefined(probe.pollUnsafe()); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + yield* Fiber.join(probe); + assert.strictEqual(query.mock.calls.length, 1); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/ClaudeProvider.ts b/apps/server/src/provider/ClaudeProvider.ts index 8bc961834f270..2d1e3d0aea2f8 100644 --- a/apps/server/src/provider/ClaudeProvider.ts +++ b/apps/server/src/provider/ClaudeProvider.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import { type ClaudeSettings, type ModelCapabilities, @@ -446,6 +447,10 @@ const probeClaudeCapabilities = ( if (!abort.signal.aborted) abort.abort(); }), ), + (effect) => + Effect.flatMap(Maintenance.ProviderMaintenanceAdmission, (admission) => + admission.withLaunchPermit(effect), + ), Effect.result, Effect.map((result) => (Result.isSuccess(result) ? result.success : undefined)), ); diff --git a/apps/server/src/provider/CodexInstallation.test.ts b/apps/server/src/provider/CodexInstallation.test.ts index abc67ad55ea18..c3c089361ddf1 100644 --- a/apps/server/src/provider/CodexInstallation.test.ts +++ b/apps/server/src/provider/CodexInstallation.test.ts @@ -1,3 +1,5 @@ +import * as Fiber from "effect/Fiber"; +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as ModelManifest from "./ModelManifest.ts"; import { expect, it } from "@effect/vitest"; @@ -358,3 +360,28 @@ it("publishes complete packages for all supported platforms", () => { ); expect(CodexInstallation.resolveCodexReleaseAsset("linux", "riscv64")).toBeNull(); }); + +it.effect("Codex first installs wait for admission before changing state or downloading", () => + Effect.gen(function* () { + const h = yield* makeHarness(); + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const install = yield* h.installation.start.pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(install.pollUnsafe() === undefined).toBe(true); + expect(h.downloads()).toBe(0); + expect((yield* h.installation.state).phase).toBe("idle"); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + yield* Fiber.join(install); + expect((yield* terminalState(h.installation)).phase).toBe("succeeded"); + expect(h.downloads()).toBe(1); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/CodexInstallation.ts b/apps/server/src/provider/CodexInstallation.ts index fa7876b1e92a5..a89d94a11fad6 100644 --- a/apps/server/src/provider/CodexInstallation.ts +++ b/apps/server/src/provider/CodexInstallation.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; // @effect-diagnostics nodeBuiltinImport:off - Effect has no incremental digest. import { ProviderDriverKind, type ProviderInstallState } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -302,31 +303,33 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function }), ); }); - const runCommand = Effect.fn("CodexInstallation.runCommand")(function* ( - command: string, - args: ReadonlyArray, - ) { - const resolved = yield* resolveSpawnCommand(command, args).pipe( - Effect.provideService(HostProcess.Platform, platform), - ); - const child = yield* spawner.spawn( - ChildProcess.make(resolved.command, resolved.args, { shell: resolved.shell }), - ); - const [output, , exitCode] = yield* Effect.all( - [ - child.stdout.pipe(Stream.decodeText(), Stream.mkString), - child.stderr.pipe(Stream.runDrain), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - if (exitCode !== 0) - return yield* installationError( - "verify", - "Could not unpack or start the downloaded Codex package.", + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const runCommand = Effect.fn("CodexInstallation.runCommand")( + function* (command: string, args: ReadonlyArray) { + const resolved = yield* resolveSpawnCommand(command, args).pipe( + Effect.provideService(HostProcess.Platform, platform), + ); + const child = yield* spawner.spawn( + ChildProcess.make(resolved.command, resolved.args, { shell: resolved.shell }), ); - return output; - }, Effect.scoped); + const [output, , exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText(), Stream.mkString), + child.stderr.pipe(Stream.runDrain), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (exitCode !== 0) + return yield* installationError( + "verify", + "Could not unpack or start the downloaded Codex package.", + ); + return output; + }, + Effect.scoped, + admission.withLaunchPermit, + ); const localCache = new Map(); const resolveLocal = Effect.fn("CodexInstallation.resolveLocal")( function* () { @@ -640,7 +643,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ); }); - const remove = Effect.fn("CodexInstallation.remove")( + const remove: CodexInstallationService["remove"] = Effect.fn("CodexInstallation.remove")( function* (protectedBinaryPaths: ReadonlyArray = []) { yield* gate.withPermit( Effect.gen(function* () { @@ -723,12 +726,28 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function return CodexInstallation.of({ managedDirectory, - resolve, - acquire, - start, + resolve: (...args) => admission.withLaunchPermit(resolve(...args)), + acquire: (...args) => admission.withLaunchPermit(acquire(...args)), + start: SubscriptionRef.get(state).pipe( + Effect.flatMap((current) => + isRunning(current) ? Effect.succeed(current) : admission.withPermit(start), + ), + ), cancel, state: SubscriptionRef.get(state), changes: SubscriptionRef.changes(state), - remove, + remove: (protectedBinaryPaths) => + SubscriptionRef.get(state).pipe( + Effect.flatMap((current) => + isRunning(current) + ? Effect.fail( + installationError( + "remove", + "Stop provider installation before removing its managed runtime.", + ), + ) + : admission.withPermit(remove(protectedBinaryPaths)), + ), + ), }); }); diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index 2c67b6f144dbb..9438fa6d6f1fb 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -1,3 +1,10 @@ +import * as Layer from "effect/Layer"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; +import { withCodexAppServerClient } from "./CodexProvider.ts"; import { assert, it } from "@effect/vitest"; import { applyPreferredCodexDefaultModel, mapCodexModelCapabilities } from "./CodexProvider.ts"; @@ -171,3 +178,30 @@ it("ignores custom models that shadow a preferred slug", () => { assert.deepStrictEqual(models.find((model) => model.isDefault)?.slug, "gpt-5.4"); }); + +it.effect("Codex account model and skills clients wait before resolving or launching a CLI", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + // No process or home service exists: reaching either before admission is a defect. + const client = yield* Effect.scoped( + withCodexAppServerClient({ + binaryPath: "codex", + homePath: "~/.codex", + cwd: "/workspace", + }), + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isTrue(client.pollUnsafe() === undefined); + yield* Fiber.interrupt(client); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + }).pipe(Effect.provide(Layer.mock(ChildProcessSpawner.ChildProcessSpawner)({}))), +); diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 5109bb840f561..e9816055b7608 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -366,57 +367,63 @@ export function buildCodexInitializeParams(): CodexSchema.V1InitializeParams { * the caller's scope closes. Shared by the status probe, the skills probe, * and account-level requests such as reset-credit redemption. */ -export const withCodexAppServerClient = Effect.fn("withCodexAppServerClient")(function* (input: { - readonly binaryPath: string; - readonly homePath?: string | undefined; - readonly launchArgs?: string | undefined; - readonly cwd: string; - readonly environment?: NodeJS.ProcessEnv | undefined; -}) { - // `~` is not shell-expanded when env vars are set via `child_process.spawn`, - // so `CODEX_HOME=~/.codex_work` would reach codex verbatim and trip - // "CODEX_HOME points to '~/.codex_work', but that path does not exist". - // Expand here for parity with `CodexTextGeneration`. - const resolvedHomePath = input.homePath - ? expandHomePath(input.homePath, yield* HostProcess.HomeDirectory) - : undefined; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const environment = { - ...input.environment, - ...(resolvedHomePath ? { CODEX_HOME: resolvedHomePath } : {}), - }; - const spawnCommand = yield* resolveSpawnCommand( - input.binaryPath, - codexAppServerArgs(input.launchArgs), - { env: environment, extendEnv: true }, - ); - const child = yield* spawner - .spawn( - ChildProcess.make(spawnCommand.command, spawnCommand.args, { - cwd: input.cwd, - env: environment, - extendEnv: true, - forceKillAfter: CODEX_APP_SERVER_PROBE_FORCE_KILL_AFTER, - shell: spawnCommand.shell, - }), - ) - .pipe( - Effect.mapError( - (cause) => - new CodexErrors.CodexAppServerSpawnError({ - command: `${input.binaryPath} app-server`, - cause, - }), - ), +export const withCodexAppServerClient = Effect.fn("withCodexAppServerClient")( + function* (input: { + readonly binaryPath: string; + readonly homePath?: string | undefined; + readonly launchArgs?: string | undefined; + readonly cwd: string; + readonly environment?: NodeJS.ProcessEnv | undefined; + }) { + // `~` is not shell-expanded when env vars are set via `child_process.spawn`, + // so `CODEX_HOME=~/.codex_work` would reach codex verbatim and trip + // "CODEX_HOME points to '~/.codex_work', but that path does not exist". + // Expand here for parity with `CodexTextGeneration`. + const resolvedHomePath = input.homePath + ? expandHomePath(input.homePath, yield* HostProcess.HomeDirectory) + : undefined; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const environment = { + ...input.environment, + ...(resolvedHomePath ? { CODEX_HOME: resolvedHomePath } : {}), + }; + const spawnCommand = yield* resolveSpawnCommand( + input.binaryPath, + codexAppServerArgs(input.launchArgs), + { env: environment, extendEnv: true }, ); - const clientContext = yield* Layer.build(CodexClient.layerChildProcess(child)); - const client = yield* Effect.service(CodexClient.CodexAppServerClient).pipe( - Effect.provide(clientContext), - ); - const initialize = yield* client.request("initialize", buildCodexInitializeParams()); - yield* client.notify("initialized", undefined); - return { client, initialize }; -}); + const child = yield* spawner + .spawn( + ChildProcess.make(spawnCommand.command, spawnCommand.args, { + cwd: input.cwd, + env: environment, + extendEnv: true, + forceKillAfter: CODEX_APP_SERVER_PROBE_FORCE_KILL_AFTER, + shell: spawnCommand.shell, + }), + ) + .pipe( + Effect.mapError( + (cause) => + new CodexErrors.CodexAppServerSpawnError({ + command: `${input.binaryPath} app-server`, + cause, + }), + ), + ); + const clientContext = yield* Layer.build(CodexClient.layerChildProcess(child)); + const client = yield* Effect.service(CodexClient.CodexAppServerClient).pipe( + Effect.provide(clientContext), + ); + const initialize = yield* client.request("initialize", buildCodexInitializeParams()); + yield* client.notify("initialized", undefined); + return { client, initialize }; + }, + (effect) => + Effect.flatMap(Maintenance.ProviderMaintenanceAdmission, (admission) => + admission.withScopedLaunchPermit(effect), + ), +); const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(function* (input: { readonly binaryPath: string; diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index 95c6ad5d9027a..c2d3cb5511d1a 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -1,16 +1,10 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; -import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Ref from "effect/Ref"; -import * as Semaphore from "effect/Semaphore"; import * as HostLock from "./ProviderMaintenanceHostLock.ts"; -/** One permit per process: background installs hold it, provider turn starts wait for it. */ -export class ProviderMaintenanceAdmission extends Context.Reference<{ - readonly withPermit: (effect: Effect.Effect) => Effect.Effect; -}>("t3/provider/providerMaintenanceCommandCoordinator/ProviderMaintenanceAdmission", { - defaultValue: () => ({ withPermit: Semaphore.makeUnsafe(1).withPermits(1) }), -}) {} +export { ProviderMaintenanceAdmission } from "@t3tools/provider-core/server/maintenance"; export interface ProviderMaintenanceCommandCoordinatorShape { readonly withCommandLock: (input: { @@ -25,6 +19,7 @@ export interface ProviderMaintenanceCommandCoordinatorShape { export const makeProviderMaintenanceCommandCoordinator = Effect.fn( "makeProviderMaintenanceCommandCoordinator", )(function* (input: { readonly makeAlreadyRunningError: (targetKey: string) => E }) { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; const runningTargetsRef = yield* Ref.make>(new Set()); const locks = yield* KeyedLock.make(); const hostLock = yield* HostLock.ProviderMaintenanceHostLock; @@ -61,7 +56,9 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( } return yield* (onQueued ?? Effect.void).pipe( - Effect.andThen(locks.withLock(lockKey, hostLock.withWrite(run))), + // Host leases drain before admission closes: a session holding a lease + // may still need admission to finish launching its CLI. + Effect.andThen(locks.withLock(lockKey, hostLock.withWrite(admission.withPermit(run)))), Effect.onInterrupt(() => onInterrupted ?? Effect.void), Effect.ensuring(releaseTarget(targetKey)), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 975c5fc11a4e2..3f6e26229239d 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -1,4 +1,11 @@ // @effect-diagnostics nodeBuiltinImport:off - Separate processes exercise host-wide maintenance leases. +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as Option from "effect/Option"; +import * as ThreadContracts from "@t3tools/contracts"; import { describe, it, assert } from "@effect/vitest"; import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; @@ -1646,6 +1653,30 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live("drains host leases before an install closes CLI admission", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const release = yield* HostLock.make(directory).acquireRead; + const entered = yield* Deferred.make(); + const gate = waitForGate(directory); + const install = yield* coordinatedWrite( + directory, + Deferred.succeed(entered, undefined), + ).pipe(Effect.forkScoped); + yield* gate; + // A session holding a lease can still launch its CLI while the install waits. + assert.equal(yield* admission.withLaunchPermit(Effect.succeed("launched")), "launched"); + assert.isFalse(yield* Deferred.isDone(entered)); + yield* release; + yield* Fiber.join(install); + assert.isTrue(yield* Deferred.isDone(entered)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("reclaims a dead reader whose PID has been reused", () => Effect.scoped( Effect.gen(function* () { @@ -1690,3 +1721,180 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); }); +describe("update window short circuit", () => { + it.effect("does not query threads, providers or tasks during the quiet period", () => + Effect.gen(function* () { + const context = yield* Layer.build( + UpdateWindow.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({}), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.succeed(Duration.minutes(14)), + }), + SqlitePersistence.layerMemory, + Layer.mock(ServerSettings.ServerSettingsService)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + ), + ), + ), + ); + const window = yield* Effect.service(UpdateWindow.UpdateWindow).pipe(Effect.provide(context)); + let ran = false; + assert.isTrue( + Option.isNone( + yield* window.runIfOpen( + Effect.sync(() => { + ran = true; + }), + ), + ), + ); + assert.isFalse(ran); + }), + ); + + it.effect("stops at active work after exactly fifteen quiet minutes", () => + Effect.gen(function* () { + let reads = 0; + const context = yield* Layer.build( + UpdateWindow.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => + Effect.sync(() => { + reads++; + return [ThreadContracts.ThreadId.make("busy")]; + }), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.succeed(Duration.minutes(15)), + }), + SqlitePersistence.layerMemory, + Layer.mock(ServerSettings.ServerSettingsService)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + ), + ), + ), + ); + const window = yield* Effect.service(UpdateWindow.UpdateWindow).pipe(Effect.provide(context)); + assert.isTrue(Option.isNone(yield* window.runIfOpen(Effect.die("must not install")))); + assert.strictEqual(reads, 1); + }), + ); + + it.effect("stops at provider background work before reading update state or SQL", () => + Effect.gen(function* () { + const context = yield* Layer.build( + UpdateWindow.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([]), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + hasPendingBackgroundWork: Effect.succeed(true), + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.succeed(Duration.minutes(15)), + }), + SqlitePersistence.layerMemory, + Layer.mock(ServerSettings.ServerSettingsService)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + ), + ), + ), + ); + const window = yield* Effect.service(UpdateWindow.UpdateWindow).pipe(Effect.provide(context)); + assert.isTrue(Option.isNone(yield* window.runIfOpen(Effect.die("must not install")))); + }), + ); +}); + +it.effect("manual updates wait for admission and verify under the same permit", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const { registry } = yield* makeRegistry(baseNativeCliProvider); + const updater = yield* makeTestRunner({ + ...registry, + refreshInstance: () => admission.withPermit(registry.getProviders), + }); + const update = yield* updater.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isTrue(update.pollUnsafe() === undefined); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + const result = yield* Fiber.join(update); + assert.strictEqual(result.providers[0]?.updateState?.status, "succeeded"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner(() => ({ stdout: "updated" })), + ), + ), + ), +); + +it.effect.each([ + { + name: "open provider window", + interaction: false, + expected: true, + }, + { + name: "interaction arriving during blocker reads", + interaction: true, + expected: false, + }, +])("rechecks the update window for $name", (scenario) => + Effect.gen(function* () { + let quietReads = 0; + let installs = 0; + const context = yield* Layer.build( + UpdateWindow.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([]), + getLimitRecoveryCandidates: () => Effect.succeed([]), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + hasPendingBackgroundWork: Effect.succeed(false), + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.sync(() => + Duration.minutes(++quietReads > 1 && scenario.interaction ? 0 : 15), + ), + }), + SqlitePersistence.layerMemory, + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.succeed(ThreadContracts.DEFAULT_SERVER_SETTINGS), + }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + ), + ), + ), + ); + const window = yield* Effect.service(UpdateWindow.UpdateWindow).pipe(Effect.provide(context)); + const result = yield* window.runIfOpen( + Effect.sync(() => { + installs++; + }), + ); + assert.strictEqual(Option.isSome(result), scenario.expected); + assert.strictEqual(installs, scenario.expected ? 1 : 0); + }), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 1c95fecdea795..18599719fbd25 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -351,3 +351,34 @@ it.effect( ); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect("update windows exclude frequent due intervals but retain their running tasks", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-09-09T12:00:00.000Z"; + for (const row of [ + { id: "frequent", next: now, enabled: 1, status: "never" }, + { id: "running", next: null, enabled: 0, status: "running" }, + { + id: "slow", + next: now, + enabled: 1, + status: "never", + scheduleJson: '{"type":"interval","everyMs":900000}', + }, + { + id: "later", + next: "2026-09-09T13:00:00.000Z", + enabled: 1, + status: "never", + scheduleJson: '{"type":"interval","everyMs":900000}', + }, + ]) + yield* insertRow(sql, row, now); + const tasks = yield* ScheduledTaskService.listRunningOrDueTasks( + DateTime.makeUnsafe(now), + 900000, + ); + assert.deepEqual(tasks.map((task) => task.id).sort(), ["running", "slow"]); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 749dace7ef464..775247b48c5dd 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -396,12 +396,16 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct /** Tasks running now or due by `by`, for background work that must not start under them. */ export const listRunningOrDueTasks = Effect.fn("ScheduledTaskService.listRunningOrDueTasks")( - function* (by: DateTime.DateTime) { + function* (by: DateTime.DateTime, minimumIntervalMs = 0) { const sql = yield* SqlClient.SqlClient; const rows = yield* sql` SELECT * FROM scheduled_tasks WHERE last_run_status = 'running' - OR (enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ${iso(by)}) + OR (enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ${iso(by)} + AND CASE WHEN json_valid(schedule_json) THEN + json_extract(schedule_json, '$.type') IS NOT 'interval' + OR json_extract(schedule_json, '$.everyMs') >= ${minimumIntervalMs} + ELSE 1 END) `; return yield* decodeTaskRowsLeniently(rows); }, diff --git a/apps/server/src/textGeneration/TextGeneration.test.ts b/apps/server/src/textGeneration/TextGeneration.test.ts index 49f445526d688..5f75c8b168327 100644 --- a/apps/server/src/textGeneration/TextGeneration.test.ts +++ b/apps/server/src/textGeneration/TextGeneration.test.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import { it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Deferred from "effect/Deferred"; @@ -235,3 +236,51 @@ it.effect("holds the host lease until the text generation provider completes", ( expect(yield* Deferred.isDone(installed)).toBe(true); }), ); + +it.effect("holds shared admission through CLI-backed text generation", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const instanceId = ProviderInstanceId.make("codex"); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const instance = makeStubInstance( + instanceId, + makeStubTextGeneration({ + generateBranchName: () => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as({ branch: "generated" }), + ), + }), + ); + const generation = yield* TextGeneration.make.pipe( + Effect.provideService( + ProviderInstanceRegistry.ProviderInstanceRegistry, + makeStubRegistry([instance]), + ), + Effect.provide( + Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ + resolveLink: () => undefined, + }), + ), + ); + const running = yield* generation + .generateBranchName({ + cwd: "/workspace", + message: "make a branch", + modelSelection: createModelSelection(instanceId, "gpt-5"), + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + // Launches share admission, so a long generation never delays a turn start. + expect(yield* admission.withLaunchPermit(Effect.succeed("turn"))).toBe("turn"); + const install = yield* admission + .withPermit(Effect.succeed("installed")) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(install.pollUnsafe() === undefined).toBe(true); + yield* Deferred.succeed(release, undefined); + expect(yield* Fiber.join(running)).toEqual({ branch: "generated" }); + expect(yield* Fiber.join(install)).toBe("installed"); + }), +); diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index 7ab79ed3fa0df..0986dab050eba 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -6,7 +6,7 @@ import type { ProviderTextGeneration } from "@t3tools/provider-core/server/textG import { TextGenerationError } from "@t3tools/contracts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; -import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; +import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; import * as ThreadTitleLinks from "./ThreadTitleLinks.ts"; @@ -60,29 +60,23 @@ export const make = Effect.gen(function* () { const sourceControl = yield* SourceControlProviderRegistry.SourceControlProviderRegistry; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const hostLock = yield* HostLock.ProviderMaintenanceHostLock; - // Waits out a background install so a provider CLI is never launched mid-replacement. + // The host lease is taken first: an install drains leases before it closes admission. + const launch = (effect: Effect.Effect) => + hostLock.withRead(admission.withLaunchPermit(effect)); const instanceFor = (operation: TextGenerationOp, instanceId: ProviderInstanceId) => - admission - .withPermit(Effect.void) - .pipe(Effect.andThen(resolveInstance(registry, operation, instanceId))); + resolveInstance(registry, operation, instanceId); return TextGeneration.of({ generateCommitMessage: (input) => instanceFor("generateCommitMessage", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => - hostLock.withRead(textGeneration.generateCommitMessage(input)), - ), + Effect.flatMap((textGeneration) => launch(textGeneration.generateCommitMessage(input))), ), generatePrContent: (input) => instanceFor("generatePrContent", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => - hostLock.withRead(textGeneration.generatePrContent(input)), - ), + Effect.flatMap((textGeneration) => launch(textGeneration.generatePrContent(input))), ), generateBranchName: (input) => instanceFor("generateBranchName", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => - hostLock.withRead(textGeneration.generateBranchName(input)), - ), + Effect.flatMap((textGeneration) => launch(textGeneration.generateBranchName(input))), ), generateThreadTitle: (input) => instanceFor("generateThreadTitle", input.modelSelection.instanceId).pipe( @@ -96,9 +90,7 @@ export const make = Effect.gen(function* () { sourceControl, ), )); - return yield* hostLock.withRead( - textGeneration.generateThreadTitle({ ...input, linkedContext }), - ); + return yield* launch(textGeneration.generateThreadTitle({ ...input, linkedContext })); }), ), ), diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index 952d4e33e687e..881ec26e22ec4 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -55,7 +55,7 @@ export const layer = Layer.effectDiscard( if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; const installations = new Set(); for (const { instanceId } of yield* providers.getProviders) { - const install = Effect.gen(function* () { + const pick = Effect.gen(function* () { const settings = yield* serverSettings.getSettings; if (!settings.automaticUpdates || !settings.enableProviderUpdateChecks) return; const current = (yield* providers.getProviders).find( @@ -85,22 +85,21 @@ export const layer = Layer.effectDiscard( fromVersion: current.version, targetVersion, }); - yield* runner - .updateProvider({ - provider: current.driver, - instanceId, - targetVersion, - quiet: true, - background: true, - }) - .pipe( - Effect.catchCause((cause) => - Effect.logWarning("Background provider update failed", { instanceId, cause }), - ), - ); + return { provider: current.driver, targetVersion }; }); // Rechecked per install: the previous one may have run for minutes. - if (Option.isNone(yield* updateWindow.runIfOpen(install))) return; + const picked = yield* updateWindow.runIfOpen(pick); + if (Option.isNone(picked)) return; + if (picked.value === undefined) continue; + // Installed after the window releases admission: the runner first drains host + // leases, and a session holding one may still need admission to launch its CLI. + yield* runner + .updateProvider({ ...picked.value, instanceId, quiet: true, background: true }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("Background provider update failed", { instanceId, cause }), + ), + ); } }).pipe( Effect.catchCause((cause) => diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 9f29e92a70ee6..f9e3566354571 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -9,7 +9,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; -import * as ProviderMaintenanceCoordinator from "../provider/providerMaintenanceCommandCoordinator.ts"; +import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -18,7 +18,6 @@ import * as ServerSettings from "../serverSettings.ts"; const UPDATE_LOOKAHEAD = Duration.minutes(5); /** * How long every client must be untouched before an update may interrupt it. - * Clients report activity about every 25s, so this must stay well above that. */ const QUIET_PERIOD = Duration.minutes(15); /** @@ -59,7 +58,10 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const blockers = Effect.fn("updates.UpdateWindow.blockers")(function* () { const now = yield* DateTime.now; const horizon = DateTime.addDuration(now, UPDATE_LOOKAHEAD); - const found: Array = []; + + if (Duration.isLessThan(yield* backgroundPolicy.sinceLastClientInteraction, QUIET_PERIOD)) { + return ["recent-interaction"]; + } // A failed read keeps the window closed: guessing "idle" could cut off a turn. const activeThreadIds = yield* projections.getRecoveryThreadIds("active-work").pipe( @@ -69,37 +71,36 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { Effect.option, ); if (Option.isNone(activeThreadIds)) return ["active-threads"]; - if (activeThreadIds.value.length > 0) found.push("active-threads"); + if (activeThreadIds.value.length > 0) return ["active-threads"]; // Work a provider keeps running after its turn ends, such as background commands. - if (yield* sessions.hasPendingBackgroundWork) found.push("background-work"); + if (yield* sessions.hasPendingBackgroundWork) return ["background-work"]; // Includes installs someone started by hand. const providerUpdating = (yield* providers.getProviders).some( (provider) => provider.updateState?.status === "queued" || provider.updateState?.status === "running", ); - if (providerUpdating) found.push("provider-update"); - - if (Duration.isLessThan(yield* backgroundPolicy.sinceLastClientInteraction, QUIET_PERIOD)) { - found.push("recent-interaction"); - } + if (providerUpdating) return ["provider-update"]; // Corrupt task rows are skipped, so one bad row cannot close the window for good. - const tasks = yield* ScheduledTaskService.listRunningOrDueTasks(horizon).pipe( + const tasks = yield* ScheduledTaskService.listRunningOrDueTasks( + horizon, + Duration.toMillis(FREQUENT_TASK_INTERVAL), + ).pipe( Effect.provideService(SqlClient.SqlClient, sql), Effect.tapError((cause) => Effect.logWarning("Update window could not read scheduled tasks", { cause }), ), Effect.option, ); - if (Option.isNone(tasks)) return [...found, "scheduled-task"]; + if (Option.isNone(tasks)) return ["scheduled-task"]; for (const task of tasks.value) { if ( task.lastRunStatus === "running" || task.schedule.type !== "interval" || task.schedule.everyMs >= Duration.toMillis(FREQUENT_TASK_INTERVAL) ) { - found.push("scheduled-task"); + return ["scheduled-task"]; } } @@ -109,7 +110,7 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { ), Effect.option, ); - if (Option.isNone(settings)) return [...found, "settings"]; + if (Option.isNone(settings)) return ["settings"]; const limited = yield* projections .getLimitRecoveryCandidates({ now: horizon, @@ -122,7 +123,7 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { ), Effect.option, ); - if (Option.isNone(limited)) return [...found, "usage-limit-resume"]; + if (Option.isNone(limited)) return ["usage-limit-resume"]; for (const thread of limited.value) { if (thread.usageLimitResetAt === null || thread.usageLimitResetAt === undefined) continue; const resetAt = DateTime.make(thread.usageLimitResetAt); @@ -138,22 +139,29 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { DateTime.isGreaterThan(resumesAt.value, now) && !DateTime.isGreaterThan(resumesAt.value, horizon) ) { - found.push("usage-limit-resume"); + return ["usage-limit-resume"]; } } - return found; + return []; }); const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (install) => admission.withPermit( blockers().pipe( Effect.flatMap((found) => - found.length === 0 - ? Effect.asSome(install) - : Effect.logDebug("Background update waiting for the update window", { - blockers: [...new Set(found)], - }).pipe(Effect.as(Option.none())), + found.length > 0 + ? Effect.logDebug("Background update waiting for the update window", { + blockers: found, + }).pipe(Effect.as(Option.none())) + : // Interaction may have arrived during the blocker reads. + backgroundPolicy.sinceLastClientInteraction.pipe( + Effect.flatMap((quiet) => + Duration.isLessThan(quiet, QUIET_PERIOD) + ? Effect.succeed(Option.none()) + : Effect.asSome(install), + ), + ), ), ), ); diff --git a/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.test.ts b/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.test.ts index 6fd3d627f05fa..422175847cac4 100644 --- a/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.test.ts +++ b/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.test.ts @@ -5,11 +5,34 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import * as TestClock from "effect/testing/TestClock"; +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import type { AcpRegistryAvailableCommands, AcpRegistryLiveConfiguration } from "./probe.ts"; import * as AcpRegistryRuntimeCoordinator from "./AcpRegistryRuntimeCoordinator.ts"; describe("AcpRegistryRuntimeCoordinator", () => { + it.effect("cancels background admission waits when foreground startup begins", () => + Effect.gen(function* () { + const coordinator = yield* AcpRegistryRuntimeCoordinator.AcpRegistryRuntimeCoordinator; + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const probe = yield* coordinator + .runBackgroundProbe("kilo", Effect.die("must not probe")) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* coordinator.withForegroundStartup("kilo", Effect.void); + expect(Option.isNone(yield* Fiber.join(probe))).toBe(true); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + }).pipe(Effect.provide(AcpRegistryRuntimeCoordinator.layer)), + ); it.effect("suppresses a background probe while foreground startup is active", () => Effect.gen(function* () { const coordinator = yield* AcpRegistryRuntimeCoordinator.AcpRegistryRuntimeCoordinator; diff --git a/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.ts b/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.ts index ef86c87fa1134..c18e537090f6a 100644 --- a/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.ts +++ b/packages/provider-acp-registry/src/server/AcpRegistryRuntimeCoordinator.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import { type AcpRegistryAcceptUrlAuthInput, type AcpRegistryUrlAuthAction, @@ -99,6 +100,7 @@ export class AcpRegistryRuntimeCoordinator extends Context.Service< >()("@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator") {} const make = Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; const foregroundStarts = yield* PubSub.unbounded(); const activeForegroundCounts = yield* Ref.make(new Map()); const availableCommandsUpdates = yield* PubSub.unbounded(); @@ -342,8 +344,9 @@ const make = Effect.gen(function* () { return AcpRegistryRuntimeCoordinator.of({ withForegroundStartup, - runBackgroundProbe, - withSessionMutation, + runBackgroundProbe: (agentId, effect) => + runBackgroundProbe(agentId, admission.withLaunchPermit(effect)), + withSessionMutation: (effect) => admission.withLaunchPermit(withSessionMutation(effect)), clearAvailableCommands, publishAvailableCommands, getAvailableCommands, diff --git a/packages/provider-acp-registry/src/server/auth.test.ts b/packages/provider-acp-registry/src/server/auth.test.ts index 5a9fc049c4937..0ea5ff22bab6f 100644 --- a/packages/provider-acp-registry/src/server/auth.test.ts +++ b/packages/provider-acp-registry/src/server/auth.test.ts @@ -1,3 +1,5 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; +import * as Fiber from "effect/Fiber"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { ProviderInstanceId, type ProviderAuthState } from "@t3tools/contracts"; @@ -362,3 +364,29 @@ it.effect("refreshes advertised methods after a registry version change without assert.deepEqual(h.closed, [1, 2]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + +it.effect("ACP auth discovery waits for the install permit before constructing its runtime", () => + Effect.gen(function* () { + const h = yield* makeHarness(browserMethod); + h.setVersion("2.0.0"); + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const before = h.runtimes(); + const refresh = yield* h.controller.refreshMethods!.pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.strictEqual(h.runtimes(), before); + assert.isTrue(refresh.pollUnsafe() === undefined); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + yield* Fiber.join(refresh); + yield* h.state((value) => value.methods?.[0]?.name === "Browser 2.0.0"); + assert.strictEqual(h.runtimes(), before + 1); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/packages/provider-acp-registry/src/server/auth.ts b/packages/provider-acp-registry/src/server/auth.ts index ded5117c37b52..b03c07f5ca18a 100644 --- a/packages/provider-acp-registry/src/server/auth.ts +++ b/packages/provider-acp-registry/src/server/auth.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import { ProviderSetupError, type ProviderAuthMethod, @@ -38,6 +39,7 @@ export const makeAcpRegistryAuth = Effect.fn("makeAcpRegistryAuth")(function* (o spawn: AcpSessionRuntime.AcpSpawnInput, ) => Effect.Effect; }) { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; const catalog = yield* AcpRegistrySupport.AcpRegistryCatalog; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const crypto = yield* Crypto.Crypto; @@ -362,9 +364,10 @@ export const makeAcpRegistryAuth = Effect.fn("makeAcpRegistryAuth")(function* (o // ACP doesn't advertise its credential scope. Conservatively treat all // instances of the same agent on this environment as sharing credentials. credentialBinding: { owner: "provider", key: `acp:${agentKey}` }, - methods, + methods: admission.withLaunchPermit(methods), ...(options.settings.authMethodId ? { defaultMethodId: options.settings.authMethodId } : {}), - authenticate, - logout: signOut, + authenticate: (methodId, context) => + admission.withLaunchPermit(authenticate(methodId, context)), + logout: admission.withLaunchPermit(signOut), }); }); diff --git a/packages/provider-acp-registry/src/server/probe.ts b/packages/provider-acp-registry/src/server/probe.ts index e8cead21804fa..d6684dc21bedb 100644 --- a/packages/provider-acp-registry/src/server/probe.ts +++ b/packages/provider-acp-registry/src/server/probe.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import { AcpRegistryListProvidersResult, AcpRegistryOperationError, @@ -450,6 +451,10 @@ export const probeAcpRegistryConfiguration = Effect.fn("AcpRegistryProbe.probeCo skills: result.commands.skills, }; }, + (effect) => + Effect.flatMap(Maintenance.ProviderMaintenanceAdmission, (admission) => + admission.withLaunchPermit(effect), + ), ); const MANAGEMENT_TIMEOUT = Duration.seconds(60); @@ -520,6 +525,10 @@ const makeAcpRegistryManagementRuntime = Effect.fn("AcpRegistryProbe.makeManagem ); return runtime; }, + (effect) => + Effect.flatMap(Maintenance.ProviderMaintenanceAdmission, (admission) => + admission.withScopedLaunchPermit(effect), + ), ); function managementFailure( diff --git a/packages/provider-acp/src/server/AcpSessionRuntime.ts b/packages/provider-acp/src/server/AcpSessionRuntime.ts index 683bf8c90f41e..f78fc2a67b43c 100644 --- a/packages/provider-acp/src/server/AcpSessionRuntime.ts +++ b/packages/provider-acp/src/server/AcpSessionRuntime.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; // @effect-diagnostics nodeBuiltinImport:off import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; @@ -1601,6 +1602,10 @@ export const make = ( }), ) .pipe( + (effect) => + Effect.flatMap(Maintenance.ProviderMaintenanceAdmission, (admission) => + admission.withLaunchPermit(effect), + ), Effect.provideService(Scope.Scope, runtimeScope), Effect.mapError( (cause) => diff --git a/packages/provider-core/src/server/maintenance.ts b/packages/provider-core/src/server/maintenance.ts index 06846b18bc40f..0a27fed1392a0 100644 --- a/packages/provider-core/src/server/maintenance.ts +++ b/packages/provider-core/src/server/maintenance.ts @@ -1,4 +1,79 @@ import type { ProviderDriverKind } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; + +interface AdmissionLease { + readonly owner: object; + readonly permits: number; + users: number; +} +interface AdmissionToken { + readonly lease: AdmissionLease; + active: boolean; +} +const HeldAdmission = Context.Reference( + "@t3tools/provider-core/server/maintenance/HeldAdmission", + { defaultValue: () => null }, +); +const INSTALL_PERMITS = Number.MAX_SAFE_INTEGER; + +/** + * Installs exclude CLI launches; launches share admission with each other. + * Work already holding admission, such as an install's verification, reenters it. + */ +export class ProviderMaintenanceAdmission extends Context.Reference<{ + readonly withPermit: (effect: Effect.Effect) => Effect.Effect; + readonly withLaunchPermit: (effect: Effect.Effect) => Effect.Effect; + readonly withScopedLaunchPermit: ( + effect: Effect.Effect, + ) => Effect.Effect; +}>("t3/provider/providerMaintenanceCommandCoordinator/ProviderMaintenanceAdmission", { + defaultValue: () => { + const semaphore = Semaphore.makeUnsafe(INSTALL_PERMITS); + const owner = {}; + const take = (permits: number) => + Effect.flatMap(HeldAdmission, (inherited) => { + if (inherited?.lease.owner === owner && inherited.active) { + inherited.lease.users++; + return Effect.succeed({ lease: inherited.lease, active: true }); + } + return semaphore.take(permits).pipe( + Effect.map(() => ({ + lease: { owner, permits, users: 1 }, + active: true, + })), + ); + }); + const release = (token: AdmissionToken) => + Effect.suspend(() => { + token.active = false; + token.lease.users--; + return token.lease.users === 0 + ? Effect.asVoid(semaphore.release(token.lease.permits)) + : Effect.void; + }); + const withPermits = + (permits: number) => + (effect: Effect.Effect) => + Effect.uninterruptibleMask((restore) => + Effect.acquireUseRelease( + restore(take(permits)), + (token) => restore(effect.pipe(Effect.provideService(HeldAdmission, token))), + release, + ), + ); + return { + withPermit: withPermits(INSTALL_PERMITS), + withLaunchPermit: withPermits(1), + withScopedLaunchPermit: (effect) => + Effect.acquireRelease(take(1), release, { interruptible: true }).pipe( + Effect.flatMap((token) => effect.pipe(Effect.provideService(HeldAdmission, token))), + ), + }; + }, +}) {} export interface ProviderMaintenanceCapabilities { readonly provider: ProviderDriverKind; diff --git a/packages/provider-core/src/server/managedProvider.test.ts b/packages/provider-core/src/server/managedProvider.test.ts index 1d73e112b815f..4ae9eb11eef8b 100644 --- a/packages/provider-core/src/server/managedProvider.test.ts +++ b/packages/provider-core/src/server/managedProvider.test.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "./maintenance.ts"; import { describe, it, assert } from "@effect/vitest"; import { DEFAULT_SERVER_SETTINGS, @@ -658,3 +659,47 @@ describe("makeManagedServerProvider", () => { ).pipe(Effect.provide(layerAlwaysRunTest)), ); }); + +it.effect("snapshot refresh waits outside its mutex so updater verification cannot deadlock", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const queued = yield* Deferred.make(); + const initialDone = yield* Deferred.make(); + let checks = 0; + const provider = yield* makeManagedServerProvider({ + resolveMaintenance: () => Effect.succeed(maintenanceCapabilities), + getSettings: Effect.succeed({ enabled: true }), + streamSettings: Stream.empty, + haveSettingsChanged: () => false, + initialSnapshot: () => Effect.succeed(initialSnapshot), + checkProvider: Effect.sync(() => { + checks++; + }).pipe( + Effect.andThen(Deferred.succeed(initialDone, undefined)), + Effect.as(refreshedSnapshot), + ), + refreshOnInterval: false, + }); + yield* Deferred.await(initialDone); + const pending = yield* Deferred.await(entered).pipe( + Effect.andThen(Deferred.succeed(queued, undefined)), + Effect.andThen(provider.refresh), + Effect.forkScoped, + ); + const verifying = yield* admission + .withPermit( + Effect.gen(function* () { + yield* Deferred.succeed(entered, undefined); + yield* Deferred.await(queued); + yield* Effect.yieldNow; + assert.isTrue(pending.pollUnsafe() === undefined); + return yield* provider.refresh; + }), + ) + .pipe(Effect.forkScoped); + assert.deepEqual(yield* Fiber.join(verifying), refreshedSnapshot); + assert.deepEqual(yield* Fiber.join(pending), refreshedSnapshot); + assert.strictEqual(checks, 3); + }).pipe(Effect.provide(layerAlwaysRunTest)), +); diff --git a/packages/provider-core/src/server/managedProvider.ts b/packages/provider-core/src/server/managedProvider.ts index effd399a9ab39..fa73e07d5e83a 100644 --- a/packages/provider-core/src/server/managedProvider.ts +++ b/packages/provider-core/src/server/managedProvider.ts @@ -16,6 +16,7 @@ import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; import * as ProviderHost from "./ProviderHost.ts"; +import * as Maintenance from "./maintenance.ts"; import { applyUsageLimitsUpdate, resolveUsageLimitsAfterProbe } from "./usageLimits.ts"; import type { ManagedServerProvider } from "./snapshot.ts"; @@ -59,6 +60,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( Scope.Scope | ProviderHost.ProviderHost > { const host = yield* ProviderHost.ProviderHost; + const admission = yield* Maintenance.ProviderMaintenanceAdmission; const refreshSemaphore = yield* Semaphore.make(1); const changesPubSub = yield* Effect.acquireRelease( PubSub.unbounded(), @@ -117,7 +119,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( getSnapshot: Ref.get(snapshotStateRef).pipe(Effect.map((state) => state.snapshot)), publishSnapshot: (nextSnapshot) => publishEnrichedSnapshot(generation, nextSnapshot), }) - .pipe(Effect.ignoreCause({ log: true }), Effect.forkIn(scope)); + .pipe(admission.withLaunchPermit, Effect.ignoreCause({ log: true }), Effect.forkIn(scope)); yield* Ref.set(enrichmentFiberRef, fiber); }); @@ -191,7 +193,9 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( return nextSnapshot; }); const applySnapshot = (nextSettings: Settings, options?: { readonly forceRefresh?: boolean }) => - refreshSemaphore.withPermits(1)(applySnapshotBase(nextSettings, options)); + admission.withLaunchPermit( + refreshSemaphore.withPermits(1)(applySnapshotBase(nextSettings, options)), + ); /** * Runtime usage updates arrive between probes. They patch only diff --git a/packages/provider-core/src/server/snapshotProbe.test.ts b/packages/provider-core/src/server/snapshotProbe.test.ts index fc207f90f3409..9ac89ee8e552c 100644 --- a/packages/provider-core/src/server/snapshotProbe.test.ts +++ b/packages/provider-core/src/server/snapshotProbe.test.ts @@ -1,3 +1,6 @@ +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as Maintenance from "./maintenance.ts"; import { describe, expect, it } from "@effect/vitest"; import type { ModelCapabilities } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -180,3 +183,150 @@ describe("ProviderCommandNotFoundError", () => { }); }); }); + +describe("provider maintenance admission", () => { + it.effect("keeps CLI probes behind an install and permits its verification", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + let launches = 0; + const spawner = ChildProcessSpawner.make(() => + Effect.sync(() => { + launches++; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + const probe = spawnAndCollect("codex", ChildProcess.make("codex", ["--version"])).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const install = yield* admission + .withPermit( + Effect.gen(function* () { + yield* Deferred.succeed(entered, undefined); + yield* Deferred.await(release); + yield* probe; + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const waiting = yield* probe.pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(launches).toBe(0); + expect(waiting.pollUnsafe() === undefined).toBe(true); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + yield* Fiber.join(waiting); + expect(launches).toBe(2); + }), + ); + + it.effect("revokes inherited verification admission when the holder exits", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const proceed = yield* Deferred.make(); + const child = yield* admission.withPermit( + Deferred.await(proceed).pipe( + Effect.andThen(admission.withPermit(Effect.succeed("child"))), + Effect.forkScoped, + ), + ); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* Deferred.succeed(proceed, undefined); + yield* Effect.yieldNow; + expect(child.pollUnsafe() === undefined).toBe(true); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + expect(yield* Fiber.join(child)).toBe("child"); + }), + ); + + it.effect("releases scoped CLI admission on interruption", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const probe = yield* Effect.scoped( + admission + .withScopedLaunchPermit(Effect.void) + .pipe(Effect.andThen(Deferred.succeed(entered, undefined)), Effect.andThen(Effect.never)), + ).pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const install = yield* admission + .withPermit(Effect.succeed("installed")) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(install.pollUnsafe() === undefined).toBe(true); + yield* Fiber.interrupt(probe); + expect(yield* Fiber.join(install)).toBe("installed"); + }), + ); +}); + +it.effect("keeps an install excluded until borrowed scoped probe admission closes", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const child = yield* admission.withPermit( + Effect.gen(function* () { + const child = yield* Effect.scoped( + admission.withScopedLaunchPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ), + ).pipe(Effect.forkScoped); + yield* Deferred.await(entered); + return child; + }), + ); + const install = yield* admission + .withPermit(Effect.succeed("installed")) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(install.pollUnsafe() === undefined).toBe(true); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(child); + expect(yield* Fiber.join(install)).toBe("installed"); + }), +); + +it.effect("cancels an admission waiter while an install still holds the permit", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const probe = yield* admission + .withPermit(Effect.die("canceled CLI must not launch")) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* Fiber.interrupt(probe); + expect(probe.pollUnsafe() !== undefined).toBe(true); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + expect(yield* admission.withPermit(Effect.succeed("next"))).toBe("next"); + }), +); diff --git a/packages/provider-core/src/server/snapshotProbe.ts b/packages/provider-core/src/server/snapshotProbe.ts index 071d9fd5ae07e..ba7a5c666339b 100644 --- a/packages/provider-core/src/server/snapshotProbe.ts +++ b/packages/provider-core/src/server/snapshotProbe.ts @@ -22,6 +22,7 @@ import { readCustomModelEntries } from "@t3tools/shared/model"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { createProviderVersionAdvisory } from "./maintenanceResolver.ts"; import { collectUint8StreamText } from "./collectStreamText.ts"; +import * as Maintenance from "./maintenance.ts"; // Localized cmd.exe "not recognized" messages for a missing executable. const WINDOWS_COMMAND_NOT_FOUND_PATTERNS = [ @@ -130,7 +131,11 @@ export const spawnAndCollect = (binaryPath: string, command: ChildProcess.Comman }); } return result; - }).pipe(Effect.scoped); + }).pipe(Effect.scoped, (effect) => + Effect.flatMap(Maintenance.ProviderMaintenanceAdmission, (admission) => + admission.withLaunchPermit(effect), + ), + ); /** * Return the first semantic version found in CLI output, or null. Accepts a diff --git a/packages/provider-opencode/src/server/OpenCodeRuntime.inventory.test.ts b/packages/provider-opencode/src/server/OpenCodeRuntime.inventory.test.ts index d9cdfa9e64414..24cc93ae9f49d 100644 --- a/packages/provider-opencode/src/server/OpenCodeRuntime.inventory.test.ts +++ b/packages/provider-opencode/src/server/OpenCodeRuntime.inventory.test.ts @@ -1,3 +1,5 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; +import * as Deferred from "effect/Deferred"; import * as NodeAssert from "node:assert/strict"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -283,4 +285,26 @@ it.layer(layerTest)("OpenCodeRuntime inventory", (it) => { NodeAssert.equal(result.code, 0); }), ); + it.effect("OpenCode version and auth CLI probes wait for maintenance admission", () => + Effect.gen(function* () { + const runtime = yield* OpenCodeRuntime.OpenCodeRuntime; + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const probe = yield* runtime + .runOpenCodeCommand({ binaryPath: "/definitely-missing-opencode", args: ["--version"] }) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + NodeAssert.ok(probe.pollUnsafe() === undefined); + yield* Fiber.interrupt(probe); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + }), + ); }); diff --git a/packages/provider-opencode/src/server/OpenCodeRuntime.ts b/packages/provider-opencode/src/server/OpenCodeRuntime.ts index 5e87afe5ed598..c2feb6043dab3 100644 --- a/packages/provider-opencode/src/server/OpenCodeRuntime.ts +++ b/packages/provider-opencode/src/server/OpenCodeRuntime.ts @@ -1,3 +1,4 @@ +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as NodeURL from "node:url"; import type { ChatAttachment, ProviderApprovalDecision, RuntimeMode } from "@t3tools/contracts"; @@ -594,6 +595,7 @@ function ensureRuntimeError( } const makeOpenCodeRuntime = Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const netService = yield* NetService.NetService; const hostPlatform = yield* HostProcess.Platform; @@ -648,6 +650,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () { } satisfies OpenCodeCommandResult; }).pipe( Effect.scoped, + admission.withLaunchPermit, Effect.mapError((cause) => ensureRuntimeError( "runOpenCodeCommand", @@ -733,6 +736,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () { }), ) .pipe( + admission.withLaunchPermit, Effect.provideService(Scope.Scope, runtimeScope), Effect.mapError( (cause) => From e55e0fd7c63100a5b0f3f6832dc4f761837ee005 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:24:09 +0000 Subject: [PATCH 072/149] fix(web): background provider updates stay out of the way in clients Update state records whether an install was manual or background. Background progress, success and unchanged results stay out of the sidebar and mobile settings, and failures show neutrally. The launch toast follows a background install that starts, and settings updates check the destination's permissions. Docs note that ACP agents are not updated automatically. --- .../SettingsEnvironmentDetailRouteScreen.tsx | 8 +- .../SettingsServerControlsRouteScreen.tsx | 25 ++++-- .../settings/environment-maintenance.test.ts | 25 ++++++ .../settings/environment-maintenance.ts | 12 +++ .../providerMaintenanceRunner.test.ts | 43 ++++++++-- .../src/provider/providerMaintenanceRunner.ts | 9 +- .../server/src/updates/ProviderAutoUpdater.ts | 2 +- .../ProviderUpdateEnvironmentRows.test.tsx | 84 +++++++++++++++++++ ...iderUpdateLaunchNotification.logic.test.ts | 57 +++++++++++++ .../ProviderUpdateLaunchNotification.logic.ts | 13 ++- .../ProviderUpdatePrimaryNotification.tsx | 33 +++++++- .../components/settings/SettingsPanels.tsx | 2 +- .../components/settings/useScopedSettings.ts | 21 +++-- docs/user/updating.md | 3 + .../src/state/commandPermissions.test.ts | 53 ++++++++++++ .../contracts/src/clientRpcPermissions.ts | 8 ++ packages/contracts/src/server.test.ts | 18 ++++ packages/contracts/src/server.ts | 1 + 18 files changed, 383 insertions(+), 34 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx index 1065a860a5e4c..b76223177d95d 100644 --- a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx @@ -29,6 +29,7 @@ import { canMaintainEnvironment, canUpdateEnvironmentProvider, findEnvironmentUpdate, + providerUpdateSettingsNotice, supportsEnvironmentUpdate, } from "./environment-maintenance"; @@ -330,17 +331,16 @@ function EnvironmentDetail({ environmentId }: { readonly environmentId: Environm ? ` · Latest ${provider.versionAdvisory.latestVersion}` : ""} - {provider.updateState && provider.updateState.status !== "idle" ? ( + {providerUpdateSettingsNotice(provider) ? ( - {provider.updateState.message ?? - `Update ${provider.updateState.status}`} + {providerUpdateSettingsNotice(provider)?.message} ) : null} {provider.compatibilityAdvisory?.message ? ( diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 964f2a3f8c888..7a0ff111af609 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -1,7 +1,9 @@ import { useNavigation } from "@react-navigation/native"; import { SettingsRow } from "./components/SettingsRow"; import { AuthSettingsWriteScope } from "@t3tools/contracts"; -import { readEnvironmentScope, useEnvironmentsWithScope } from "../../state/session"; +import { readEnvironmentScope } from "../../state/session"; +import { useAtomValue } from "@effect/atom-react"; +import { Atom } from "effect/reactivity"; import { ScreenScrollView as ScrollView } from "../../components/ScreenScrollView"; import { AppText as Text } from "../../components/AppText"; import { @@ -13,7 +15,7 @@ import { PROJECT_SCOPED_SERVER_SETTING_KEYS, type ProjectScopedServerSettingKey, } from "@t3tools/contracts"; -import { useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { View } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; @@ -143,7 +145,19 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { const navigation = useNavigation(); const { selectedTargets, projectGroups, selectedProjectKey } = useSettingsEnvironmentFilter(); const selectedProject = projectGroups.find((group) => group.key === selectedProjectKey); - const writableEnvironments = useEnvironmentsWithScope(selectedTargets, AuthSettingsWriteScope); + const canWriteSettings = useAtomValue( + useMemo( + () => + Atom.make( + (get) => + selectedTargets.length > 0 && + selectedTargets.every((target) => + get(serverEnvironment.updateSettings.permissionAtom(target.environmentId)), + ), + ), + [selectedTargets], + ), + ); const projectSelected = selectedProjectKey !== null; const targets = resolveMobileSettingsTargets( selectedTargets, @@ -154,9 +168,6 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { const [pendingTargets, setPendingTargets] = useState< readonly ScopedMobileSettingsTarget[] | null >(null); - const canWriteSettings = - targets.length > 0 && - targets.every((target) => writableEnvironments.has(target.environment.environmentId)); const displayTargets = pendingWrites > 0 && pendingTargets !== null ? pendingTargets : targets; const hasConnectedSelection = targets.length > 0; const reference = displayTargets[0] ?? null; @@ -473,7 +484,7 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { projectSelected ? "Environment-wide setting. Select All projects to change it." : supportsAutomaticUpdates - ? "Install provider CLI updates while nothing is running." + ? "Install updates for provider CLIs that support one-click updates on the selected environments while nothing is running." : "Update older servers to control automatic updates." } value={uniform("automaticUpdates")} diff --git a/apps/mobile/src/features/settings/environment-maintenance.test.ts b/apps/mobile/src/features/settings/environment-maintenance.test.ts index e2e1ebdf99a4f..5cfcf1ca5ed53 100644 --- a/apps/mobile/src/features/settings/environment-maintenance.test.ts +++ b/apps/mobile/src/features/settings/environment-maintenance.test.ts @@ -6,6 +6,7 @@ import { canMaintainEnvironment, canUpdateEnvironmentProvider, findEnvironmentUpdate, + providerUpdateSettingsNotice, supportsEnvironmentUpdate, } from "./environment-maintenance"; @@ -33,6 +34,30 @@ const provider = Schema.decodeUnknownSync(ServerProvider)({ afterEach(() => vi.unstubAllGlobals()); describe("environment maintenance access", () => { + it.each(["queued", "running", "succeeded", "failed", "unchanged"] as const)( + "keeps a background %s quiet and leaves settings diagnostics neutral", + (status) => { + const updateState = { + status, + origin: "background" as const, + startedAt: null, + finishedAt: null, + message: "Update outcome", + output: null, + }; + expect(providerUpdateSettingsNotice({ ...provider, updateState })).toEqual( + status === "failed" || status === "unchanged" + ? { message: "Update outcome", failed: false } + : null, + ); + expect( + providerUpdateSettingsNotice({ + ...provider, + updateState: { ...updateState, origin: "manual" }, + }), + ).toEqual({ message: "Update outcome", failed: status === "failed" }); + }, + ); it("requires a connected authenticated session with operate permission", () => { const session = { authenticated: true, diff --git a/apps/mobile/src/features/settings/environment-maintenance.ts b/apps/mobile/src/features/settings/environment-maintenance.ts index 3dd1d559c71d4..e3cab5d2c3e63 100644 --- a/apps/mobile/src/features/settings/environment-maintenance.ts +++ b/apps/mobile/src/features/settings/environment-maintenance.ts @@ -45,6 +45,18 @@ export function canUpdateEnvironmentProvider(provider: ServerProvider) { ); } +/** Background outcomes stay quiet except for a neutral diagnostic in settings. */ +export function providerUpdateSettingsNotice(provider: ServerProvider) { + const state = provider.updateState; + if (!state || state.status === "idle") return null; + const background = state.origin === "background"; + if (background && state.status !== "failed" && state.status !== "unchanged") return null; + return { + message: state.message ?? `Update ${state.status}`, + failed: !background && state.status === "failed", + }; +} + const Releases = Schema.Array( Schema.Struct({ tag_name: Schema.String, diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 3f6e26229239d..dbbcea7c5ff21 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -274,6 +274,34 @@ const makeTestRunner = ( ); describe("providerMaintenanceRunner", () => { + it.effect.each([ + { origin: "manual" as const, exitCode: 0 }, + { origin: "manual" as const, exitCode: 1 }, + ])("carries $origin origin through every update state ($exitCode)", ({ origin, exitCode }) => + Effect.gen(function* () { + const { registry, updateStatesRef } = yield* makeRegistry(baseNativeCliProvider); + const runner = yield* makeTestRunner(registry); + const result = yield* runner.updateProvider({ + provider: NATIVE_CLI_DRIVER, + origin, + }); + assert.strictEqual( + result.providers[0]?.updateState?.status, + exitCode === 0 ? "succeeded" : "failed", + ); + const states = yield* Ref.get(updateStatesRef); + assert.isAtLeast(states.length, 3); + assert.isTrue(states.every((state) => state.origin === origin)); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner(() => ({ stdout: "installer output", code: exitCode })), + ), + ), + ), + ); it.effect("runs the allowlisted provider update command and records success", () => { const calls: Array<{ command: string; args: ReadonlyArray }> = []; return Effect.gen(function* () { @@ -288,6 +316,7 @@ describe("providerMaintenanceRunner", () => { }, ]); assert.strictEqual(result.providers[0]?.updateState?.status, "succeeded"); + assert.strictEqual(result.providers[0]?.updateState?.origin, "manual"); assert.deepStrictEqual( (yield* Ref.get(updateStatesRef)).map((state) => [state.status, state.message]), [ @@ -1125,7 +1154,7 @@ it.effect.each([ const sibling = ProviderInstanceId.make("codex-work"); const unrelated = ProviderInstanceId.make("codex-other"); return Effect.gen(function* () { - const { registry, providersRef } = yield* makeRegistry([ + const { registry, providersRef, updateStatesRef } = yield* makeRegistry([ baseProvider, { ...baseProvider, instanceId: sibling }, { ...baseProvider, instanceId: unrelated }, @@ -1165,7 +1194,7 @@ it.effect.each([ provider: CODEX_DRIVER, ...(exitCode === 0 ? { targetVersion } : {}), quiet: true, - background: true, + origin: "background", }); assert.deepStrictEqual(calls, [`install -g --prefix /owned @openai/codex@${targetVersion}`]); assert.deepStrictEqual(refreshed, [CODEX_INSTANCE_ID, CODEX_INSTANCE_ID, sibling]); @@ -1175,6 +1204,8 @@ it.effect.each([ ); assert.strictEqual(result.providers[1]?.version, targetVersion); assert.strictEqual(result.providers[2]?.version, "0.0.0"); + // Background installs need proven ownership, so their origin is covered here. + assert.isTrue((yield* Ref.get(updateStatesRef)).every((s) => s.origin === "background")); }).pipe( Effect.provide( Layer.mergeAll( @@ -1218,7 +1249,7 @@ it.effect.each(["2.0.0", "3.0.0", "v3.0.0", "unknown"])( }); const result = yield* runner.updateProvider({ provider: CODEX_DRIVER, - background: true, + origin: "background", }); assert.strictEqual(result.providers[0]?.updateState?.status, "unchanged"); }).pipe( @@ -1262,7 +1293,7 @@ it.effect.each([false, true])( }); const result = yield* runner.updateProvider({ provider: CODEX_DRIVER, - background: true, + origin: "background", }); assert.strictEqual(result.providers[0]?.updateState?.status, "failed"); assert.deepStrictEqual(calls, []); @@ -1384,7 +1415,7 @@ it.effect( instanceId: target.instanceId, targetVersion: target.targetVersion, }); - assert.strictEqual(target.background, true); + assert.strictEqual(target.origin, "background"); assert.strictEqual(target.quiet, true); return yield* new ServerProviderUpdateError({ provider: CODEX_DRIVER, @@ -1497,7 +1528,7 @@ it.effect( const result = yield* runner.updateProvider({ provider: CODEX_DRIVER, targetVersion: "2.0.0", - background: true, + origin: "background", }); assert.strictEqual(result.providers[0]?.updateState?.status, "failed"); assert.deepStrictEqual(calls, []); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 3f55abf719119..296863b2ce576 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -69,8 +69,8 @@ export interface ProviderMaintenanceRunnerShape { readonly targetVersion?: string | undefined; /** Skips installer output progress, for background installs nobody watches. */ readonly quiet?: boolean | undefined; - /** Requires proven ownership and pins the checked release when supported. */ - readonly background?: boolean | undefined; + /** Background installs require proven ownership and pin the checked release. */ + readonly origin?: ServerProviderUpdateState["origin"]; }, ) => Effect.Effect; } @@ -431,7 +431,8 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { : (target.instanceId ?? defaultInstanceIdForDriver(provider)); const targetVersion = typeof target === "string" ? undefined : target.targetVersion; const quiet = typeof target !== "string" && target.quiet === true; - const background = typeof target !== "string" && target.background === true; + const origin = typeof target === "string" ? "manual" : (target.origin ?? "manual"); + const background = origin === "background"; const targetKey = `instance:${instanceId}`; const capabilities = yield* providerRegistry.getProviderMaintenanceCapabilitiesForInstance( instanceId, @@ -449,7 +450,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { providerRegistry.setProviderMaintenanceActionState({ instanceId, action: "update", - state, + state: state === null ? null : { ...state, origin }, }); const setQueuedState = setUpdateState( makeUpdateState({ diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index 881ec26e22ec4..7bb90eb36f76c 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -94,7 +94,7 @@ export const layer = Layer.effectDiscard( // Installed after the window releases admission: the runner first drains host // leases, and a session holding one may still need admission to launch its CLI. yield* runner - .updateProvider({ ...picked.value, instanceId, quiet: true, background: true }) + .updateProvider({ ...picked.value, instanceId, quiet: true, origin: "background" }) .pipe( Effect.catchCause((cause) => Effect.logWarning("Background provider update failed", { instanceId, cause }), diff --git a/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx b/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx index 6f6fcb4ef0c60..10748a5553700 100644 --- a/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx +++ b/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx @@ -20,11 +20,15 @@ const testState = vi.hoisted(() => ({ groups: [] as LocalEnvironmentUpdateGroup[], permittedEnvironmentIds: new Set(), updateProvider: vi.fn(), + primaryProviders: [] as ServerProvider[], + addToast: vi.fn(), + updateToast: vi.fn(), })); const hooks = vi.hoisted(() => { let cursor = 0; let slots: unknown[] = []; + let effects: Array<() => void> = []; const nextIndex = () => cursor++; @@ -35,6 +39,20 @@ const hooks = vi.hoisted(() => { reset() { cursor = 0; slots = []; + effects = []; + }, + useEffect(effect: () => void, dependencies: readonly unknown[]) { + const index = nextIndex(); + const previous = slots[index] as readonly unknown[] | undefined; + if (!previous || dependencies.some((value, i) => !Object.is(value, previous[i]))) { + slots[index] = dependencies; + effects.push(effect); + } + }, + flushEffects() { + const pending = effects; + effects = []; + pending.forEach((effect) => effect()); }, useCallback(callback: T): T { nextIndex(); @@ -79,6 +97,8 @@ vi.mock("react", async (importOriginal) => { return { ...actual, useCallback: hooks.useCallback, + useEffect: hooks.useEffect, + useLayoutEffect: hooks.useEffect, useMemo: hooks.useMemo, useRef: hooks.useRef, useState: hooks.useState, @@ -91,6 +111,24 @@ vi.mock("react/compiler-runtime", () => ({ vi.mock("~/state/server", () => ({ serverEnvironment: { updateProvider: Symbol("updateProvider") }, + primaryServerProvidersAtom: Symbol("providers"), +})); + +vi.mock("@effect/atom-react", () => ({ useAtomValue: () => testState.primaryProviders })); +vi.mock("@tanstack/react-router", () => ({ useNavigate: () => vi.fn() })); +vi.mock("~/state/environments", () => ({ + usePrimaryEnvironment: () => ({ environmentId: "env-wsl" }), +})); +vi.mock("~/providerUpdateDismissal", () => ({ + useDismissedProviderUpdateNotificationKeys: () => ({ + dismissedNotificationKeys: new Set(), + dismissNotificationKey: vi.fn(), + }), +})); +vi.mock("./ui/toast", () => ({ + toastManager: { add: testState.addToast, update: testState.updateToast, close: vi.fn() }, + hiddenToastActionProps: { hidden: true }, + stackedThreadToast: (options: unknown) => options, })); vi.mock("~/state/session", () => ({ @@ -114,6 +152,7 @@ vi.mock("./ProviderUpdateLaunchNotification.environments", () => ({ })); import { ProviderUpdateEnvironmentRows } from "./ProviderUpdateEnvironmentRows"; +import { ProviderUpdatePrimaryNotification } from "./ProviderUpdatePrimaryNotification"; const environmentId = "env-wsl" as EnvironmentId; const pendingExpiryMs = 6 * 60_000; @@ -184,6 +223,51 @@ async function flushPromises(): Promise { await Promise.resolve(); } +describe("ProviderUpdatePrimaryNotification", () => { + it("removes the stale launch Update action when an unsolicited update starts", () => { + hooks.reset(); + testState.permittedEnvironmentIds.add(environmentId); + testState.updateProvider.mockReset(); + testState.addToast.mockReset().mockReturnValue("launch-toast"); + testState.updateToast.mockReset(); + const initial = provider(); + testState.primaryProviders = [initial]; + const render = () => { + hooks.beginRender(); + ProviderUpdatePrimaryNotification(); + hooks.flushEffects(); + }; + render(); + const staleAction = testState.addToast.mock.calls[0]![0].actionProps; + expect(staleAction.children).toBe("Update"); + testState.primaryProviders = [ + { + ...initial, + updateState: { + status: "running", + origin: "background", + startedAt: initial.checkedAt, + finishedAt: null, + message: "Updating", + output: null, + }, + }, + ]; + render(); + expect(testState.updateToast.mock.calls.at(-1)?.[1].actionProps.children).toBe("Settings"); + staleAction.onClick(); + expect(testState.updateProvider).not.toHaveBeenCalled(); + testState.primaryProviders = [ + { + ...testState.primaryProviders[0]!, + updateState: { ...testState.primaryProviders[0]!.updateState!, status: "failed" }, + }, + ]; + render(); + expect(testState.updateToast.mock.calls.at(-1)?.[1].actionProps.children).toBe("Update"); + }); +}); + describe("ProviderUpdateEnvironmentRows", () => { beforeEach(() => { vi.useFakeTimers(); diff --git a/apps/web/src/components/ProviderUpdateLaunchNotification.logic.test.ts b/apps/web/src/components/ProviderUpdateLaunchNotification.logic.test.ts index d388f9f036cb9..845a6d790ccde 100644 --- a/apps/web/src/components/ProviderUpdateLaunchNotification.logic.test.ts +++ b/apps/web/src/components/ProviderUpdateLaunchNotification.logic.test.ts @@ -29,6 +29,7 @@ import { isTerminalProviderUpdatePhase, localEnvironmentUpdateNotificationKey, providerUpdateNotificationKey, + providerUpdatePromptActionKey, resolveEnvironmentUpdateRowStatus, shouldShowPrimaryProviderUpdateToast, type LocalEnvironmentProvidersInput, @@ -92,6 +93,62 @@ function updateCandidate(input: Parameters[0]): ProviderUpdateC } describe("provider update launch notification logic", () => { + it("refreshes the launch action when a background update starts and finishes", () => { + const candidate = updateCandidate({ driver: driver("codex") }); + const initial = providerUpdatePromptActionKey([candidate]); + const running = { + ...candidate, + updateState: { + status: "running" as const, + origin: "background" as const, + startedAt: checkedAt, + finishedAt: null, + message: null, + output: null, + }, + }; + expect(providerUpdateNotificationKey([running])).toBe( + providerUpdateNotificationKey([candidate]), + ); + expect(providerUpdatePromptActionKey([running])).not.toBe(initial); + expect(canOneClickUpdateProviderCandidate(running, [running])).toBe(false); + expect( + providerUpdatePromptActionKey([ + { ...running, updateState: { ...running.updateState, status: "failed" } }, + ]), + ).toBe(initial); + }); + + it.each(["queued", "running", "failed", "unchanged", "succeeded"] as const)( + "keeps background %s out of the sidebar, including beside a manual sibling", + (status) => { + const background = provider({ + driver: driver("codex"), + instanceId: instanceId("personal"), + updateState: { + status, + origin: "background", + startedAt: checkedAt, + finishedAt: checkedAt, + message: "Automatic outcome", + output: null, + }, + }); + expect(getProviderUpdateSidebarPillView([background])).toBeNull(); + const manual = provider({ + driver: driver("codex"), + updateState: { + ...background.updateState!, + origin: "manual", + status: "failed", + message: "Manual failure", + }, + }); + expect(getProviderUpdateSidebarPillView([background, manual])?.description).toContain( + "Manual failure", + ); + }, + ); it("detects enabled providers with a latest-version advisory", () => { expect(isProviderUpdateCandidate(provider({ driver: driver("codex") }))).toBe(true); expect(isProviderUpdateCandidate(provider({ driver: driver("codex"), enabled: false }))).toBe( diff --git a/apps/web/src/components/ProviderUpdateLaunchNotification.logic.ts b/apps/web/src/components/ProviderUpdateLaunchNotification.logic.ts index e245fcb7cf30a..9f787fe1c50c6 100644 --- a/apps/web/src/components/ProviderUpdateLaunchNotification.logic.ts +++ b/apps/web/src/components/ProviderUpdateLaunchNotification.logic.ts @@ -212,6 +212,15 @@ export function canOneClickUpdateProviderCandidate( ); } +/** Refresh a launch prompt when live progress changes which updates it can start. */ +export function providerUpdatePromptActionKey(providers: ReadonlyArray): string { + return collectProviderUpdateCandidates(providers) + .filter((provider) => canOneClickUpdateProviderCandidate(provider, providers)) + .map((provider) => provider.instanceId) + .toSorted() + .join("|"); +} + export function providerUpdateNotificationKey( providers: ReadonlyArray, ): string | null { @@ -452,7 +461,9 @@ export function getProviderUpdateSidebarPillView( providers: ReadonlyArray, options?: ProviderUpdateSidebarPillOptions, ): ProviderUpdateSidebarPillView | null { - const dedupedProviders = dedupeProvidersByDriver(providers); + const dedupedProviders = dedupeProvidersByDriver( + providers.filter((provider) => provider.updateState?.origin !== "background"), + ); const activeProviders = dedupedProviders.filter(isProviderUpdateActive); if (activeProviders.length > 0) { const activeProvider = activeProviders[0]!; diff --git a/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx b/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx index 98b3e71dac212..6cc541de78609 100644 --- a/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx +++ b/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx @@ -1,7 +1,7 @@ import { useNavigate } from "@tanstack/react-router"; import { useAtomValue } from "@effect/atom-react"; import { DownloadIcon } from "lucide-react"; -import { useCallback, useEffect, useMemo, useRef } from "react"; +import { useCallback, useEffect, useLayoutEffect, useMemo, useRef } from "react"; import { AuthProvidersManageScope, type ServerProvider, @@ -18,10 +18,12 @@ import { collectProviderUpdateCandidates, collectUpdatedProviderSnapshots, firstFailedProviderUpdateMessage, + isProviderUpdateCandidate, getProviderUpdateInitialToastView, getProviderUpdateProgressToastView, getProviderUpdateRejectedToastView, providerUpdateNotificationKey, + providerUpdatePromptActionKey, shouldShowPrimaryProviderUpdateToast, type ProviderUpdateToastView, } from "./ProviderUpdateLaunchNotification.logic"; @@ -37,6 +39,7 @@ type ActiveProviderUpdateToast = readonly key: string; readonly toastId: ProviderUpdateToastId; readonly canManageProviders: boolean; + readonly actionKey: string; } | { readonly kind: "update"; @@ -143,6 +146,11 @@ export function ProviderUpdatePrimaryNotification() { updateProviders.filter((provider) => canOneClickUpdateProviderCandidate(provider, providers)), [providers, updateProviders], ); + const actionKey = providerUpdatePromptActionKey(providers); + const latestProvidersRef = useRef(providers); + useLayoutEffect(() => { + latestProvidersRef.current = providers; + }, [providers]); const openProviderSettings = useCallback( (toastId?: ProviderUpdateToastId) => { @@ -195,7 +203,8 @@ export function ProviderUpdatePrimaryNotification() { const updateExistingPrompt = activeToast?.kind === "prompt" && activeToast.key === notificationKey && - activeToast.canManageProviders !== canManageProviders; + (activeToast.canManageProviders !== canManageProviders || + activeToast.actionKey !== actionKey); if ( !notificationKey || dismissedNotificationKeys.has(notificationKey) || @@ -220,6 +229,7 @@ export function ProviderUpdatePrimaryNotification() { if ( updateStarted || oneClickProviders.length === 0 || + actionKey !== providerUpdatePromptActionKey(latestProvidersRef.current) || !primaryEnvironment || !readEnvironmentScope(primaryEnvironment.environmentId, AuthProvidersManageScope) ) { @@ -242,6 +252,16 @@ export function ProviderUpdatePrimaryNotification() { void (async () => { const results = []; for (const provider of oneClickProviders) { + const current = latestProvidersRef.current.find( + (candidate) => candidate.instanceId === provider.instanceId, + ); + if ( + !current || + !isProviderUpdateCandidate(current) || + !canOneClickUpdateProviderCandidate(current, latestProvidersRef.current) + ) { + continue; + } if (!readEnvironmentScope(primaryEnvironment.environmentId, AuthProvidersManageScope)) { if (activeToastRef.current === activeUpdate) { addProviderUpdateToast({ @@ -339,8 +359,15 @@ export function ProviderUpdatePrimaryNotification() { } else { toastId = toastManager.add(toastOptions); } - activeToastRef.current = { kind: "prompt", key: notificationKey, toastId, canManageProviders }; + activeToastRef.current = { + kind: "prompt", + key: notificationKey, + toastId, + canManageProviders, + actionKey, + }; }, [ + actionKey, canManageProviders, updateProvider, dismissNotificationKey, diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index c63da912eb68e..0aec59922ba52 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -2925,7 +2925,7 @@ export function GeneralSettingsPanel() { serverScoped settingKeys={["automaticUpdates"]} {...searchableSetting("automatic-updates")} - description="Install provider CLI updates in the background on the selected environments while nothing is running." + description="Install updates for provider CLIs that support one-click updates in the background on the selected environments while nothing is running." status={ !supportsAutomaticUpdates ? "All selected connected environments must support automatic updates." diff --git a/apps/web/src/components/settings/useScopedSettings.ts b/apps/web/src/components/settings/useScopedSettings.ts index 4bed07267b133..3092c331d5c08 100644 --- a/apps/web/src/components/settings/useScopedSettings.ts +++ b/apps/web/src/components/settings/useScopedSettings.ts @@ -1,12 +1,11 @@ -import { AuthSettingsWriteScope } from "@t3tools/contracts"; -import { useEnvironmentsWithScope } from "../../state/session"; +import { useAtomValue } from "@effect/atom-react"; import { requiredScopesForServerSettingsPatch, EnvironmentAuthorizationError, } from "@t3tools/contracts"; import { readEnvironmentScope } from "../../state/session"; import * as Cause from "effect/Cause"; -import { AsyncResult } from "effect/reactivity"; +import { AsyncResult, Atom } from "effect/reactivity"; import { DEFAULT_SERVER_SETTINGS, type ProjectScopedServerSettingKey, @@ -152,9 +151,17 @@ export function useClearProjectOverrides() { export function useScopedSettingsWriteAllowed() { const { connectedEnvironments } = useSettingsScope(); - const writable = useEnvironmentsWithScope(connectedEnvironments, AuthSettingsWriteScope); - return ( - connectedEnvironments.length > 0 && - connectedEnvironments.every((target) => writable.has(target.environmentId)) + return useAtomValue( + useMemo( + () => + Atom.make( + (get) => + connectedEnvironments.length > 0 && + connectedEnvironments.every((target) => + get(serverEnvironment.updateSettings.permissionAtom(target.environmentId)), + ), + ), + [connectedEnvironments], + ), ); } diff --git a/docs/user/updating.md b/docs/user/updating.md index 76e25ec164595..bda372495cf43 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -84,6 +84,9 @@ run. A failed update is retried after 6 hours, when a newer version is published, or after the server restarts. While `settings.json` cannot be read, automatic updates stay off. +ACP Registry agents, including Antigravity, and local ACP commands do not offer +one-click updates and are not updated by this setting. + **Settings → Providers** shows provider updates for the selected environment. **Update all** updates every outdated provider on every connected environment at once. Hover it to see which providers it will update. Providers that only diff --git a/packages/client-runtime/src/state/commandPermissions.test.ts b/packages/client-runtime/src/state/commandPermissions.test.ts index b2c6933fabb7c..290fe4ebcfcef 100644 --- a/packages/client-runtime/src/state/commandPermissions.test.ts +++ b/packages/client-runtime/src/state/commandPermissions.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import { AuthOrchestrationOperateScope, + AuthProvidersManageScope, AuthSettingsWriteScope, AuthSourceControlWriteScope, ThreadId, @@ -18,6 +19,7 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; +import * as Duration from "effect/Duration"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as TestClock from "effect/testing/TestClock"; @@ -59,6 +61,57 @@ const setup = Effect.gen(function* () { }); describe("command permissions", () => { + it.effect("guards settings updates with the destination grant for every changed domain", () => + Effect.scoped( + Effect.gen(function* () { + const registry = yield* setup; + const command = createEnvironmentRpcCommand(runtime, { + label: "test.settings", + tag: WS_METHODS.serverUpdateSettings, + execute: () => Effect.die("must not dispatch a denied update"), + }); + registry.set(sessions(env), AsyncResult.success(grant(false))); + expect(registry.get(command.permissionAtom(env))).toBe(false); + const denied = yield* Effect.promise(() => + command.run(registry, { + environmentId: env, + input: { patch: { automaticUpdates: false } }, + }), + ); + expect(denied._tag).toBe("Failure"); + const settings = createCommandPermissions(runtime, WS_METHODS.serverUpdateSettings); + registry.set( + sessions(env), + AsyncResult.success({ ...grant(false), permissions: [AuthSettingsWriteScope] }), + ); + expect(registry.get(command.permissionAtom(env))).toBe(true); + yield* settings.authorize(registry, env, { patch: { automaticUpdates: false } }); + yield* settings.authorize(registry, env, { + patch: { providerHealthRefreshInterval: Duration.minutes(10) }, + }); + const providerPatch = { patch: { providerInstances: {} } }; + expect(registry.get(settings.permissionAtom(env, providerPatch))).toBe(false); + registry.set( + sessions(env), + AsyncResult.success({ ...grant(false), permissions: [AuthProvidersManageScope] }), + ); + yield* settings.authorize(registry, env, providerPatch); + expect(registry.get(command.permissionAtom(env))).toBe(false); + const mixed = { patch: { automaticUpdates: false, providerInstances: {} } }; + expect(registry.get(settings.permissionAtom(env, mixed))).toBe(false); + registry.set( + sessions(env), + AsyncResult.success({ + ...grant(false), + permissions: [AuthProvidersManageScope, AuthSettingsWriteScope], + }), + ); + yield* settings.authorize(registry, env, mixed); + registry.set(sessions(other), AsyncResult.success(grant(false))); + expect(registry.get(command.permissionAtom(other))).toBe(false); + }), + ), + ); it.effect("requires the destination settings grant to run storage cleanup", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/contracts/src/clientRpcPermissions.ts b/packages/contracts/src/clientRpcPermissions.ts index c84169e6e33f6..7978aab471f00 100644 --- a/packages/contracts/src/clientRpcPermissions.ts +++ b/packages/contracts/src/clientRpcPermissions.ts @@ -7,9 +7,11 @@ import { type AuthEnvironmentScope, } from "./auth.ts"; import { WS_METHODS } from "./rpc.ts"; +import { ServerSettingsPatch, requiredScopesForServerSettingsPatch } from "./settings.ts"; /** Incremental client enforcement; the server still authorizes every request. */ export const CLIENT_GUARDED_RPC_SCOPES = { + [WS_METHODS.serverUpdateSettings]: AuthSettingsWriteScope, [WS_METHODS.serverRunStorageCleanup]: AuthSettingsWriteScope, [WS_METHODS.pullRequestsRunAction]: AuthSourceControlWriteScope, [WS_METHODS.pullRequestsUpdate]: AuthSourceControlWriteScope, @@ -45,11 +47,17 @@ export const CLIENT_GUARDED_RPC_SCOPES = { export type ClientGuardedRpcTag = keyof typeof CLIENT_GUARDED_RPC_SCOPES; const decodePrepareThread = Schema.decodeUnknownSync(GitPreparePullRequestThreadInput); +const decodeSettingsInput = Schema.decodeUnknownSync( + Schema.Struct({ patch: Schema.toType(ServerSettingsPatch) }), +); export function clientRpcRequiredScopes( method: string, input: unknown, ): readonly AuthEnvironmentScope[] { + if (method === WS_METHODS.serverUpdateSettings && input !== undefined) { + return requiredScopesForServerSettingsPatch(decodeSettingsInput(input).patch); + } if (method === WS_METHODS.gitPreparePullRequestThread && input !== undefined) { const payload = decodePrepareThread(input); if (payload.mode === "worktree" && payload.threadId !== undefined) diff --git a/packages/contracts/src/server.test.ts b/packages/contracts/src/server.test.ts index fdb6b82b3dbcb..b4a5043952e02 100644 --- a/packages/contracts/src/server.test.ts +++ b/packages/contracts/src/server.test.ts @@ -31,6 +31,24 @@ const baseProviderSnapshot = { }; describe("ServerProvider", () => { + it.each([undefined, "manual", "background"])( + "preserves update origin %s across the wire", + (origin) => { + const parsed = decodeServerProvider({ + ...baseProviderSnapshot, + updateState: { + status: "failed", + ...(origin === undefined ? {} : { origin }), + startedAt: null, + finishedAt: null, + message: "Installer failed", + output: null, + }, + }); + expect(parsed.updateState?.origin).toBe(origin); + expect(parsed.updateState?.message).toBe("Installer failed"); + }, + ); it.each([undefined, true, false])("decodes workspace command discovery pending=%s", (pending) => { const workspace = { cwd: "/workspace/project", diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index a726f05a311c8..6c4c0faf6a93d 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -236,6 +236,7 @@ export type ServerProviderUpdateStatus = typeof ServerProviderUpdateStatus.Type; export const ServerProviderUpdateState = Schema.Struct({ status: ServerProviderUpdateStatus, + origin: Schema.optionalKey(Schema.Literals(["manual", "background"])), startedAt: Schema.NullOr(IsoDateTime), finishedAt: Schema.NullOr(IsoDateTime), message: Schema.NullOr(TrimmedNonEmptyString), From 0226f3ce329304681d1c7d635f7b4d38fa638fe6 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:24:09 +0000 Subject: [PATCH 073/149] fix(server): a standalone Codex reached through a linked home still updates automatically The configured CODEX_HOME is resolved like the command path before ownership is compared, so a home behind a symlink keeps its key. --- .../provider-core/src/server/maintenanceResolver.test.ts | 7 +++++++ packages/provider-core/src/server/maintenanceResolver.ts | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/provider-core/src/server/maintenanceResolver.test.ts b/packages/provider-core/src/server/maintenanceResolver.test.ts index 3db55d621ad5b..b74400aec42c0 100644 --- a/packages/provider-core/src/server/maintenanceResolver.test.ts +++ b/packages/provider-core/src/server/maintenanceResolver.test.ts @@ -1203,6 +1203,13 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { expect( (yield* resolve(NodePath.join(root, "other"))).update?.installationKey, ).toBeUndefined(); + // The default home is often reached through a link, such as macOS's /var -> /private/var. + const linkedHome = `${root}-linked`; + NodeFS.symlinkSync(root, linkedHome); + yield* Effect.addFinalizer(() => Effect.sync(() => NodeFS.rmSync(linkedHome))); + expect((yield* resolve(linkedHome)).update?.installationKey).toBe( + owned.update?.installationKey, + ); }).pipe(Effect.scoped), { skip: windowsHost }, ); diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index c91c398a47418..24cca5259c2a4 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -421,7 +421,11 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( .replaceAll("\\", "/") .replace(/(\/packages\/standalone)\/.*$/i, "$1") .replace(/(\/\.local\/share\/claude)\/.*$/i, "$1"); - const codexHome = nativeUpdate.env?.CODEX_HOME; + // Compared after resolving links, like the command path: a symlinked home is still owned. + const configuredHome = nativeUpdate.env?.CODEX_HOME; + const codexHome = configuredHome + ? yield* fileSystem.realPath(configuredHome).pipe(Effect.orElseSucceed(() => configuredHome)) + : undefined; const ownsHome = definition.provider !== "codex" || !codexHome || From 9c22c2a6ef512ad65b4afdf39f9a41cb5cd72478 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:01:04 +0000 Subject: [PATCH 074/149] fix(server): only servers share the user-wide provider install gate The server layer now provides the gate in the user cache. Anything that runs without it, such as a test, gets a private temporary gate, so a test run never waits on or blocks the turns and installs of a live T3. --- .../ProviderSessionManager.test.ts | 2 ++ .../provider/ProviderMaintenanceHostLock.ts | 32 +++++++++++++------ .../providerMaintenanceRunner.test.ts | 14 ++++++++ apps/server/src/server.ts | 3 ++ .../src/textGeneration/TextGeneration.test.ts | 1 + 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index c4a22376ba919..527a2a3047b80 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -54,6 +54,7 @@ import * as HostLock from "../provider/ProviderMaintenanceHostLock.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const layerHostLock = Layer.succeed(HostLock.ProviderMaintenanceHostLock, { + directory: "", acquireRead: Effect.succeed(Effect.void), withRead: (effect) => effect, withWrite: (effect) => effect, @@ -4740,6 +4741,7 @@ it.effect("ProviderSessionManagerV2 holds the host lease through terminal backgr const pending = yield* Ref.make(true); const held = yield* Ref.make(0); const hostLock = { + directory: "", withRead: (effect: Effect.Effect) => effect, withWrite: (effect: Effect.Effect) => effect, acquireRead: Ref.update(held, (value) => value + 1).pipe( diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index e594389ef3388..6a5e4b1c99e85 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -7,6 +7,7 @@ import * as NodePath from "node:path"; import { lock } from "proper-lockfile"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; import * as Deferred from "effect/Deferred"; import * as Semaphore from "effect/Semaphore"; @@ -129,19 +130,32 @@ export const make = (directory: string) => { return yield* effect; }), ); - return { acquireRead, withRead, withWrite }; + return { directory, acquireRead, withRead, withWrite }; }; +/** Every T3 server on the host shares the user's provider CLIs, so servers share one gate. */ +export const userDirectory = () => + NodePath.join( + process.env.XDG_CACHE_HOME || NodePath.join(NodeOS.homedir(), ".cache"), + "t3", + "provider-maintenance", + ); + +/** + * Servers provide the user-wide gate through `layer`. Anything else, such as a + * test, gets a private gate so it never waits on or blocks a live T3. + */ export class ProviderMaintenanceHostLock extends Context.Reference>( "t3/provider/ProviderMaintenanceHostLock", { - defaultValue: () => - make( - NodePath.join( - process.env.XDG_CACHE_HOME || NodePath.join(NodeOS.homedir(), ".cache"), - "t3", - "provider-maintenance", - ), - ), + defaultValue: () => { + const directory = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "t3-provider-maintenance-"), + ); + process.once("exit", () => NodeFS.rmSync(directory, { recursive: true, force: true })); + return make(directory); + }, }, ) {} + +export const layer = Layer.sync(ProviderMaintenanceHostLock, () => make(userDirectory())); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index dbbcea7c5ff21..49844f4f965ab 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -9,6 +9,7 @@ import * as ThreadContracts from "@t3tools/contracts"; import { describe, it, assert } from "@effect/vitest"; import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as FileSystem from "effect/FileSystem"; @@ -68,6 +69,7 @@ const encoder = new TextEncoder(); const layerNonWindowsPlatform = Layer.merge( Layer.succeed(HostProcess.Platform, "linux"), Layer.succeed(HostLock.ProviderMaintenanceHostLock, { + directory: "", acquireRead: Effect.succeed(Effect.void), withRead: (effect) => effect, withWrite: (effect) => effect, @@ -1708,6 +1710,18 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.effect("keeps anything but a server off the user-wide gate", () => + Effect.gen(function* () { + const fallback = yield* HostLock.ProviderMaintenanceHostLock; + assert.isTrue(fallback.directory.startsWith(NodeOS.tmpdir())); + assert.notStrictEqual(fallback.directory, HostLock.userDirectory()); + const served = yield* HostLock.ProviderMaintenanceHostLock.pipe( + Effect.provide(HostLock.layer), + ); + assert.strictEqual(served.directory, HostLock.userDirectory()); + }), + ); + it.live("reclaims a dead reader whose PID has been reused", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index cbd7ec5fd238a..af6fe123f8992 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -162,6 +162,7 @@ import * as UsageService from "./usage/UsageService.ts"; import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts"; import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; import * as ProviderAutoUpdater from "./updates/ProviderAutoUpdater.ts"; +import * as ProviderMaintenanceHostLock from "./provider/ProviderMaintenanceHostLock.ts"; import * as UpdateWindow from "./updates/UpdateWindow.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; @@ -1115,6 +1116,8 @@ const layerMakeServer = Layer.unwrap( Layer.provide(VcsProcess.layer), // Every agent and terminal spawn reads this, so it sits below everything. Layer.provideMerge(AgentScopeLive.layer), + // Provider installs and CLI launches coordinate with every T3 server on the host. + Layer.provideMerge(ProviderMaintenanceHostLock.layer), Layer.provideMerge(layerPlatformServices), ); }), diff --git a/apps/server/src/textGeneration/TextGeneration.test.ts b/apps/server/src/textGeneration/TextGeneration.test.ts index 5f75c8b168327..412d87f8c8012 100644 --- a/apps/server/src/textGeneration/TextGeneration.test.ts +++ b/apps/server/src/textGeneration/TextGeneration.test.ts @@ -201,6 +201,7 @@ it.effect("holds the host lease until the text generation provider completes", ( }), ); const host = { + directory: "", acquireRead: Effect.succeed(Effect.void), withRead: permit.withPermits(1), withWrite: permit.withPermits(1), From dbbc727dfa27968729db1024b40ff795099eb722 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:01:04 +0000 Subject: [PATCH 075/149] fix(server): a manual update waiting for running turns says so When turns on this or another server must finish first, the update stays queued with "Waiting for running turns to finish." instead of the previous-update message. Interrupting it while it waits ends it as canceled before installation started. --- .../provider/ProviderMaintenanceHostLock.ts | 11 +++- .../providerMaintenanceCommandCoordinator.ts | 7 ++- .../providerMaintenanceRunner.test.ts | 51 +++++++++++++++++++ .../src/provider/providerMaintenanceRunner.ts | 8 +++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index 6a5e4b1c99e85..1d6dacf23c38e 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -123,10 +123,17 @@ export const make = (directory: string) => { } return !busy; }); - const withWrite = (effect: Effect.Effect) => + /** `onBusy` runs once if active turns or CLI calls, here or on another server, must drain first. */ + const withWrite = ( + effect: Effect.Effect, + onBusy?: Effect.Effect, + ) => withGate( Effect.gen(function* () { - while (!(yield* drained)) yield* Effect.sleep("100 millis"); + if (!(yield* drained)) { + if (onBusy) yield* onBusy; + while (!(yield* drained)) yield* Effect.sleep("100 millis"); + } return yield* effect; }), ); diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index c2d3cb5511d1a..663ebe281f08b 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -11,6 +11,8 @@ export interface ProviderMaintenanceCommandCoordinatorShape { readonly targetKey: string; readonly lockKey: string; readonly onQueued?: Effect.Effect; + /** Runs once if turns on this or another server must finish first. */ + readonly onWaitingForTurns?: Effect.Effect; readonly onInterrupted?: Effect.Effect; readonly run: Effect.Effect; }) => Effect.Effect; @@ -46,6 +48,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( targetKey, lockKey, onQueued, + onWaitingForTurns, onInterrupted, run, }) => @@ -58,7 +61,9 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( return yield* (onQueued ?? Effect.void).pipe( // Host leases drain before admission closes: a session holding a lease // may still need admission to finish launching its CLI. - Effect.andThen(locks.withLock(lockKey, hostLock.withWrite(admission.withPermit(run)))), + Effect.andThen( + locks.withLock(lockKey, hostLock.withWrite(admission.withPermit(run), onWaitingForTurns)), + ), Effect.onInterrupt(() => onInterrupted ?? Effect.void), Effect.ensuring(releaseTarget(targetKey)), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 49844f4f965ab..ae07283f3d63a 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -1722,6 +1722,57 @@ describe("provider maintenance host leases", () => { }), ); + it.effect("shows a manual update waiting for turns and cancels it before installing", () => { + const calls: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const hostLock = HostLock.make(yield* fs.makeTempDirectoryScoped()); + const releaseTurn = yield* hostLock.acquireRead; + const waiting = yield* Deferred.make(); + const { registry, updateStatesRef } = yield* makeRegistry(baseNativeCliProvider); + const runner = yield* makeTestRunner({ + ...registry, + setProviderMaintenanceActionState: (input) => + registry + .setProviderMaintenanceActionState(input) + .pipe( + Effect.tap(() => + input.state?.message === "Waiting for running turns to finish." + ? Deferred.succeed(waiting, undefined) + : Effect.void, + ), + ), + }).pipe(Effect.provideService(HostLock.ProviderMaintenanceHostLock, hostLock)); + const update = yield* runner.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkScoped); + yield* Deferred.await(waiting); + yield* Fiber.interrupt(update); + yield* releaseTurn; + assert.deepStrictEqual( + (yield* Ref.get(updateStatesRef)).map((state) => [state.status, state.message]), + [ + ["queued", "Waiting for another provider update to finish."], + ["queued", "Waiting for running turns to finish."], + ["failed", "Update canceled before installation started."], + ], + ); + assert.deepStrictEqual(calls, []); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + layerNonWindowsPlatform, + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner((command) => { + calls.push(command); + return {}; + }), + ), + ), + ); + }); + it.live("reclaims a dead reader whose PID has been reused", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 296863b2ce576..5193fc0b91e7f 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -690,6 +690,14 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { targetKey, lockKey: update.lockKey, onQueued: setQueuedState, + onWaitingForTurns: setUpdateState( + makeUpdateState({ + status: "queued", + startedAt: null, + finishedAt: null, + message: "Waiting for running turns to finish.", + }), + ).pipe(Effect.asVoid), onInterrupted: Effect.gen(function* () { const current = (yield* providerRegistry.getProviders).find( (candidate) => candidate.instanceId === instanceId, From b74de4728456a7e72f67ec7fc666a8b987bb65d2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:01:04 +0000 Subject: [PATCH 076/149] test(server): other providers keep launching during a Codex first install The managed install holds exclusive admission only while it starts. The download, extraction and verification run in a staging directory that Codex cannot resolve until the atomic activation. --- .../src/provider/CodexInstallation.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/apps/server/src/provider/CodexInstallation.test.ts b/apps/server/src/provider/CodexInstallation.test.ts index c3c089361ddf1..a04dc8b5105a8 100644 --- a/apps/server/src/provider/CodexInstallation.test.ts +++ b/apps/server/src/provider/CodexInstallation.test.ts @@ -385,3 +385,27 @@ it.effect("Codex first installs wait for admission before changing state or down expect(h.downloads()).toBe(1); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + +it.effect("other providers keep launching while a Codex first install downloads", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const downloading = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const h = yield* makeHarness({ + body: Stream.fromEffect( + Deferred.succeed(downloading, undefined).pipe( + Effect.andThen(Deferred.await(finish)), + Effect.as(new Uint8Array(archive)), + ), + ), + }); + yield* h.installation.start; + yield* Deferred.await(downloading); + expect(yield* admission.withLaunchPermit(Effect.succeed("launched"))).toBe("launched"); + // Codex itself only resolves a completed, activated release. + expect(Exit.isFailure(yield* h.installation.resolve().pipe(Effect.exit))).toBe(true); + yield* Deferred.succeed(finish, undefined); + expect((yield* terminalState(h.installation)).phase).toBe("succeeded"); + expect((yield* h.installation.resolve()).source).toBe("managed"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); From 15911e9ee2a02f89875ee489e66fcb24f9bd24ed Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:03:43 +0000 Subject: [PATCH 077/149] fix(claude): unusable transcripts fall back to saved context and the CLI ends with a crashed server --- .../Adapters/ClaudeAdapterV2.test.ts | 102 +++++++++++++++++- .../Adapters/ClaudeAdapterV2.ts | 60 +++++++++-- 2 files changed, 149 insertions(+), 13 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index e59b69b5d0e3f..fca4212700638 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -1,6 +1,9 @@ -// @effect-diagnostics nodeBuiltinImport:off - the fake Claude CLI hands the SDK Node streams. +// @effect-diagnostics nodeBuiltinImport:off - the fake Claude CLI hands the SDK Node streams and exercises process death. +import * as NodeChildProcess from "node:child_process"; import * as NodeEvents from "node:events"; +import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import * as NodeStream from "node:stream"; import type { @@ -2034,7 +2037,82 @@ describe("ClaudeAdapterV2 MCP credential channel", () => { }); describe("ClaudeAdapterV2 native session identity", () => { - it.effect("rejects a missing transcript before accepting a resume", () => + it.live( + "ends the native CLI when its hosting process crashes", + () => + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform !== "linux") return; + const executablePath = yield* HostProcess.ExecutablePath; + const directory = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "t3-claude-parent-death-"), + ); + const readyPath = NodePath.join(directory, "ready"); + const stoppedPath = NodePath.join(directory, "stopped"); + const waitForFile = (file: string) => + Effect.callback((resume) => { + const check = () => { + if (NodeFS.existsSync(file)) resume(Effect.void); + }; + const watcher = NodeFS.watch(directory, check); + check(); + return Effect.sync(() => watcher.close()); + }).pipe( + Effect.timeoutOrElse({ + duration: "5 seconds", + orElse: () => Effect.die(new Error(`No process milestone at ${file}`)), + }), + ); + const cliSource = [ + "const fs = require('node:fs');", + `process.on('SIGTERM', () => { fs.writeFileSync(${JSON.stringify(stoppedPath)}, 'stopped'); process.exit(0); });`, + `fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid));`, + "setInterval(() => {}, 1000);", + ].join("\n"); + const parentSource = [ + `import * as Effect from ${JSON.stringify(import.meta.resolve("effect/Effect"))};`, + `import * as NodeServices from ${JSON.stringify(import.meta.resolve("@effect/platform-node/NodeServices"))};`, + `import * as Adapter from ${JSON.stringify(import.meta.resolve("./ClaudeAdapterV2.ts"))};`, + `import * as Loggers from ${JSON.stringify(import.meta.resolve("@t3tools/provider-core/server/ProviderEventLoggers"))};`, + "await Effect.runPromise(Effect.scoped(Effect.gen(function* () {", + "const runner = yield* Adapter.ClaudeAgentSdkQueryRunner;", + "yield* runner.open({ threadId: 'parent-death', providerSessionId: 'parent-death', options: {", + `pathToClaudeCodeExecutable: process.execPath, executableArgs: ['-e', ${JSON.stringify(cliSource)}, '--'],`, + "sessionId: '12345678-1234-4123-8123-123456789012', model: 'sonnet', tools: [], permissionMode: 'default', env: process.env }});", + "yield* Effect.never;", + "}).pipe(Effect.provide(Adapter.layerQueryRunner), Effect.provide(NodeServices.layer),", + "Effect.provideService(Loggers.ProviderEventLoggers, Loggers.NoOpProviderEventLoggers))));", + ].join("\n"); + const ready = yield* waitForFile(readyPath).pipe(Effect.forkScoped); + const parent = NodeChildProcess.spawn( + executablePath, + ["--input-type=module", "-e", parentSource], + { + stdio: ["ignore", "ignore", "inherit"], + }, + ); + let cliPid: number | undefined; + try { + yield* Fiber.join(ready); + cliPid = Number(NodeFS.readFileSync(readyPath, "utf8")); + const stopped = yield* waitForFile(stoppedPath).pipe(Effect.forkScoped); + parent.kill("SIGKILL"); + yield* Fiber.join(stopped); + assert.equal(NodeFS.readFileSync(stoppedPath, "utf8"), "stopped"); + } finally { + parent.kill("SIGKILL"); + if (cliPid !== undefined) { + try { + process.kill(cliPid, "SIGKILL"); + } catch {} + } + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }), + 15000, + ); + + it.effect("rejects missing or unusable transcripts before accepting a resume", () => Effect.scoped( Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -2078,9 +2156,25 @@ describe("ClaudeAdapterV2 native session identity", () => { assert.equal(missing.failure._tag, "ProviderAdapterResumeThreadError"); const directory = path.join(configDir, "projects", "-workspace"); yield* fileSystem.makeDirectory(directory, { recursive: true }); + const transcriptPath = path.join( + directory, + `${providerThread.nativeThreadRef!.nativeId}.jsonl`, + ); + for (const transcript of [ + "", + "{broken\n", + '{"type":"summary","summary":"summary only"}\n', + '{"type":"user","uuid":"sidechain","isSidechain":true,"message":{"role":"user","content":"subagent only"}}\n', + ]) { + yield* fileSystem.writeFileString(transcriptPath, transcript); + const unusable = yield* runtime.resumeThread(resumeInput).pipe(Effect.result); + assert.equal(unusable._tag, "Failure", transcript); + if (unusable._tag === "Failure") + assert.equal(unusable.failure._tag, "ProviderAdapterResumeThreadError"); + } yield* fileSystem.writeFileString( - path.join(directory, `${providerThread.nativeThreadRef!.nativeId}.jsonl`), - '{"type":"user","message":{"role":"user","content":"previous prompt"}}\n', + transcriptPath, + '{broken\n{"type":"user","uuid":"saved-prompt","message":{"role":"user","content":"previous prompt"}}\n{incomplete', ); const resumed = yield* runtime.resumeThread(resumeInput); assert.deepEqual(resumed.nativeThreadRef, providerThread.nativeThreadRef); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 2f93d53561b3a..6847fb5c5014c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -33,6 +33,7 @@ import type { import { parseCliArgs } from "@t3tools/shared/cliArgs"; import { AgentScope } from "@t3tools/shared/AgentScope"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { resolveCommandPath } from "@t3tools/shared/shell"; import { applyClaudePromptEffortPrefix } from "@t3tools/shared/model"; import { CLAUDE_RESUME_COMPACTION_NEVER_ANSWER, @@ -659,13 +660,40 @@ export const layerQueryRunner: Layer.Layer< () => Effect.succeed([]), ), ); + const isConversationMessage = Schema.is( + Schema.Struct({ + type: Schema.Literals(["user", "assistant"]), + uuid: Schema.String, + isSidechain: Schema.optional(Schema.Boolean), + message: Schema.Struct({ + role: Schema.Literals(["user", "assistant"]), + content: Schema.Union([Schema.String, Schema.Array(Schema.Unknown)]), + }), + }), + ); for (const directory of directories) { - if ( - yield* fileSystem.exists( - path.join(projectsDir, directory, `${input.sessionId}.jsonl`), - ) - ) - return true; + // The CLI cannot resume an empty or metadata-only transcript. + // Ignore damaged lines as it does, and stop at the first usable message. + const message = yield* fileSystem + .stream(path.join(projectsDir, directory, `${input.sessionId}.jsonl`)) + .pipe( + Stream.decodeText, + Stream.splitLines, + Stream.filter((line) => { + try { + const entry: unknown = JSON.parse(line); + return isConversationMessage(entry) && entry.isSidechain !== true; + } catch { + return false; + } + }), + Stream.runHead, + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + ); + if (Option.isSome(message)) return true; } return false; }, @@ -698,13 +726,27 @@ export const layerQueryRunner: Layer.Layer< // SDK keeps its own stderr capture and exit reporting. The SDK runs a // script entry through node instead, so those launch unwrapped. const binaryPath = queryOptions.pathToClaudeCodeExecutable; + const parentDeathSignal = + (yield* HostProcess.Platform) === "linux" + ? yield* resolveCommandPath("setpriv", { + env: queryOptions.env ?? (yield* HostProcess.Environment), + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.option, + ) + : Option.none(); const launch = binaryPath === undefined || CLAUDE_SCRIPT_EXTENSIONS.some((ext) => binaryPath.endsWith(ext)) ? undefined : yield* agentScope.wrap({ - command: binaryPath, - args: [], + // stdin EOF does not stop Claude's background work. A crashed + // server must end its CLI before recovery resumes the transcript. + command: Option.isSome(parentDeathSignal) ? parentDeathSignal.value : binaryPath, + args: Option.isSome(parentDeathSignal) + ? ["--pdeathsig", "TERM", "--", binaryPath] + : [], name: "claude", threadId: input.threadId, env: queryOptions.env, @@ -8311,7 +8353,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if (!(yield* queryRunner.hasSession({ sessionId: nativeThreadId, configDir }))) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver: CLAUDE_PROVIDER, - detail: "The saved Claude session transcript is missing.", + detail: "The saved Claude session transcript is missing or unusable.", }); } } From 26231f823d0dac3433ae2c5fe138fce64390709b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:03:44 +0000 Subject: [PATCH 078/149] fix(opencode,pi): restarts keep finished turns, replayed text and subagents, and reject lost pi sessions --- .../Adapters/OpenCode2AdapterV2.test.ts | 399 +++++++++++++++++- .../src/server/v2/adapter.ts | 104 ++++- .../provider-pi/src/server/adapter.test.ts | 100 +++++ packages/provider-pi/src/server/adapter.ts | 13 + 4 files changed, 600 insertions(+), 16 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index daf722f19fac2..470aa46869b76 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -447,21 +447,83 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); - it.effect("prompts after a restart when the native execution did not survive", () => + it.effect.each(["empty", "already settled", "failed"] as const)( + "prompts after a restart with %s history instead of following an earlier completion", + (previous) => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.active"), + replyData("session.active", {}), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", { + data: + previous === "empty" + ? [] + : history.data + .toReversed() + .map((message) => + message.type === "idle" && previous === "failed" + ? { ...message, outcome: "failed" } + : message, + ), + cursor: {}, + }), + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); + yield* runtime.startTurn({ + ...turnInput( + previous === "already settled" + ? { + ...thread, + nativeConversationHeadRef: { + driver: OPENCODE_PROVIDER, + nativeId: history.data[0]!.id, + strength: "weak", + }, + } + : thread, + ), + restartContinuationOfRunId: RunId.make("interrupted-run"), + }); + assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + }).pipe(Effect.scoped), + ); + + it.effect("settles an execution that finished during a T3 restart without another prompt", () => Effect.gen(function* () { - const { runtime, thread } = yield* resumed([ - out("session.active"), - replyData("session.active", {}), - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.succeeded", { sessionID: SESSION }), - ]); - const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); + const finished = { data: history.data.toReversed(), cursor: {} }; + const { runtime, thread } = yield* resumed( + [ + out("session.active"), + replyData("session.active", {}), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", finished), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", finished), + ], + { external: true }, + ); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); yield* runtime.startTurn({ ...turnInput(thread), - restartContinuationOfRunId: RunId.make("interrupted-run"), + restartContinuationOfRunId: RunId.make("cut-run"), }); - assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + const collected = yield* Fiber.join(events); + assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); + const updated = collected.findLast((event) => event.type === "provider_thread.updated"); + assert.equal( + updated?.type === "provider_thread.updated" + ? updated.providerThread.nativeConversationHeadRef?.nativeId + : undefined, + history.data[0]!.id, + ); }).pipe(Effect.scoped), ); @@ -757,6 +819,266 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { } as OrchestrationV2AppThread, }); const CHILD = "ses_f1485c529ffea4URrYruwEg0Ja"; + + it.effect.each(["prompt", "follow-up", "joined"] as const)( + "restores a subagent card and its pending question after a T3 restart during a %s", + (execution) => + Effect.gen(function* () { + const restored = { + data: [ + { + id: "msg_subagent", + type: "assistant", + time: { created: 2 }, + agent: "build", + model: { providerID: "opencode", id: "big-pickle", variant: "default" }, + content: [ + { + type: "tool", + id: "call-restored", + name: "subagent", + executed: true, + state: { + status: "running", + input: { prompt: "Choose a color", description: "Color" }, + metadata: { sessionID: CHILD }, + }, + time: { created: 2, ran: 2 }, + }, + ], + }, + { id: PROMPT_ID, type: "user", time: { created: 1 }, text: "Delegate" }, + ], + cursor: {}, + }; + if (execution === "joined") { + restored.data[0]!.content!.push({ + ...restored.data[0]!.content![0]!, + id: "call-restored-again", + }); + } + const childHistory = { + ...restored, + data: [ + { + ...history.data[1], + id: "msg_child", + time: { created: 2 }, + finish: undefined, + content: [{ type: "text", text: "Pre" }], + }, + ...(execution === "follow-up" + ? [{ id: "msg_child_idle", type: "idle", time: { created: 1 }, outcome: "succeeded" }] + : []), + { id: "msg_child_prompt", type: "user", time: { created: 1 }, text: "Choose a color" }, + ], + }; + const { runtime, thread } = yield* resumed( + [ + out("session.active"), + replyData("session.active", { + [SESSION]: { type: "running" }, + [CHILD]: { type: "running" }, + }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", restored), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", restored), + out("session.update", { sessionID: CHILD, permissions: t3Rules }), + reply("session.update", null), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", childHistory), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", childHistory), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", [{ ...colorForm, sessionID: CHILD }]), + ...(execution === "joined" + ? [ + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", childHistory), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", childHistory), + out("permission.list", { sessionID: CHILD }), + replyData("permission.list", []), + out("session.form.list", { sessionID: CHILD }), + replyData("session.form.list", [{ ...colorForm, sessionID: CHILD }]), + ] + : []), + out("session.form.reply", { + sessionID: CHILD, + formID: colorForm.id, + answer: { q0: "Red" }, + }), + reply("session.form.reply", null), + event("session.text.delta", { + sessionID: CHILD, + assistantMessageID: "msg_child", + ordinal: 0, + delta: "fix", + }), + event("session.text.ended", { + sessionID: CHILD, + assistantMessageID: "msg_child", + ordinal: 0, + text: "Prefix", + }), + event("session.execution.succeeded", { sessionID: CHILD }), + ...(execution === "joined" + ? [ + event("session.execution.started", { sessionID: CHILD }), + event("session.execution.succeeded", { sessionID: CHILD }), + ] + : []), + ...["call-restored", ...(execution === "joined" ? ["call-restored-again"] : [])].map( + (id): ProviderReplayEntry => ({ + type: "emit_inbound", + frame: { + type: "sdk.event", + event: { + id: "evt_restoredsuccess", + created: 3, + type: "session.tool.success", + durable: { aggregateID: SESSION, seq: 1, version: 2 }, + data: { + sessionID: SESSION, + assistantMessageID: "msg_subagent", + id, + executed: true, + content: [{ type: "text", text: "Red" }], + metadata: { sessionID: CHILD }, + }, + }, + }, + }), + ), + event("session.execution.succeeded", { sessionID: SESSION }), + ], + { external: true }, + ); + const requested = yield* Deferred.make< + Extract, + Error + >(); + const events = yield* runtime.events.pipe( + Stream.tap((event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending" + ? Deferred.succeed(requested, event) + : event.type === "turn.terminal" + ? Deferred.fail(requested, new Error(JSON.stringify(event))) + : Effect.void, + ), + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn({ + ...withLineage(thread), + restartContinuationOfRunId: RunId.make("cut-run"), + }); + const question = (yield* Deferred.await(requested)).runtimeRequest; + assert.equal(question.nativeRequestRef?.nativeId, colorForm.id); + yield* runtime.respondToRuntimeRequest({ + requestId: question.id, + decision: "accept", + answers: { q0: "Red" }, + }); + const collected = yield* Fiber.join(events); + assert.isTrue( + collected.some( + (event) => + event.type === "subagent.updated" && + event.subagent.childThreadId !== null && + event.subagent.result === "Red", + ), + ); + assert.isTrue( + collected.some( + (event) => + event.type === "message.updated" && + event.message.threadId !== threadId && + event.message.text === "Prefix", + ), + ); + assert.isFalse( + collected.some( + (event) => event.type === "message.updated" && event.message.text === "fix", + ), + ); + if (execution === "joined") { + const childTurns = collected.flatMap((event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId.startsWith(CHILD) + ? [event.providerTurn.ordinal] + : [], + ); + assert.include(childTurns, 2); + } + assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); + }).pipe(Effect.scoped), + ); + + it.effect("retires a restored background subagent that ended while T3 was down", () => + Effect.gen(function* () { + const finished = { + data: [ + { id: "msg_parent_idle", type: "idle", time: { created: 3 }, outcome: "succeeded" }, + { + ...history.data[1], + id: "msg_parent", + content: [ + { + type: "tool", + id: "call-retired", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { prompt: "Work", background: true }, + content: [{ type: "text", text: "Running in the background" }], + metadata: { sessionID: CHILD, status: "running" }, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + }, + { id: PROMPT_ID, type: "user", time: { created: 1 }, text: "Delegate" }, + ], + cursor: {}, + }; + const { runtime, thread } = yield* resumed( + [ + out("session.active"), + replyData("session.active", {}), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", finished), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", finished), + out("session.update", { sessionID: CHILD, permissions: t3Rules }), + reply("session.update", null), + ], + { external: true }, + ); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn({ + ...withLineage(thread), + restartContinuationOfRunId: RunId.make("cut-run"), + }); + const collected = yield* Fiber.join(events); + assert.isFalse(yield* runtime.hasPendingBackgroundWork!); + assert.isTrue( + collected.some( + (event) => event.type === "subagent.updated" && event.subagent.status === "interrupted", + ), + ); + assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); + }).pipe(Effect.scoped), + ); /** * The single reader of the runtime's events: resolves `attached` once the * running background child has its thread, then returns the turn's terminal. @@ -2927,6 +3249,61 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); + it.effect.each(["text", "reasoning"] as const)( + "preserves backfilled %s before the next delta and end event", + (kind) => + Effect.gen(function* () { + const reconnectEntries = reconnected({ [SESSION]: { type: "running" } }); + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, id: PROMPT_ID, text: "" }), + promptAccepted, + ...reconnectEntries.slice(0, -1), + reply("message.list", { + data: [ + { + id: "msg_assistant_gap", + time: { created: 2 }, + type: "assistant", + agent: "build", + model: { id: "big-pickle", providerID: "opencode", variant: "default" }, + content: [{ type: kind, text: "prefix" }], + }, + { id: PROMPT_ID, time: { created: 1 }, text: "hi", type: "user" }, + ], + cursor: {}, + }), + ...openRequests([]), + event(`session.${kind}.delta`, { + sessionID: SESSION, + assistantMessageID: "msg_assistant_gap", + ordinal: 0, + delta: " suffix", + }), + event(`session.${kind}.ended`, { + sessionID: SESSION, + assistantMessageID: "msg_assistant_gap", + ordinal: 0, + text: "prefix suffix", + }), + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn(turnInput(thread)); + const texts = (yield* Fiber.join(events)).flatMap((event) => + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ? [event.turnItem.text] + : [], + ); + assert.include(texts, "prefix suffix"); + assert.notInclude(texts, " suffix"); + }).pipe(Effect.scoped), + ); + it.effect("shows a permission asked while the stream was down, and answers it", () => Effect.gen(function* () { const { runtime, thread } = yield* resumed( diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index e586bbccd1806..9477820c76409 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -1788,6 +1788,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } state.providerThread = { ...state.providerThread, + ...(terminal.status === "completed" && promptOf(turn.providerTurn) !== undefined + ? { nativeConversationHeadRef: ref(promptOf(turn.providerTurn)!, "weak") } + : {}), status: threadDisposition === "broken" ? "error" : "idle", updatedAt: completedAt, }; @@ -2689,7 +2692,20 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr * OpenCode appends after each one, or undefined when there is none after * the turn's start. */ - const backfill = Effect.fnUntraced(function* (sessionId: string, state: ThreadState) { + const backfill = Effect.fnUntraced(function* ( + sessionId: string, + state: ThreadState, + activeSessions?: ReadonlySet, + ): Effect.fn.Return< + { outcome: string } | undefined, + Effect.Error< + | ReturnType + | ReturnType + | ReturnType + | ReturnType + | ReturnType + > + > { const turn = state.active; if (turn === undefined) return undefined; const promptId = promptOf(turn.providerTurn); @@ -2716,10 +2732,79 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr for (const part of message.content) { if (part.type === "text" || part.type === "reasoning") { const block = { assistantMessageID: message.id, ordinal: ordinals[part.type]++ }; - yield* emitText(state, turn, block, part.type, () => part.text, true); + yield* emitText( + state, + turn, + block, + part.type, + () => part.text, + message.time.completed !== undefined || idle !== undefined, + ); continue; } if (part.type !== "tool") continue; + if (part.name === "subagent") { + if (!state.calls.has(part.id)) yield* startCall(state, turn, part.id); + const call = state.calls.get(part.id)!; + const input = typeof part.state.input === "object" ? part.state.input : undefined; + const metadata = "metadata" in part.state ? part.state.metadata : undefined; + call.prompt = stringField(input, "prompt") ?? call.prompt; + call.title = stringField(input, "description") ?? call.title; + call.agent = stringField(input, "agent") ?? call.agent; + call.model = stringField(input, "model") ?? call.model; + call.background = + call.background || + stringField(metadata, "status") === "running" || + input?.["background"] === true; + const childId = stringField(metadata, "sessionID"); + const restoringChild = childId !== undefined && call.child === undefined; + if (childId !== undefined) yield* attachChild(call, childId); + if (part.state.status === "error") { + yield* settleCall(call, "failed", part.state.error.message); + } else if (part.state.status === "completed" && !call.background) { + yield* settleCall(call, "completed", subagentOutput(textOf(part.state.content))); + } else { + yield* emitSubagent(call); + if (restoringChild) { + if ( + call.background && + activeSessions !== undefined && + !activeSessions.has(childId!) + ) { + yield* settleCall(call, "interrupted", LOST_BACKGROUND); + continue; + } + const sessionID = Session.ID.make(childId!); + if (activeSessions?.has(childId!) && call.child !== undefined) { + const history = yield* OpenCode2Client.paginate( + { sessionID, order: "desc" as const, limit: 50 }, + client.message.list, + ).pipe(Stream.runCollect); + const child = call.child; + child.subagent!.turns = Math.max( + child.subagent!.turns, + history.filter((entry) => entry.type === "idle").length, + ); + yield* startChildTurn(child); + child.active!.before = + history.find( + (entry) => + entry.type === "user" || + entry.type === "synthetic" || + entry.type === "idle", + )?.id ?? null; + yield* backfill(childId!, child, activeSessions); + } + const [permissions, forms] = yield* Effect.all([ + client.permission.list({ sessionID }), + client.session.form.list({ sessionID }), + ]); + for (const request of permissions) yield* onAsked({ type: "permission", request }); + for (const form of forms) yield* onAsked({ type: "form", form }); + } + } + continue; + } if (!turn.tools.has(part.id)) { turn.tools.set(part.id, { name: part.name, input: {} }); turn.startedAt.set(part.id, yield* DateTime.now); @@ -2784,7 +2869,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr for (const [sessionId, state] of running) { const turn = state.active; if (turn === undefined) continue; - const ended = yield* backfill(sessionId, state); + const ended = yield* backfill(sessionId, state, new Set(Object.keys(active))); if (sessionId in active || state.active !== turn) continue; yield* finishTurn( state, @@ -3787,7 +3872,6 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const active = yield* client.session .active() .pipe(Effect.timeout(ACTIVE_CHECK_TIMEOUT)); - if (!(sessionId in active)) return false; // A surviving server is still executing the interrupted // run. Attach its events without enqueuing another prompt. const history = yield* OpenCode2Client.paginate( @@ -3798,6 +3882,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr Stream.runCollect, ); const prompt = history.at(-1); + const idle = history.find((message) => message.type === "idle"); + const finishedWhileDown = + prompt?.type === "user" && + prompt.id !== turnInput.providerThread.nativeConversationHeadRef?.nativeId && + idle?.type === "idle" && + idle.outcome === "succeeded"; + if (!(sessionId in active) && !finishedWhileDown) return false; const turn = yield* beginTurn(state, turnInput, false); yield* takeRunningWake(state); if (state.active !== turn) return true; @@ -3807,7 +3898,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr nativeTurnRef: ref(prompt.id, "weak"), }; yield* emitProviderTurn(state, turn, turn.providerTurn); - yield* backfill(sessionId, state).pipe( + yield* backfill(sessionId, state, new Set(Object.keys(active))).pipe( Effect.tapError((cause) => finishTurn(state, { status: "failed", @@ -3816,6 +3907,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr ), ); } + if (finishedWhileDown && !(sessionId in active)) { + yield* finishTurn(state, { status: "completed" }); + } return true; }), ); diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index cab0043ab185f..05e4ba1c3444a 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -1408,6 +1408,106 @@ describe("PiAdapterV2", () => { }).pipe(Effect.scoped, Effect.provide(layerTest)), ); + it.effect.each(["settled", "interrupted first", "model entries"] as const)( + "rejects a resumed session that lost its persisted conversation after a %s turn", + (previous) => + Effect.gen(function* () { + const fake = yield* makeFakePi; + const { runtime } = yield* openRuntime(fake); + const thread = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + if (previous === "model entries") { + fake.queueEntries({ + entries: [ + { + type: "model_change", + id: "new-model", + parentId: null, + provider: "cliproxyapi", + modelId: "gpt-6-luna", + }, + { + type: "thinking_level_change", + id: "new-thinking", + parentId: "new-model", + thinkingLevel: "medium", + }, + ], + leafId: "new-thinking", + }); + } + const error = yield* runtime + .resumeThread({ + providerThread: { + ...thread, + firstRunOrdinal: 1, + nativeConversationHeadRef: + previous === "interrupted first" + ? null + : { + driver: PI_PROVIDER, + nativeId: "saved-leaf", + strength: "strong", + }, + }, + }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterResumeThreadError"); + assert.match(String(error.cause), /persisted conversation/); + const replacement = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + existingProviderThread: { ...thread, nativeThreadRef: null }, + }); + assert.notEqual(replacement.nativeThreadRef?.nativeId, thread.nativeThreadRef?.nativeId); + yield* startTurn(runtime, replacement); + yield* fake.takeRequest("prompt"); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + + it.effect("resumes a compacted conversation whose older entries were pruned", () => + Effect.gen(function* () { + const fake = yield* makeFakePi; + const { runtime } = yield* openRuntime(fake); + const thread = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + fake.queueEntries({ + entries: [ + { + type: "compaction", + id: "summary", + parentId: null, + summary: "Remember PAPAYA", + firstKeptEntryId: "pruned", + tokensBefore: 1000, + }, + ], + leafId: "summary", + }); + const resumed = yield* runtime.resumeThread({ + providerThread: { + ...thread, + firstRunOrdinal: 1, + nativeConversationHeadRef: { + driver: PI_PROVIDER, + nativeId: "pruned", + strength: "strong", + }, + }, + }); + assert.equal(resumed.nativeThreadRef?.nativeId, thread.nativeThreadRef?.nativeId); + yield* startTurn(runtime, resumed); + yield* fake.takeRequest("prompt"); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + it.effect.each(["veto", "same identity"] as const)( "rejects a replacement with %s", (invalidReplacement) => diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 96d2783ecf3f1..5245fbfda5128 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -2412,6 +2412,19 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( const baselineEntries = yield* request({ type: "get_entries" }).pipe( Effect.orElseSucceed(() => undefined), ); + const entries = recordField(baselineEntries, "entries"); + if ( + resumeId != null && + (existing?.nativeConversationHeadRef != null || existing?.firstRunOrdinal != null) && + Array.isArray(entries) && + !entries.some((entry) => { + const type = recordField(entry, "type"); + return type === "message" || type === "compaction" || type === "custom_message"; + }) + ) { + threadState = null; + return yield* protocolError("Pi lost the persisted conversation for this session"); + } lastKnownLeaf = recordString(baselineEntries, "leafId") ?? null; // A successful full baseline makes the cursor trustworthy again. Only // a failed one leaves it stale, so a recovered session does not keep From ecd61638776c7d2252fc929841bdab37f2d646af Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:03:50 +0000 Subject: [PATCH 079/149] fix(acp): respawned sessions renegotiate capabilities and fall back to saved context --- .../Adapters/AcpAdapterV2.test.ts | 355 ++++++++++++++++++ .../provider/acp/AcpJsonRpcConnection.test.ts | 38 ++ .../src/server/AcpRegistrySupport.test.ts | 45 +++ .../src/server/AcpSessionRuntime.ts | 9 + packages/provider-acp/src/server/adapter.ts | 66 ++-- 5 files changed, 492 insertions(+), 21 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 369f1c29a569c..1e3a395a86210 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -13,6 +13,8 @@ import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import { + ChatAttachmentId, + ChatImageAttachment, CheckpointId, EnvironmentId, MessageId, @@ -12093,6 +12095,359 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.live.each([ + { before: "load", after: "resume", preferResume: false, action: "resume" }, + { before: "resume", after: "load", preferResume: false, action: "resume" }, + { before: "resume", after: "none", preferResume: false, action: "resume" }, + { before: "load", after: "stale", preferResume: false, action: "resume" }, + { before: "both", after: "both", preferResume: true, action: "resume" }, + { before: "resume", after: "load", preferResume: false, action: "snapshot" }, + { before: "load", after: "resume", preferResume: false, action: "snapshot" }, + { before: "resume", after: "load", preferResume: false, action: "fork" }, + { before: "load", after: "resume", preferResume: false, action: "fork" }, + ] as const)( + "renegotiates ACP session activation after respawn: $before -> $after ($action)", + ({ before, after, preferResume, action }) => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const path = yield* Path.Path; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.unbounded(); + const instanceId = ProviderInstanceId.make("acp-renegotiate"); + const threadId = ThreadId.make("thread-acp-renegotiate"); + const runtimeInputs: Array = []; + const activationMethods: string[] = []; + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const setCredentials = (token: string) => + mcpSessions.set({ + environmentId: EnvironmentId.make("environment-acp-renegotiate"), + threadId, + providerSessionId: "mcp-session-acp-renegotiate", + providerInstanceId: instanceId, + endpoint: "http://127.0.0.1:43123/mcp", + authorizationHeader: token, + browserToolsAvailable: false, + }); + yield* setCredentials("Bearer before-restart"); + const makeRuntime = makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + protocolEvents, + environment: (ordinal) => + ordinal === 1 + ? { T3_ACP_HANG_FIRST_PROMPT_FOREVER: "1" } + : after === "stale" + ? { T3_ACP_FAIL_LOAD_SESSION: "1" } + : {}, + ownDetachedProcessGroup: true, + wrapRuntime: (nativeRuntime, ordinal) => { + const method = ordinal === 1 ? before : after; + const initialized = nativeRuntime.initialize().pipe( + Effect.map((result) => { + const { + resume: _resume, + close: _close, + fork: _fork, + ...sessionCapabilities + } = result.agentCapabilities?.sessionCapabilities ?? {}; + return { + ...result, + agentCapabilities: { + ...result.agentCapabilities, + loadSession: method === "load" || method === "both" || method === "stale", + promptCapabilities: { + ...result.agentCapabilities?.promptCapabilities, + image: method === "load", + }, + sessionCapabilities: { + ...sessionCapabilities, + ...(method === "load" || method === "stale" ? { close: {}, fork: {} } : {}), + ...(method === "resume" || method === "both" ? { resume: {} } : {}), + }, + }, + }; + }), + ); + const unavailable = (unsupportedMethod: string) => + Effect.fail( + new EffectAcpErrors.AcpRequestError({ + code: -32601, + method: unsupportedMethod, + errorMessage: "Replacement agent no longer supports this method", + }), + ); + return { + ...nativeRuntime, + initialize: () => initialized, + start: () => + nativeRuntime + .start() + .pipe( + Effect.flatMap((result) => + initialized.pipe( + Effect.map((initializeResult) => ({ ...result, initializeResult })), + ), + ), + ), + loadSession: (id, options) => { + activationMethods.push("load"); + return method === "load" || method === "both" || method === "stale" + ? nativeRuntime + .loadSession(id, options) + .pipe( + Effect.flatMap((result) => + initialized.pipe( + Effect.map((initializeResult) => ({ ...result, initializeResult })), + ), + ), + ) + : unavailable("session/load"); + }, + forkSession: (id, options) => { + activationMethods.push("fork"); + return nativeRuntime + .forkSession(id, options) + .pipe( + Effect.flatMap((result) => + initialized.pipe( + Effect.map((initializeResult) => ({ ...result, initializeResult })), + ), + ), + ); + }, + resumeSession: (id, options) => { + activationMethods.push("resume"); + return method === "resume" || method === "both" + ? nativeRuntime + .resumeSession(id, options) + .pipe( + Effect.flatMap((result) => + initialized.pipe( + Effect.map((initializeResult) => ({ ...result, initializeResult })), + ), + ), + ) + : unavailable("session/resume"); + }, + }; + }, + }); + const adapter = yield* makeAcpAdapterV2({ + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preferResumeSession: preferResume, + makeRuntime: (input) => + Effect.sync(() => runtimeInputs.push(input)).pipe(Effect.andThen(makeRuntime(input))), + }, + selfInvocation, + }); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const sessionScope = yield* Scope.make(); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-renegotiate"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const started = yield* runtime.events.pipe( + Stream.filter((event) => event.type === "provider_turn.updated"), + Stream.runHead, + ); + assert.isTrue(Option.isSome(started)); + if (Option.isNone(started) || started.value.type !== "provider_turn.updated") return; + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: started.value.providerTurn.id, + requestRuntimeRestart: true, + }); + yield* pollProtocolMethods(protocolEvents); + yield* setCredentials("Bearer after-restart"); + const resumed = yield* ( + action === "resume" + ? runtime.resumeThread({ providerThread, modelSelection, runtimePolicy }) + : action === "snapshot" + ? runtime + .readThreadSnapshot({ providerThread }) + .pipe(Effect.map((result) => result.providerThread)) + : runtime.forkThread({ + sourceProviderThread: providerThread, + targetThreadId: threadId, + modelSelection, + runtimePolicy, + }) + ).pipe(Effect.result); + if (after === "none" || after === "stale") { + assert.equal(resumed._tag, "Failure"); + if (resumed._tag === "Failure") + assert.include( + String(resumed.failure.cause), + after === "none" ? "cannot load or resume" : "Mock load session failure", + ); + const methods = yield* pollProtocolMethods(protocolEvents); + assert.notInclude(methods, "session/load"); + if (after === "none") assert.notInclude(methods, "session/resume"); + else assert.include(methods, "session/resume"); + assert.deepEqual(activationMethods, after === "none" ? [] : ["load"]); + const replacement = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + assert.isNotNull(replacement.nativeThreadRef); + assert.equal(replacement.status, "idle"); + const next = makeTurnInput({ + threadId, + providerThread: replacement, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 2, + messageText: "portable context: pf05-amber-sparrow", + }); + yield* runtime.startTurn(next); + const terminal = yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "turn.terminal" && + event.providerTurnId !== started.value.providerTurn.id, + ), + Stream.runHead, + ); + assert.isTrue(Option.isSome(terminal)); + if (Option.isSome(terminal) && terminal.value.type === "turn.terminal") + assert.equal(terminal.value.status, "completed"); + } else if (action !== "resume" && after === "resume") { + assert.equal(resumed._tag, "Failure"); + assert.deepEqual(activationMethods, []); + } else { + assert.equal(resumed._tag, "Success"); + if (resumed._tag === "Success") assert.isNotNull(resumed.success.nativeThreadRef); + assert.deepEqual(activationMethods, [ + action === "fork" + ? "fork" + : action === "snapshot" + ? "load" + : preferResume + ? "resume" + : after, + ]); + // ACP v2 carries both replaying load and native resume through session/resume. + assert.include( + yield* pollProtocolMethods(protocolEvents), + action === "fork" ? "session/fork" : "session/resume", + ); + assert.equal( + runtime.providerSession.capabilities.threads.canReadThreadSnapshot, + after === "load" || after === "both", + ); + } + if (action === "resume" && (after === "resume" || after === "load")) { + const snapshot = yield* runtime + .readThreadSnapshot({ + providerThread: { + ...providerThread, + nativeThreadRef: { + driver: ACP_TEST_DRIVER, + nativeId: "another-session", + strength: "strong", + }, + }, + }) + .pipe(Effect.result); + assert.equal(snapshot._tag, after === "load" ? "Success" : "Failure"); + const fork = yield* runtime + .forkThread({ + sourceProviderThread: providerThread, + targetThreadId: ThreadId.make("thread-acp-renegotiate-fork"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.result); + assert.equal(fork._tag, after === "load" ? "Success" : "Failure"); + } + if (action === "resume" && after === "resume") { + const imageTurn = makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 2, + }); + const rejected = yield* runtime + .startTurn({ + ...imageTurn, + message: { + ...imageTurn.message, + attachments: [ + ChatImageAttachment.make({ + type: "image", + id: ChatAttachmentId.make("pf05-image-12345678-1234-1234-1234-123456789abc"), + name: "image.png", + mimeType: "image/png", + sizeBytes: 1, + }), + ], + }, + }) + .pipe(Effect.result); + assert.equal(rejected._tag, "Failure"); + if (rejected._tag === "Failure") + assert.include( + String(rejected.failure.cause), + "did not negotiate image prompt support", + ); + } + yield* pollProtocolMethods(protocolEvents); + yield* Scope.close(sessionScope, Exit.void); + const finalizerMethods = yield* pollProtocolMethods(protocolEvents); + assert.equal( + finalizerMethods.filter((method) => method === "session/close").length, + after === "load" || after === "stale" ? 1 : 0, + ); + assert.lengthOf(runtimeInputs, 2); + assert.equal( + runtimeInputs[1]?.processEnvironment?.T3_ACP_MCP_AUTHORIZATION, + "Bearer after-restart", + ); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.effect("restarts the ACP child process before the next prompt after interrupt", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; diff --git a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts index b11fb5baec3a0..ab32d95c63394 100644 --- a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts +++ b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts @@ -1258,6 +1258,44 @@ describe("AcpSessionRuntime", () => { ); }); + it.effect.each(["loadSession", "resumeSession", "forkSession"] as const)( + "preserves extra workspace roots and fresh MCP credentials on %s", + (method) => + Effect.gen(function* () { + const requests: Array = []; + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { ...mockRuntimeOptions.spawn, env: { T3_ACP_SESSION_LIFECYCLE: "1" } }, + additionalDirectories: ["/tmp/acp-extra-root"], + mcpServers: [ + { name: "t3-code", command: "node", args: [], env: [{ name: "TOKEN", value: "old" }] }, + ], + requestLogger: (event) => + Effect.sync(() => { + requests.push(event); + }), + }); + const started = yield* runtime.start(); + const freshMcpServers = [ + { name: "t3-code", command: "node", args: [], env: [{ name: "TOKEN", value: "fresh" }] }, + ]; + yield* runtime[method](started.sessionId, { mcpServers: freshMcpServers }); + const operation = + method === "loadSession" + ? "session/load" + : method === "resumeSession" + ? "session/resume" + : "session/fork"; + expect( + requests.find((event) => event.method === operation && event.status === "started") + ?.payload, + ).toMatchObject({ + additionalDirectories: ["/tmp/acp-extra-root"], + mcpServers: freshMcpServers, + }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("supports negotiated ACP session list, fork, resume, and close methods", () => { const requestEvents: Array = []; return Effect.gen(function* () { diff --git a/packages/provider-acp-registry/src/server/AcpRegistrySupport.test.ts b/packages/provider-acp-registry/src/server/AcpRegistrySupport.test.ts index f95286d004af9..2722be6d9cce2 100644 --- a/packages/provider-acp-registry/src/server/AcpRegistrySupport.test.ts +++ b/packages/provider-acp-registry/src/server/AcpRegistrySupport.test.ts @@ -639,6 +639,51 @@ describe("AcpRegistrySupport", () => { ); }); + it.effect("resolves the refreshed registry version after recreating the catalog", () => { + let version = "1.2.3"; + const agent = () => ({ + ...makeAgent({ binary: { "linux-x86_64": { archive: archiveUrl, cmd: "agent" } } }), + version, + }); + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cacheDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-acp-registry-restart-", + }); + const options = { cacheDir, toolsDir: `${cacheDir}/tools`, registryUrl }; + const original = yield* AcpRegistrySupport.makeAcpRegistryCatalog(options); + const first = yield* original.resolve(settings(), "/workspace"); + version = "2.0.0"; + yield* original.search({ query: "example-agent" }); + const restarted = yield* AcpRegistrySupport.makeAcpRegistryCatalog(options); + const replacement = yield* restarted.resolve(settings(), "/workspace"); + expect(first.agent?.version).toBe("1.2.3"); + expect(replacement.agent?.version).toBe("2.0.0"); + expect(replacement.spawn.command).toContain("/tools/example-agent/2.0.0/"); + expect(replacement.spawn.command).not.toBe(first.spawn.command); + expect(yield* fileSystem.readFileString(first.spawn.command)).toBe("#!/bin/sh\necho 1.2.3\n"); + expect(yield* fileSystem.readFileString(replacement.spawn.command)).toBe( + "#!/bin/sh\necho 2.0.0\n", + ); + }).pipe( + Effect.scoped, + Effect.provide( + layerResolver((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + request.url === registryUrl + ? makeRegistry(agent()) + : `#!/bin/sh\necho ${version}\n`, + ), + ), + ), + ), + ), + ); + }); + it.effect( "prepares the registry version despite an older PATH binary and permits explicit overrides", () => { diff --git a/packages/provider-acp/src/server/AcpSessionRuntime.ts b/packages/provider-acp/src/server/AcpSessionRuntime.ts index fc1822e93408c..695fc71b38d29 100644 --- a/packages/provider-acp/src/server/AcpSessionRuntime.ts +++ b/packages/provider-acp/src/server/AcpSessionRuntime.ts @@ -2540,6 +2540,9 @@ export const make = ( const requestPayload = { sessionId, cwd: options.cwd, + ...(options.additionalDirectories?.length + ? { additionalDirectories: options.additionalDirectories } + : {}), mcpServers: sessionMcpServers(started.initializeResult, activationOptions), } satisfies EffectAcpSchema.LoadSessionRequest; return runLoadSessionWithReplayIdle(requestPayload, started.initializeResult); @@ -2552,6 +2555,9 @@ export const make = ( const requestPayload = { sessionId, cwd: options.cwd, + ...(options.additionalDirectories?.length + ? { additionalDirectories: options.additionalDirectories } + : {}), mcpServers: sessionMcpServers(started.initializeResult, activationOptions), } satisfies EffectAcpSchema.ResumeSessionRequest; return runLoggedRequest( @@ -2568,6 +2574,9 @@ export const make = ( const requestPayload = { sessionId, cwd: options.cwd, + ...(options.additionalDirectories?.length + ? { additionalDirectories: options.additionalDirectories } + : {}), mcpServers: sessionMcpServers(started.initializeResult, activationOptions), } satisfies EffectAcpSchema.ForkSessionRequest; return runLoggedRequest( diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index c1208b49556e8..e420cf34f35a5 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -6166,6 +6166,10 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ); yield* wireAcpRuntimeTerminalHandlers(replacementExit.value.replacementRuntime); yield* commitSessionState(replacementExit.value.started); + capabilities = negotiatedCapabilities( + flavor.capabilities, + replacementExit.value.started, + ); while (true) { const buffered = stagedSessionUpdates.splice(0, stagedSessionUpdates.length); if (buffered.length === 0) { @@ -6243,15 +6247,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* Ref.set(activeSessionId, started.sessionId); yield* Ref.set(activeSessionSetup, started); rememberTerminalEnvironment(started.sessionId, yield* readMcpContext(input.threadId)); - const capabilities = negotiatedCapabilities(flavor.capabilities, started); - const canLoadSession = started.initializeResult.agentCapabilities?.loadSession === true; - const canResumeSession = - started.initializeResult.agentCapabilities?.sessionCapabilities?.resume != null; - const supportsImagePrompts = acpSupportsImagePrompts({ - flavorSupportsImagePrompts: flavor.supportsImagePrompts, - negotiatedImage: - started.initializeResult.agentCapabilities?.promptCapabilities?.image === true, - }); + let capabilities = negotiatedCapabilities(flavor.capabilities, started); const activateSession = Effect.fnUntraced(function* ( sessionId: string, @@ -6265,15 +6261,31 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } const mcpContext = yield* readMcpContext(threadId); const activationOptions = acpMcpActivation(mcpContext); + // A respawn may run a newer agent with different session capabilities. + const initialized = yield* runtime.initialize(); + if ((yield* Ref.get(activeSessionId)) === null) { + // Keep the respawn's fresh session available for portable recovery + // if the saved native session cannot be activated. + prepareClaimableTerminalEnvironment(mcpContext); + const fresh = yield* runtime.start(); + yield* Ref.set(activeSessionId, fresh.sessionId); + yield* Ref.set(activeSessionSetup, fresh); + capabilities = negotiatedCapabilities(flavor.capabilities, fresh); + rememberTerminalEnvironment(fresh.sessionId, mcpContext); + } prepareTerminalEnvironment(mcpContext, sessionId); - const activated = canLoadSession - ? yield* runtime.loadSession(sessionId, activationOptions) - : canResumeSession + const canLoad = initialized.agentCapabilities?.loadSession === true; + const canResume = initialized.agentCapabilities?.sessionCapabilities?.resume != null; + const activated = + canResume && (flavor.preferResumeSession === true || !canLoad) ? yield* runtime.resumeSession(sessionId, activationOptions) - : yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver, - detail: `ACP driver cannot load or resume session ${sessionId}`, - }); + : canLoad + ? yield* runtime.loadSession(sessionId, activationOptions) + : yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: `ACP driver cannot load or resume session ${sessionId}`, + }); + capabilities = negotiatedCapabilities(flavor.capabilities, activated); rememberTerminalEnvironment(activated.sessionId, yield* readMcpContext(threadId)); return activated; }); @@ -6492,7 +6504,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( status: "ready", cwd: input.runtimePolicy.cwd ?? process.cwd(), model: input.modelSelection.model, - capabilities, + get capabilities() { + return capabilities; + }, createdAt, updatedAt: createdAt, lastError: null, @@ -6806,6 +6820,11 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const imageAttachments = turnInput.message.attachments.filter( isProviderNativeImageAttachment, ); + const supportsImagePrompts = acpSupportsImagePrompts({ + flavorSupportsImagePrompts: flavor.supportsImagePrompts, + negotiatedImage: + (yield* runtime.initialize()).agentCapabilities?.promptCapabilities?.image === true, + }); if (imageAttachments.length > 0 && !supportsImagePrompts) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver, @@ -7286,8 +7305,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ); const transportHadOutstandingResponses = yield* closeNativeTransport; yield* options.testHooks?.afterNativeResponseTransportClosed?.() ?? Effect.void; - const sessionCapabilities = - started.initializeResult.agentCapabilities?.sessionCapabilities; + const sessionCapabilities = (yield* Ref.get(activeSessionSetup))?.initializeResult + .agentCapabilities?.sessionCapabilities; if (sessionCapabilities?.close != null) { yield* runtimeTransitionPermit.withPermitsIfAvailable(1)( Effect.gen(function* () { @@ -7821,7 +7840,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ); const sessionId = yield* nativeThreadId(driver, snapshotInput.providerThread); if ((yield* Ref.get(activeSessionId)) !== sessionId) { - if (!capabilities.threads.canReadThreadSnapshot) { + if ((yield* runtime.initialize()).agentCapabilities?.loadSession !== true) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver, detail: "ACP driver does not support session/load snapshots", @@ -7846,6 +7865,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( activated.sessionId, yield* readMcpContext(snapshotInput.providerThread.appThreadId), ); + capabilities = negotiatedCapabilities(flavor.capabilities, activated); yield* Ref.set(activeSessionId, activated.sessionId); yield* Ref.set(activeSessionSetup, activated); yield* Ref.set(activeSelection, null); @@ -7986,7 +8006,10 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* restartRuntimeAfterTeardownIfRequired( forkInput.sourceProviderThread.appThreadId, ); - if (!capabilities.threads.canForkThread) { + if ( + (yield* runtime.initialize()).agentCapabilities?.sessionCapabilities?.fork == + null + ) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver, detail: "ACP driver did not negotiate session/fork", @@ -8012,6 +8035,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( forked.sessionId, yield* readMcpContext(forkInput.targetThreadId), ); + capabilities = negotiatedCapabilities(flavor.capabilities, forked); yield* Ref.set(activeSessionId, forked.sessionId); yield* Ref.set(activeSessionSetup, forked); yield* Ref.set(activeSelection, null); From 5d9e98ac5c60ad4747f93abcfdfb4a8ed9eebe89 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:05:14 +0000 Subject: [PATCH 080/149] fix(opencode): restarts admit each prompt once, rebind waiting questions, and keep external turns across server restarts --- apps/server/src/cloud/bootService.test.ts | 25 +- apps/server/src/cloud/bootService.ts | 17 +- apps/server/src/cloud/serviceProtocol.ts | 2 + .../ProviderSessionManager.test.ts | 81 ++ .../ProviderSessionManager.ts | 23 + .../ProviderTurnStartService.ts | 48 + .../RestartContinuation.test.ts | 81 ++ .../orchestration-v2/RestartContinuation.ts | 36 +- .../orchestration-v2/RunExecutionService.ts | 5 + apps/server/src/provider/ProviderHostLive.ts | 29 + apps/server/src/server.ts | 10 +- .../src/server/ProviderAdapter.ts | 12 + .../provider-core/src/server/ProviderHost.ts | 2 + .../src/server/adapter.test.ts | 843 +++++++++++++++++- .../provider-opencode/src/server/adapter.ts | 375 ++++++-- 15 files changed, 1514 insertions(+), 75 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 65bf0818a9706..957c25ae0ec34 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -17,6 +17,7 @@ import { parseServiceState, SERVICE_LAUNCHER_PROTOCOL, SERVICE_RESTART_PENDING_FILE, + SERVICE_RESTART_MARKER_FILE, serviceStateHasPendingUpdate, } from "./serviceProtocol.ts"; @@ -150,6 +151,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( linger: string; enabled: boolean; active: boolean; + restartingAtStop?: boolean; } = { failCommand: undefined, linger: "yes", @@ -164,6 +166,11 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( commands.push(command); timeouts.set(command, input.timeout); const failed = command === control.failCommand; + if (command === "systemctl --user stop t3code.service") { + control.restartingAtStop = yield* fs + .exists(path.join(baseDir, "runtime", SERVICE_RESTART_MARKER_FILE)) + .pipe(Effect.orDie); + } if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; @@ -562,12 +569,21 @@ it.layer(NodeServices.layer)("boot service install", (it) => { it.effect("restart stops and starts an installed service, and is a no-op otherwise", () => Effect.gen(function* () { - const { service, commands } = yield* makeHarness(); + const { service, commands, control, fs, statePath } = yield* makeHarness(); expect(yield* service.restart).toBe(false); yield* service.install(); commands.length = 0; expect(yield* service.restart).toBe(true); + expect(control.restartingAtStop).toBe(true); + expect( + yield* fs.exists( + (yield* Path.Path).join( + (yield* Path.Path).dirname(statePath), + SERVICE_RESTART_MARKER_FILE, + ), + ), + ).toBe(false); expect( commands.filter( (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), @@ -597,13 +613,18 @@ it.layer(NodeServices.layer)("boot service install", (it) => { it.effect("restart brings the service back when activation fails", () => Effect.gen(function* () { - const { service, commands, control } = yield* makeHarness(); + const { service, commands, control, fs, statePath } = yield* makeHarness(); yield* service.install(); commands.length = 0; control.failCommand = "systemctl --user daemon-reload"; const error = yield* service.restart.pipe(Effect.flip); expect(error._tag).toBe("BootServiceCommandError"); + expect(control.restartingAtStop).toBe(true); + const path = yield* Path.Path; + expect( + yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_MARKER_FILE)), + ).toBe(false); expect( commands.filter( (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 1d7d50b13185f..4a914fd6826d1 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -23,6 +23,7 @@ import { import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_RESTART_PENDING_FILE, + SERVICE_RESTART_MARKER_FILE, SERVICE_STATE_FILE, compareExactServiceVersions, parseServiceState, @@ -916,12 +917,16 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ) { return false; } - yield* runSteps(manager.stop); - yield* runSteps(manager.activate).pipe( - // Same recovery as a failed repair: a service that was running should - // not be left stopped because daemon-reload or enable failed. - Effect.tapError(() => runSteps(manager.restart).pipe(Effect.ignore)), - ); + const restartMarkerPath = path.join(input.baseDir, "runtime", SERVICE_RESTART_MARKER_FILE); + yield* writeDurably(restartMarkerPath, ""); + yield* Effect.gen(function* () { + yield* runSteps(manager.stop); + yield* runSteps(manager.activate).pipe( + // Same recovery as a failed repair: a service that was running should + // not be left stopped because daemon-reload or enable failed. + Effect.tapError(() => runSteps(manager.restart).pipe(Effect.ignore)), + ); + }).pipe(Effect.ensuring(fs.remove(restartMarkerPath, { force: true }).pipe(Effect.ignore))); yield* fs.remove(restartPendingPath, { force: true }); return true; }).pipe( diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 2d32a996ee2c2..a4a96672fcd15 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -9,6 +9,8 @@ export const SERVICE_STATE_FILE = "service-state.json"; the child can tell "the service is going away" from "the launcher is about to start my replacement" while a pending update is recorded. */ export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; +/** Written before a requested service restart stops the launcher. */ +export const SERVICE_RESTART_MARKER_FILE = ".service-restarting"; /** Written by `t3 update` when the unit was repointed at a new version but the running service was deliberately left on the old one. The launcher removes it when it starts (whoever restarted the service), so while it exists the diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index ceb230b008940..263e29df8fe92 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { AgentScope } from "@t3tools/shared/AgentScope"; import { + DEFAULT_SERVER_SETTINGS, EnvironmentId, type ModelSelection, type OrchestrationV2AppThread, @@ -51,6 +52,7 @@ import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; const layerTestDatabase = SqlitePersistence.layerMemory; const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( @@ -321,6 +323,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; + readonly prepareForServerShutdown?: (restarting: boolean) => Effect.Effect; /** Registers the process's closeCount finalizer before `beforeOpen` runs. */ readonly spawnBeforeOpen?: boolean; /** Completed when a hanging scope close reaches its wedged finalizer. */ @@ -415,6 +418,11 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( resumeCount: current.resumeCount + 1, })).pipe(Effect.as(threadInput.providerThread)), startTurn: () => options.startTurn ?? Effect.void, + ...(options.prepareForServerShutdown === undefined + ? {} + : { + prepareForServerShutdown: options.prepareForServerShutdown, + }), steerTurn: () => Effect.void, interruptTurn: () => Ref.update(state, (current) => ({ @@ -468,6 +476,8 @@ function layerTest(input: { readonly hangSessionScopeClose?: boolean; readonly startTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; + readonly prepareForServerShutdown?: (restarting: boolean) => Effect.Effect; + readonly serverRestartPending?: Effect.Effect; readonly spawnBeforeOpen?: boolean; readonly scopeCloseReached?: Deferred.Deferred; readonly serverSettingsLayer?: ReturnType; @@ -498,6 +508,11 @@ function layerTest(input: { : { hangSessionScopeClose: input.hangSessionScopeClose }), ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), + ...(input.prepareForServerShutdown === undefined + ? {} + : { + prepareForServerShutdown: input.prepareForServerShutdown, + }), ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), ...(input.scopeCloseReached === undefined ? {} @@ -536,6 +551,27 @@ function layerTest(input: { layerProviderEventIngestorTest, layerConfiguredMcpRegistry, layerTestStores, + ...(input.serverRestartPending === undefined + ? [] + : [ + Layer.mock(ProviderHost.ProviderHost)({ + paths: { + cwd: process.cwd(), + baseDir: "/tmp", + stateDir: "/tmp", + providerStatusCacheDir: "/tmp", + attachmentsDir: "/tmp", + }, + settings: { + get: Effect.succeed(DEFAULT_SERVER_SETTINGS), + withSnapshot: (use) => use(DEFAULT_SERVER_SETTINGS), + changes: Stream.empty, + subscribe: Effect.succeed(Stream.empty), + }, + resolveAttachmentPath: () => null, + serverRestartPending: input.serverRestartPending, + }), + ]), ...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]), ...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]), ), @@ -1681,6 +1717,51 @@ it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts }), ); +it.effect.each(["quit", "restart", "session-stop"] as const)( + "ProviderSessionManagerV2 captures %s intent before marker cleanup", + (action) => + Effect.gen(function* () { + const restarting = action !== "quit"; + const state = yield* Ref.make(emptyState); + const marker = yield* Ref.make(restarting); + const prepared: boolean[] = []; + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("shutdown-marker"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + yield* manager.prepareForServerShutdown!; + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Ref.set(marker, false); + if (action === "session-stop") yield* manager.close(providerSessionId); + else yield* manager.shutdown; + assert.deepEqual(prepared, [restarting, action === "session-stop" ? false : restarting]); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + serverRestartPending: Ref.get(marker), + prepareForServerShutdown: (intent) => + Effect.sync(() => { + prepared.push(intent); + }), + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 closes event subscriptions normally on server shutdown", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 99bcc097d244d..027776899aea0 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -50,6 +50,7 @@ import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; @@ -150,6 +151,7 @@ export const ProviderSessionManagerV2Error = Schema.Union([ export type ProviderSessionManagerV2Error = typeof ProviderSessionManagerV2Error.Type; export interface ProviderSessionManagerV2Shape { + readonly prepareForServerShutdown?: Effect.Effect; readonly shutdown: Effect.Effect; readonly open: (input: { readonly threadId: ThreadId; @@ -347,6 +349,7 @@ export const layerWithOptions = ( ProviderSessionManagerV2, Effect.gen(function* () { const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; + const providerHost = yield* Effect.serviceOption(ProviderHost.ProviderHost); const fileSystem = yield* FileSystem.FileSystem; const mcpSessionRegistry = yield* McpSessionRegistry.McpSessionRegistry; const mcpSessions = yield* McpProviderSessions.McpProviderSessions; @@ -953,6 +956,9 @@ export const layerWithOptions = ( onNone: () => Effect.void, onSome: (entry) => Effect.gen(function* () { + if (input.reason !== "server_shutdown") { + yield* entry.runtime.prepareForServerShutdown?.(false) ?? Effect.void; + } if (input.cancelIdleFiber !== false) { yield* cancelIdleFiber(entry.idleFiber); } @@ -2007,7 +2013,20 @@ export const layerWithOptions = ( // Parallel, so one session whose close hangs does not stop the rest from // closing within the time box. const sessionScopes = yield* Scope.make("parallel"); + let shutdownRestartIntent: boolean | undefined; + const prepareForServerShutdown = Effect.gen(function* () { + // Tunnel cleanup consumes the desktop marker. Capture intent first, + // and keep it for sessions whose open completes during teardown. + shutdownRestartIntent ??= yield* Option.match(providerHost, { + onNone: () => Effect.succeed(false), + onSome: (host) => host.serverRestartPending ?? Effect.succeed(false), + }); + for (const entry of (yield* Ref.get(sessions)).values()) { + yield* entry.runtime.prepareForServerShutdown?.(shutdownRestartIntent) ?? Effect.void; + } + }); const shutdown = Effect.gen(function* () { + yield* prepareForServerShutdown; const activeSessions = [...(yield* Ref.get(sessions)).values()]; yield* Effect.forEach( activeSessions, @@ -2045,6 +2064,7 @@ export const layerWithOptions = ( }); return ProviderSessionManagerV2.of({ + prepareForServerShutdown, shutdown, hasPendingBackgroundWork, open: (input) => @@ -2174,6 +2194,9 @@ export const layerWithOptions = ( attributes: { provider: adapter.driver, operation: "open" }, }), ); + if (shutdownRestartIntent !== undefined) { + yield* runtime.prepareForServerShutdown?.(shutdownRestartIntent) ?? Effect.void; + } const eventSubscribers = yield* Ref.make< ReadonlyMap> >(new Map()); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3fd3755d0426f..79f268c2d1789 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -39,6 +39,7 @@ import { } from "@t3tools/provider-core/server/handoffBudget"; import { deliverContextHandoffs } from "./ContextHandoffDelivery.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { hasRestorableOpenCodeRequest } from "./RestartContinuation.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; @@ -1230,6 +1231,25 @@ export const layer: Layer.Layer< !noteContinuation ? session : makeDeliverySession(session, startWithHandoffs); + const restartSourceRun = projection.runs.find( + (source) => source.id === run.restartContinuationOfRunId, + ); + const restartSourceTurn = projection.providerTurns.findLast( + (turn) => turn.runAttemptId === restartSourceRun?.activeAttemptId, + ); + const restartSourceRecords = + restartSourceRun === undefined + ? undefined + : yield* projectionStore.getThreadRecords( + input.threadId, + ["messages", "runtimeRequests"], + { + messageIds: [restartSourceRun.userMessageId], + }, + ); + const restartSourceMessage = restartSourceRecords?.messages.find( + (message) => message.id === restartSourceRun?.userMessageId, + ); yield* runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -1262,6 +1282,34 @@ export const layer: Layer.Layer< (legacyInputRunIds.size > 0 && sameNativeThread && !acceptedAttempts.some((source) => source.nativeThreadId !== undefined)), + ...(restartSourceMessage === undefined + ? {} + : { + restartContinuationSource: { + attemptId: restartSourceRun?.activeAttemptId ?? null, + nativeTurnRef: restartSourceTurn?.nativeTurnRef ?? null, + awaitingUser: + restartSourceRun !== undefined && + restartSourceRecords !== undefined && + hasRestorableOpenCodeRequest( + { + providerTurns: projection.providerTurns, + runtimeRequests: restartSourceRecords.runtimeRequests, + }, + restartSourceRun, + ), + ...(restartSourceRun === undefined + ? {} + : { startedAt: restartSourceRun.startedAt }), + message: { + messageId: restartSourceMessage.id, + text: restartSourceMessage.text, + attachments: restartSourceMessage.attachments, + createdBy: restartSourceMessage.createdBy, + creationSource: restartSourceMessage.creationSource, + }, + }, + }), shouldStartProviderTurn: runControls.shouldStartProviderTurn, shouldFinalizeRun: runControls.shouldFinalizeRun, hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest, diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index a01f78b8c238a..d97e75b2928e6 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -194,6 +194,87 @@ it.effect.each([ }), ); +it.effect.each(["pending", "cancelled"] as const)( + "rebinds OpenCode's %s question after restart", + (status) => + Effect.gen(function* () { + const base = makeProjection(); + const opencode = ProviderDriverKind.make("opencode"); + const projection = { + ...base, + runs: [{ ...base.runs[0]!, status: "waiting" }], + providerThreads: [ + { + ...base.providerThreads[0]!, + driver: opencode, + nativeThreadRef: { driver: opencode, nativeId: "native", strength: "strong" }, + }, + ], + providerSessions: [{ ...base.providerSessions[0]!, driver: opencode }], + providerTurns: [ + { + ...base.providerTurns[0]!, + nativeTurnRef: { driver: opencode, nativeId: "msg_question", strength: "weak" }, + }, + ], + runtimeRequests: [ + { + id: RuntimeRequestId.make("request:opencode"), + nodeId: NodeId.make("question-node"), + providerTurnId: base.providerTurns[0]!.id, + nativeRequestRef: { driver: opencode, nativeId: "que_question", strength: "strong" }, + kind: "user_input", + status, + responseCapability: + status === "pending" + ? { type: "live", providerSessionId: sessionId } + : { type: "not_resumable", reason: "server restart" }, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + assert.equal(restartContinuationRun(projection, testNow)?.id, runId); + const dispatched: Parameters< + ThreadManagementService.ThreadManagementService["Service"]["dispatch"] + >[0][] = []; + yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe( + Effect.provide( + Layer.merge( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => + Effect.succeed({ + ...projection, + runs: [{ ...projection.runs[0]!, status: "cancelled" as const }], + providerTurns: [ + { ...projection.providerTurns[0]!, status: "cancelled" as const }, + ], + }), + dispatch: (command) => { + dispatched.push(command); + return Effect.succeed({} as never); + }, + recoverDelegatedTask: () => Effect.void, + }), + ), + ), + ); + assert.lengthOf(dispatched, 1); + const withOtherRequest = { + ...projection, + runtimeRequests: [ + ...projection.runtimeRequests, + { + ...projection.runtimeRequests[0]!, + id: RuntimeRequestId.make("other-provider-request"), + status: "pending" as const, + nativeRequestRef: { driver, nativeId: "other", strength: "strong" as const }, + }, + ], + }; + assert.isUndefined(restartContinuationRun(withOtherRequest, testNow)); + }), +); + it("continues a live turn whose session the adapter never marked running", () => { const projection = makeProjection(); // Codex, Claude, Cursor and ACP sessions stay "ready" for their whole life. diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index bce2dd9ab4edc..967b0aef0dffa 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -25,7 +25,33 @@ import { const CONTINUE_PROMPT = "Continue where you left off."; -/** A recorded intent must not wake held threads, unanswered requests, or settled provider turns. */ +/** OpenCode keeps these requests on its server and can list and rebind them. */ +export function hasRestorableOpenCodeRequest( + projection: Pick, + source: OrchestrationV2Run, +): boolean { + return ( + source.activeAttemptId !== null && + projection.runtimeRequests.some( + (request) => + request.nativeRequestRef?.driver === "opencode" && + request.kind !== "dynamic_tool_call" && + request.kind !== "auth_refresh" && + request.responseCapability.type !== "message" && + (request.status === "pending" || + ((request.status === "expired" || request.status === "cancelled") && + request.responseCapability.type === "not_resumable")) && + projection.providerTurns.some( + (turn) => + turn.id === request.providerTurnId && + turn.nativeTurnRef?.driver === "opencode" && + turn.runAttemptId === source.activeAttemptId, + ), + ) + ); +} + +/** A recorded intent must not wake held threads, unbindable requests, or settled provider turns. */ export function restartContinuationBlocked( projection: Pick< ProjectionRuntimeRecoveryState, @@ -51,6 +77,7 @@ export function restartContinuationBlocked( (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")) || projection.runtimeRequests.some( (request) => + !hasRestorableOpenCodeRequest({ ...projection, runtimeRequests: [request] }, source) && // Tool calls and auth refreshes do not wait on a person, and a // message-capable question stays answerable after the restart. request.kind !== "dynamic_tool_call" && @@ -111,7 +138,12 @@ export function restartContinuationRun( (run.restartContinuationOfRunId !== undefined || restartPromptSource(run, projection.runs, projection.providerTurns, projection.attempts) !== undefined); - if (run.status !== "running" && !preparedContinuation) return; + if ( + run.status !== "running" && + !preparedContinuation && + !(run.status === "waiting" && hasRestorableOpenCodeRequest(projection, run)) + ) + return; if (restartContinuationBlocked(projection, run, now)) return; const liveTurnRequired = !preparedContinuation; if (projection.thread.providerInstanceId !== run.providerInstanceId) return; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index fa6d0fbd7f125..91dd29d74cc05 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -509,6 +509,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly attemptId: RunAttemptId; readonly providerTurnOrdinal: number; readonly nativeThreadHasTurns?: boolean; + readonly restartContinuationSource?: ProviderAdapter.ProviderAdapterV2TurnInput["restartContinuationSource"]; readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< ReadonlyArray, unknown @@ -1387,6 +1388,7 @@ export const layer: Layer.Layer< runId: input.run.id, runOrdinal: input.run.ordinal, providerTurnOrdinal: input.providerTurnOrdinal, + runStartedAt: input.run.startedAt, ...(input.nativeThreadHasTurns === undefined ? {} : { nativeThreadHasTurns: input.nativeThreadHasTurns }), @@ -1394,6 +1396,9 @@ export const layer: Layer.Layer< ? {} : { restartContinuationOfRunId: input.run.restartContinuationOfRunId, + ...(input.restartContinuationSource === undefined + ? {} + : { restartContinuationSource: input.restartContinuationSource }), }), attemptId: input.attemptId, rootNodeId: input.rootNode.id, diff --git a/apps/server/src/provider/ProviderHostLive.ts b/apps/server/src/provider/ProviderHostLive.ts index 14d08de50ee97..633f03589ad9f 100644 --- a/apps/server/src/provider/ProviderHostLive.ts +++ b/apps/server/src/provider/ProviderHostLive.ts @@ -9,6 +9,11 @@ import { ProviderCredentialError } from "@t3tools/provider-core/server/errors"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Option from "effect/Option"; +import * as Clock from "effect/Clock"; +import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -16,6 +21,8 @@ import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerConfig from "../config.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as ProviderCredentialStore from "./ProviderCredentialStore.ts"; +import { pendingUpdateHandoffExists } from "../cloud/updateHandoff.ts"; +import { SERVICE_RESTART_MARKER_FILE } from "../cloud/serviceProtocol.ts"; export const layer = Layer.effect( ProviderHost.ProviderHost, @@ -25,6 +32,8 @@ export const layer = Layer.effect( const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const secrets = yield* ServerSecretStore.ServerSecretStore; const crypto = yield* Crypto.Crypto; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; return ProviderHost.ProviderHost.of({ paths: { cwd: config.cwd, @@ -40,6 +49,26 @@ export const layer = Layer.effect( subscribe: serverSettings.subscribeChanges, }, shouldRunBackgroundWork: backgroundPolicy.shouldRunScopeWork, + serverRestartPending: Effect.gen(function* () { + if (yield* pendingUpdateHandoffExists) return true; + const now = yield* Clock.currentTimeMillis; + for (const name of [DESKTOP_UPDATE_RESTART_MARKER_FILE, SERVICE_RESTART_MARKER_FILE]) { + const marker = yield* fs + .stat(path.join(config.baseDir, "runtime", name)) + .pipe(Effect.option); + if ( + Option.isSome(marker) && + Option.isSome(marker.value.mtime) && + now - marker.value.mtime.value.getTime() < 60_000 + ) + return true; + } + return false; + }).pipe( + Effect.provideService(ServerConfig.ServerConfig, config), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), resolveAttachmentPath: (attachment) => resolveAttachmentPath({ attachmentsDir: config.attachmentsDir, attachment }), credentials: (namespace, bindingId) => diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 190ec87a437e0..ae6ffb36799a8 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -50,6 +50,7 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as OpenCodeRuntime from "@t3tools/provider-opencode/server/OpenCodeRuntime"; import * as OpenCodeServerLedger from "@t3tools/provider-opencode/server/OpenCodeServerLedger"; import * as ProviderHostLive from "./provider/ProviderHostLive.ts"; +import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import * as AcpRegistrySupport from "@t3tools/provider-acp-registry/server/AcpRegistrySupport"; import * as CheckpointDiffQuery from "./checkpointing/CheckpointDiffQuery.ts"; import * as CheckpointStore from "./checkpointing/CheckpointStore.ts"; @@ -486,6 +487,7 @@ const layerScheduledTaskWebhookOrigin = Layer.effect( ); const layerOrchestrationV2Runtime = RuntimeLayer.layerProduction.pipe( + Layer.provide(ProviderHostLive.layer), Layer.provide(layerScheduledTaskWebhookOrigin), Layer.provide(ProviderEventIngestor.layerAnalytics), Layer.provide(layerCheckpointStore), @@ -858,7 +860,13 @@ const layerMakeServer = Layer.unwrap( const layerCloudDesiredLinkReconcile = Layer.effectDiscard( Effect.gen(function* () { const cloudLink = yield* CloudLink.CloudLink; - const releaseManagedTunnel = cloudLink.releaseManagedTunnelOnShutdown().pipe( + const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + // Provider cleanup reads the desktop restart marker before tunnel + // cleanup consumes it. + const releaseManagedTunnel = ( + providerSessions.prepareForServerShutdown ?? Effect.void + ).pipe( + Effect.andThen(cloudLink.releaseManagedTunnelOnShutdown()), Effect.timeout("10 seconds"), Effect.tap((released) => released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 9094e8989fa46..50b311a21163d 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -34,6 +34,7 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; +import type * as DateTime from "effect/DateTime"; import * as Schema from "effect/Schema"; import type * as Effect from "effect/Effect"; import type * as Scope from "effect/Scope"; @@ -410,7 +411,16 @@ export interface ProviderAdapterV2TurnInput { /** Whether the current native session has an accepted turn; omitted when unknown. */ readonly nativeThreadHasTurns?: boolean; readonly providerTurnOrdinal: number; + readonly runStartedAt?: DateTime.Utc | null; readonly restartContinuationOfRunId?: RunId; + /** Persisted native admission and original input for an interrupted turn. */ + readonly restartContinuationSource?: { + readonly attemptId?: RunAttemptId | null; + readonly nativeTurnRef: OrchestrationV2ProviderTurn["nativeTurnRef"]; + readonly message: ProviderAdapterV2TurnMessage; + readonly awaitingUser?: boolean; + readonly startedAt?: DateTime.Utc | null; + }; readonly attemptId: RunAttemptId; readonly rootNodeId: NodeId; readonly providerThread: OrchestrationV2ProviderThread; @@ -549,6 +559,8 @@ export interface ProviderAdapterV2SessionRuntime { * here so the session manager defers idle release while it is pending. */ readonly hasPendingBackgroundWork?: Effect.Effect; + /** Records restart intent before scope cleanup; an explicit stop clears it. */ + readonly prepareForServerShutdown?: (restarting: boolean) => Effect.Effect; /** * Per-provider-thread pending work for root-run ingestion stop gates. When * present, RunExecutionService uses only this probe (never the session-wide diff --git a/packages/provider-core/src/server/ProviderHost.ts b/packages/provider-core/src/server/ProviderHost.ts index 858a7c1e22d1a..d291f06421bba 100644 --- a/packages/provider-core/src/server/ProviderHost.ts +++ b/packages/provider-core/src/server/ProviderHost.ts @@ -62,6 +62,8 @@ export class ProviderHost extends Context.Service< }; /** Whether background work for `scope` should run now (client demand and host power). */ readonly shouldRunBackgroundWork: (scope: BackgroundScope) => Effect.Effect; + /** True only when shutdown hands off to a replacement server. */ + readonly serverRestartPending?: Effect.Effect; /** * Absolute path of a stored chat attachment, or `null` when its id does not * resolve inside the attachments directory. diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index 70daeac9c2223..d4ee4f34ac63c 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -187,9 +187,11 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct text = "hello", startProviderThread: OrchestrationV2ProviderThread = providerThread, restartContinuationOfRunId?: RunId, + restartContinuationSource?: ProviderAdapter.ProviderAdapterV2TurnInput["restartContinuationSource"], ) => runtime.startTurn({ ...(restartContinuationOfRunId === undefined ? {} : { restartContinuationOfRunId }), + ...(restartContinuationSource === undefined ? {} : { restartContinuationSource }), appThread: { id: threadId, projectId: ProjectId.make(`project-opencode-${suffix}`), @@ -217,6 +219,7 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct runId: RunId.make(`run-opencode-${suffix}`), runOrdinal: 1, providerTurnOrdinal: 1, + runStartedAt: now, attemptId: RunAttemptId.make(`attempt-opencode-${suffix}`), rootNodeId: NodeId.make(`node-opencode-${suffix}`), providerThread: startProviderThread, @@ -243,7 +246,97 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct }); describe("OpenCodeAdapterV2", () => { - it.effect.each(["busy", "retry", "idle"] as const)( + it.effect("announces native admission before the prompt can be recorded by OpenCode", () => + Effect.gen(function* () { + const nativeEvents = asyncEventStream(); + const admission = promiseGate(); + const response = promiseGate(); + const harness = yield* makeOpenCodeRuntimeHarness("admission-persistence", "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + promptAsync: async (input: { messageID: string }) => { + admission.resolve(input.messageID); + await response.promise; + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + }); + const announced = yield* harness.runtime.events.pipe( + Stream.filter((event) => event.type === "provider_turn.updated"), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + const starting = yield* harness.startTurn().pipe(Effect.forkScoped); + const messageID = yield* Effect.promise(() => admission.promise); + const event = (yield* Fiber.join(announced))[0]!; + assert.equal(event.providerTurn.nativeTurnRef?.nativeId, messageID); + response.resolve(); + yield* Fiber.join(starting); + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + + it.effect("recreates admission when neither OpenCode nor T3 recorded the prompt", () => + Effect.gen(function* () { + const prompts: Array<{ messageID: string; parts: Array<{ type: string; text?: string }> }> = + []; + const client = { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + const nativeEvents = asyncEventStream(); + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + get: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + status: async () => ({ data: {} }), + messages: async () => ({ data: [] }), + promptAsync: async (input: (typeof prompts)[number]) => { + prompts.push(input); + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + }; + const original = yield* makeOpenCodeRuntimeHarness("durable-admission", "root", client); + yield* original.startTurn("original task"); + const recovered = yield* makeOpenCodeRuntimeHarness("durable-recovery", "root", client); + yield* recovered.startTurn("Continue", recovered.providerThread, original.runId, { + nativeTurnRef: null, + attemptId: RunAttemptId.make("attempt-opencode-durable-admission"), + startedAt: original.now, + message: { + messageId: MessageId.make("original-input"), + text: "original task", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + }); + assert.lengthOf(prompts, 2); + assert.equal(prompts[1]!.messageID, prompts[0]!.messageID); + assert.deepEqual(prompts[1]!.parts, prompts[0]!.parts); + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + + it.effect.each(["busy", "retry", "idle", "raced-busy"] as const)( "does not prompt a surviving %s session again after a T3 restart", (status) => Effect.gen(function* () { @@ -257,12 +350,26 @@ describe("OpenCodeAdapterV2", () => { }; let prompts = 0; const harness = yield* makeOpenCodeRuntimeHarness(`restart-${status}`, "root", { - event: { subscribe: async () => ({ stream: nativeEvents.stream }) }, + permission: { list: async () => ({ data: [] }) }, + question: { list: async () => ({ data: [] }) }, + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, session: { create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), get: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), - status: async () => ({ data: { root: { type: status } } }), + status: async () => ({ + data: { root: { type: status === "raced-busy" ? "idle" : status } }, + }), messages: async () => { + if (status === "raced-busy") + await nativeEvents.push({ + type: "session.status", + properties: { sessionID: "root", status: { type: "busy" } }, + }); // A live update lands while history loads; the older snapshot must not undo it. await nativeEvents.push({ type: "message.part.updated", @@ -344,6 +451,736 @@ describe("OpenCodeAdapterV2", () => { ), ); + it.effect.each([ + "finished", + "unrecorded-admission", + "unrecorded-clock-skew", + "unrecorded-missing", + "note-survivor", + "waiting-user-gone", + "automatic-compaction-finished", + "automatic-compaction-unfinished", + "imported", + "missing-admission", + "already-settled", + "aborted", + "errored", + "tool-calls", + "length", + "missing-finish", + "wrong-parent", + "reverted-idle", + "reverted-busy", + "empty", + "unavailable-history", + "unavailable-status", + "unavailable-session", + "unavailable-permission", + "unavailable-question", + ] as const)("reconciles restart history for %s", (ending) => + Effect.gen(function* () { + const nativeEvents = asyncEventStream(); + const prompts: Array<{ parts: Array<{ type: string; text?: string }>; messageID: string }> = + []; + const busy = + ending === "reverted-busy" || + ending === "note-survivor" || + ending === "unavailable-permission" || + ending === "unavailable-question"; + const oldHistory = [ + { + info: { id: "msg-old", role: "user", sessionID: "root", time: { created: 1 } }, + parts: [], + }, + { + info: { + id: "old-answer", + role: "assistant", + sessionID: "root", + parentID: "msg-old", + time: { created: 2, completed: 3 }, + finish: "stop", + }, + parts: [ + { + id: "old-part", + type: "text", + messageID: "old-answer", + sessionID: "root", + text: "old answer", + }, + ], + }, + ]; + const cutMessageId = + ending === "unrecorded-clock-skew" ? `msg_000000004001${"a".repeat(28)}` : "msg_cut"; + const cutHistory = [ + { + info: { + id: cutMessageId, + role: "user", + sessionID: "root", + time: { created: ending === "unrecorded-clock-skew" ? -1_000 : 4 }, + }, + parts: [{ type: "text", text: "original task" }], + }, + { + info: { + id: "cut-answer", + role: "assistant", + sessionID: "root", + parentID: ending === "wrong-parent" ? "msg-other" : cutMessageId, + time: { created: 5, completed: 6 }, + finish: + ending === "missing-finish" || ending === "waiting-user-gone" + ? undefined + : ending === "tool-calls" + ? "tool-calls" + : ending === "length" + ? "length" + : "stop", + ...(ending === "aborted" + ? { error: { name: "MessageAbortedError", data: { message: "aborted" } } } + : {}), + ...(ending === "errored" + ? { error: { name: "APIError", data: { message: "failed" } } } + : {}), + }, + parts: [ + { + id: "cut-part", + type: "text", + messageID: "cut-answer", + sessionID: "root", + text: "cut answer", + }, + ], + }, + ]; + const history = + ending === "empty" + ? [] + : ending === "imported" || + ending === "missing-admission" || + ending === "unrecorded-missing" + ? oldHistory + : [ + ...oldHistory, + ...cutHistory, + ...(ending.startsWith("automatic-compaction-") + ? [ + { + info: { + id: "msg_auto_compact", + role: "user", + sessionID: "root", + time: { created: 7 }, + }, + parts: [{ type: "compaction", auto: true }], + }, + { + info: { + id: "summary", + role: "assistant", + sessionID: "root", + parentID: "msg_auto_compact", + summary: true, + time: { created: 8, completed: 9 }, + finish: "stop", + }, + parts: [], + }, + ...(ending === "automatic-compaction-finished" + ? [ + { + info: { + id: "msg_synthetic_continue", + role: "user", + sessionID: "root", + time: { created: 10 }, + }, + parts: [{ type: "text", text: "Continue", synthetic: true }], + }, + { + info: { + id: "after-compact", + role: "assistant", + sessionID: "root", + parentID: "msg_synthetic_continue", + time: { created: 11, completed: 12 }, + finish: "stop", + }, + parts: [ + { + id: "after-compact-text", + messageID: "after-compact", + sessionID: "root", + type: "text", + text: "after compaction", + }, + ], + }, + ] + : []), + ] + : []), + ]; + const harness = yield* makeOpenCodeRuntimeHarness(`restart-history-${ending}`, "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, + permission: { + list: async () => { + if (ending === "unavailable-permission") throw new Error("permission unavailable"); + return { data: [] }; + }, + }, + question: { + list: async () => { + if (ending === "unavailable-question") throw new Error("question unavailable"); + return { data: [] }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + get: async () => { + if (ending === "unavailable-session") throw new Error("session unavailable"); + return { + data: { + id: "root", + time: { created: 1, updated: 6 }, + ...(ending.startsWith("reverted-") ? { revert: { messageID: "msg_cut" } } : {}), + }, + }; + }, + status: async () => { + if (ending === "unavailable-status") throw new Error("status unavailable"); + return { data: busy ? { root: { type: "busy" } } : {} }; + }, + messages: async () => { + if (ending === "unavailable-history") throw new Error("history unavailable"); + return { data: history }; + }, + promptAsync: async (input: (typeof prompts)[number]) => { + prompts.push(input); + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + }); + const terminal = yield* harness.runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + const providerThread = { + ...harness.providerThread, + nativeConversationHeadRef: + ending === "imported" || ending === "finished" + ? null + : { + driver: OPENCODE_PROVIDER, + nativeId: ending === "already-settled" ? "msg_cut" : "msg-old", + strength: "weak" as const, + }, + }; + const source = + ending === "imported" || ending === "already-settled" + ? undefined + : { + nativeTurnRef: ending.startsWith("unrecorded-") + ? null + : { + driver: OPENCODE_PROVIDER, + nativeId: "msg_cut", + strength: "weak" as const, + }, + startedAt: DateTime.makeUnsafe(4), + awaitingUser: ending === "waiting-user-gone", + message: { + messageId: MessageId.make("original-input"), + text: "original task", + attachments: [], + createdBy: "user" as const, + creationSource: "web" as const, + }, + }; + const started = yield* Effect.exit( + harness.startTurn( + "Continue where you left off.", + providerThread, + ending === "note-survivor" ? undefined : RunId.make("cut-run"), + source, + ), + ); + const unavailable = ending.startsWith("unavailable-"); + assert.equal(Exit.isFailure(started), unavailable); + assert.equal( + prompts.length, + ending === "finished" || + ending === "automatic-compaction-finished" || + ending === "unrecorded-admission" || + ending === "unrecorded-clock-skew" || + ending === "waiting-user-gone" || + busy || + unavailable + ? 0 + : 1, + ); + if (ending === "missing-admission" || ending === "empty" || ending === "reverted-idle") { + assert.deepEqual(prompts[0]?.parts, [{ type: "text", text: "original task" }]); + if (ending === "reverted-idle") assert.notEqual(prompts[0]?.messageID, "msg_cut"); + else assert.equal(prompts[0]?.messageID, "msg_cut"); + } + if (prompts.length > 0) + yield* Effect.promise(() => + nativeEvents.push({ + type: "message.updated", + properties: { + sessionID: "root", + info: { + id: prompts[0]!.messageID, + role: "user", + sessionID: "root", + time: { created: 7 }, + }, + }, + }), + ); + if (prompts.length > 0 || busy) + yield* Effect.promise(() => + nativeEvents.push({ type: "session.idle", properties: { sessionID: "root" } }), + ); + const collected = yield* Fiber.join(terminal); + assert.deepInclude(collected.at(-1), { + type: "turn.terminal", + status: unavailable + ? "failed" + : ending === "waiting-user-gone" + ? "interrupted" + : "completed", + }); + const texts = collected.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "assistant_message" + ? [event.turnItem.text] + : [], + ); + if (ending === "finished") assert.include(texts, "cut answer"); + if (ending === "automatic-compaction-finished") assert.include(texts, "after compaction"); + if (ending.startsWith("reverted-")) assert.notInclude(texts, "cut answer"); + if (ending === "reverted-busy") assert.include(texts, "old answer"); + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + + it.effect.each([ + "permission-root", + "permission-child", + "permission-linked-child", + "question-root", + "question-child", + "question-linked-child", + ] as const)("recovers a pending %s after restart without prompting", (kind) => + Effect.gen(function* () { + const nativeEvents = asyncEventStream(); + const child = kind.endsWith("child"); + const sessionID = child ? "child" : "root"; + const permission = { + id: "permission-cut", + sessionID, + permission: "bash", + patterns: ["*"], + always: [], + metadata: {}, + }; + const question = { + id: "question-cut", + sessionID, + questions: [ + { + header: "Choice", + question: "Which?", + options: [{ label: "Yes", description: "Proceed" }], + }, + ], + }; + const replies: unknown[] = []; + let prompts = 0; + const harness = yield* makeOpenCodeRuntimeHarness(kind, "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + get: async ({ sessionID }: { sessionID: string }) => ({ + data: { + id: sessionID, + ...(sessionID === "child" ? { parentID: "root" } : {}), + time: { created: 1, updated: 1 }, + }, + }), + status: async () => ({ data: { root: { type: "busy" }, child: { type: "busy" } } }), + messages: async ({ sessionID: nativeSessionId }: { sessionID: string }) => ({ + data: + nativeSessionId === "child" + ? [] + : [ + { + info: { + id: "msg_cut", + role: "user", + sessionID: "root", + time: { created: 1 }, + }, + parts: [], + }, + ...(kind.includes("linked") + ? [ + { + info: { + id: "task-message", + role: "assistant", + sessionID: "root", + parentID: "msg_cut", + time: { created: 2 }, + }, + parts: [ + { + id: "task-part", + type: "tool", + tool: "task", + callID: "task-call", + messageID: "task-message", + sessionID: "root", + state: { + status: "running", + input: { prompt: "child work", description: "Child" }, + title: "Child", + metadata: { sessionId: "child" }, + time: { start: 2 }, + }, + }, + ], + }, + ] + : []), + ], + }), + update: async () => ({ data: true }), + promptAsync: async () => { + prompts++; + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + permission: { + list: async () => ({ data: kind.startsWith("permission") ? [permission] : [] }), + reply: async (input: unknown) => { + replies.push(input); + return { data: true }; + }, + }, + question: { + list: async () => ({ data: kind.startsWith("question") ? [question] : [] }), + reply: async (input: unknown) => { + replies.push(input); + return { data: true }; + }, + }, + }); + const requests = yield* harness.runtime.events.pipe( + Stream.filter((event) => event.type === "runtime_request.updated"), + Stream.take(1), + Stream.runCollect, + Effect.forkScoped, + ); + yield* harness.startTurn("Continue", harness.providerThread, RunId.make("cut-run")); + const request = (yield* Fiber.join(requests))[0]!.runtimeRequest; + assert.equal(prompts, 0); + assert.equal(request.status, "pending"); + yield* harness.runtime.respondToRuntimeRequest({ + requestId: request.id, + ...(kind.startsWith("permission") + ? { decision: "accept" as const } + : { answers: { Choice: "Yes" } }), + }); + assert.lengthOf(replies, 1); + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + + it.effect.each(["busy", "idle", "aborted", "errored", "raced-idle"] as const)( + "replays a surviving task's %s child history and restores pending work", + (childStatus) => + Effect.gen(function* () { + const nativeEvents = asyncEventStream(); + let prompts = 0; + const task = { + id: "task-part", + sessionID: "root", + messageID: "root-answer", + type: "tool", + tool: "task", + callID: "task-call", + state: { + status: "completed", + input: { prompt: "child work", description: "Child" }, + title: "Child", + metadata: { sessionId: "child", background: true }, + time: { start: 2, end: 3 }, + output: "Background task started", + }, + }; + const history = (sessionID: string) => [ + { + info: { id: `msg-${sessionID}`, sessionID, role: "user", time: { created: 1 } }, + parts: [ + { + id: `${sessionID}-input`, + sessionID, + messageID: `msg-${sessionID}`, + type: "text", + text: "child work", + }, + ], + }, + { + info: { + id: `${sessionID}-answer`, + sessionID, + role: "assistant", + parentID: `msg-${sessionID}`, + time: { created: 2 }, + ...(sessionID === "child" && (childStatus === "aborted" || childStatus === "errored") + ? { + error: { + name: childStatus === "aborted" ? "MessageAbortedError" : "APIError", + data: { message: "cut child" }, + }, + } + : {}), + }, + parts: + sessionID === "root" + ? [task] + : [ + { + id: "child-text", + sessionID, + messageID: "child-answer", + type: "text", + text: "work while T3 was down", + }, + ], + }, + ]; + const harness = yield* makeOpenCodeRuntimeHarness("restart-child", "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + get: async ({ sessionID }: { sessionID: string }) => ({ + data: { + id: sessionID, + ...(sessionID === "child" ? { parentID: "root" } : {}), + permission: [], + time: { created: 1, updated: 2 }, + }, + }), + update: async () => ({ data: true }), + status: async () => ({ + data: { + root: { type: "busy" }, + child: { + type: childStatus === "busy" || childStatus === "raced-idle" ? "busy" : "idle", + }, + unrelated: { type: "busy" }, + }, + }), + messages: async ({ sessionID }: { sessionID: string }) => { + if (sessionID === "child" && childStatus === "raced-idle") + await nativeEvents.push({ type: "session.idle", properties: { sessionID } }); + return { data: history(sessionID) }; + }, + promptAsync: async () => { + prompts++; + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + permission: { list: async () => ({ data: [] }) }, + question: { list: async () => ({ data: [] }) }, + }); + const events = yield* harness.runtime.events.pipe( + Stream.takeUntil( + (event) => + event.type === "turn.terminal" && + event.providerThreadId === harness.providerThread.id, + ), + Stream.runCollect, + Effect.forkScoped, + ); + yield* harness.startTurn("Continue", harness.providerThread, RunId.make("cut-root")); + assert.equal(prompts, 0); + assert.isTrue(yield* harness.runtime.hasPendingBackgroundWork!); + yield* Effect.promise(() => + nativeEvents.push({ type: "session.idle", properties: { sessionID: "root" } }), + ); + const collected = yield* Fiber.join(events); + if (childStatus !== "busy") { + const childTerminal = collected.findLast( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.providerThreadId !== harness.providerThread.id, + ); + assert.deepInclude( + childTerminal?.type === "provider_turn.updated" + ? childTerminal.providerTurn + : undefined, + { + status: + childStatus === "aborted" + ? "interrupted" + : childStatus === "errored" + ? "failed" + : "completed", + }, + ); + } + assert.isTrue( + collected.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text === "work while T3 was down", + ), + ); + assert.equal(yield* harness.runtime.hasPendingBackgroundWork!, childStatus === "busy"); + if (childStatus === "busy") { + yield* Effect.promise(() => + nativeEvents.push({ type: "session.idle", properties: { sessionID: "child" } }), + ); + assert.isFalse(yield* harness.runtime.hasPendingBackgroundWork!); + } + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + + it.effect("follows surviving compaction without submitting summarize again", () => + Effect.gen(function* () { + const nativeEvents = asyncEventStream(); + let summarize = 0; + const harness = yield* makeOpenCodeRuntimeHarness("restart-compact", "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => nativeEvents.close(), { once: true }); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + get: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + status: async () => ({ data: { root: { type: "busy" } } }), + messages: async () => ({ data: [] }), + summarize: async () => { + summarize++; + return { data: true }; + }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + }, + permission: { list: async () => ({ data: [] }) }, + question: { list: async () => ({ data: [] }) }, + }); + yield* harness.startTurn("/compact", harness.providerThread, RunId.make("cut-compact")); + const terminal = yield* harness.runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Effect.promise(() => + nativeEvents.push({ type: "session.compacted", properties: { sessionID: "root" } }), + ); + yield* Effect.promise(() => + nativeEvents.push({ type: "session.idle", properties: { sessionID: "root" } }), + ); + const events = yield* Fiber.join(terminal); + assert.equal(summarize, 0); + assert.isTrue( + events.some( + (event) => event.type === "turn_item.updated" && event.turnItem.type === "compaction", + ), + ); + assert.deepInclude(events.at(-1), { type: "turn.terminal", status: "completed" }); + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + + it.effect.each([false, true])( + "external shutdown preserves work only for restart=%s", + (restarting) => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const nativeEvents = asyncEventStream(); + const calls: string[] = []; + const harness = yield* makeOpenCodeRuntimeHarness("shutdown-intent", "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener( + "abort", + () => { + calls.push("stream.close"); + nativeEvents.close(); + }, + { once: true }, + ); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + abort: async ({ sessionID }: { sessionID: string }) => { + calls.push(`abort:${sessionID}`); + return { data: true }; + }, + children: async ({ sessionID }: { sessionID: string }) => ({ + data: sessionID === "root" ? [{ id: "child" }] : [], + }), + }, + }).pipe(Effect.provideService(Scope.Scope, scope)); + yield* harness.runtime.prepareForServerShutdown!(restarting); + yield* Scope.close(scope, Exit.void); + assert.deepEqual( + calls, + restarting ? ["stream.close"] : ["abort:root", "abort:child", "stream.close"], + ); + }).pipe( + Effect.provide(Layer.merge(IdAllocator.layer, McpProviderSessions.layer)), + Effect.scoped, + ), + ); + it.effect.each(["completed", "failed", "unresolved", "unavailable", "reconnect"] as const)( "normalizes OpenCode step usage for %s turns", (ending) => diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 1b3316d9f4793..840c73b7b3a27 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -53,6 +53,7 @@ import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; +import * as Hex from "effect/encoding/Hex"; import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import { mcpToolPresentation } from "@t3tools/provider-core/server/mcpToolPresentation"; @@ -89,6 +90,19 @@ export const OPENCODE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(OPENCODE_ export const OPENCODE_SDK_PROTOCOL = "opencode-sdk.sse" as const; const DEFAULT_OPENCODE_SETTINGS = Schema.decodeSync(OpenCodeSettings)({}); +// Recovery can recreate admission even if its provider event never committed. +const durableOpenCodeMessageId = Effect.fnUntraced(function* ( + attemptId: string, + startedAt: DateTime.Utc, +) { + const digest = yield* Effect.promise(() => + globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(attemptId)), + ); + return `msg_${BigInt.asUintN(48, BigInt(DateTime.toEpochMillis(startedAt)) * 0x1000n + 1n) + .toString(16) + .padStart(12, "0")}${Hex.encode(new Uint8Array(digest)).slice(0, 28)}`; +}); + const makeOpenCodeMessageId = Effect.fnUntraced(function* ( clock: Ref.Ref<{ readonly epochMillis: number; readonly counter: number }>, ) { @@ -290,7 +304,7 @@ interface ActiveOpenCodeTurn { readonly partIdsByMessage: Map>; readonly toolNamesByCallId: Map; mcpServerNames?: ReadonlyArray; - readonly providerTurn: OrchestrationV2ProviderTurn; + providerTurn: OrchestrationV2ProviderTurn; nextItemOrdinal: number; nativeUserMessageId: string | null; admissionMessageId: string | null; @@ -1021,6 +1035,11 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function // task child keeps running after its parent turn settles, so idle // release must not close the server under it. const busySessionIds = new Set(); + const observedStatuses = new Map< + string, + { type: "busy" | "retry" | "idle"; revision: number } + >(); + let statusRevision = 0; // Requests settled before their session relation resolved: a late // routing attempt must never resurrect them. Bounded because entries // only matter for the seconds a routing retry can still be running. @@ -1032,6 +1051,7 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function }; const abortController = new AbortController(); let closing = false; + let preserveExternalTurns = false; let hasConnected = false; const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => @@ -2631,9 +2651,15 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function case "session.deleted": relatedSessionOwners.delete(event.properties.info.id); busySessionIds.delete(event.properties.info.id); + observedStatuses.delete(event.properties.info.id); return; case "session.status": { const sessionId = event.properties.sessionID; + if (threads.has(sessionId) || relatedSessionOwners.has(sessionId)) + observedStatuses.set(sessionId, { + type: event.properties.status.type, + revision: ++statusRevision, + }); switch (event.properties.status.type) { case "busy": case "retry": @@ -2667,6 +2693,11 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function case "session.idle": { busySessionIds.delete(event.properties.sessionID); const state = threads.get(event.properties.sessionID); + if (state !== undefined || relatedSessionOwners.has(event.properties.sessionID)) + observedStatuses.set(event.properties.sessionID, { + type: "idle", + revision: ++statusRevision, + }); if (state?.activeTurn !== null && state?.activeTurn !== undefined) { if (state.activeTurn.admissionPending) { advanceOpenCodePromptAdmission(state.activeTurn, "idle"); @@ -2795,23 +2826,25 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function yield* Scope.addFinalizer( scope, Effect.suspend(() => - Effect.forEach( - Array.from(threads.values()).filter((state) => state.parentSubagent === null), - (state) => - sdkCall("session.abort", { sessionID: state.nativeSessionId }, (signal) => - client.session.abort({ sessionID: state.nativeSessionId }, { signal }), - ).pipe( - Effect.timeout("1 second"), - Effect.ignore({ log: true }), - Effect.andThen( - abortDescendants(state.nativeSessionId).pipe( + preserveExternalTurns + ? Effect.void + : Effect.forEach( + Array.from(threads.values()).filter((state) => state.parentSubagent === null), + (state) => + sdkCall("session.abort", { sessionID: state.nativeSessionId }, (signal) => + client.session.abort({ sessionID: state.nativeSessionId }, { signal }), + ).pipe( Effect.timeout("1 second"), Effect.ignore({ log: true }), + Effect.andThen( + abortDescendants(state.nativeSessionId).pipe( + Effect.timeout("1 second"), + Effect.ignore({ log: true }), + ), + ), ), - ), + { concurrency: 8, discard: true }, ), - { concurrency: 8, discard: true }, - ), ), ); } @@ -3039,6 +3072,10 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function providerSession: sessionEntity, events: Stream.fromEffectRepeat(Queue.take(events)), hasPendingBackgroundWork: Effect.sync(() => busySessionIds.size > 0), + prepareForServerShutdown: (restarting) => + Effect.sync(() => { + preserveExternalTurns = restarting; + }), ensureThread: (threadInput) => Effect.gen(function* () { // Only a row that already carries a native session can be @@ -3155,25 +3192,36 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function const isCompaction = turnInput.message.text.trim() === "/compact" && turnInput.message.attachments.length === 0; - const parts = isCompaction ? [] : resolvePromptParts(turnInput); + // A cancellation note must not prompt an external server whose + // task sessions survived. Delivery keeps the source even when + // it removes the continuation flag to force other providers to prompt. + const reconcileRestart = + turnInput.restartContinuationOfRunId !== undefined || + (connection.external && turnInput.restartContinuationSource !== undefined); + let parts = isCompaction ? [] : resolvePromptParts(turnInput); const startedAt = yield* DateTime.now; const syntheticNativeTurnId = `${sessionId}:attempt:${turnInput.attemptId}`; const providerTurnId = idAllocator.derive.providerTurn({ driver: OPENCODE_PROVIDER, nativeTurnId: syntheticNativeTurnId, }); + let admissionMessageId = + turnInput.runStartedAt == null + ? yield* makeOpenCodeMessageId(messageIdClock) + : yield* durableOpenCodeMessageId(turnInput.attemptId, turnInput.runStartedAt); const providerTurn: OrchestrationV2ProviderTurn = { id: providerTurnId, providerThreadId: turnInput.providerThread.id, nodeId: turnInput.rootNodeId, runAttemptId: turnInput.attemptId, - nativeTurnRef: providerRef(syntheticNativeTurnId, "weak"), + nativeTurnRef: + turnInput.restartContinuationSource?.nativeTurnRef ?? + providerRef(admissionMessageId, "weak"), ordinal: turnInput.providerTurnOrdinal, status: "running", startedAt, completedAt: null, }; - const admissionMessageId = yield* makeOpenCodeMessageId(messageIdClock); // No Effect may be yielded between this check and installing the // turn. If the event stream ended while IDs were being prepared, // registering afterward would leave a running turn that the EOF @@ -3237,8 +3285,9 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function lastRunOrdinal: turnInput.runOrdinal, }); yield* updateProviderSession("running", null); - if (turnInput.restartContinuationOfRunId !== undefined) { + if (reconcileRestart) { turn.usage.followingRestart = true; + const snapshotRevision = statusRevision; const statuses = unwrapData( "session.status", yield* sdkCall("session.status", { sessionID: sessionId }, () => @@ -3251,35 +3300,142 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function ), ), ); - const readHistory = sdkCall("session.messages", { sessionID: sessionId }, () => - client.session.messages({ sessionID: sessionId }), - ).pipe( - Effect.map((response) => unwrapData("session.messages", response)), - Effect.tapError((cause) => - finalizeTurn(state, turn, "failed", { - failure: makeProviderFailure({ cause, class: "provider_error" }), - }), - ), - ); - const idle = - statuses[sessionId] === undefined || statuses[sessionId]?.type === "idle"; - const idleHistory = idle ? yield* readHistory : undefined; + const revertedMessageIds = new Set(); + const readHistory = (nativeSessionId = sessionId) => + Effect.gen(function* () { + const nativeSession = unwrapData( + "session.get", + yield* sdkCall("session.get", { sessionID: nativeSessionId }, () => + client.session.get({ sessionID: nativeSessionId }), + ), + ); + const history = unwrapData( + "session.messages", + yield* sdkCall("session.messages", { sessionID: nativeSessionId }, () => + client.session.messages({ sessionID: nativeSessionId }), + ), + ); + const boundary = history.findIndex( + (entry) => entry.info.id === nativeSession.revert?.messageID, + ); + if (nativeSessionId === sessionId && boundary >= 0) + for (const entry of history.slice(boundary)) + revertedMessageIds.add(entry.info.id); + return boundary < 0 ? history : history.slice(0, boundary); + }).pipe( + Effect.tapError((cause) => + finalizeTurn(state, turn, "failed", { + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ); + const statusFor = (nativeSessionId: string) => { + const observed = observedStatuses.get(nativeSessionId); + return observed !== undefined && observed.revision > snapshotRevision + ? observed.type + : (statuses[nativeSessionId]?.type ?? "idle"); + }; + const idleHistory = yield* readHistory(); + const isPrompt = (entry: (typeof idleHistory)[number]) => + entry.info.role === "user" && + !( + entry.parts.length > 0 && + entry.parts.every( + (part) => + (part.type === "compaction" && part.auto) || + (part.type === "text" && part.synthetic === true), + ) + ); // Live events may have finished the surviving run meanwhile. if (turn.finalized || state.activeTurn !== turn) return; // An idle session whose newest prompt T3 never settled, answered by // a clean final reply, finished the cut run while T3 was down. - const lastPrompt = idleHistory?.findLast((entry) => entry.info.role === "user"); + const lastPrompt = idleHistory.findLast(isPrompt); const lastInfo = idleHistory?.at(-1)?.info; + const source = turnInput.restartContinuationSource; + const recordedMessageId = source?.nativeTurnRef?.nativeId; + const savedMessageId = + recordedMessageId?.startsWith("msg_") === true + ? recordedMessageId + : source?.attemptId != null && source.startedAt != null + ? yield* durableOpenCodeMessageId(source.attemptId, source.startedAt) + : recordedMessageId; + if (turn.finalized || state.activeTurn !== turn) return; + const idle = statusFor(sessionId) === "idle"; + const originalParts = + source === undefined + ? [] + : resolvePromptParts({ ...turnInput, message: source.message }); + const matchingPrompt = idleHistory?.findLast((entry) => { + if (entry.info.role !== "user") return false; + if (savedMessageId?.startsWith("msg_") === true) { + if (entry.info.id === savedMessageId) return true; + if (recordedMessageId?.startsWith("msg_") === true) return false; + } + if (source?.startedAt == null) return false; + const sourceEpoch = DateTime.toEpochMillis(source.startedAt); + // Our admission IDs encode T3's clock; a remote server's + // message timestamp may be behind it. + const afterStart = /^msg_[0-9a-f]{12}[A-Za-z0-9]{28}$/.test(entry.info.id) + ? BigInt.asIntN( + 36, + BigInt(`0x${entry.info.id.slice(4, 16)}`) / 0x1000n - BigInt(sourceEpoch), + ) >= 0n + : entry.info.time.created >= sourceEpoch; + if (!afterStart) return false; + return ( + originalParts.length > 0 && + originalParts.every((expected) => + entry.parts.some((actual) => + expected.type === "text" + ? actual.type === "text" && + actual.text.trim().endsWith(expected.text.trim()) + : actual.type === "file" && actual.url === expected.url, + ), + ) + ); + }); + const sourceMessageId = matchingPrompt?.info.id ?? savedMessageId; const finishedWhileDown = lastPrompt !== undefined && - lastPrompt.info.id !== - turnInput.providerThread.nativeConversationHeadRef?.nativeId && + (sourceMessageId === undefined + ? source === undefined && + turnInput.providerThread.nativeConversationHeadRef !== null && + lastPrompt.info.id !== + turnInput.providerThread.nativeConversationHeadRef.nativeId + : lastPrompt.info.id === sourceMessageId) && lastInfo?.role === "assistant" && + lastInfo.parentID === + idleHistory.findLast((entry) => entry.info.role === "user")?.info.id && + lastInfo.summary !== true && lastInfo.time.completed !== undefined && lastInfo.error === undefined && - lastInfo.finish !== "tool-calls"; + lastInfo.finish === "stop"; if (idle && !finishedWhileDown) { turn.usage.followingRestart = false; + // Rebinding a durable question is not permission to start + // another generation if the native callback disappeared. + if (source?.awaitingUser === true) { + yield* finalizeTurn(state, turn, "interrupted"); + return; + } + if ( + source !== undefined && + !idleHistory?.some((entry) => entry.info.id === sourceMessageId) + ) { + parts = resolvePromptParts({ ...turnInput, message: source.message }); + // A delayed original request and this retry must share + // OpenCode's admission ID. Reverted records need a new ID. + if ( + savedMessageId?.startsWith("msg_") === true && + !revertedMessageIds.has(savedMessageId) + ) { + turn.usage.promptMessageIds.delete(admissionMessageId); + admissionMessageId = savedMessageId; + turn.admissionMessageId = admissionMessageId; + turn.usage.promptMessageIds.add(admissionMessageId); + } + } } else { // A server that outlived T3 still owns this turn; follow it // instead of adding another prompt to its running execution. @@ -3288,41 +3444,131 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function const admissionAction = advanceOpenCodePromptAdmission(turn, "accepted"); yield* Deferred.succeed(admissionSettled, undefined); turn.admissionAbortController = null; - const history = idleHistory ?? (yield* readHistory); - const userIndex = history.findLastIndex((entry) => entry.info.role === "user"); - for (const entry of userIndex < 0 ? [] : history.slice(userIndex)) { - if (turn.finalized) break; - yield* handleMessageUpdated({ - id: entry.info.id, - type: "message.updated", - properties: { sessionID: sessionId, info: entry.info }, - }); - for (const part of entry.parts) { - // Live events since this turn became active already - // hold a newer copy; the snapshot would roll it back. - if ( - part.type === "tool" - ? turn.toolNamesByCallId.has(part.callID) - : turn.parts.has(part.id) - ) - continue; - yield* handlePartUpdated({ - id: part.id, - type: "message.part.updated", - properties: { - sessionID: sessionId, - part, - time: DateTime.toEpochMillis(startedAt), - }, + const history = idleHistory; + // Keep the original admission identity if T3 is restarted + // again while following this same surviving execution. + const followedPrompt = history.findLast(isPrompt); + if (followedPrompt !== undefined) { + turn.nativeUserMessageId = followedPrompt.info.id; + yield* emitProviderTurn(state, turn, "running", null); + } + const replayHistory = Effect.fnUntraced(function* ( + replayState: OpenCodeThreadState, + entries: typeof history, + ) { + const userIndex = entries.findLastIndex(isPrompt); + for (const entry of userIndex < 0 ? [] : entries.slice(userIndex)) { + if (replayState.activeTurn?.finalized) break; + yield* handleMessageUpdated({ + id: entry.info.id, + type: "message.updated", + properties: { sessionID: replayState.nativeSessionId, info: entry.info }, }); + for (const part of entry.parts) { + // Live events since this turn became active already + // hold a newer copy; the snapshot would roll it back. + if ( + part.type === "tool" + ? replayState.activeTurn?.toolNamesByCallId.has(part.callID) + : replayState.activeTurn?.parts.has(part.id) + ) + continue; + yield* handlePartUpdated({ + id: part.id, + type: "message.part.updated", + properties: { + sessionID: replayState.nativeSessionId, + part, + time: DateTime.toEpochMillis(startedAt), + }, + }); + } + } + }); + yield* replayHistory(state, history); + const replayed = new Set([sessionId]); + // Task parts reconnect their child sessions. Replay those + // too, including nested tasks created while T3 was down. + for (const child of threads.values()) { + if (child.parentSubagent === null || replayed.has(child.nativeSessionId)) + continue; + if (!replayed.has(child.parentSubagent.parentState.nativeSessionId)) continue; + replayed.add(child.nativeSessionId); + const childHistory = yield* readHistory(child.nativeSessionId); + yield* replayHistory(child, childHistory); + const status = statusFor(child.nativeSessionId); + if (status !== "idle") { + busySessionIds.add(child.nativeSessionId); + } else if (child.activeTurn !== null) { + const lastMessage = childHistory.at(-1)?.info; + if (lastMessage?.role === "assistant" && lastMessage.error !== undefined) { + yield* handleEvent({ + id: lastMessage.id, + type: "session.error", + properties: { + sessionID: child.nativeSessionId, + error: lastMessage.error, + }, + }); + } else { + yield* finalizeTurn(child, child.activeTurn, "completed"); + } } } + if ( + statusFor(sessionId) !== "idle" && + state.activeTurn === turn && + !turn.finalized + ) + busySessionIds.add(sessionId); turn.usage.followingRestart = false; if (idle) { if (!turn.finalized) yield* finalizeTurn(state, turn, "completed"); } else if (admissionAction === "reconcile-idle") { yield* reconcilePromptAdmission(state, turn); } + if (!turn.finalized) { + const permissions = unwrapData( + "permission.list", + yield* sdkCall("permission.list", {}, (signal) => + client.permission.list(undefined, { signal }), + ).pipe( + Effect.tapError((cause) => + finalizeTurn(state, turn, "failed", { + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ), + ); + const questions = unwrapData( + "question.list", + yield* sdkCall("question.list", {}, (signal) => + client.question.list(undefined, { signal }), + ).pipe( + Effect.tapError((cause) => + finalizeTurn(state, turn, "failed", { + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ), + ); + for (const request of permissions) { + const owner = yield* resolveSessionOwner(request.sessionID); + if (owner !== undefined && topLevelRequestOwner(owner)?.state === state) + yield* routeRuntimeRequest(request.id, request.sessionID, { + type: "permission", + value: request, + }); + } + for (const request of questions) { + const owner = yield* resolveSessionOwner(request.sessionID); + if (owner !== undefined && topLevelRequestOwner(owner)?.state === state) + yield* routeRuntimeRequest(request.id, request.sessionID, { + type: "question", + value: request, + }); + } + } return; } } @@ -3369,6 +3615,13 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function turnInput.modelSelection, "variant", ); + if (reconcileRestart) { + turn.providerTurn = { + ...turn.providerTurn, + nativeTurnRef: providerRef(admissionMessageId, "weak"), + }; + yield* emitProviderTurn(state, turn, "running", null); + } yield* submitPrompt( state, turn, From 7c5c624cc10f7c03b3f1cd3812759d4e7d851f8f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:06:14 +0000 Subject: [PATCH 081/149] fix(server): accepted webhooks survive restarts and stale scheduled deliveries never run --- .../ThreadLaunchService.test.ts | 149 +++++++ apps/server/src/persistence/Migrations.ts | 4 + .../Migrations/055_OrchestrationV2.test.ts | 4 + .../061_ScheduledTaskRunningIndex.ts | 8 + .../Migrations/062_WebhookDispatchQueue.ts | 15 + .../reconcileV2PreviewMigration.test.ts | 4 + .../src/scheduledTasks/Schedule.test.ts | 26 ++ .../ScheduledTaskService.test.ts | 92 ++++ .../scheduledTasks/ScheduledTaskService.ts | 395 ++++++++++++------ .../ScheduledTaskService.webhook.test.ts | 274 ++++++++++++ .../scheduling/Scheduler.integration.test.ts | 81 ++++ 11 files changed, 929 insertions(+), 123 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/061_ScheduledTaskRunningIndex.ts create mode 100644 apps/server/src/persistence/Migrations/062_WebhookDispatchQueue.ts diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index f7eede36b8e41..19e4337e37a52 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -27,17 +27,20 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; +import * as SqlClient from "effect/sql/SqlClient"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -2530,3 +2533,149 @@ it.effect.each([ }).pipe(Effect.provide(harness.layer)); }), ); + +it.effect.each( + (["fresh", "bound"] as const).flatMap((binding) => + [false, true].map((edit) => ({ binding, edit })), + ), +)( + "replays a committed $binding webhook dispatch through receipts without another run (edited $edit)", + ({ binding, edit }) => { + const harness = makeHarness({ runSetup: () => Effect.never }); + const cleanupFailed = Deferred.makeUnsafe(); + let poll: Effect.Effect | undefined; + const layer = ScheduledTasks.layer.pipe( + Layer.provide( + Layer.mergeAll( + harness.layer, + NodeCrypto.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.succeed(Scheduler.Scheduler, { + register: (_name: string, work: Effect.Effect) => + Effect.context().pipe( + Effect.map((context) => { + poll = work.pipe(Effect.provideContext(context), Effect.orDie); + }), + ), + }), + ), + ), + ); + return Effect.gen(function* () { + const tasks = yield* ScheduledTasks.ScheduledTaskService; + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const sql = yield* SqlClient.SqlClient; + const bound = + binding === "bound" + ? yield* launches.launch( + launchInput({ + command: "command:bound", + thread: "thread:bound", + message: "First message", + }), + ) + : null; + const { task } = yield* tasks.upsert({ + id: ScheduledTaskId.make("scheduled-task:durable"), + title: "Webhook", + prompt: "Handle {{body}}", + enabled: true, + schedule: { type: "webhook" }, + projectId, + threadId: bound?.threadId ?? null, + workspaceStrategy: { type: "root" }, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + }); + yield* sql`CREATE TRIGGER hold_delivery BEFORE DELETE ON scheduled_task_webhook_queue + BEGIN SELECT RAISE(FAIL, 'disk write failed'); END`; + const request = { + hookId: task.id, + token: task.webhook!.path.split("/").at(-1)!, + method: "POST", + path: "/hook", + query: "", + headers: {}, + body: new TextEncoder().encode('{"number":7}'), + bodyText: '{"number":7}', + relayDeliveryId: "committed", + }; + const subscribed = yield* Deferred.make(); + const completion = yield* tasks.subscribeList().pipe( + Stream.tap(() => Deferred.succeed(subscribed, undefined)), + Stream.filter((list) => list.tasks[0]?.lastRunStatus === "succeeded"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + yield* tasks.triggerWebhook(request); + yield* Fiber.join(completion); + yield* Deferred.await(cleanupFailed); + // The provider handoff committed, but cleanup failed. A new tick must use the same receipts. + const pending = yield* sql<{ + delivery_id: string; + }>`SELECT delivery_id FROM scheduled_task_webhook_queue`; + assert.lengthOf(pending, 1); + const changedTarget = + edit && binding === "fresh" + ? yield* launches.launch( + launchInput({ command: "command:changed-target", thread: "thread:changed-target" }), + ) + : null; + if (edit) + yield* tasks.upsert({ + id: task.id, + title: "Edited task", + prompt: "Edited prompt", + enabled: true, + schedule: { type: "webhook" }, + projectId, + threadId: changedTarget?.threadId ?? null, + workspaceStrategy: { type: "root" }, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + }); + yield* sql`DROP TRIGGER hold_delivery`; + const replaySubscribed = yield* Deferred.make(); + const replayed = yield* tasks.subscribeList().pipe( + Stream.tap(() => Deferred.succeed(replaySubscribed, undefined)), + Stream.drop(1), + Stream.filter((list) => list.tasks[0]?.lastRunStatus !== "running"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Deferred.await(replaySubscribed); + yield* poll!; + yield* Fiber.join(replayed); + const projectThreads = yield* threads.listProjectThreads({ + projectId, + includeSubagents: false, + }); + assert.lengthOf(projectThreads, changedTarget === null ? 1 : 2); + const original = projectThreads.find((thread) => thread.id !== changedTarget?.threadId)!; + const projection = yield* threads.getThreadProjection(original.id); + const wire = yield* encodeThreadProjection(projection); + const automated = wire.messages.filter((message) => message.scheduledTaskId === task.id); + assert.lengthOf(automated, 1); + assert.equal(automated[0]?.text, 'Handle {"number":7}'); + assert.equal(wire.runs.length, binding === "bound" ? 2 : 1); + assert.equal((yield* tasks.list()).tasks[0]?.lastRunStatus, "succeeded"); + }).pipe( + Effect.provide( + Layer.mergeAll(harness.layer, layer).pipe( + Layer.provideMerge( + Logger.layer([ + Logger.make(({ message }) => { + if (Array.isArray(message) && message[0] === "Webhook dispatch failed; will retry") + Deferred.doneUnsafe(cleanupFailed, Effect.void); + }), + ]), + ), + ), + ), + ); + }, +); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 35c1a3fdce4bb..d4739c20276a2 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -74,6 +74,8 @@ import Migration0057 from "./Migrations/057_ScheduledTaskWebhooks.ts"; import Migration0058 from "./Migrations/058_WebhookRelayDeliveries.ts"; import Migration0059 from "./Migrations/059_McpAppModelContext.ts"; import Migration0060 from "./Migrations/060_ThreadSnapshotWindowIndexes.ts"; +import Migration0061 from "./Migrations/061_ScheduledTaskRunningIndex.ts"; +import Migration0062 from "./Migrations/062_WebhookDispatchQueue.ts"; /** * Migration loader with all migrations defined inline. @@ -148,6 +150,8 @@ export const migrationEntries = [ [58, "WebhookRelayDeliveries", Migration0058], [59, "McpAppModelContext", Migration0059], [60, "ThreadSnapshotWindowIndexes", Migration0060], + [61, "ScheduledTaskRunningIndex", Migration0061], + [62, "WebhookDispatchQueue", Migration0062], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts index 5e45ed05d3024..74704d8ed17c9 100644 --- a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts +++ b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts @@ -32,6 +32,8 @@ layer("055_OrchestrationV2", (it) => { [58, "WebhookRelayDeliveries"], [59, "McpAppModelContext"], [60, "ThreadSnapshotWindowIndexes"], + [61, "ScheduledTaskRunningIndex"], + [62, "WebhookDispatchQueue"], ]); assert.deepStrictEqual(yield* runMigrations(), []); @@ -58,6 +60,8 @@ layer("055_OrchestrationV2", (it) => { { migration_id: 58, name: "WebhookRelayDeliveries" }, { migration_id: 59, name: "McpAppModelContext" }, { migration_id: 60, name: "ThreadSnapshotWindowIndexes" }, + { migration_id: 61, name: "ScheduledTaskRunningIndex" }, + { migration_id: 62, name: "WebhookDispatchQueue" }, ]); const tables = yield* sql<{ readonly name: string }>` diff --git a/apps/server/src/persistence/Migrations/061_ScheduledTaskRunningIndex.ts b/apps/server/src/persistence/Migrations/061_ScheduledTaskRunningIndex.ts new file mode 100644 index 0000000000000..865a808cec04f --- /dev/null +++ b/apps/server/src/persistence/Migrations/061_ScheduledTaskRunningIndex.ts @@ -0,0 +1,8 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_running + ON scheduled_tasks(last_run_status) WHERE last_run_status = 'running'`; +}); diff --git a/apps/server/src/persistence/Migrations/062_WebhookDispatchQueue.ts b/apps/server/src/persistence/Migrations/062_WebhookDispatchQueue.ts new file mode 100644 index 0000000000000..f0c15739cbb51 --- /dev/null +++ b/apps/server/src/persistence/Migrations/062_WebhookDispatchQueue.ts @@ -0,0 +1,15 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE scheduled_task_webhook_queue ( + delivery_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + task_created_at TEXT NOT NULL, + prompt TEXT NOT NULL, + target_json TEXT + )`; + yield* sql`CREATE INDEX idx_scheduled_task_webhook_queue_task + ON scheduled_task_webhook_queue(task_id, task_created_at)`; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 675d5f5db0215..5eec788d996f0 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -41,6 +41,8 @@ describe("V2 preview upgrade", () => { [58, "WebhookRelayDeliveries"], [59, "McpAppModelContext"], [60, "ThreadSnapshotWindowIndexes"], + [61, "ScheduledTaskRunningIndex"], + [62, "WebhookDispatchQueue"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -124,6 +126,8 @@ describe("V2 preview upgrade", () => { [58, "WebhookRelayDeliveries"], [59, "McpAppModelContext"], [60, "ThreadSnapshotWindowIndexes"], + [61, "ScheduledTaskRunningIndex"], + [62, "WebhookDispatchQueue"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/scheduledTasks/Schedule.test.ts b/apps/server/src/scheduledTasks/Schedule.test.ts index c6f48599d1942..754023817b00a 100644 --- a/apps/server/src/scheduledTasks/Schedule.test.ts +++ b/apps/server/src/scheduledTasks/Schedule.test.ts @@ -134,3 +134,29 @@ describe("scheduled task schedule calculation", () => { ).toBe(true); }); }); + +it.each([ + ["spring gap", "2026-03-08T08:00:00.000Z", "02:30", "2026-03-08T10:30:00.000Z"], + ["after spring gap", "2026-03-08T10:31:00.000Z", "02:30", "2026-03-09T09:30:00.000Z"], + ["fall fold", "2026-11-01T07:00:00.000Z", "01:30", "2026-11-01T08:30:00.000Z"], + ["after first fall slot", "2026-11-01T08:31:00.000Z", "01:30", "2026-11-02T09:30:00.000Z"], + ["after second fall slot", "2026-11-01T09:31:00.000Z", "01:30", "2026-11-02T09:30:00.000Z"], +])("preserves daily wall-clock scheduling across %s", (_name, instant, timeOfDay, expected) => { + const from = DateTime.makeZonedUnsafe(instant, { timeZone: "America/Los_Angeles" }); + const next = nextScheduledRunAt({ type: "fixed_time", timeOfDay }, from); + expect(next && DateTime.formatIso(DateTime.toUtc(next))).toBe(expected); +}); + +it("keeps interval elapsed time across DST and applies weekdays in the server zone", () => { + const from = DateTime.makeZonedUnsafe("2026-03-08T09:55:00.000Z", { + timeZone: "America/Los_Angeles", + }); + const next = nextScheduledRunAt({ type: "interval", everyMs: 15 * 60_000 }, from)!; + expect(DateTime.formatIso(DateTime.toUtc(next))).toBe("2026-03-08T10:10:00.000Z"); + expect(DateTime.toParts(next).hour).toBe(3); + const sunday = nextScheduledRunAt( + { type: "fixed_time", timeOfDay: "23:30", weekdays: [0] }, + DateTime.makeZonedUnsafe("2026-07-06T01:00:00.000Z", { timeZone: "America/Los_Angeles" }), + ); + expect(sunday && DateTime.formatIso(DateTime.toUtc(sunday))).toBe("2026-07-06T06:30:00.000Z"); +}); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 83bf6902dca5e..bd621a94c8802 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -11,6 +11,7 @@ import * as Logger from "effect/Logger"; import * as SqlClient from "effect/sql/SqlClient"; import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -475,3 +476,94 @@ it.effect( ); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect("retries failed crash recovery without resetting a live dispatch", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-09-09T12:00:00.000Z"; + yield* TestClock.setTime(Date.parse(now)); + yield* insertRow(sql, { id: "stuck", next: now, status: "running", enabled: 1 }, now); + yield* insertRow(sql, { id: "stuck-second", next: now, status: "running", enabled: 1 }, now); + yield* sql`CREATE TRIGGER fail_recovery BEFORE UPDATE ON scheduled_tasks + WHEN OLD.task_id = 'stuck' BEGIN SELECT RAISE(FAIL, 'disk write failed'); END`; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + let poll: Effect.Effect | undefined; + const dependencies = Layer.mergeAll( + NodeCrypto.layer, + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: () => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as({} as never), + ), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.succeed(Scheduler.Scheduler, { + register: (_name: string, work: Effect.Effect) => + Effect.context().pipe( + Effect.map((context) => { + poll = work.pipe(Effect.provideContext(context), Effect.orDie); + }), + ), + }), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const initial = (yield* service.list()).tasks; + assert.equal(initial.find((task) => task.id === "stuck")?.lastRunStatus, "running"); + assert.equal(initial.find((task) => task.id === "stuck-second")?.lastRunStatus, "failed"); + yield* sql`DROP TRIGGER fail_recovery`; + yield* poll!; + const recovered = (yield* service.list()).tasks.find((task) => task.id === "stuck")!; + assert.equal(recovered.lastRunStatus, "failed"); + assert.equal(recovered.runCount, 1); + assert.equal(recovered.nextRunAt, "2026-09-09T12:01:00.000Z"); + yield* poll!; + assert.equal((yield* service.list()).tasks.find((task) => task.id === "stuck")?.runCount, 1); + yield* insertRow(sql, { id: "live", next: now, status: "never", enabled: 1 }, now); + const dispatch = yield* poll!.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* poll!; + const live = (yield* service.list()).tasks.find((task) => task.id === "live"); + assert.equal(live?.lastRunStatus, "running"); + assert.equal(live?.runCount, 0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(dispatch); + assert.equal( + (yield* service.list()).tasks.find((task) => task.id === "live")?.lastRunStatus, + "succeeded", + ); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("uses both partial indexes for the inclusive update horizon", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-09-09T12:00:00.000Z"; + const horizon = "2026-09-09T12:05:00.000Z"; + for (const row of [ + { id: "running-disabled", enabled: 0, next: null, status: "running" }, + { id: "disabled", enabled: 0, next: now, status: "succeeded" }, + { id: "exact-horizon", enabled: 1, next: horizon, status: "succeeded" }, + { id: "after-horizon", enabled: 1, next: "2026-09-09T12:05:00.001Z", status: "succeeded" }, + ]) + yield* insertRow(sql, row, now); + const selected = yield* ScheduledTaskService.listRunningOrDueTasks( + DateTime.makeUnsafe(horizon), + ); + assert.deepEqual(selected.map((task) => task.id).sort(), ["exact-horizon", "running-disabled"]); + const plan = yield* sql<{ detail: string }>`EXPLAIN QUERY PLAN SELECT * FROM scheduled_tasks + WHERE last_run_status = 'running' + OR (enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ${horizon}) + UNION SELECT task.* FROM scheduled_task_webhook_queue AS delivery + CROSS JOIN scheduled_tasks AS task + WHERE task.task_id = delivery.task_id AND task.created_at = delivery.task_created_at + AND task.enabled = 1`; + assert.isTrue(plan.some((row) => row.detail.includes("idx_scheduled_tasks_running"))); + assert.isTrue(plan.some((row) => row.detail.includes("idx_scheduled_tasks_due"))); + assert.isFalse(plan.some((row) => /SCAN (scheduled_tasks|task)\b/.test(row.detail))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index e2bf272e7ab37..189289fe6a469 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -38,6 +38,7 @@ import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import * as Scope from "effect/Scope"; import * as SqlClient from "effect/sql/SqlClient"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; @@ -45,6 +46,7 @@ import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; +import { forkParked } from "../serverActivation.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; import { redactHeaders, @@ -173,6 +175,21 @@ const decodeHeadersJson = Schema.decodeUnknownOption(HeadersJson); const decodeMissingFieldsJson = Schema.decodeUnknownOption(MissingFieldsJson); const encodeHeadersJson = Schema.encodeSync(HeadersJson); const encodeMissingFieldsJson = Schema.encodeSync(MissingFieldsJson); +const WebhookDispatchTargetJson = Schema.fromJsonString( + Schema.Struct({ + projectId: ScheduledTask.fields.projectId, + threadId: ScheduledTask.fields.threadId, + title: ScheduledTask.fields.title, + modelSelection: ScheduledTask.fields.modelSelection, + runtimeMode: ScheduledTask.fields.runtimeMode, + interactionMode: ScheduledTask.fields.interactionMode, + workspaceStrategy: ScheduledTask.fields.workspaceStrategy, + createdBy: ScheduledTask.fields.createdBy, + creationSource: ScheduledTask.fields.creationSource, + }), +); +const decodeWebhookDispatchTarget = Schema.decodeUnknownEffect(WebhookDispatchTargetJson); +const encodeWebhookDispatchTarget = Schema.encodeSync(WebhookDispatchTargetJson); interface ScheduledTaskRow { readonly task_id: string; @@ -402,6 +419,11 @@ export const listRunningOrDueTasks = Effect.fn("ScheduledTaskService.listRunning SELECT * FROM scheduled_tasks WHERE last_run_status = 'running' OR (enabled = 1 AND next_run_at IS NOT NULL AND next_run_at <= ${iso(by)}) + UNION + SELECT task.* FROM scheduled_task_webhook_queue AS delivery + CROSS JOIN scheduled_tasks AS task + WHERE task.task_id = delivery.task_id AND task.created_at = delivery.task_created_at + AND task.enabled = 1 `; return yield* decodeTaskRowsLeniently(rows); }, @@ -422,9 +444,7 @@ export const layer = Layer.effect( const webhookPermits = yield* Ref.make>( new Map(), ); - // Keyed by task id and creation time, so deliveries of a deleted task that - // finish late release their own count, never a recreated task's. - const webhookQueued = yield* Ref.make>(new Map()); + const dispatchingDeliveries = yield* Ref.make>(new Set()); const webhookRateWindows = yield* Ref.make>(new Map()); const activeRuns = yield* Ref.make>(new Set()); // Sliding(1) coalesces the dirty-signal: every notification triggers a @@ -639,6 +659,7 @@ export const layer = Layer.effect( sql .withTransaction( sql`DELETE FROM scheduled_task_webhook_deliveries WHERE task_id = ${id}`.pipe( + Effect.andThen(sql`DELETE FROM scheduled_task_webhook_queue WHERE task_id = ${id}`), Effect.andThen(sql`DELETE FROM scheduled_tasks WHERE task_id = ${id}`), ), ) @@ -651,14 +672,17 @@ export const layer = Layer.effect( // Run-state transitions use targeted UPDATEs (never the full-row upsert) so // a completing run cannot resurrect a deleted task or clobber concurrent // edits to the task definition. - const markRunning = (id: ScheduledTaskId, startedAtIso: string) => + const markRunning = (id: ScheduledTaskId, startedAtIso: string, deliveryId?: string) => sql<{ task_id: string }>` UPDATE scheduled_tasks SET updated_at = ${startedAtIso}, last_run_at = ${startedAtIso}, last_run_status = 'running', last_run_error = NULL - WHERE task_id = ${id} + WHERE task_id = ${id} AND last_run_status <> 'running' + AND (${deliveryId ?? null} IS NULL OR EXISTS ( + SELECT 1 FROM scheduled_task_webhook_queue WHERE delivery_id = ${deliveryId ?? null} + )) RETURNING task_id `.pipe( Effect.mapError((cause) => @@ -730,7 +754,7 @@ export const layer = Layer.effect( const runTask = Effect.fn("ScheduledTaskService.runTask")(function* ( task: ScheduledTask, trigger: "scheduled" | "manual" | "webhook", - webhook?: { readonly deliveryId: string; readonly prompt: string }, + webhook?: { readonly deliveryId: string }, ) { const reserved = yield* Ref.modify(activeRuns, (active) => { if (active.has(task.id)) return [false, active] as const; @@ -750,9 +774,33 @@ export const layer = Layer.effect( // state. The task may have been deleted, paused, or postponed since // the poll loaded it — none of those may fire. let active = yield* findTask(task.id); + const delivery = + webhook === undefined + ? [] + : yield* sql<{ target_json: string | null; prompt: string }>` + SELECT target_json, prompt FROM scheduled_task_webhook_queue WHERE delivery_id = ${webhook.deliveryId} + `.pipe( + Effect.mapError((cause) => + taskError("Could not read webhook dispatch target.", { taskId: task.id, cause }), + ), + ); + if (webhook !== undefined && delivery.length === 0) { + return yield* new WebhookDeliverySkipped({ + reason: "The task was deleted before this delivery ran.", + }); + } + const storedTarget = delivery[0]?.target_json; + const frozen = + storedTarget == null + ? null + : yield* decodeWebhookDispatchTarget(storedTarget).pipe( + Effect.mapError((cause) => + taskError("Could not read webhook dispatch target.", { taskId: task.id, cause }), + ), + ); // Recovery cancels the old run before its continuation is dispatched. // In that gap, queue mode would start a new run and suppress the continuation. - while (active !== null && active.threadId !== null) { + while (active !== null && (frozen === null ? active.threadId : frozen.threadId) !== null) { if ( trigger === "manual" || !active.enabled || @@ -762,7 +810,7 @@ export const layer = Layer.effect( break; const pending = yield* sql` SELECT 1 FROM orchestration_v2_effect_outbox - WHERE thread_id = ${active.threadId} + WHERE thread_id = ${frozen === null ? active.threadId : frozen.threadId} AND effect_type = 'provider-runtime.continue' AND status IN ('pending', 'running') LIMIT 1 @@ -822,7 +870,19 @@ export const layer = Layer.effect( if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); } - yield* markRunning(active.id, startedAtIso); + // A partially committed dispatch must replay its original target even + // if the task was edited between attempts. + const dispatchTask = frozen ?? active; + if (webhook !== undefined && frozen === null) + yield* sql` + UPDATE scheduled_task_webhook_queue SET target_json = ${encodeWebhookDispatchTarget(dispatchTask)} + WHERE delivery_id = ${webhook.deliveryId} AND target_json IS NULL + `.pipe( + Effect.mapError((cause) => + taskError("Could not save webhook dispatch target.", { taskId: task.id, cause }), + ), + ); + yield* markRunning(active.id, startedAtIso, webhook?.deliveryId); yield* notifyChanged; // A webhook run is keyed by its delivery so the same delivery can @@ -836,49 +896,57 @@ export const layer = Layer.effect( // Dispatch from the fresh row so prompt/model/binding edits made // after the poll read are honoured. A webhook prompt was rendered // from the row when the request arrived. - const prompt = webhook?.prompt ?? active.prompt; + const prompt = delivery[0]?.prompt ?? active.prompt; // Effect.exit (not Effect.result) so defects and interruptions in the // dispatch are also captured and recorded as a failed run instead of // aborting before markCompleted. const result = - active.threadId === null + dispatchTask.threadId === null ? yield* Effect.exit( threadLaunch.launch({ commandId, - projectId: active.projectId, - title: active.title, - modelSelection: active.modelSelection, - runtimeMode: active.runtimeMode, - interactionMode: active.interactionMode, - workspaceStrategy: active.workspaceStrategy, + projectId: dispatchTask.projectId, + title: dispatchTask.title, + modelSelection: dispatchTask.modelSelection, + runtimeMode: dispatchTask.runtimeMode, + interactionMode: dispatchTask.interactionMode, + workspaceStrategy: dispatchTask.workspaceStrategy, initialMessage: { messageId, scheduledTaskId: active.id, text: prompt, attachments: [], }, - createdBy: active.createdBy, - creationSource: active.creationSource, + createdBy: dispatchTask.createdBy, + creationSource: dispatchTask.creationSource, }), ) : yield* Effect.exit( threadManagement.sendToThread({ - projectId: active.projectId, + projectId: dispatchTask.projectId, commandId, - threadId: ThreadId.make(active.threadId), + threadId: ThreadId.make(dispatchTask.threadId), messageId, scheduledTaskId: active.id, text: prompt, attachments: [], - modelSelection: active.modelSelection, + modelSelection: dispatchTask.modelSelection, // Scheduled prompts must not interrupt tools in the bound thread. mode: "queue", - createdBy: active.createdBy, - creationSource: active.creationSource, + createdBy: dispatchTask.createdBy, + creationSource: dispatchTask.creationSource, }), ); + if ( + webhook !== undefined && + result._tag === "Failure" && + Cause.hasInterrupts(result.cause) + ) { + return yield* Effect.interrupt; + } + const completedAt = yield* localNow; const runSucceeded = result._tag === "Success"; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); @@ -948,30 +1016,6 @@ export const layer = Layer.effect( yield* notifyChanged; }); - const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () { - const now = yield* localNow; - const tasks = yield* listDueTasks(now).pipe( - Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), - ); - const due = tasks.flatMap((task) => - task.nextRunAt === null ? [] : [{ task, dueAt: DateTime.makeUnsafe(task.nextRunAt) }], - ); - yield* Effect.forEach( - due, - ({ task, dueAt }) => - Effect.suspend(() => - isMissedFixedTimeRun(task.schedule, dueAt, now) - ? rescheduleMissedRun(task, now) - : runTask(task, "scheduled"), - ).pipe( - Effect.catch((cause) => - Effect.logWarning("Scheduled task run failed", { taskId: task.id, cause }), - ), - ), - { concurrency: 1, discard: true }, - ); - }); - // Recover from a crash or hard shutdown mid-run: rows stuck in 'running' // would otherwise be skipped by the due-task filter forever. The dispatch // may already have gone out before the crash, so next_run_at must advance @@ -979,9 +1023,14 @@ export const layer = Layer.effect( // every restart re-fires the interrupted task (same rationale as // releaseStuckRun). Schedules are JSON, so this is per-row Effect work // rather than a single UPDATE. - yield* Effect.gen(function* () { - const rows = yield* selectAllRows(); - const stuck = rows.filter((row) => row.last_run_status === "running"); + const recoverInterruptedRuns = Effect.gen(function* () { + const rows = + yield* sql`SELECT * FROM scheduled_tasks WHERE last_run_status = 'running'`; + const active = yield* Ref.get(activeRuns); + const stuck = rows.filter((row) => { + const id = decodeTaskId(row.task_id); + return Option.isNone(id) || !active.has(id.value); + }); if (stuck.length === 0) return; const now = yield* localNow; yield* Effect.forEach( @@ -998,6 +1047,7 @@ export const layer = Layer.effect( updated_at = ${iso(now)}, run_count = run_count + 1 WHERE task_id IS ${row.task_id} AND last_run_status = 'running' + AND created_at = ${row.created_at} AND last_run_at IS ${row.last_run_at} `; return; } @@ -1015,8 +1065,16 @@ export const layer = Layer.effect( updated_at = ${iso(now)}, run_count = run_count + 1 WHERE task_id IS ${row.task_id} AND last_run_status = 'running' + AND created_at = ${row.created_at} AND last_run_at IS ${row.last_run_at} `; - }), + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not reset interrupted schedule task run", { + taskId: row.task_id, + cause, + }), + ), + ), { concurrency: 1, discard: true }, ); yield* notifyChanged; @@ -1026,7 +1084,33 @@ export const layer = Layer.effect( ), ); - yield* scheduler.register("scheduled-tasks", runDueTasks()); + yield* recoverInterruptedRuns; + + const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () { + yield* recoverInterruptedRuns; + yield* replayWebhookDeliveries(); + const now = yield* localNow; + const tasks = yield* listDueTasks(now).pipe( + Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), + ); + const due = tasks.flatMap((task) => + task.nextRunAt === null ? [] : [{ task, dueAt: DateTime.makeUnsafe(task.nextRunAt) }], + ); + yield* Effect.forEach( + due, + ({ task, dueAt }) => + Effect.suspend(() => + isMissedFixedTimeRun(task.schedule, dueAt, now) + ? rescheduleMissedRun(task, now) + : runTask(task, "scheduled"), + ).pipe( + Effect.catch((cause) => + Effect.logWarning("Scheduled task run failed", { taskId: task.id, cause }), + ), + ), + { concurrency: 1, discard: true }, + ); + }); const list: ScheduledTaskService["Service"]["list"] = () => listRows().pipe( @@ -1307,6 +1391,7 @@ export const layer = Layer.effect( const recordDelivery = (input: { readonly id: string; readonly taskId: ScheduledTaskId; + readonly taskCreatedAt: string; readonly receivedAt: string; readonly request: WebhookTriggerRequest; readonly outcome: ScheduledTaskWebhookDeliveryOutcome; @@ -1349,9 +1434,19 @@ export const layer = Layer.effect( rendered_prompt = excluded.rendered_prompt, error = excluded.error `; + if (input.outcome === "accepted" && input.renderedPrompt !== null) { + yield* sql` + INSERT INTO scheduled_task_webhook_queue (delivery_id, task_id, task_created_at, prompt) + SELECT ${input.id}, ${input.taskId}, ${input.taskCreatedAt}, ${input.renderedPrompt} + WHERE EXISTS (SELECT 1 FROM scheduled_tasks + WHERE task_id = ${input.taskId} AND created_at = ${input.taskCreatedAt}) + ON CONFLICT (delivery_id) DO NOTHING + `; + } yield* sql` DELETE FROM scheduled_task_webhook_deliveries WHERE task_id = ${input.taskId} + AND delivery_id NOT IN (SELECT delivery_id FROM scheduled_task_webhook_queue) AND delivery_id NOT IN ( SELECT delivery_id FROM scheduled_task_webhook_deliveries WHERE task_id = ${input.taskId} @@ -1374,7 +1469,7 @@ export const layer = Layer.effect( UPDATE scheduled_task_webhook_deliveries SET outcome = 'dispatch_failed', error = ${message} WHERE delivery_id = ${deliveryId} - `.pipe(Effect.ignore); + `; /** * Sliding one-minute window counting every request with a valid token, @@ -1450,6 +1545,90 @@ export const layer = Layer.effect( } }); + // The full rendered prompt lives outside the trimmed delivery log. Replaying + // the stable command id is safe even when dispatch committed before shutdown. + const replayWebhookDeliveries = Effect.fn("ScheduledTaskService.replayWebhookDeliveries")( + function* () { + const pending = yield* sql<{ + delivery_id: string; + task_id: string; + task_created_at: string; + }>`SELECT delivery_id, task_id, task_created_at FROM scheduled_task_webhook_queue ORDER BY rowid`; + for (const delivery of pending) { + const reserved = yield* Ref.modify(dispatchingDeliveries, (active) => + active.has(delivery.delivery_id) + ? ([false, active] as const) + : ([true, new Set(active).add(delivery.delivery_id)] as const), + ); + if (!reserved) continue; + const permit = yield* webhookPermit(ScheduledTaskId.make(delivery.task_id)); + yield* Effect.gen(function* () { + const runOutcome = (outcome: "started" | "skipped" | "failed") => + Effect.all([ + Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), + Metrics.increment(Metrics.webhookRunsTotal, { outcome }), + ]); + const task = yield* findTask(ScheduledTaskId.make(delivery.task_id)); + if (task === null || task.createdAt !== delivery.task_created_at) { + yield* runOutcome("skipped"); + yield* markDeliveryFailed( + delivery.delivery_id, + task === null + ? "The task was deleted before this delivery ran." + : "The task was replaced before this delivery ran.", + ); + } else { + const completed = yield* runTask(task, "webhook", { + deliveryId: delivery.delivery_id, + }).pipe( + Effect.catchTags({ + WebhookDeliverySkipped: (skipped) => + markDeliveryFailed(delivery.delivery_id, skipped.reason).pipe( + Effect.as(undefined), + ), + }), + ); + const outcome = + completed === undefined + ? "skipped" + : completed.lastRunStatus === "failed" + ? "failed" + : "started"; + yield* runOutcome(outcome); + if (outcome === "failed") + yield* markDeliveryFailed(delivery.delivery_id, "The run failed to start."); + } + yield* sql`DELETE FROM scheduled_task_webhook_queue WHERE delivery_id = ${delivery.delivery_id}`; + }).pipe( + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.failCause(cause) + : Effect.logWarning("Webhook dispatch failed; will retry", { + taskId: delivery.task_id, + cause, + }), + ), + permit.withPermits(1), + Effect.withSpan("ScheduledTaskService.runWebhookDelivery", { + root: true, + attributes: { + "scheduled_task.id": delivery.task_id, + "scheduled_task.webhook.delivery_id": delivery.delivery_id, + }, + }), + Effect.ensuring( + Ref.update(dispatchingDeliveries, (active) => { + const next = new Set(active); + next.delete(delivery.delivery_id); + return next; + }), + ), + (effect) => forkParked(effect).pipe(Effect.provideService(Scope.Scope, serviceScope)), + ); + } + }, + ); + const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) => triggerWebhookUnobserved(request).pipe( Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)), @@ -1522,13 +1701,18 @@ export const layer = Layer.effect( : sql` DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${request.relayDeliveryId} - `.pipe(Effect.ignore, Effect.as(result)); - // From the claim until the run is forked nothing may interrupt: a - // request dropped in between (the relay hangs up after its timeout) - // would leave a claimed delivery that never runs, or a queue slot - // that is never released. Everything in here is local and quick. + `.pipe(Effect.as(result)); + // Commit the claim and durable dispatch together; a dropped request + // or failed acceptance write must still allow delivery or a relay retry. return yield* Effect.uninterruptible( Effect.gen(function* () { + const current = yield* findTask(task.id); + if ( + current === null || + current.createdAt !== task.createdAt || + current.webhook?.path !== task.webhook?.path + ) + return yield* notFound; if (request.relayDeliveryId !== undefined) { const claimed = yield* sql<{ relay_delivery_id: string }>` INSERT INTO scheduled_task_webhook_relay_deliveries @@ -1564,6 +1748,7 @@ export const layer = Layer.effect( recordDelivery({ id: deliveryId, taskId: task.id, + taskCreatedAt: task.createdAt, receivedAt: iso(receivedAt), request, outcome, @@ -1628,86 +1813,50 @@ export const layer = Layer.effect( error: "The filled-in prompt is too long.", }); yield* observe("prompt_too_long"); - return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const }; + return { + _tag: "accepted" as const, + deliveryId, + outcome: "prompt_too_long" as const, + }; } - // Bound the deliveries one task holds, so steady traffic to a stuck - // task cannot pile up parked fibers. A refused request is not logged, - // so it cannot push real deliveries out of the log. - const queueKey = `${task.id}\u0000${task.createdAt}`; - const queued = yield* Ref.modify(webhookQueued, (counts) => { - const count = counts.get(queueKey) ?? 0; - return count >= WEBHOOK_MAX_QUEUED_PER_TASK - ? ([false, counts] as const) - : ([true, new Map(counts).set(queueKey, count + 1)] as const); - }); - if (!queued) { - // The task is busy with WEBHOOK_MAX_QUEUED_PER_TASK deliveries already. + const queued = yield* sql<{ count: number }>` + SELECT COUNT(*) AS count FROM scheduled_task_webhook_queue + WHERE task_id = ${task.id} AND task_created_at = ${task.createdAt} + `; + if ((queued[0]?.count ?? 0) >= WEBHOOK_MAX_QUEUED_PER_TASK) { yield* observe("queue_full"); return yield* releaseClaim({ _tag: "rate_limited" as const, outcome: "queue_full" as const, }); } - // Entries leave the map when their count reaches zero, so a deleted - // task's key does not linger once its last delivery finishes. - const release = Ref.update(webhookQueued, (counts) => { - const next = new Map(counts); - const count = (next.get(queueKey) ?? 1) - 1; - if (count <= 0) next.delete(queueKey); - else next.set(queueKey, count); - return next; - }); yield* log("accepted", { signatureVerified: signature !== null, missing: rendered.missing, renderedPrompt: rendered.prompt, - }).pipe(Effect.onError(() => release)); + }); yield* observe("accepted"); - const permit = yield* webhookPermit(task.id); - const runOutcome = (outcome: "started" | "skipped" | "failed") => - Effect.all([ - Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), - Metrics.increment(Metrics.webhookRunsTotal, { outcome }), - ]); - yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe( - Effect.flatMap((completed) => - completed.lastRunStatus === "failed" - ? runOutcome("failed").pipe( - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - ) - : runOutcome("started"), - ), - Effect.catchTags({ - WebhookDeliverySkipped: (skipped) => - runOutcome("skipped").pipe( - Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)), - ), - }), - // The log is readable over RPC, so it gets a fixed reason; the - // cause, which can carry request data, stays in the server log. - Effect.catchCause((cause) => - Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe( - Effect.andThen(runOutcome("failed")), - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - ), - ), - permit.withPermits(1), - // Its own trace: the request that triggered it has already been answered. - Effect.withSpan("ScheduledTaskService.runWebhookDelivery", { - root: true, - attributes: { - "scheduled_task.id": task.id, - "scheduled_task.webhook.delivery_id": deliveryId, - }, - }), - Effect.ensuring(release), - Effect.forkIn(serviceScope), - ); return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const }; - }), + }).pipe( + sql.withTransaction, + Effect.mapError((cause) => + taskError("Could not record webhook delivery.", { taskId: task.id, cause }), + ), + Effect.tap((result) => + result._tag === "accepted" && result.outcome === "accepted" + ? replayWebhookDeliveries().pipe( + Effect.catch((cause) => + Effect.logWarning("Webhook replay will retry on the next tick", { cause }), + ), + ) + : Effect.void, + ), + ), ); }); + yield* scheduler.register("scheduled-tasks", runDueTasks()); + return ScheduledTaskService.of({ list, subscribeList, diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index a5685d2ab43b6..a86121f351544 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -5,6 +5,7 @@ import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it } from "@effect/vitest"; import { ScheduledTaskUpsertInput, SecretRequestError } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Crypto from "effect/Crypto"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -14,6 +15,7 @@ import * as Queue from "effect/Queue"; import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import { ServerActivation } from "../serverActivation.ts"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; @@ -977,3 +979,275 @@ it.effect("counts each handled request by what happened to it", () => }), ), ); + +it.effect.each([null, "thread:bound"])( + "replays an accepted webhook and its untruncated body after shutdown (binding %s)", + (threadId) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const deliveries = yield* Queue.unbounded<{ commandId: string; text: string }>(); + const dispatchStarted = yield* Deferred.make(); + const accepted = yield* Deferred.make(); + const bodyText = JSON.stringify({ text: "x".repeat(70_000) }); + const dependencies = (hold: boolean) => + Layer.mergeAll( + NodePlatformCrypto.layer, + Scheduler.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: (input) => + Queue.offer(deliveries, { + commandId: input.commandId, + text: input.initialMessage!.text, + }).pipe( + Effect.andThen(Deferred.succeed(dispatchStarted, undefined)), + Effect.andThen(hold ? Effect.never : Effect.void), + Effect.as({} as never), + ), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Queue.offer(deliveries, { commandId: input.commandId, text: input.text }).pipe( + Effect.andThen(Deferred.succeed(dispatchStarted, undefined)), + Effect.andThen(hold ? Effect.never : Effect.void), + Effect.as({} as never), + ), + }), + ); + let request: ScheduledTaskService.WebhookTriggerRequest; + const first = yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert( + yield* webhookTaskInput({ threadId, prompt: "Handle {{body}}" }), + ); + request = requestFor(task, { + relayDeliveryId: "durable", + bodyText, + body: new TextEncoder().encode(bodyText), + }); + assert.equal((yield* service.triggerWebhook(request))._tag, "accepted"); + yield* Deferred.succeed(accepted, undefined); + return yield* Effect.never; + }).pipe( + Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies(true)))), + Effect.forkScoped, + ); + yield* Deferred.await(accepted); + yield* Deferred.await(dispatchStarted); + const initial = yield* Queue.take(deliveries); + yield* Fiber.interrupt(first); + const pending = yield* sql<{ + prompt: string; + }>`SELECT prompt FROM scheduled_task_webhook_queue`; + assert.equal(pending[0]?.prompt, "Handle " + bodyText); + const replayed = yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const duplicate = yield* service.triggerWebhook(request!); + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate"); + const ready = yield* Deferred.make(); + const completed = yield* service.subscribeList().pipe( + Stream.tap(() => Deferred.succeed(ready, undefined)), + Stream.filter((list) => list.tasks[0]?.lastRunStatus === "succeeded"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Deferred.await(ready); + yield* TestClock.adjust("5 seconds"); + const delivered = yield* Queue.take(deliveries); + yield* Fiber.join(completed); + assert.equal((yield* service.list()).tasks[0]?.lastRunStatus, "succeeded"); + yield* TestClock.adjust("5 seconds"); + assert.equal(yield* Queue.size(deliveries), 0); + return delivered; + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies(false))))); + assert.deepEqual(replayed, initial); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_queue`).length, 0); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("keeps acceptance durable when shutdown precedes the dispatch fork", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const launches = yield* Queue.unbounded(); + const dependencies = Layer.mergeAll( + NodePlatformCrypto.layer, + Scheduler.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: (input) => Queue.offer(launches, input).pipe(Effect.as({} as never)), + }), + ); + let request: ScheduledTaskService.WebhookTriggerRequest; + // The request has committed, but no background dispatch may run until activation. + const activation = yield* Deferred.make(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(yield* webhookTaskInput()); + request = requestFor(task, { relayDeliveryId: "before-dispatch" }); + assert.equal((yield* service.triggerWebhook(request))._tag, "accepted"); + }).pipe( + Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies))), + Effect.provideService(ServerActivation, Deferred.await(activation)), + ); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_queue`).length, 1); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const duplicate = yield* service.triggerWebhook(request!); + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate"); + yield* TestClock.adjust("5 seconds"); + const launch = yield* Queue.take(launches); + assert.equal( + launch.initialMessage?.text, + "Review this PR: https://github.com/org/repo/pull/45", + ); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("rolls back relay deduplication when the acceptance write fails", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const dependencies = Layer.mergeAll( + NodePlatformCrypto.layer, + Scheduler.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: () => Effect.succeed({} as never), + }), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(yield* webhookTaskInput()); + yield* sql`CREATE TRIGGER fail_accept BEFORE INSERT ON scheduled_task_webhook_deliveries + BEGIN SELECT RAISE(FAIL, 'disk write failed'); END`; + const request = requestFor(task, { relayDeliveryId: "write-failure" }); + yield* service.triggerWebhook(request).pipe(Effect.flip); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0); + yield* sql`DROP TRIGGER fail_accept`; + const retried = yield* service.triggerWebhook(request); + assert.equal(retried._tag === "accepted" && retried.outcome, "accepted"); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("rolls back a relay claim when a full queue cannot release it", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const gate = yield* Deferred.make(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + for (let index = 0; index < 20; index += 1) { + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted"); + } + yield* Queue.take(launches); + yield* sql`CREATE TRIGGER fail_claim_release BEFORE DELETE ON scheduled_task_webhook_relay_deliveries + BEGIN SELECT RAISE(FAIL, 'disk write failed'); END`; + const request = requestFor(task, { relayDeliveryId: "full-queue-retry" }); + yield* service.triggerWebhook(request).pipe(Effect.flip); + assert.equal( + (yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, + 0, + ); + yield* sql`DROP TRIGGER fail_claim_release`; + const retried = yield* service.triggerWebhook(request); + assert.equal(retried._tag, "rate_limited"); + assert.equal( + (yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, + 0, + ); + yield* Deferred.succeed(gate, undefined); + }), + { gate }, + ); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("rejects a webhook whose task was replaced before acceptance commits", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const allocating = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + let holdAllocation = false; + const crypto = Layer.effect( + Crypto.Crypto, + Effect.gen(function* () { + const underlying = yield* Crypto.Crypto; + return { + ...underlying, + randomUUIDv4: Effect.suspend(() => + holdAllocation + ? Deferred.succeed(allocating, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + Effect.andThen(underlying.randomUUIDv4), + ) + : underlying.randomUUIDv4, + ), + }; + }), + ).pipe(Layer.provide(NodePlatformCrypto.layer)); + const dependencies = Layer.mergeAll( + crypto, + Scheduler.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: () => Effect.succeed({} as never), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({}), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(yield* webhookTaskInput()); + holdAllocation = true; + const receiving = yield* service.triggerWebhook(requestFor(task)).pipe(Effect.forkScoped); + yield* Deferred.await(allocating); + holdAllocation = false; + yield* service.delete({ id: task.id }); + const replacement = yield* service.upsert(yield* webhookTaskInput()); + assert.equal(replacement.task.createdAt, task.createdAt); + yield* Deferred.succeed(proceed, undefined); + assert.equal((yield* Fiber.join(receiving))._tag, "not_found"); + assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0); + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_queue`).length, 0); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("does not dispatch a deleted delivery into a task recreated at the same instant", () => + Effect.gen(function* () { + const gate = yield* Deferred.make(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + yield* service.triggerWebhook(requestFor(task)); + yield* Queue.take(launches); + yield* service.triggerWebhook(requestFor(task)); + yield* service.delete({ id: task.id }); + const replacement = yield* service.upsert( + yield* webhookTaskInput({ prompt: "Replacement task" }), + ); + assert.equal(replacement.task.createdAt, task.createdAt); + const subscribed = yield* Deferred.make(); + const oldCompletion = yield* service.subscribeList().pipe( + Stream.tap(() => Deferred.succeed(subscribed, undefined)), + Stream.drop(1), + Stream.runHead, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + yield* Deferred.succeed(gate, undefined); + yield* Fiber.join(oldCompletion); + yield* TestClock.adjust("5 seconds"); + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]?.lastRunStatus, "never"); + assert.equal((yield* service.list()).tasks[0]?.runCount, 0); + }).pipe(Effect.scoped), + { gate }, + ); + }), +); diff --git a/apps/server/src/scheduling/Scheduler.integration.test.ts b/apps/server/src/scheduling/Scheduler.integration.test.ts index 1409697a79886..e360543be6ec7 100644 --- a/apps/server/src/scheduling/Scheduler.integration.test.ts +++ b/apps/server/src/scheduling/Scheduler.integration.test.ts @@ -11,6 +11,15 @@ import { type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Option from "effect/Option"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as CodexInstallation from "../provider/CodexInstallation.ts"; +import * as AntigravityInstallation from "../provider/AntigravityInstallation.ts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import * as UpdateWindow from "../updates/UpdateWindow.ts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; @@ -154,3 +163,75 @@ it.effect.each(["on time", "after restart"])( }).pipe(Effect.provide(layerWorkers)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect.each([ + ["exact five-minute horizon", 15 * 60_000, 5 * 60_000, true, "succeeded", false], + ["one millisecond beyond horizon", 15 * 60_000, 5 * 60_000 + 1, true, "succeeded", true], + ["interval just under fifteen minutes", 15 * 60_000 - 1, 60_000, true, "succeeded", true], + ["interval exactly fifteen minutes", 15 * 60_000, 60_000, true, "succeeded", false], + ["disabled due task", 15 * 60_000, 60_000, false, "succeeded", true], + ["disabled running task", 60_000, 60_000, false, "running", false], + ["frequent running task", 60_000, 60_000, true, "running", false], + ["accepted webhook waiting for dispatch", 0, 0, true, "never", false], + ["paused webhook waiting for dispatch", 0, 0, false, "never", true], +] as const)("update window handles %s", ([, everyMs, dueIn, enabled, status, open]) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const sql = yield* SqlClient.SqlClient; + const timestamp = DateTime.formatIso(now); + yield* sql`INSERT INTO scheduled_tasks ${sql.insert({ + task_id: "task:window", + title: "Work", + prompt: "Run", + enabled: enabled ? 1 : 0, + schedule_json: JSON.stringify( + everyMs === 0 ? { type: "webhook", signature: null } : { type: "interval", everyMs }, + ), + project_id: "project:test", + thread_id: null, + workspace_strategy_json: '{"type":"root"}', + model_selection_json: '{"instanceId":"codex","model":"gpt-5"}', + runtime_mode: "full-access", + interaction_mode: "default", + created_by: "user", + creation_source: "web", + created_at: timestamp, + updated_at: timestamp, + next_run_at: + everyMs === 0 ? null : DateTime.formatIso(DateTime.add(now, { milliseconds: dueIn })), + last_run_at: timestamp, + last_run_status: status, + last_run_error: null, + run_count: 1, + })}`; + if (everyMs === 0) + yield* sql`INSERT INTO scheduled_task_webhook_queue (delivery_id, task_id, task_created_at, prompt) + VALUES ('pending', 'task:window', ${timestamp}, 'Handle body')`; + const dependencies = Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([]), + getLimitRecoveryCandidates: () => Effect.succeed([]), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + hasPendingBackgroundWork: Effect.succeed(false), + }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.succeed(Duration.minutes(15)), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS), + }), + Layer.mock(TerminalManager.TerminalManager)({}), + Layer.mock(CodexInstallation.CodexInstallation)({ managedDirectory: "/tmp/codex" }), + Layer.mock(AntigravityInstallation.AntigravityInstallation)({ + managedDirectory: "/tmp/antigravity", + }), + ); + const result = yield* Effect.gen(function* () { + const window = yield* UpdateWindow.UpdateWindow; + return yield* window.runIfOpen(Effect.succeed("installed")); + }).pipe(Effect.provide(UpdateWindow.layer.pipe(Layer.provide(dependencies)))); + assert.equal(Option.isSome(result), open); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); From c54651b5aeba2420eccc38894907c1f7617b7f7b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:06:20 +0000 Subject: [PATCH 082/149] fix(server): restart recovery continues every thread on a shared session and notes child cancellations on the parent --- .../ProviderRuntimeRecoveryService.test.ts | 8 +- .../ProviderRuntimeRecoveryService.ts | 37 +++++++- .../RestartContinuation.test.ts | 84 +++++++++++++++++++ 3 files changed, 124 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index ae6e7bc962541..444ad0ce929e5 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -1346,7 +1346,7 @@ it.effect.each([true, false])( }, ); -it.effect("leaves delegated tasks to their own child threads after process loss", () => { +it.effect("notes native child cancellation on its parent", () => { const threadId = ThreadId.make("thread_recovery_delegation"); const settledRunId = RunId.make("run_recovery_delegation_settled"); const runningRunId = RunId.make("run_recovery_delegation_running"); @@ -1431,7 +1431,11 @@ it.effect("leaves delegated tasks to their own child threads after process loss" turnItems: [ subagentItem(settledTaskId, settledRunId, "app_owned"), subagentItem(runningTaskId, runningRunId, "app_owned"), - subagentItem(nativeSubagentId, settledRunId, "provider_native"), + { + ...subagentItem(nativeSubagentId, settledRunId, "provider_native"), + // Codex points this item at the child, while its run belongs to the parent. + providerThreadId: ProviderThreadId.make("provider_thread_recovery_native_child"), + }, ], } as unknown as OrchestrationV2ThreadProjection; const layer = ProviderRuntimeRecovery.layer.pipe( diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 0a8389878c1b5..f88105d17c530 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -191,6 +191,10 @@ export const make = Effect.gen(function* () { projection: ProjectionStore.ProjectionRuntimeRecoveryState, trigger: "startup" | "shutdown", continueAfterRestart: boolean, + initialSessions: ReadonlyMap< + string, + OrchestrationV2ThreadProjection["providerSessions"][number] + >, ) { const now = yield* DateTime.now; const runs = [] as Array; @@ -276,8 +280,17 @@ export const make = Effect.gen(function* () { if (!isBackgroundCapableTurnItemType(item.type)) return; const work = cancelledTurnItemWork(item); if (work === undefined) return; + // A native subagent item can point at its child thread. The parent + // that spawned it must receive the note about its missing result. + const parentThreadId = + item.type === "subagent" + ? (projection.providerTurns.find((turn) => turn.id === item.providerTurnId) + ?.providerThreadId ?? + projection.runs.find((run) => run.id === item.runId)?.providerThreadId) + : undefined; recordCancelledBackgroundWork( - item.providerThreadId ?? + parentThreadId ?? + item.providerThreadId ?? projection.runs.find((run) => run.id === item.runId)?.providerThreadId, work, ); @@ -646,7 +659,16 @@ export const make = Effect.gen(function* () { } const continuationRun = continueAfterRestart && trigger === "startup" - ? (handoffContinuation ?? restartContinuationRun(projection, now)) + ? (handoffContinuation ?? + restartContinuationRun( + { + ...projection, + providerSessions: projection.providerSessions.map( + (session) => initialSessions.get(session.id) ?? session, + ), + }, + now, + )) : undefined; const effects: Array = continuationRun ? [ @@ -744,6 +766,12 @@ export const make = Effect.gen(function* () { let stoppedSessions = 0; let closedRequests = 0; let retiredEffects = 0; + // Recovering one thread stops its shared session. Later threads must + // judge continuation against that session's state before recovery. + const initialSessions = new Map< + string, + OrchestrationV2ThreadProjection["providerSessions"][number] + >(); for (const threadId of threadIds) { const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe( Effect.mapError( @@ -755,11 +783,14 @@ export const make = Effect.gen(function* () { }), ), ); + for (const session of projection.providerSessions) { + if (!initialSessions.has(session.id)) initialSessions.set(session.id, session); + } const enabled = continueAfterRestart !== null && resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings .continueThreadsAfterServerUpdate; - const result = yield* reconcileProjection(projection, trigger, enabled); + const result = yield* reconcileProjection(projection, trigger, enabled, initialSessions); terminalizedRuns += result.terminalizedRuns; stoppedSessions += result.stoppedSessions; closedRequests += result.closedRequests; diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index d97e75b2928e6..70f7f9924ca18 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -1340,3 +1340,87 @@ it.effect("continues a cut goal turn after an earlier phase of the attempt compl ); }), ); + +it.effect.each(["settled", "running", "stopped"] as const)( + "preserves startup session liveness after a shared-session %s thread is recovered", + (firstStatus) => + Effect.gen(function* () { + const secondThreadId = ThreadId.make("thread:restart-second"); + const secondRunId = RunId.make("run:restart-second"); + const base = makeProjection(); + let session = { + ...base.providerSessions[0]!, + status: firstStatus === "stopped" ? "stopped" : "ready", + } as OrchestrationV2ThreadProjection["providerSessions"][number]; + const writes: Parameters[0][] = []; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId, secondThreadId]), + getRuntimeRecoveryProjection: (id) => { + const second = id === secondThreadId; + const settled = !second && firstStatus === "settled"; + return Effect.succeed({ + ...base, + thread: { ...base.thread, id }, + runs: [ + { + ...base.runs[0]!, + id: second ? secondRunId : runId, + status: settled ? "completed" : "running", + }, + ], + providerThreads: [ + { + ...base.providerThreads[0]!, + appThreadId: id, + status: settled ? "idle" : "active", + }, + ], + providerSessions: [session], + providerTurns: [ + { ...base.providerTurns[0]!, status: settled ? "completed" : "running" }, + ], + } as OrchestrationV2ThreadProjection); + }, + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + writes.push(input); + for (const event of input.events) { + if (event.type === "provider-session.updated") session = event.payload; + } + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + yield* recovery.reconcile("startup"); + const sources = writes + .flatMap((write) => write.effects ?? []) + .flatMap((effect) => + effect.request.type === "provider-runtime.continue" ? [effect.request.sourceRunId] : [], + ); + assert.deepEqual( + sources, + firstStatus === "stopped" + ? [] + : firstStatus === "running" + ? [runId, secondRunId] + : [secondRunId], + ); + assert.lengthOf( + writes + .flatMap((write) => write.events) + .filter((event) => event.type === "provider-session.updated"), + firstStatus === "stopped" ? 0 : 1, + ); + }), +); From 6d9ac3de5bc3fe4aef9f4573637df74a12084090 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:06:49 +0000 Subject: [PATCH 083/149] fix(server): steering and delegated launches survive restarts without duplicates --- apps/server/scripts/acp-mock-agent.ts | 2 +- .../src/mcp/toolkits/project/handlers.test.ts | 44 ++++- .../src/mcp/toolkits/project/handlers.ts | 15 +- apps/server/src/mcp/toolkits/project/tools.ts | 3 +- .../DelegatedCompletionDelivery.test.ts | 106 +++++++++++ .../src/orchestration-v2/Orchestrator.ts | 12 +- .../ProjectionRecovery.test.ts | 136 ++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 68 ++++++- .../ProviderRuntimeRecoveryService.test.ts | 35 +++- .../ProviderRuntimeRecoveryService.ts | 66 ++++++- .../RestartBackgroundNote.test.ts | 18 ++ .../orchestration-v2/RestartBackgroundNote.ts | 37 +++- .../RestartContinuation.test.ts | 171 +++++++++++++++++- .../orchestration-v2/RestartContinuation.ts | 67 +++++-- packages/contracts/src/orchestrationV2.ts | 2 + .../src/server/orchestrationInstructions.ts | 2 +- 16 files changed, 743 insertions(+), 41 deletions(-) diff --git a/apps/server/scripts/acp-mock-agent.ts b/apps/server/scripts/acp-mock-agent.ts index 5548bebac84c0..05daa6ea474c9 100644 --- a/apps/server/scripts/acp-mock-agent.ts +++ b/apps/server/scripts/acp-mock-agent.ts @@ -121,7 +121,7 @@ const permissionRequestCount = Math.max( 1, Number(process.env.T3_ACP_PERMISSION_REQUEST_COUNT ?? "1") || 1, ); -const sessionId = "mock-session-1"; +const sessionId = process.env.T3_ACP_SESSION_ID ?? "mock-session-1"; let currentModeId = antigravityProfile ? "default" : "ask"; let currentModelId = antigravityProfile ? "gemini-test-low" : "default"; diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index b74df9dae1f99..68fd41a9420b6 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -28,7 +28,7 @@ import * as McpToolAccess from "../../McpToolAccess.ts"; import * as ProjectHandlers from "./handlers.ts"; import { ProjectToolkit } from "./tools.ts"; -it.effect("attributes a launched thread's first message to the calling thread", () => +it.effect("attributes launch messages and keeps restart retry identities", () => Effect.gen(function* () { const sourceThreadId = ThreadId.make("source-thread"); const projectId = ProjectId.make("project"); @@ -46,6 +46,7 @@ it.effect("attributes a launched thread's first message to the calling thread", deletedAt: null, } as OrchestrationV2ThreadShell; let launchedSender: ThreadId | undefined; + const launches: Array = []; const layerDependencies = Layer.mergeAll( NodeCrypto.layer, Layer.succeed(McpInvocationContext.McpInvocationContext, { @@ -65,6 +66,7 @@ it.effect("attributes a launched thread's first message to the calling thread", }), Layer.mock(ThreadLaunch.ThreadLaunchService)({ launch: (input) => { + launches.push(input); launchedSender = input.initialMessage?.senderThreadId; return Effect.succeed({ threadId: input.threadId, @@ -96,6 +98,46 @@ it.effect("attributes a launched thread's first message to the calling thread", .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(layerDependencies)); expect(result.at(-1)?.result).toMatchObject({ projectId, modelSelection }); expect(launchedSender).toBe(sourceThreadId); + const invoke = ( + clientRequestId: string | undefined, + providerSessionId: string, + requestNamespace = "session", + ) => + Effect.gen(function* () { + const dependencies = Layer.mergeAll( + layerDependencies, + Layer.succeed(McpInvocationContext.McpInvocationContext, { + environmentId: EnvironmentId.make("environment"), + requestNamespace, + thread: { threadId: sourceThreadId, providerInstanceId, providerSessionId }, + client: undefined, + issuedAt: 0, + capabilities: new Set(["orchestration" as const]), + }), + ); + const retryToolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe( + Layer.provide(dependencies), + ), + ), + ); + return yield* retryToolkit + .handle("t3_thread_launch", { + title: "Audit", + message: "Review the change", + ...(clientRequestId ? { clientRequestId } : {}), + }) + .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies)); + }); + yield* invoke("retry-launch", "before-restart"); + yield* invoke("retry-launch", "after-restart"); + expect(launches[1]?.commandId).toBe(launches[2]?.commandId); + expect(launches[1]?.threadId).toBe(launches[2]?.threadId); + expect(launches[1]?.initialMessage?.messageId).toBe(launches[2]?.initialMessage?.messageId); + yield* invoke("retry-launch", "other-parent", "other-thread"); + yield* invoke(undefined, "after-restart"); + expect(new Set(launches.map((input) => input.threadId)).size).toBe(4); }), ); diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index edb732e400bc5..ef3e70b11dfe8 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -1,4 +1,10 @@ -import { MessageId, ThreadId, OrchestratorMcpFailure, ProjectId } from "@t3tools/contracts"; +import { + CommandId, + MessageId, + ThreadId, + OrchestratorMcpFailure, + ProjectId, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; @@ -59,7 +65,12 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { Effect.gen(function* () { const context = yield* readCaller(); const { caller } = context; - const commandId = yield* newCommandId(); + const commandId = + input.clientRequestId === undefined + ? yield* newCommandId() + : CommandId.make( + `command:mcp:${encodeURIComponent(context.scope.requestNamespace)}:thread-launch:${encodeURIComponent(input.clientRequestId)}`, + ); const threadId = ThreadId.make(commandId); const messageId = MessageId.make(commandId); const attachments = input.attachments ?? []; diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index ee489d4e104c0..ed1c7f2888d1f 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -103,8 +103,9 @@ const ProjectCloneTool = Tool.make("t3_project_clone", { const ThreadLaunchTool = Tool.make("t3_thread_launch", { ...shared, description: - 'Create an ordinary TOP-LEVEL thread with an explicit workspace binding before its agent starts. Use this when the user requests independent work, a new thread, or a PR stack in its own worktree; use delegate_task for child subagents. Set workspaceStrategy to {type:"worktree",baseRef:"parent-branch",branch:"new-branch",startFromOrigin:false} for a new worktree based on local commits, or {type:"existing_worktree",worktreePath:"/absolute/path",branch:"existing-branch"} to use an existing checkout. For upstream commits, set startFromOrigin:true. Omitted workspaceStrategy means the project root, NOT the caller\'s worktree. Omit projectId/modelSelection/modes to inherit those settings from the calling thread; a caller outside a T3 thread must pass projectId and gets the project\'s default model. Set scratch:true instead of projectId for a thread without a project: it runs in a fresh folder of its own, outside any repository. Put the task in message. Do not ask the agent to create its own worktree via shell: that does not update the thread binding. Each call creates a new launch with no retry key; retain threadId and use t3_thread_read/t3_thread_wait to follow preparation. To link a thread for the user, write `[title](t3-thread://v1/)` with the threadId exactly as returned, not URL-encoded; T3 Code shows the thread\'s current title. After errors or lost responses, inspect t3_thread_list before retrying. Attachments must be pending uploads. The new thread may not run with broader runtime or interaction modes than the caller: the calling T3 thread\'s own modes, or the permission mode an outside agent was approved with.', + 'Create an ordinary TOP-LEVEL thread with an explicit workspace binding before its agent starts. Use this when the user requests independent work, a new thread, or a PR stack in its own worktree; use delegate_task for child subagents. Set workspaceStrategy to {type:"worktree",baseRef:"parent-branch",branch:"new-branch",startFromOrigin:false} for a new worktree based on local commits, or {type:"existing_worktree",worktreePath:"/absolute/path",branch:"existing-branch"} to use an existing checkout. For upstream commits, set startFromOrigin:true. Omitted workspaceStrategy means the project root, NOT the caller\'s worktree. Omit projectId/modelSelection/modes to inherit those settings from the calling thread; a caller outside a T3 thread must pass projectId and gets the project\'s default model. Set scratch:true instead of projectId for a thread without a project: it runs in a fresh folder of its own, outside any repository. Put the task in message. Do not ask the agent to create its own worktree via shell: that does not update the thread binding. Pass a stable clientRequestId to retry the same launch after errors or restarts; retain threadId and use t3_thread_read/t3_thread_wait to follow preparation. To link a thread for the user, write `[title](t3-thread://v1/)` with the threadId exactly as returned, not URL-encoded; T3 Code shows the thread\'s current title. After errors or lost responses, inspect t3_thread_list before retrying. Attachments must be pending uploads. The new thread may not run with broader runtime or interaction modes than the caller: the calling T3 thread\'s own modes, or the permission mode an outside agent was approved with.', parameters: Schema.Struct({ + clientRequestId: Schema.optional(TrimmedNonEmptyString.check(Schema.isMaxLength(256))), projectId: Schema.optional(ProjectId), scratch: Schema.optional( Schema.Boolean.annotate({ diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index a037354b8905d..554225fcaaa4f 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -1843,4 +1843,110 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { ); }), ); + + it.effect.each(["mixed-handoff", "settled-steering"] as const)( + "dispatches a %s restart continuation through the real orchestrator once", + (scenario) => + Effect.gen(function* () { + const projects = yield* ProjectService.ProjectService; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:restart-dispatch:${scenario}`); + const projectId = ProjectId.make(`project:restart-dispatch:${scenario}`); + yield* projects.create({ + commandId: CommandId.make(`command:${scenario}:project`), + projectId, + title: scenario, + workspaceRoot: `/workspace/${scenario}`, + }); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command:${scenario}:thread`), + threadId, + projectId, + title: scenario, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: `/workspace/${scenario}`, + }); + const firstMessageId = MessageId.make(`message:${scenario}:first`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`command:${scenario}:first`), + threadId, + messageId: firstMessageId, + text: "Pending steering input.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + if (scenario === "mixed-handoff") { + for (const [messageId, text] of [ + [ + MessageId.make("message:mcp:test:worktree-continuation:mixed"), + "Handoff continuation.", + ], + [MessageId.make("message:mixed:held"), "Held user input."], + ]) + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`command:${messageId}`), + threadId, + messageId: MessageId.make(messageId!), + text: text!, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + } + const before = yield* orchestrator.getThreadProjection(threadId); + const source = before.runs[scenario === "mixed-handoff" ? 1 : 0]!; + yield* eventSink.write({ + commandId: CommandId.make(`command:${scenario}:cut`), + events: before.runs.map((run) => ({ + id: EventId.make(`event:${scenario}:${run.ordinal}`), + type: "run.updated" as const, + threadId, + runId: run.id, + occurredAt: now, + payload: + scenario === "settled-steering" + ? { + ...run, + status: "completed" as const, + completedAt: now, + restartPendingMessageIds: [firstMessageId], + } + : run.ordinal === 1 + ? { ...run, status: "cancelled" as const, completedAt: now } + : run.ordinal === 3 + ? { ...run, queueHeld: true } + : run, + })), + }); + const continuation = continueRestartedRun({ threadId, sourceRunId: source.id }).pipe( + Effect.provide(ServerSettings.layerTest()), + ); + yield* continuation; + yield* continuation; + const after = yield* orchestrator.getThreadProjection(threadId); + const resumed = after.runs.filter((run) => run.restartContinuationOfRunId === source.id); + assert.lengthOf(resumed, 1); + assert.equal( + after.messages.find((message) => message.id === resumed[0]!.userMessageId)?.text, + scenario === "mixed-handoff" ? "Handoff continuation." : "Pending steering input.", + ); + if (scenario === "mixed-handoff") { + assert.equal(after.runs.find((run) => run.id === source.id)?.status, "cancelled"); + assert.isTrue(after.runs.find((run) => run.ordinal === 3)?.queueHeld); + } + }), + ); }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index cf88d9b3e1a17..fb8f59c79ce21 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -4571,17 +4571,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio source.queueHeld !== true && isWorktreeContinuationRun(source) && projection.thread.worktreePath != null && - projection.runs.filter((run) => run.status === "queued").length === 1 && + projection.runs.filter( + (run) => + run.status === "queued" && run.queueHeld !== true && isWorktreeContinuationRun(run), + ).length === 1 && !projection.runs.some(isBlockingRun); if ( !source || - (source.status !== "cancelled" && !worktreeContinuation) || + (source.status !== "cancelled" && + !worktreeContinuation && + (source.restartPendingMessageIds?.length ?? 0) === 0) || (isWorktreeContinuationRun(source) && source.status === "cancelled" && !projection.providerTurns.some( (turn) => turn.runAttemptId === source.activeAttemptId, )) || - isRestartNoteSource(source, projection.providerTurns) || + (isRestartNoteSource(source, projection.providerTurns) && + (source.restartPendingMessageIds?.length ?? 0) === 0) || restartContinuationBlocked( { ...projection, turnItems: secretRequests.turnItems }, source, diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index 158d82740cafb..e2814be016724 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -794,3 +794,139 @@ it.effect("marks fork descendants unreadable when their source is missing or cor ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("retains a native subagent's owning run when no turn item arrived", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = yield* createThread("native-before-item"); + const runId = yield* createRun(threadId, "completed"); + yield* createRun(threadId, "completed", { ordinal: 2 }); + yield* projections.apply({ + id: EventId.make("event:native-before-item"), + type: "subagent.updated", + threadId, + occurredAt: now, + payload: { + id: NodeId.make("node:native-before-item"), + threadId, + runId, + parentNodeId: NodeId.make("node:native-parent"), + origin: "provider_native", + createdBy: "agent", + driver, + providerInstanceId, + providerThreadId: null, + childThreadId: null, + nativeTaskRef: null, + prompt: "unfinished task", + title: null, + model: null, + status: "running", + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }); + assert.deepEqual( + (yield* projections.getRuntimeRecoveryProjection(threadId)).runs.map((run) => run.id), + [runId, RunId.make(`run:${threadId}:2`)], + ); + }).pipe(Effect.provide(layerTest)), +); + +it.effect( + "snapshots unsettled native steering and excludes delivered input and mailbox deliveries", + () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const threadId = yield* createThread("steering-recovery"); + const runId = yield* createRun(threadId, "running"); + const ids: Array = []; + for (const [index, mailbox] of [false, false, true, false].entries()) { + const messageId = MessageId.make(`message:steering:${index}`); + ids.push(messageId); + yield* projections.apply({ + id: EventId.make(`event:${messageId}`), + type: "message.updated", + threadId, + runId, + occurredAt: now, + payload: { + id: messageId, + threadId, + runId, + nodeId: null, + role: "user", + text: `steer ${index}`, + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + ...(mailbox + ? { delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [] } } + : {}), + }, + }); + yield* outbox.enqueue([ + { + id: `effect:steering:${index}`, + commandId: CommandId.make(`command:steering:${index}`), + threadId, + request: { + type: "provider-turn.steer", + messageId, + providerSessionId: ProviderSessionId.make("session:steering"), + providerThreadId: ProviderThreadId.make("provider-thread:steering"), + providerTurnId: ProviderTurnId.make("provider-turn:steering"), + }, + }, + ]); + } + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'succeeded' WHERE effect_id = 'effect:steering:1'`; + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'failed' WHERE effect_id = 'effect:steering:3'`; + const recovered = yield* projections.getRuntimeRecoveryProjection(threadId); + assert.deepEqual(recovered.pendingSteering, [{ runId, messageId: ids[0]! }]); + let committed: Parameters[0] | undefined; + yield* ProviderRuntimeRecovery.make.pipe( + Effect.flatMap((recovery) => recovery.reconcile("startup")), + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest(), + IdAllocator.layer, + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committed = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + ), + ), + ); + const event = committed?.events.find((event) => event.type === "run.updated"); + assert.deepEqual( + event?.type === "run.updated" ? event.payload.restartPendingMessageIds : [], + [ids[0]!], + ); + const stale = (yield* projections.getThreadProjection(threadId)).runs[0]!; + if (event !== undefined) yield* projections.apply(event); + yield* projections.apply({ + id: EventId.make("event:steering-stale"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { ...stale, status: "cancelled" }, + }); + assert.deepEqual( + (yield* projections.getThreadProjection(threadId)).runs[0]?.restartPendingMessageIds, + [ids[0]!], + ); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 61b3744eb5114..5540d632a1480 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -278,7 +278,12 @@ export type ProjectionRuntimeRecoveryState = Pick< | "runtimeRequests" | "messages" | "turnItems" ->; +> & { + readonly pendingSteering?: ReadonlyArray<{ + readonly runId: RunId; + readonly messageId: MessageId; + }>; +}; export type ProjectionPendingUserInputs = Pick< OrchestrationV2ThreadProjection, @@ -679,6 +684,10 @@ function preserveRunRecordedFields( ...(next.delegatedCompletion === undefined && current.delegatedCompletion !== undefined ? { delegatedCompletion: current.delegatedCompletion } : {}), + ...(next.restartPendingMessageIds === undefined && + current.restartPendingMessageIds !== undefined + ? { restartPendingMessageIds: current.restartPendingMessageIds } + : {}), ...(next.restartCancelledBackgroundWork === undefined && current.restartCancelledBackgroundWork !== undefined ? { restartCancelledBackgroundWork: current.restartCancelledBackgroundWork } @@ -1961,7 +1970,10 @@ export const layer: Layer.Layer = requested_at = excluded.requested_at, completed_at = excluded.completed_at, payload_json = ${keepRecordedRunField( - keepRecordedRunField(sql`excluded.payload_json`, "$.delegatedCompletion"), + keepRecordedRunField( + keepRecordedRunField(sql`excluded.payload_json`, "$.delegatedCompletion"), + "$.restartPendingMessageIds", + ), "$.restartCancelledBackgroundWork", )} `; @@ -4085,6 +4097,18 @@ export const layer: Layer.Layer = THEN json_array_length(roster.payload_json, '$.pendingBackgroundTasks') > 0 ELSE 0 END ) + OR run.run_id IN ( + SELECT subagent.run_id FROM orchestration_v2_projection_subagents AS subagent + WHERE subagent.thread_id = ${threadId} AND subagent.origin = 'provider_native' + AND subagent.status IN ('pending', 'running', 'waiting') + ) + OR run.run_id IN ( + SELECT message.run_id FROM orchestration_v2_projection_messages AS message + JOIN orchestration_v2_effect_outbox AS effect + ON json_extract(effect.payload_json, '$.messageId') = message.message_id + WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' + AND effect.status IN ('pending', 'running') + ) OR run.run_id IN ( SELECT item.run_id FROM orchestration_v2_projection_turn_items AS item WHERE item.thread_id = ${threadId} @@ -4172,7 +4196,16 @@ export const layer: Layer.Layer = SELECT run.provider_thread_id FROM orchestration_v2_projection_runs AS run WHERE run.thread_id = ${threadId} - AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + AND ( + run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + OR run.run_id IN ( + SELECT message.run_id FROM orchestration_v2_projection_messages AS message + JOIN orchestration_v2_effect_outbox AS effect + ON json_extract(effect.payload_json, '$.messageId') = message.message_id + WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' + AND effect.status IN ('pending', 'running') + ) + ) ) ) ) @@ -4202,7 +4235,16 @@ export const layer: Layer.Layer = OR provider_thread.provider_thread_id IN ( SELECT run.provider_thread_id FROM orchestration_v2_projection_runs AS run WHERE run.thread_id = ${threadId} - AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + AND ( + run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + OR run.run_id IN ( + SELECT message.run_id FROM orchestration_v2_projection_messages AS message + JOIN orchestration_v2_effect_outbox AS effect + ON json_extract(effect.payload_json, '$.messageId') = message.message_id + WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' + AND effect.status IN ('pending', 'running') + ) + ) ) OR provider_thread.provider_thread_id IN ( SELECT item.provider_thread_id @@ -4294,7 +4336,25 @@ export const layer: Layer.Layer = decodeRows(decodeMessagePayload, threadId)(messageRows), decodeRows(decodeTurnItemPayload, threadId)(turnItemRows), ]); + const pendingSteering = yield* sql<{ + readonly run_id: string; + readonly message_id: string; + }>` + SELECT message.run_id, message.message_id + FROM orchestration_v2_effect_outbox AS effect + JOIN orchestration_v2_projection_messages AS message + ON message.thread_id = effect.thread_id + AND message.message_id = json_extract(effect.payload_json, '$.messageId') + WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' + AND effect.status IN ('pending', 'running') + AND json_extract(message.payload_json, '$.delegatedCompletion') IS NULL + ORDER BY message.created_at ASC, message.message_id ASC + `; return { + pendingSteering: pendingSteering.map((row) => ({ + runId: RunId.make(row.run_id), + messageId: MessageId.make(row.message_id), + })), thread, runs, attempts, diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 444ad0ce929e5..f43f2a10a8727 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -1212,12 +1212,29 @@ it.effect.each([true, false])( driver: ProviderDriverKind.make("claude"), providerInstanceId: claudeInstanceId, status: "idle", - pendingBackgroundTasks: [], + ownerNodeId: null, + pendingBackgroundTasks: hasTurnItem + ? [] + : [ + { + taskId: "native-before-item", + kind: "subagent", + status: "running", + description: "recover native child", + }, + ], }, ], providerTurns: [], // Settled run: the stale-item loop owns it, not the nonterminal loop. - runs: [{ id: settledRunId, status: "completed", providerInstanceId: claudeInstanceId }], + runs: [ + { + id: settledRunId, + status: "completed", + providerThreadId, + providerInstanceId: claudeInstanceId, + }, + ], attempts: [], nodes: [ { id: staleSubagentNodeId, runId: settledRunId, status: "running" }, @@ -1226,6 +1243,9 @@ it.effect.each([true, false])( subagents: [ { id: staleSubagentNodeId, + title: "recover native child", + prompt: "native task prompt", + nativeTaskRef: { driver: "claude", nativeId: "native-before-item", strength: "strong" }, runId: settledRunId, origin: "provider_native", driver: ProviderDriverKind.make("claude"), @@ -1331,6 +1351,17 @@ it.effect.each([true, false])( staleSubagentNodeId, ); + const note = events.find((event) => event.type === "run.background-work-cancelled"); + assert.equal( + note?.type === "run.background-work-cancelled" + ? note.payload.restartCancelledBackgroundWork.length + : 0, + 1, + ); + if (!hasTurnItem && note?.type === "run.background-work-cancelled") { + assert.equal(note.payload.restartCancelledBackgroundWork[0]?.label, "recover native child"); + } + // The already-completed subagent and node are left untouched. assert.isFalse( events.some( diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index f88105d17c530..dec8aa13548e7 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -28,6 +28,7 @@ import { } from "@t3tools/provider-core/server/subagentProjection"; import { cancelledRosterTaskWork, + cancelledSubagentWork, cancelledTurnItemWork, isWorktreeContinuationRun, mergeRestartCancelledBackgroundWork, @@ -180,6 +181,26 @@ function latestStartedRun( ); } +function withPendingSteering(projection: ProjectionStore.ProjectionRuntimeRecoveryState) { + return { + ...projection, + runs: projection.runs.map((run) => { + const pending = (projection.pendingSteering ?? []).filter((item) => item.runId === run.id); + return pending.length === 0 + ? run + : { + ...run, + restartPendingMessageIds: [ + ...new Set([ + ...(run.restartPendingMessageIds ?? []), + ...pending.map((item) => item.messageId), + ]), + ], + }; + }), + }; +} + export const make = Effect.gen(function* () { const settings = yield* ServerSettings.ServerSettingsService; const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -197,6 +218,7 @@ export const make = Effect.gen(function* () { >, ) { const now = yield* DateTime.now; + projection = withPendingSteering(projection); const runs = [] as Array; for (const run of nonterminalRuns(projection)) { if (run.status === "waiting") { @@ -300,18 +322,31 @@ export const make = Effect.gen(function* () { }; // Queued runs have not started provider work. Preserve their execution // identities and order, but require explicit consent before draining them. - // A handoff's sole continuation already has that consent, and the old + // A handoff's continuation already has that consent, and the old // session was detached to run it in the newly bound worktree. const queuedRuns = projection.runs.filter((run) => run.status === "queued"); const handoffContinuation = continueAfterRestart && projection.thread.worktreePath != null && - queuedRuns.length === 1 && - queuedRuns[0]!.queueHeld !== true && - isWorktreeContinuationRun(queuedRuns[0]!) - ? queuedRuns[0] + queuedRuns.filter((run) => run.queueHeld !== true && isWorktreeContinuationRun(run)) + .length === 1 + ? queuedRuns.find((run) => run.queueHeld !== true && isWorktreeContinuationRun(run)) : undefined; for (const run of projection.runs) { + if ( + (projection.pendingSteering ?? []).some((item) => item.runId === run.id) && + !runs.some((candidate) => candidate.id === run.id) + ) { + events.push({ + id: yield* allocateEventId(), + type: "run.updated", + threadId: projection.thread.id, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: run, + }); + } if (run.status !== "queued" || run.queueHeld === true || run === handoffContinuation) continue; events.push({ @@ -496,6 +531,23 @@ export const make = Effect.gen(function* () { // No provider process outlives the server, so a native subagent still open // here never reports its end, whatever state its run and item are in. for (const subagent of projection.subagents ?? []) { + if ( + subagent.origin === "provider_native" && + isNonterminalNodeStatus(subagent.status) && + !projection.turnItems.some( + (item) => item.type === "subagent" && item.subagentId === subagent.id, + ) + ) { + const run = projection.runs.find((candidate) => candidate.id === subagent.runId); + recordCancelledBackgroundWork( + run?.providerThreadId ?? + projection.nodes.find((node) => node.id === subagent.parentNodeId)?.providerThreadId, + cancelledSubagentWork(subagent), + ); + if (subagent.nativeTaskRef?.nativeId != null) { + cancelledBackgroundNativeIds.add(subagent.nativeTaskRef.nativeId); + } + } events.push( ...(yield* endOrphanedNativeSubagent({ projection, @@ -812,7 +864,7 @@ export const make = Effect.gen(function* () { // Snapshot intent only while providers are live. A provider may finish while // this commits; reconciliation reads fresh state after shutdown, and delivery - // rejects any source run that actually completed. + // rejects a completed source unless it still has undelivered steering. const prepareForShutdown = Effect.gen(function* () { const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null)); if (!enabled) return; @@ -825,7 +877,7 @@ export const make = Effect.gen(function* () { .continueThreadsAfterServerUpdate ) return; - const run = restartContinuationRun(projection, yield* DateTime.now); + const run = restartContinuationRun(withPendingSteering(projection), yield* DateTime.now); if (!run) return; const commandId = CommandId.make(`command:restart-prepare:${run.id}`); yield* eventSink.writeWithEffects({ diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts index 40246f64e29c4..dc4d7b3966480 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts @@ -256,3 +256,21 @@ it("counts a prompted mid-turn or chained continuation as delivering the note", [], ); }); + +it("carries pending native steering through repeated crashes until its prompt completes", () => { + const messageId = MessageId.make("message:pending-native-steer"); + const source = run(1, codexThread, { + status: "cancelled", + restartPendingMessageIds: [messageId], + }); + const continuation = run(2, codexThread, { + status: "cancelled", + restartContinuationOfRunId: source.id, + }); + const again = run(3, codexThread, { restartContinuationOfRunId: continuation.id }); + const runs = [source, continuation, again]; + assert.isTrue(isRestartNoteContinuation(again, runs, [], [])); + assert.isFalse(isRestartNoteContinuation(again, runs, [turnFor(continuation)], [])); + // A pre-existing completed turn cannot have delivered a still-pending steer. + assert.isTrue(isRestartNoteContinuation(continuation, runs, [turnFor(source)], [])); +}); diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index 28db9481bdcca..b88dc9be9f38b 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -4,6 +4,8 @@ import type { OrchestrationV2RestartCancelledBackgroundWork, OrchestrationV2Run, OrchestrationV2RunAttempt, + OrchestrationV2Subagent, + MessageId, OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; @@ -52,6 +54,15 @@ export function cancelledTurnItemWork(item: OrchestrationV2TurnItem): Work | und } } +/** A native subagent can be recorded before its turn item arrives. */ +export function cancelledSubagentWork(subagent: OrchestrationV2Subagent): Work { + return { + kind: "subagent", + label: compactLabel(subagent.title) ?? compactLabel(subagent.prompt) ?? "subagent", + id: subagent.id, + }; +} + function restartWorkKind(task: OrchestrationV2PendingBackgroundTask): Work["kind"] { switch (task.kind) { case "command": @@ -187,6 +198,29 @@ export function restartPromptSource( } } +/** Unsettled steering follows a continuation until a completed turn proves delivery. */ +export function restartPendingMessageIds( + source: OrchestrationV2Run, + runs: ReadonlyArray, + providerTurns: ReadonlyArray>, +): ReadonlyArray { + const visited = new Set(); + const messages: Array = []; + let current: OrchestrationV2Run | undefined = source; + while (current !== undefined && !visited.has(current.id)) { + visited.add(current.id); + messages.unshift(...(current.restartPendingMessageIds ?? [])); + if ( + providerTurns.some( + (turn) => turn.runAttemptId === current?.activeAttemptId && turn.status === "completed", + ) + ) + break; + current = runs.find((run) => run.id === current?.restartContinuationOfRunId); + } + return [...new Set(messages)]; +} + /** * A restart continuation prompted with the note or an undelivered steering restart. * A turn cut mid-way that lost background work is prompted too, so the note @@ -204,7 +238,8 @@ export function isRestartNoteContinuation( : runs.find((candidate) => candidate.id === run.restartContinuationOfRunId); return ( source !== undefined && - (restartContinuationNote(source, runs, providerTurns, attempts).work.length > 0 || + (restartPendingMessageIds(source, runs, providerTurns).length > 0 || + restartContinuationNote(source, runs, providerTurns, attempts).work.length > 0 || restartPromptSource(source, runs, providerTurns, attempts) !== undefined) ); } diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 70f7f9924ca18..79980586ab4fa 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; import { + ComposerContextId, MessageId, NodeId, ProjectId, @@ -1016,12 +1017,14 @@ it.effect.each([ yield* recovery.reconcile(trigger); const held = alreadyHeld || - events.some((event) => event.type === "run.updated" && event.payload.queueHeld === true); - assert.equal(held, !enabled || alreadyHeld || extraQueue); - assert.lengthOf( - effects, - trigger === "startup" && enabled && !alreadyHeld && !extraQueue ? 1 : 0, - ); + events.some( + (event) => + event.type === "run.updated" && + event.payload.id === queuedFollowUp.id && + event.payload.queueHeld === true, + ); + assert.equal(held, !enabled || alreadyHeld); + assert.lengthOf(effects, trigger === "startup" && enabled && !alreadyHeld ? 1 : 0); if (effects.length > 0) assert.equal(effects[0]!.request.type, "provider-runtime.continue"); }), ); @@ -1045,6 +1048,16 @@ it.effect("delivers a queued handoff prompt once and respects cancellation befor providerTurns: [], } as unknown as OrchestrationV2ThreadProjection; assert.deepEqual(yield* continuationTexts(projection), [message.text]); + assert.deepEqual( + yield* continuationTexts({ + ...projection, + runs: [ + ...projection.runs, + { ...queuedFollowUp, id: RunId.make("run:mixed-queue"), queueHeld: true }, + ], + } as OrchestrationV2ThreadProjection), + [message.text], + ); for (const status of ["queued", "cancelled"] as const) { assert.deepEqual( yield* continuationTexts({ @@ -1424,3 +1437,149 @@ it.effect.each(["settled", "running", "stopped"] as const)( ); }), ); + +it.effect.each(["cancelled", "completed"] as const)( + "delivers pending native steering after a %s turn with attachments and context", + (status) => + Effect.gen(function* () { + const base = cutMidTurn(); + const messageId = MessageId.make("message:pending-steer"); + const attachment = { + type: "file" as const, + id: "file_steer", + name: "steering.txt", + mimeType: "text/plain", + sizeBytes: 4, + }; + const context = { + version: 1 as const, + records: [ + { + version: 1 as const, + contextId: ComposerContextId.make("ctx_steer"), + kind: "file" as const, + label: "steering.txt", + attachmentId: attachment.id, + name: attachment.name, + mimeType: attachment.mimeType, + sizeBytes: attachment.sizeBytes, + }, + ], + }; + const projection = { + ...base, + runs: [{ ...base.runs[0]!, status, restartPendingMessageIds: [messageId] }], + messages: [ + { + id: messageId, + threadId, + runId, + nodeId: null, + role: "user" as const, + text: "Use the steering input.", + attachments: [attachment], + context, + createdBy: "user" as const, + creationSource: "web" as const, + streaming: false, + createdAt: testNow, + updatedAt: testNow, + }, + ], + } as OrchestrationV2ThreadProjection; + const commands = yield* continuationDispatches(projection); + assert.lengthOf(commands, 1); + const command = commands[0]!; + assert.equal(command.type, "message.dispatch"); + if (command.type !== "message.dispatch") return; + assert.equal(command.text, "Use the steering input."); + assert.deepEqual(command.attachments, [attachment]); + assert.deepEqual(command.context, context); + const earlierId = MessageId.make("message:earlier-steer"); + const earlierRunId = RunId.make("run:earlier-steer"); + assert.deepEqual( + yield* continuationTexts({ + ...projection, + runs: [ + { + ...projection.runs[0]!, + id: earlierRunId, + ordinal: 0, + restartPendingMessageIds: [earlierId], + }, + { ...projection.runs[0]!, restartContinuationOfRunId: earlierRunId }, + ], + messages: [ + { ...projection.messages[0]!, id: earlierId, text: "Earlier input." }, + ...projection.messages, + ], + }), + ["Earlier input.\n\nUse the steering input."], + ); + for (const turnStatus of ["failed", "interrupted"] as const) { + assert.deepEqual( + yield* continuationTexts({ + ...projection, + providerTurns: projection.providerTurns.map((turn) => ({ + ...turn, + status: turnStatus, + })), + }), + [], + ); + } + }), +); + +it.effect("prepares pending native steering on a settled run before graceful shutdown", () => + Effect.gen(function* () { + const base = makeProjection(); + const messageId = MessageId.make("message:shutdown-pending-steer"); + const projection = { + ...base, + runs: [{ ...base.runs[0]!, status: "completed" }], + providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }], + pendingSteering: [{ runId, messageId }], + } as unknown as ProjectionStore.ProjectionRuntimeRecoveryState; + const effects: Array = []; + const events: Array< + Parameters[0]["events"][number] + > = []; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest(), + IdAllocator.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + writeWithEffects: (input) => { + effects.push(...input.effects); + return Effect.succeed([]); + }, + commitCommand: (input) => { + events.push(...input.events); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + signalCancellations: () => Effect.void, + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + yield* recovery.prepareForShutdown; + assert.lengthOf(effects, 1); + yield* recovery.reconcile("shutdown"); + assert.isTrue( + events.some( + (event) => + event.type === "run.updated" && + event.payload.restartPendingMessageIds?.includes(messageId), + ), + ); + }), +); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 967b0aef0dffa..6d00380394a79 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -21,6 +21,7 @@ import { restartCancelledBackgroundWorkNote, restartContinuationNote, restartPromptSource, + restartPendingMessageIds, } from "./RestartBackgroundNote.ts"; const CONTINUE_PROMPT = "Continue where you left off."; @@ -51,7 +52,7 @@ export function hasRestorableOpenCodeRequest( ); } -/** A recorded intent must not wake held threads, unbindable requests, or settled provider turns. */ +/** A recorded intent must not wake held threads, unbindable requests, or turns with no new input. */ export function restartContinuationBlocked( projection: Pick< ProjectionRuntimeRecoveryState, @@ -74,7 +75,9 @@ export function restartContinuationBlocked( const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId); return ( (turn !== undefined && - (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")) || + (turn.status === "failed" || + turn.status === "interrupted" || + (turn.status === "completed" && (source.restartPendingMessageIds?.length ?? 0) === 0))) || projection.runtimeRequests.some( (request) => !hasRestorableOpenCodeRequest({ ...projection, runtimeRequests: [request] }, source) && @@ -109,6 +112,7 @@ export function restartContinuationBlocked( /** * Resume only an unfinished root turn. Leftover background work is cleaned up * separately and reported on the next user turn; it must not wake a settled run. + * Undelivered steering is unfinished input even when the root turn settled. */ export function restartContinuationRun( projection: Pick< @@ -138,14 +142,16 @@ export function restartContinuationRun( (run.restartContinuationOfRunId !== undefined || restartPromptSource(run, projection.runs, projection.providerTurns, projection.attempts) !== undefined); + const pendingSteering = (run.restartPendingMessageIds?.length ?? 0) > 0; if ( run.status !== "running" && !preparedContinuation && + !pendingSteering && !(run.status === "waiting" && hasRestorableOpenCodeRequest(projection, run)) ) return; if (restartContinuationBlocked(projection, run, now)) return; - const liveTurnRequired = !preparedContinuation; + const liveTurnRequired = !preparedContinuation && !pendingSteering; if (projection.thread.providerInstanceId !== run.providerInstanceId) return; const providerThread = projection.providerThreads.find( (thread) => thread.id === run.providerThreadId, @@ -216,7 +222,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta source.queueHeld !== true && isWorktreeContinuationRun(source) && projection.thread.worktreePath != null && - projection.runs.filter((run) => run.status === "queued").length === 1 && + projection.runs.filter( + (run) => + run.status === "queued" && run.queueHeld !== true && isWorktreeContinuationRun(run), + ).length === 1 && !projection.runs.some((run) => ["preparing", "starting", "running", "waiting"].includes(run.status), ); @@ -224,12 +233,15 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta // including waiting runs that reconciliation subsequently cancelled. if ( !source || - (source.status !== "cancelled" && !worktreeContinuation) || + (source.status !== "cancelled" && + !worktreeContinuation && + (source.restartPendingMessageIds?.length ?? 0) === 0) || (isWorktreeContinuationRun(source) && source.status === "cancelled" && !projection.providerTurns.some((turn) => turn.runAttemptId === source.activeAttemptId)) || restartContinuationBlocked(projection, source, yield* DateTime.now) || - isRestartNoteSource(source, projection.providerTurns) + (isRestartNoteSource(source, projection.providerTurns) && + (source.restartPendingMessageIds?.length ?? 0) === 0) ) return; // A user submission after reconciliation takes precedence over an automatic @@ -266,11 +278,20 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta projection.providerTurns, projection.attempts, ); + const pendingMessageIds = restartPendingMessageIds( + source, + projection.runs, + projection.providerTurns, + ); const sourceRecords = yield* threads.getThreadRecords( input.threadId, ["messages", "turnItems"], { - messageIds: [source.userMessageId, ...(promptSource ? [promptSource.userMessageId] : [])], + messageIds: [ + source.userMessageId, + ...(promptSource ? [promptSource.userMessageId] : []), + ...pendingMessageIds, + ], turnItemRunIds: [source.id], turnItemTypes: ["run_interrupt_request"], }, @@ -291,7 +312,18 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta ? undefined : sourceRecords.messages.find((message) => message.id === promptSource.userMessageId); if (promptSource !== undefined && replacementMessage === undefined) return; - const prompt = replacementMessage?.text ?? CONTINUE_PROMPT; + const pendingMessages = pendingMessageIds.map((id) => + sourceRecords.messages.find((message) => message.id === id), + ); + if (pendingMessages.some((message) => message === undefined)) return; + const promptMessages = [ + ...(replacementMessage ? [replacementMessage] : []), + ...pendingMessages.filter((message) => message !== undefined), + ]; + const prompt = + promptMessages.length > 0 + ? promptMessages.map((message) => message.text).join("\n\n") + : CONTINUE_PROMPT; const note = restartContinuationNote( source, projection.runs, @@ -301,7 +333,11 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta const noteText = note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work); const text = - noteText === undefined ? prompt : note.settled ? noteText : `${noteText}\n\n${prompt}`; + noteText === undefined + ? prompt + : note.settled && pendingMessages.length === 0 + ? noteText + : `${noteText}\n\n${prompt}`; yield* threads.dispatch({ type: "message.dispatch", commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`), @@ -313,7 +349,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta text, // Shown as a work log row; the prompt the agent got is its detail. A // replacement prompt keeps its own message, attachments and context. - ...(replacementMessage === undefined + ...(replacementMessage === undefined && pendingMessages.length === 0 ? { notification: { source: { kind: "system" as const }, @@ -325,8 +361,15 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta }, } : {}), - attachments: replacementMessage?.attachments ?? [], - ...(replacementMessage?.context === undefined ? {} : { context: replacementMessage.context }), + attachments: promptMessages.flatMap((message) => message.attachments), + ...(promptMessages.some((message) => message.context !== undefined) + ? { + context: { + version: 1 as const, + records: promptMessages.flatMap((message) => message.context?.records ?? []), + }, + } + : {}), modelSelection: source.modelSelection, dispatchMode: { type: "start_immediately" }, createdBy: "agent", diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 842f0576f2902..9a44641d9d26e 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -561,6 +561,8 @@ export const OrchestrationV2Run = Schema.Struct({ contextHandoffId: Schema.NullOr(ContextHandoffId), /** Links server-generated restart continuations to the interrupted run. */ restartContinuationOfRunId: Schema.optional(RunId), + /** Steering input lost with a provider process, carried into the restart prompt. */ + restartPendingMessageIds: Schema.optional(Schema.Array(MessageId)), /** The delegated task this run answers, including a follow-up in an existing child thread. */ delegatedTaskId: Schema.optional(NodeId), /** diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 6dcf3875ff250..226d618080f7e 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -25,7 +25,7 @@ For stacked work, set \`baseRef\` to the intended parent branch and \`startFromO \`t3_thread_launch\` is the single-thread launch tool. Use \`create_threads\` only for a batch of threads intentionally sharing the caller's checkout: it always inherits the caller's project, branch, and worktree and has no workspace override. Asking an agent to run \`git worktree add\` or \`cd\` in its prompt does not update T3's thread binding. Select the workspace in the launch call instead. \`t3_worktree_handoff\` moves the calling thread, not another thread, and cannot move a thread already attached to a worktree. -\`t3_thread_launch\` has no idempotency key. Retain its returned threadId and inspect it with \`t3_thread_read\` / \`t3_thread_wait\`; preparation can still be running after acceptance. If a launch fails or its response is lost, inspect \`t3_thread_list\` before retrying, since a thread may already exist. +Pass a stable \`clientRequestId\` to \`t3_thread_launch\` and reuse it when retrying a launch, including after a server restart. Retain its returned threadId and inspect it with \`t3_thread_read\` / \`t3_thread_wait\`; preparation can still be running after acceptance. Without a retry key, inspect \`t3_thread_list\` before retrying a lost response, since a thread may already exist. Tool names may include a harness-normalized MCP prefix, such as \`mcp__t3_code__delegate_task\`; the semantics are the same. Some harnesses attach optional MCP servers lazily: if an initial tool-catalog scan does not show T3 tools, do not conclude that cross-provider delegation is unavailable. Make one bounded direct attempt using the known T3 tool name on the next tool step. In Codex code mode, for example, call \`tools.mcp__t3_code__orchestrator_capabilities({})\` before reporting that the capability is absent. Keep polling/wait loops bounded, do not duplicate active work, and use stable \`clientRequestId\` values when retrying tools that accept them. From b6aee5500e06e26c1d0fd7c9898fef8a3ec8f524 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:11:25 +0000 Subject: [PATCH 084/149] fix(server): continuation keeps goal phases, held handoffs, clock changes and the restart cap consistent --- .../src/orchestration-v2/EffectOutbox.ts | 8 +- .../src/orchestration-v2/Orchestrator.ts | 22 +- .../src/orchestration-v2/ProjectionStore.ts | 16 +- ...rRuntimeRecoveryService.regression.test.ts | 64 +++ .../ProviderRuntimeRecoveryService.ts | 23 +- .../orchestration-v2/RestartBackgroundNote.ts | 72 +++- .../RestartContinuation.test.ts | 388 ++++++++++++++++++ .../orchestration-v2/RestartContinuation.ts | 61 ++- .../src/orchestration-v2/runtimeLayer.test.ts | 278 +++++++++++++ 9 files changed, 899 insertions(+), 33 deletions(-) diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 0cb3de2e0012f..c76501f677d3c 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -511,7 +511,13 @@ export const layer: Layer.Layer = La available_at = ${now}, updated_at = ${now}, last_error = 'Requeued after the previous server process ended.' - WHERE status = 'running' + WHERE ( + status = 'running' + -- Shutdown intent is ready on startup even if the wall clock + -- moved backwards or a previous delivery was awaiting a retry. + OR (status = 'pending' AND effect_type = 'provider-runtime.continue' + AND available_at > ${now}) + ) AND effect_type IN ${sql.in(REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS)} RETURNING effect_id `; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index fb8f59c79ce21..c6c8b42345aab 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -87,8 +87,15 @@ import { CommandPolicyV2, resolveMessageDispatchIntent } from "./CommandPolicy.t import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts"; import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts"; import { notificationTurnItem } from "@t3tools/provider-core/server/notification"; -import { isRestartNoteSource, isWorktreeContinuationRun } from "./RestartBackgroundNote.ts"; -import { restartContinuationBlocked } from "./RestartContinuation.ts"; +import { + isRestartNoteSource, + isWorktreeContinuationRun, + restartRunRanAfter, +} from "./RestartBackgroundNote.ts"; +import { + restartContinuationBlocked, + restartContinuationLimitReached, +} from "./RestartContinuation.ts"; import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; import { EventSinkV2 } from "./EventSink.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; @@ -4588,6 +4595,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio )) || (isRestartNoteSource(source, projection.providerTurns) && (source.restartPendingMessageIds?.length ?? 0) === 0) || + restartContinuationLimitReached(source, projection.runs, projection.attempts) || restartContinuationBlocked( { ...projection, turnItems: secretRequests.turnItems }, source, @@ -4599,7 +4607,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (run) => run.id !== source.id && run.status !== "queued" && - (worktreeContinuation ? run.ordinal > source.ordinal : runRanAfter(run, source)), + (worktreeContinuation + ? run.ordinal > source.ordinal + : restartRunRanAfter( + run, + source, + projection.providerTurns, + projection.runs, + projection.attempts, + )), ) || (yield* stopReachedRun(command, command.threadId, source.id)) ) { diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5540d632a1480..790fabaf7a183 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -4256,10 +4256,24 @@ export const layer: Layer.Layer = ORDER BY COALESCE(provider_thread.first_run_ordinal, 0), provider_thread.provider_thread_id ASC `, sql` + WITH latest_turns AS MATERIALIZED ( + SELECT provider_thread_id, run_attempt_id, MAX(ordinal) AS ordinal + FROM orchestration_v2_projection_provider_turns + WHERE thread_id = ${threadId} AND run_attempt_id IS NOT NULL + GROUP BY provider_thread_id, run_attempt_id + ) SELECT provider_turn.payload_json FROM orchestration_v2_projection_provider_turns AS provider_turn WHERE provider_turn.thread_id = ${threadId} - AND provider_turn.status IN ('pending', 'starting', 'running', 'waiting') + AND ( + provider_turn.status IN ('pending', 'running') + -- A terminal last phase rules out continuation even when the + -- app run has not settled yet. Older live phases still need cleanup. + OR (provider_turn.provider_thread_id, provider_turn.run_attempt_id, + provider_turn.ordinal) IN ( + SELECT provider_thread_id, run_attempt_id, ordinal FROM latest_turns + ) + ) AND ( provider_turn.run_attempt_id IS NULL OR provider_turn.run_attempt_id IN ( diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.regression.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.regression.test.ts index 126746648f970..264f05d191089 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.regression.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.regression.test.ts @@ -80,3 +80,67 @@ it("uses the thread provider for stale background work without provider threads" expect(event?.type === "turn-item.updated" ? event.payload.status : null).toBe("cancelled"); }).pipe(Effect.provide(layer), Effect.runPromise); }); + +it.each(["assistant_message", "reasoning", "proposed_plan"] as const)( + "closes the streaming flag on a cancelled %s item", + async (type) => { + const threadId = ThreadId.make(`thread:streaming:${type}`); + const runId = RunId.make(`run:streaming:${type}`); + const events: Parameters< + EventSink.EventSinkV2["Service"]["commitCommand"] + >[0]["events"][number][] = []; + const projection = { + thread: { id: threadId, providerInstanceId: ProviderInstanceId.make("codex") }, + runs: [ + { id: runId, status: "running", providerInstanceId: ProviderInstanceId.make("codex") }, + ], + runtimeRequests: [], + providerSessions: [], + providerThreads: [], + providerTurns: [], + attempts: [], + nodes: [], + subagents: [], + messages: [], + turnItems: [ + { + id: TurnItemId.make(`item:${type}`), + runId, + nodeId: null, + type, + status: "running", + streaming: true, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + await Effect.gen(function* () { + const recovery = yield* ProviderRuntimeRecovery.make; + yield* recovery.reconcile("startup"); + }).pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest(), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + events.push(...input.events); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + Effect.runPromise, + ); + const item = events.find((event) => event.type === "turn-item.updated"); + expect( + item?.type === "turn-item.updated" && "streaming" in item.payload && item.payload.streaming, + ).toBe(false); + }, +); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index dec8aa13548e7..5c48032cddd8e 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -21,7 +21,7 @@ import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ServerSettings from "../serverSettings.ts"; -import { restartContinuationRun } from "./RestartContinuation.ts"; +import { restartContinuationBlocked, restartContinuationRun } from "./RestartContinuation.ts"; import { endOrphanedNativeSubagent, endRunlessRootTurns, @@ -158,7 +158,17 @@ const cancelledItem = ( ): OrchestrationV2ThreadProjection["turnItems"][number] => item.type === "secret_request" && item.secretStatus === "pending" ? { ...item, status: "cancelled", secretStatus: "cancelled", completedAt: now, updatedAt: now } - : { ...item, status: "cancelled", completedAt: now, updatedAt: now }; + : { + ...item, + status: "cancelled", + completedAt: now, + updatedAt: now, + ...(item.type === "reasoning" || + item.type === "assistant_message" || + item.type === "proposed_plan" + ? { streaming: false } + : {}), + }; /** * A provider thread's latest started run: the last turn that provider saw. @@ -325,12 +335,15 @@ export const make = Effect.gen(function* () { // A handoff's continuation already has that consent, and the old // session was detached to run it in the newly bound worktree. const queuedRuns = projection.runs.filter((run) => run.status === "queued"); + const handoffCandidates = queuedRuns.filter( + (run) => run.queueHeld !== true && isWorktreeContinuationRun(run), + ); const handoffContinuation = continueAfterRestart && projection.thread.worktreePath != null && - queuedRuns.filter((run) => run.queueHeld !== true && isWorktreeContinuationRun(run)) - .length === 1 - ? queuedRuns.find((run) => run.queueHeld !== true && isWorktreeContinuationRun(run)) + handoffCandidates.length === 1 && + !restartContinuationBlocked(projection, handoffCandidates[0]!, now) + ? handoffCandidates[0] : undefined; for (const run of projection.runs) { if ( diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index b88dc9be9f38b..ce4574199c00d 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -111,8 +111,74 @@ export function restartCancelledBackgroundWorkNote(work: ReadonlyArray): s type ProviderTurnState = Pick< OrchestrationV2ProviderTurn, "runAttemptId" | "providerThreadId" | "status" +> & + Partial>; + +type OrderedRun = Pick< + OrchestrationV2Run, + | "id" + | "ordinal" + | "completedAt" + | "providerThreadId" + | "activeAttemptId" + | "restartContinuationOfRunId" >; +/** Native turn ordinals and continuation links retain order across wall clock changes. */ +export function restartRunRanAfter( + run: OrderedRun, + other: OrderedRun, + providerTurns: ReadonlyArray, + runs: ReadonlyArray, + attempts: ReadonlyArray = [], +): boolean { + const lineage = (source: OrderedRun) => { + const chain = [source]; + const visited = new Set([source.id]); + for (let current = source; current.restartContinuationOfRunId !== undefined;) { + const previous = runs.find( + (candidate) => candidate.id === current.restartContinuationOfRunId, + ); + if (previous === undefined || visited.has(previous.id)) break; + chain.push(previous); + visited.add(previous.id); + current = previous; + } + return chain; + }; + const runChain = lineage(run); + const otherChain = lineage(other); + if (runChain.some((ancestor) => ancestor.id === other.id)) return run.id !== other.id; + if (otherChain.some((ancestor) => ancestor.id === run.id)) return false; + if (run.providerThreadId !== null && run.providerThreadId === other.providerThreadId) { + const turnOrdinal = (chain: ReadonlyArray) => { + for (const source of chain) { + const attemptIds = new Set( + attempts.filter((attempt) => attempt.runId === source.id).map((attempt) => attempt.id), + ); + if (source.activeAttemptId != null) attemptIds.add(source.activeAttemptId); + if (attemptIds.size === 0) continue; + const ordinal = providerTurns.reduce( + (latest, turn) => + turn.providerThreadId === run.providerThreadId && + turn.runAttemptId !== null && + attemptIds.has(turn.runAttemptId) + ? Math.max(latest, turn.ordinal ?? 0) + : latest, + 0, + ); + if (ordinal > 0) return ordinal; + } + return 0; + }; + const runOrdinal = turnOrdinal(runChain); + const otherOrdinal = turnOrdinal(otherChain); + if (runOrdinal > 0 && otherOrdinal > 0 && runOrdinal !== otherOrdinal) + return runOrdinal > otherOrdinal; + } + return runRanAfter(run, other); +} + /** * A run whose turn had settled before the restart cancelled its background * work. Continuing it prompts the provider with the note. A run cut mid-turn @@ -311,9 +377,11 @@ export function pendingRestartCancelledBackgroundWork(input: { return sameThread .filter( (source) => - runRanAfter(input.run, source) && (source.restartCancelledBackgroundWork?.length ?? 0) > 0 && - !prompted.some((later) => runRanAfter(later, source)), + restartRunRanAfter(input.run, source, input.providerTurns, input.runs, input.attempts) && + !prompted.some((later) => + restartRunRanAfter(later, source, input.providerTurns, input.runs, input.attempts), + ), ) .reduce>( (work, source) => diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 79980586ab4fa..47cd1c0dbbddf 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -24,6 +24,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as ServerSettings from "../serverSettings.ts"; import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; +import { pendingRestartCancelledBackgroundWork } from "./RestartBackgroundNote.ts"; import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as EventSink from "./EventSink.ts"; @@ -925,6 +926,45 @@ it.effect("stops continuing after three automatic restart continuations in a row }), ); +it.effect.each([0, 1, 2, 3])( + "counts only %s automatic continuations after a human steering restart", + (automaticCount) => + Effect.gen(function* () { + const cut = cutMidTurn(); + const chain = Array.from({ length: automaticCount + 4 }, (_, index) => ({ + ...cut.runs[0]!, + id: index === 0 ? runId : RunId.make(`run:steered-chain:${index}`), + activeAttemptId: RunAttemptId.make(`attempt:steered-chain:${index}`), + rootNodeId: NodeId.make(`node:steered-chain:${index}`), + restartContinuationOfRunId: RunId.make(`run:steered-chain:${index + 1}`), + })); + const steered = chain[automaticCount]!; + assert.lengthOf( + yield* continuationTexts({ + ...cut, + runs: chain, + providerTurns: [{ ...cut.providerTurns[0]!, runAttemptId: chain[0]!.activeAttemptId }], + attempts: [ + { + id: steered.activeAttemptId!, + runId: steered.id, + attemptOrdinal: 1, + rootNodeId: steered.rootNodeId!, + providerInstanceId: instanceId, + providerThreadId, + providerTurnId: null, + reason: "steering_restart", + status: "cancelled", + startedAt: testNow, + completedAt: testNow, + }, + ], + }), + automaticCount < 3 ? 1 : 0, + ); + }), +); + const queuedFollowUp = { id: RunId.make("run:queued"), ordinal: 2, @@ -1583,3 +1623,351 @@ it.effect("prepares pending native steering on a settled run before graceful shu ); }), ); + +it.effect.each(["expired", "cancelled"] as const)( + "continues a later goal phase after an earlier phase's request was %s", + (status) => + Effect.gen(function* () { + const cut = cutMidTurn(); + const earlier = { ...cut.providerTurns[0]!, ordinal: 1, status: "completed" as const }; + const current = { + ...earlier, + id: ProviderTurnId.make("turn:goal-current"), + ordinal: 2, + status: "cancelled" as const, + }; + assert.lengthOf( + yield* continuationTexts({ + ...cut, + providerTurns: [earlier, current], + runtimeRequests: [ + { + kind: "user_input", + status, + providerTurnId: earlier.id, + responseCapability: { type: "not_resumable", reason: "Earlier phase ended." }, + } as never, + ], + }), + 1, + ); + }), +); + +it.effect.each(["saved", "declined"] as const)( + "continues after a secret was explicitly %s", + (secretStatus) => + Effect.gen(function* () { + assert.lengthOf( + yield* continuationTexts({ + ...cutMidTurn(), + turnItems: [{ type: "secret_request", runId, secretStatus } as never], + }), + 1, + ); + }), +); + +it.effect.each(["dynamic_tool_call", "auth_refresh"] as const)( + "continues after recovery expired a %s callback", + (kind) => + Effect.gen(function* () { + const cut = cutMidTurn(); + assert.lengthOf( + yield* continuationTexts({ + ...cut, + runtimeRequests: [ + { + kind, + status: "expired", + providerTurnId: cut.providerTurns[0]!.id, + responseCapability: { type: "not_resumable", reason: "Process lost." }, + } as never, + ], + }), + 1, + ); + }), +); + +it.effect.each(["environment", "project"] as const)( + "reads the %s setting fresh for shutdown, startup, and delivery", + (scope) => + Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService.pipe( + Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })), + ); + let projection = makeProjection(); + const intents: Array = []; + const commands: Parameters< + ThreadManagementService.ThreadManagementService["Service"]["dispatch"] + >[0][] = []; + const layer = Layer.mergeAll( + Layer.succeed(ServerSettings.ServerSettingsService, settings), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + writeWithEffects: (input) => { + intents.push(...input.effects); + return Effect.succeed([]); + }, + commitCommand: (input) => { + intents.push(...input.effects); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => Effect.succeed(projection), + recoverDelegatedTask: () => Effect.void, + dispatch: (command) => { + commands.push(command); + return Effect.succeed({} as never); + }, + }), + ); + const recovery = yield* ProviderRuntimeRecovery.make.pipe(Effect.provide(layer)); + const setEnabled = (enabled: boolean) => + settings.updateSettings( + scope === "environment" + ? { continueThreadsAfterServerUpdate: enabled } + : { + projectSettingsOverrides: { + [projection.thread.projectId]: { continueThreadsAfterServerUpdate: enabled }, + }, + }, + ); + for (const enabled of [false, true, false]) { + yield* setEnabled(enabled); + intents.length = 0; + yield* recovery.prepareForShutdown; + assert.lengthOf(intents, enabled ? 1 : 0); + intents.length = 0; + yield* recovery.reconcile("startup"); + assert.lengthOf(intents, enabled ? 1 : 0); + projection = cutMidTurn(); + commands.length = 0; + yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(Effect.provide(layer)); + assert.lengthOf(commands, enabled ? 1 : 0); + projection = makeProjection(); + } + }), +); + +it.effect.each(["archivedAt", "deletedAt"] as const)( + "declines an already recorded continuation after %s changes", + (field) => + Effect.gen(function* () { + const cut = cutMidTurn(); + assert.deepEqual( + yield* continuationTexts({ ...cut, thread: { ...cut.thread, [field]: testNow } }), + [], + ); + }), +); + +it.effect.each([-1, 0, 1])("checks snooze expiry at the current clock boundary (%s ms)", (offset) => + Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(testNow)); + const live = makeProjection(); + const thread = { + ...live.thread, + snoozedUntil: DateTime.add(testNow, { milliseconds: offset }), + }; + assert.equal(restartContinuationRun({ ...live, thread }, testNow) !== undefined, offset <= 0); + assert.lengthOf(yield* continuationTexts({ ...cutMidTurn(), thread }), offset <= 0 ? 1 : 0); + }), +); + +it.effect.each(["snoozed", "settled", "archived", "deleted", "question"] as const)( + "holds a queued worktree handoff when the thread is %s during recovery", + (state) => + Effect.gen(function* () { + const base = makeProjection(); + const projection = { + ...base, + thread: { + ...base.thread, + worktreePath: "/tmp/handoff-worktree", + ...(state === "snoozed" + ? { snoozedUntil: DateTime.makeUnsafe("2099-01-01T00:00:00.000Z") } + : {}), + ...(state === "settled" ? { settledOverride: "settled" } : {}), + ...(state === "archived" ? { archivedAt: testNow } : {}), + ...(state === "deleted" ? { deletedAt: testNow } : {}), + }, + runs: [ + { + ...queuedFollowUp, + queueHeld: false, + userMessageId: MessageId.make("message:mcp:session:worktree-continuation:held"), + }, + ], + providerThreads: [], + providerSessions: [], + providerTurns: [], + runtimeRequests: + state === "question" + ? [ + { + nodeId: NodeId.make("question-node"), + kind: "user_input", + status: "pending", + responseCapability: { type: "live", providerSessionId: sessionId }, + }, + ] + : [], + } as unknown as OrchestrationV2ThreadProjection; + const events: Parameters< + EventSink.EventSinkV2["Service"]["commitCommand"] + >[0]["events"][number][] = []; + const intents: EffectOutbox.PendingOrchestrationEffectV2[] = []; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + writeWithEffects: (input) => { + intents.push(...input.effects); + return Effect.succeed([]); + }, + commitCommand: (input) => { + events.push(...input.events); + intents.push(...input.effects); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectOutbox.EffectOutboxV2)({ + cancelUnsettled: () => Effect.succeed([]), + signalCancellations: () => Effect.void, + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + yield* recovery.reconcile("startup"); + assert.lengthOf(intents, 0); + assert.isTrue( + events.some((event) => event.type === "run.updated" && event.payload.queueHeld === true), + ); + }), +); + +it.effect.each(["earlier", "newer"] as const)( + "uses provider turn order when %s completed work has a skewed wall clock", + (order) => + Effect.gen(function* () { + const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T09:00:00.000Z") }); + const otherAttempt = RunAttemptId.make(`attempt:clock:${order}`); + const otherRun = { + ...cut.runs[0]!, + id: RunId.make(`run:clock:${order}`), + ordinal: 2, + status: "completed" as const, + activeAttemptId: otherAttempt, + completedAt: DateTime.makeUnsafe( + order === "earlier" ? "2026-10-03T10:00:00.000Z" : "2026-10-03T08:00:00.000Z", + ), + }; + const projection = { + ...cut, + runs: [...cut.runs, otherRun], + providerTurns: [ + { ...cut.providerTurns[0]!, ordinal: 2 }, + { + ...cut.providerTurns[0]!, + id: ProviderTurnId.make(`turn:clock:${order}`), + runAttemptId: otherAttempt, + ordinal: order === "earlier" ? 1 : 3, + status: "completed" as const, + }, + ], + }; + assert.lengthOf(yield* continuationTexts(projection), order === "earlier" ? 1 : 0); + }), +); + +it.each([false, true])( + "does not repeat a delivered background note after a backward clock change and steering: %s", + (steered) => { + const source = cutMidTurn({ + completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"), + restartCancelledBackgroundWork: [{ kind: "shell", label: "Cancelled command" }], + }); + const delivered = { + ...source.runs[0]!, + id: RunId.make("run:delivered-clock-note"), + ordinal: 2, + activeAttemptId: RunAttemptId.make( + steered ? "attempt:steered-clock-note" : "attempt:delivered-clock-note", + ), + status: "completed" as const, + restartCancelledBackgroundWork: [], + completedAt: DateTime.makeUnsafe("2026-10-03T09:00:00.000Z"), + }; + const current = { + ...delivered, + id: RunId.make("run:after-clock-note"), + ordinal: 3, + activeAttemptId: RunAttemptId.make("attempt:after-clock-note"), + status: "running" as const, + completedAt: null, + }; + assert.deepEqual( + pendingRestartCancelledBackgroundWork({ + runs: [...source.runs, delivered, current], + run: current, + compactionMessageIds: new Set(), + attempts: [{ id: RunAttemptId.make("attempt:delivered-clock-note"), runId: delivered.id }], + providerTurns: [ + { ...source.providerTurns[0]!, ordinal: 1 }, + { + ...source.providerTurns[0]!, + ordinal: 2, + runAttemptId: RunAttemptId.make("attempt:delivered-clock-note"), + status: "completed", + }, + ], + }), + [], + ); + }, +); + +it.effect("recovers an unstarted continuation after a second restart with a backward clock", () => + Effect.gen(function* () { + const cut = cutMidTurn({ + ordinal: 2, + restartContinuationOfRunId: RunId.make("run:clock-ancestor"), + completedAt: DateTime.makeUnsafe("2026-10-03T09:00:00.000Z"), + }); + const ancestor = { + ...cut.runs[0]!, + id: RunId.make("run:clock-ancestor"), + ordinal: 1, + restartContinuationOfRunId: undefined, + activeAttemptId: RunAttemptId.make("attempt:clock-ancestor"), + completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"), + }; + assert.lengthOf( + yield* continuationTexts({ + ...cut, + runs: [...cut.runs, ancestor], + providerTurns: [ + { ...cut.providerTurns[0]!, ordinal: 1, runAttemptId: ancestor.activeAttemptId }, + ], + }), + 1, + ); + }), +); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 6d00380394a79..3bc6ab43abc54 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -5,6 +5,7 @@ import { latestProviderTurnForAttempt, MessageId, type OrchestrationV2Run, + type OrchestrationV2RunAttempt, type RunId, type ThreadId, } from "@t3tools/contracts"; @@ -22,6 +23,7 @@ import { restartContinuationNote, restartPromptSource, restartPendingMessageIds, + restartRunRanAfter, } from "./RestartBackgroundNote.ts"; const CONTINUE_PROMPT = "Continue where you left off."; @@ -90,11 +92,8 @@ export function restartContinuationBlocked( // Recovery closes live callbacks; it does not answer their question. ((request.status === "expired" || request.status === "cancelled") && request.responseCapability.type === "not_resumable" && - source.activeAttemptId !== null && - projection.providerTurns.some( - (turn) => - turn.id === request.providerTurnId && turn.runAttemptId === source.activeAttemptId, - ))), + turn !== undefined && + turn.id === request.providerTurnId)), ) || projection.turnItems.some( (item) => @@ -181,14 +180,10 @@ export function restartContinuationRun( (liveTurnRequired && (session.status === "stopped" || session.status === "error")) ) return; + const turn = latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId); if ( liveTurnRequired && - !projection.providerTurns.some( - (turn) => - turn.providerThreadId === providerThread.id && - turn.runAttemptId === run.activeAttemptId && - turn.status === "running", - ) + (turn === undefined || turn.providerThreadId !== providerThread.id || turn.status !== "running") ) return; return run; @@ -197,6 +192,30 @@ export function restartContinuationRun( /** Consecutive automatic continuations allowed before a person has to step in. */ const MAX_CONTINUATION_CHAIN = 3; +/** Admission and delivery share the cap so stale commands cannot bypass it. */ +export function restartContinuationLimitReached( + source: OrchestrationV2Run, + runs: ReadonlyArray, + attempts: ReadonlyArray>, +): boolean { + let chainLength = 0; + for ( + let run: OrchestrationV2Run | undefined = source; + run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN; + run = runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId) + ) { + // A replacement prompt interrupts the consecutive automatic restart chain. + if ( + attempts.some( + (attempt) => attempt.id === run.activeAttemptId && attempt.reason === "steering_restart", + ) + ) + break; + chainLength += 1; + } + return chainLength >= MAX_CONTINUATION_CHAIN; +} + export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")( function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) { const settings = yield* ServerSettings.ServerSettingsService; @@ -251,21 +270,21 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta (run) => run.id !== source.id && run.status !== "queued" && - (worktreeContinuation ? run.ordinal > source.ordinal : runRanAfter(run, source)), + (worktreeContinuation + ? run.ordinal > source.ordinal + : restartRunRanAfter( + run, + source, + projection.providerTurns, + projection.runs, + projection.attempts, + )), ) ) return; if (projection.thread.providerInstanceId !== source.providerInstanceId) return; // A turn that keeps crashing the server would otherwise restart it forever. - let chainLength = 0; - for ( - let run: (typeof projection.runs)[number] | undefined = source; - run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN; - run = projection.runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId) - ) { - chainLength += 1; - } - if (chainLength >= MAX_CONTINUATION_CHAIN) { + if (restartContinuationLimitReached(source, projection.runs, projection.attempts)) { yield* Effect.logWarning("Not continuing a run restarted too many times in a row", { threadId: input.threadId, sourceRunId: input.sourceRunId, diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index c0cc714517eaf..a9f83f01a8401 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -25,6 +25,7 @@ import { PullRequestOperationError, ProviderDriverKind, ProviderInstanceId, + ProviderSessionId, ProviderThreadId, ProviderTurnId, RunId, @@ -3438,6 +3439,283 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect.each([ + ["starting", "completed", false], + ["starting", "failed", false], + ["starting", "interrupted", false], + ["running", "completed", false], + ["running", "failed", false], + ["running", "interrupted", false], + ["running", "cancelled", false], + ["running", "running", true], + ] as const)( + "recovery retains the latest root turn when the run is %s and its goal phase is %s", + ([runStatus, turnStatus, shouldContinue]) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = ThreadId.make(`recovery-goal:${runStatus}:${turnStatus}`); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make("restart-project"), + title: "Goal recovery", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:message`), + threadId, + messageId: MessageId.make(`${threadId}:message`), + text: "Review the goal phases.", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const attempt = before.attempts[0]!; + const providerThread = before.providerThreads[0]!; + const sessionId = ProviderSessionId.make(`${threadId}:session`); + const now = yield* DateTime.now; + const turns = ( + [turnStatus === "cancelled" ? "running" : "completed", turnStatus] as const + ).map((status, index) => ({ + id: ProviderTurnId.make(`${threadId}:turn:${index}`), + providerThreadId: providerThread.id, + nodeId: run.rootNodeId!, + runAttemptId: attempt.id, + nativeTurnRef: null, + ordinal: index + 1, + status, + startedAt: now, + completedAt: status === "running" ? null : now, + })); + yield* sink.commitCommand({ + commandId: CommandId.make(`${threadId}:state`), + threadId, + commandType: "provider-runtime.reconcile", + acceptedAt: now, + effects: [], + events: [ + { + id: EventId.make(`${threadId}:run`), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: runStatus, startedAt: now }, + }, + { + id: EventId.make(`${threadId}:attempt`), + type: "run-attempt.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...attempt, reason: "steering_restart", status: "running" }, + }, + { + id: EventId.make(`${threadId}:session`), + type: "provider-session.attached", + threadId, + occurredAt: now, + driver, + providerInstanceId: modelSelection.instanceId, + payload: { + id: sessionId, + driver, + providerInstanceId: modelSelection.instanceId, + status: "ready", + cwd: process.cwd(), + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + }, + { + id: EventId.make(`${threadId}:provider-thread`), + type: "provider-thread.updated", + threadId, + occurredAt: now, + driver, + providerInstanceId: modelSelection.instanceId, + payload: { + ...providerThread, + status: "active", + providerSessionId: sessionId, + nativeThreadRef: { driver, nativeId: "goal-native", strength: "strong" }, + }, + }, + ...turns.map((turn) => ({ + id: EventId.make(`${threadId}:event:${turn.id}`), + type: "provider-turn.updated" as const, + threadId, + runId: run.id, + occurredAt: now, + payload: turn, + })), + ], + }); + const projection = yield* store.getRuntimeRecoveryProjection(threadId); + assert.equal(projection.providerTurns.at(-1)?.id, turns[1]!.id); + yield* recovery.prepareForShutdown; + yield* recovery.reconcile("startup"); + const intent = yield* outbox.get(`effect:restart-continuation:${run.id}`); + assert.equal(Option.isSome(intent), shouldContinue); + // A crash while recovery itself is committing must replay the same durable intent. + yield* recovery.reconcile("startup"); + const repeated = yield* outbox.get(`effect:restart-continuation:${run.id}`); + assert.deepEqual(repeated, intent); + }), + ); + + it.effect.each([false, true])( + "enforces the restart chain cap during command admission after steering: %s", + (steered) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make(`restart-admission-cap:${steered}`); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make("restart-project"), + title: "Restart cap", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:original`), + threadId, + messageId: MessageId.make(`${threadId}:original`), + text: "Original work", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + for (let index = 1; index <= (steered ? 7 : 4); index++) { + const before = yield* orchestrator.getThreadProjection(threadId); + const source = before.runs.at(-1)!; + const now = yield* DateTime.now; + yield* sink.commitCommand({ + commandId: CommandId.make(`${threadId}:cancel:${index}`), + threadId, + commandType: "provider-runtime.reconcile", + acceptedAt: now, + effects: [], + events: [ + { + id: EventId.make(`${threadId}:cancel:${index}`), + type: "run.updated", + threadId, + runId: source.id, + occurredAt: now, + payload: { ...source, status: "cancelled", completedAt: now }, + }, + ...(steered && index === 4 + ? [ + { + id: EventId.make(`${threadId}:steered-attempt`), + type: "run-attempt.updated" as const, + threadId, + runId: source.id, + occurredAt: now, + payload: { + ...before.attempts.find( + (attempt) => attempt.id === source.activeAttemptId, + )!, + reason: "steering_restart" as const, + }, + }, + ] + : []), + ], + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:continue:${index}`), + threadId, + messageId: MessageId.make(`${threadId}:continue:${index}`), + text: "Continue where you left off.", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + createdBy: "agent", + creationSource: "server", + restartContinuationOfRunId: source.id, + }); + assert.lengthOf( + (yield* orchestrator.getThreadProjection(threadId)).runs, + Math.min(index + 1, steered ? 7 : 4), + ); + } + }), + ); + + it.effect( + "makes a pending shutdown continuation claimable after the wall clock moves backwards", + () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = ThreadId.make("restart-clock-rollback"); + yield* TestClock.setTime( + DateTime.toEpochMillis(DateTime.makeUnsafe("2026-10-03T10:00:00.000Z")), + ); + yield* outbox.enqueue([ + { + id: "effect:restart-clock-rollback", + commandId: CommandId.make("restart-clock-rollback"), + threadId, + request: { type: "provider-runtime.continue", sourceRunId: RunId.make("clock-source") }, + }, + ]); + yield* TestClock.setTime( + DateTime.toEpochMillis(DateTime.makeUnsafe("2026-10-03T09:00:00.000Z")), + ); + yield* outbox.reconcileAfterProcessLoss; + // Startup recovery must leave it for the normal worker, regardless of clock skew. + assert.isTrue( + Option.isNone( + yield* outbox.claimNext({ + workerId: "recovery", + leaseDurationMs: 1000, + excludeRestartContinuations: true, + }), + ), + ); + const claim = yield* outbox.claimNext({ workerId: "normal", leaseDurationMs: 1000 }); + assert.isTrue(Option.isSome(claim)); + if (Option.isSome(claim)) assert.equal(claim.value.id, "effect:restart-clock-rollback"); + }).pipe( + Effect.provide( + Layer.fresh(EffectOutbox.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))), + ), + ), + ); + it.effect("rejects settling a thread while a run is active", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; From 24d41808d0118af8b640f8ef8cb267ef2e52cad9 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:05:02 +0000 Subject: [PATCH 085/149] fix(server): background server updates roll back unhealthy releases and announce restarts - The launcher watches a committed update for a health period and rolls back a runtime that crashes after commit. The launcher protocol stays 3, so existing launchers keep running new runtimes. - Clients hear about the restart before it happens, so they show the update as resuming instead of a lost connection. - The release feed is checked again at the handoff, a newer release replaces a staged one, stable skips prereleases, mirrors must use HTTPS, and old runtimes are pruned. --- apps/server/src/cli/update.ts | 1 + apps/server/src/cloud/pinnedRuntime.test.ts | 66 ++++++ apps/server/src/cloud/pinnedRuntime.ts | 13 +- apps/server/src/cloud/selfUpdate.test.ts | 115 +++++++++- apps/server/src/cloud/selfUpdate.ts | 72 ++++++- .../src/cloud/serviceLauncherClient.test.ts | 8 +- .../server/src/cloud/servicePreflight.test.ts | 7 +- apps/server/src/cloud/serviceProtocol.ts | 35 ++- apps/server/src/serverLifecycleEvents.test.ts | 31 +++ apps/server/src/serverLifecycleEvents.ts | 25 ++- apps/server/src/serviceLauncher.test.ts | 204 ++++++++++++++---- apps/server/src/serviceLauncher.ts | 97 ++++++++- apps/server/src/updates/ServerAutoUpdater.ts | 31 ++- apps/server/src/ws.ts | 7 +- .../client-runtime/src/state/server.test.ts | 98 +++++++++ packages/client-runtime/src/state/server.ts | 83 +++++-- packages/contracts/src/rpc.test.ts | 14 ++ packages/contracts/src/rpc.ts | 2 +- packages/contracts/src/server.ts | 15 ++ packages/shared/src/cliRelease.test.ts | 30 +++ packages/shared/src/cliRelease.ts | 25 ++- 21 files changed, 885 insertions(+), 94 deletions(-) diff --git a/apps/server/src/cli/update.ts b/apps/server/src/cli/update.ts index 47c2748b93ea1..c5dec35494805 100644 --- a/apps/server/src/cli/update.ts +++ b/apps/server/src/cli/update.ts @@ -45,6 +45,7 @@ const ReleaseIndex = Schema.Array( Schema.Struct({ tag_name: Schema.String, draft: Schema.optional(Schema.Boolean), + prerelease: Schema.optional(Schema.Boolean), }), ); const decodeReleaseIndex = Schema.decodeUnknownEffect(Schema.fromJsonString(ReleaseIndex)); diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts index 4c75a78ff03f0..36bc26bc82e56 100644 --- a/apps/server/src/cloud/pinnedRuntime.test.ts +++ b/apps/server/src/cloud/pinnedRuntime.test.ts @@ -63,6 +63,72 @@ const extractingRunner = (fs: FileSystem.FileSystem, path: Path.Path, commands: }); it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { + it.effect("refuses a plaintext release mirror before fetching checksums or executing it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-insecure-origin-" }); + const requests: string[] = []; + const error = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + releaseBaseUrl: "http://mirror.example/releases", + httpClient: releaseHttpClient(yield* validChecksums, requests), + runner: extractingRunner(fs, path), + validate: () => Effect.void, + }).pipe(Effect.flip); + assert.equal(error._tag, "PinnedRuntimeInstallError"); + assert.equal( + error._tag === "PinnedRuntimeInstallError" ? error.step : undefined, + "validating the release download origin", + ); + assert.deepEqual(requests, []); + assert.isFalse( + yield* fs.exists(pinnedRuntimePaths(path, baseDir, version, "linux").entryPath), + ); + }), + ); + + it.effect("refuses HTTPS downloads redirected to plaintext checksums or archives", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const checksums = yield* validChecksums; + for (const asset of ["SHA256SUMS", archiveName]) { + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-redirect-" }); + const commands: string[] = []; + const error = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient: HttpClient.make((request) => { + const response = new Response( + request.url.endsWith("/SHA256SUMS") ? checksums : archiveBytes, + ); + if (request.url.endsWith(`/${asset}`)) { + Object.defineProperty(response, "url", { value: `http://mirror.example/${asset}` }); + } + return Effect.succeed(HttpClientResponse.fromWeb(request, response)); + }), + runner: extractingRunner(fs, path, commands), + validate: () => Effect.void, + }).pipe(Effect.flip); + assert.equal(error._tag, "PinnedRuntimeInstallError"); + assert.deepEqual(commands, []); + assert.isFalse( + yield* fs.exists(pinnedRuntimePaths(path, baseDir, version, "linux").entryPath), + ); + } + }), + ); + it.effect("installs the verified release archive as the runtime executable", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts index 3c04e4960d103..488a4426198c6 100644 --- a/apps/server/src/cloud/pinnedRuntime.ts +++ b/apps/server/src/cloud/pinnedRuntime.ts @@ -16,6 +16,7 @@ import { cliArchiveTarCommand, cliReleaseDownloadBaseUrl, parseChecksums, + validateCliReleaseDownloadUrl, } from "@t3tools/shared/cliRelease"; import * as ProcessRunner from "../processRunner.ts"; @@ -125,6 +126,7 @@ interface PinnedRuntimeInstallInput { readonly arch: string; readonly httpClient: HttpClient.HttpClient; readonly releaseBaseUrl?: string | undefined; + readonly cleanup?: Effect.Effect; readonly onProgress?: (progress: PinnedRuntimeProgress) => void; } @@ -140,6 +142,7 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")( Effect.flatMap(HttpClientResponse.filterStatusOk), Effect.flatMap( Effect.fn(function* (response) { + yield* Effect.try(() => validateCliReleaseDownloadUrl(response.url || url)); if (onProgress === undefined) return new Uint8Array(yield* response.arrayBuffer); const length = Number(response.headers["content-length"]); const total = Number.isFinite(length) && length > 0 ? length : undefined; @@ -193,7 +196,11 @@ const installFromArchive = Effect.fn("cloud.pinned_runtime.install_archive")(fun }); } const httpClient = input.httpClient; - const baseUrl = cliReleaseDownloadBaseUrl(input.version, input.releaseBaseUrl); + const baseUrl = yield* Effect.try({ + try: () => cliReleaseDownloadBaseUrl(input.version, input.releaseBaseUrl), + catch: (cause) => + new PinnedRuntimeInstallError({ step: "validating the release download origin", cause }), + }); const fileName = cliArchiveFileName(input.version, platformKey); input.onProgress?.({ stage: "download", received: 0, total: undefined }); @@ -377,4 +384,6 @@ const installPinnedRuntime = Effect.fn("cloud.pinned_runtime.ensure_installed")( }); export const ensurePinnedRuntimeInstalled = (input: PinnedRuntimeInstallInput) => - pinnedRuntimeInstallLock.withPermit(installPinnedRuntime(input)); + pinnedRuntimeInstallLock.withPermit( + installPinnedRuntime(input).pipe(Effect.tap(() => input.cleanup ?? Effect.void)), + ); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 125000df8da73..440c686849742 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -7,6 +7,14 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; +import * as Option from "effect/Option"; +import * as Layer from "effect/Layer"; +import * as ServerLifecycleEvents from "../serverLifecycleEvents.ts"; +import * as ServerAutoUpdater from "../updates/ServerAutoUpdater.ts"; +import * as UpdateWindow from "../updates/UpdateWindow.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; import * as Path from "effect/Path"; import { HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -82,6 +90,7 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( status: "ready", version: "1.1.0", launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION, }; return { stdout: JSON.stringify(result), @@ -110,7 +119,11 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( const config = yield* ServerConfig.ServerConfig.pipe( Effect.provide(ServerConfig.layerTest(process.cwd(), baseDir)), ); + const lifecycle = yield* ServerLifecycleEvents.ServerLifecycleEvents.pipe( + Effect.provide(ServerLifecycleEvents.layer), + ); const selfUpdate = yield* ServerSelfUpdate.make().pipe( + Effect.provideService(ServerLifecycleEvents.ServerLifecycleEvents, lifecycle), Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, launcher), Effect.provideService( @@ -125,12 +138,112 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( Effect.provideService(HostProcess.Architecture, "x64"), Effect.provide(ServerConfig.layer({ ...config, mode: options.mode ?? "web" })), ); - return { selfUpdate, order }; + return { selfUpdate, order, lifecycle }; }); const withoutStaging = { capability: null, stage: () => Effect.die("unused") } as const; it.layer(NodeServices.layer)("server self update", (it) => { + it.effect("announces an accepted service restart to every lifecycle subscriber", () => + Effect.gen(function* () { + const { selfUpdate, lifecycle } = yield* makeHarness(); + const result = yield* selfUpdate.update({ targetVersion: "1.1.0" }); + const snapshot = yield* lifecycle.snapshot; + expect(snapshot.events).toContainEqual({ + version: 1, + sequence: 1, + type: "updateRestart", + payload: { + fromVersion: "0.0.45", + targetVersion: "1.1.0", + updateId: result.updateId, + expiresAt: 300_000, + }, + }); + }), + ); + + it.effect("expires an accepted handoff without permitting concurrent updates", () => + Effect.gen(function* () { + const { selfUpdate } = yield* makeHarness(); + yield* selfUpdate.update({ targetVersion: "1.1.0" }); + expect((yield* selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason).toBe( + "A server update is already in progress.", + ); + yield* TestClock.adjust("5 minutes"); + expect((yield* selfUpdate.update({ targetVersion: "1.1.0" })).updateId).toBe("launcher-id"); + }), + ); + + it.effect("rechecks a staged release after a closed window and refuses a withdrawn target", () => + Effect.gen(function* () { + let latest = "1.1.0"; + let open = false; + const staged: string[] = []; + const installed: string[] = []; + const config = yield* ServerConfig.ServerConfig.pipe( + Effect.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-background-update-test-" }), + ), + ); + const { run: pass } = yield* ServerAutoUpdater.makeUpdatePass().pipe( + Effect.provide( + Layer.mergeAll( + ServerConfig.layer({ ...config, devUrl: undefined }), + ServerSettings.layerTest({ automaticUpdates: true }), + ), + ), + Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, { + managed: true, + lastOutcome: undefined, + requestUpdate: () => Effect.succeed("id"), + prepareTrial: Effect.undefined, + }), + Effect.provideService(ServerSelfUpdate.ServerSelfUpdate, { + capability: "boot-service", + commitDesktopUpdate: () => Effect.never, + stage: (target) => + Effect.sync(() => { + staged.push(target); + return { orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION }; + }), + update: ({ targetVersion }) => + Effect.sync(() => { + installed.push(targetVersion); + return { targetVersion, method: "boot-service" as const }; + }), + }), + Effect.provideService(UpdateWindow.UpdateWindow, { + runIfOpen: (effect) => + open ? Effect.map(effect, Option.some) : Effect.succeed(Option.none()), + }), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.sync(() => + HttpClientResponse.fromWeb( + request, + new Response(JSON.stringify([{ tag_name: `v${latest}` }])), + ), + ), + ), + ), + ); + yield* pass; + expect(staged).toEqual(["1.1.0"]); + latest = "1.2.0"; + yield* TestClock.adjust("1 hour"); + yield* pass; + expect(staged).toEqual(["1.1.0", "1.2.0"]); + latest = "1.0.0"; + open = true; + const handoff = yield* pass.pipe(Effect.forkChild); + yield* TestClock.adjust("5 minutes"); + yield* Fiber.join(handoff); + expect(installed).toEqual([]); + }), + ); + it.effect("marks running threads at the boot-service handoff", () => Effect.gen(function* () { const events: string[] = []; diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 093e0f79ba830..4f128af67bbe1 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -8,6 +8,7 @@ import { } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; @@ -22,6 +23,10 @@ import { HttpClient } from "effect/http"; import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; +import { pruneServiceRuntimes } from "../serviceLauncher.ts"; +import * as ServerLifecycleEvents from "../serverLifecycleEvents.ts"; +import packageJson from "../../package.json" with { type: "json" }; + import * as ServerConfig from "../config.ts"; import * as DesktopAppUpdate from "../desktopUpdate/DesktopAppUpdate.ts"; import * as ProcessRunner from "../processRunner.ts"; @@ -35,6 +40,8 @@ import { decodeServicePreflightResult } from "./servicePreflight.ts"; import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; import { isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; +export const HANDOFF_TIMEOUT = Duration.minutes(5); + const PREFLIGHT_TIMEOUT = Duration.seconds(30); export function resolveServerSelfUpdateCapability(input: { @@ -194,7 +201,21 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { const releaseBaseUrl = Option.getOrUndefined( yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), ); - const inFlight = yield* Ref.make(false); + const lifecycle = yield* ServerLifecycleEvents.ServerLifecycleEvents; + const inFlight = yield* Ref.make<{ readonly acceptedAt?: bigint } | undefined>(undefined); + const acquire = Effect.gen(function* () { + const now = yield* Clock.currentTimeNanos; + const acquired = yield* Ref.modify(inFlight, (current) => { + const expired = + current?.acceptedAt !== undefined && + now - current.acceptedAt >= BigInt(Duration.toMillis(HANDOFF_TIMEOUT)) * 1_000_000n; + return current === undefined || expired ? ([true, {}] as const) : ([false, current] as const); + }); + if (!acquired) + return yield* new ServerSelfUpdateError({ + reason: "A server update is already in progress.", + }); + }); const capability: ServerSelfUpdateCapability | null = serverConfig.mode === "desktop" ? "desktop-managed" : launcher.managed ? "boot-service" : null; @@ -219,6 +240,13 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { platform, arch, releaseBaseUrl, + cleanup: Effect.tryPromise(() => + pruneServiceRuntimes(serverConfig.baseDir, new Set([targetVersion, packageJson.version])), + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not prune server runtimes", { cause }), + ), + ), validate: (runtime) => runner .run({ @@ -325,9 +353,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { if (!isExactServiceVersion(targetVersion)) { return yield* failWith(`'${targetVersion}' is not an exact t3 version.`); } - if (yield* Ref.getAndSet(inFlight, true)) { - return yield* failWith("A server update is already in progress."); - } + yield* acquire; return yield* Effect.gen(function* () { yield* reportProgress("downloading"); @@ -344,7 +370,28 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { error, ), ), - Effect.tap(() => onHandoffAccepted()), + Effect.tap((updateId) => + Clock.currentTimeNanos.pipe( + Effect.flatMap((acceptedAt) => Ref.set(inFlight, { acceptedAt })), + Effect.andThen(onHandoffAccepted()), + Effect.andThen( + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + lifecycle.publish({ + version: 1, + type: "updateRestart", + payload: { + fromVersion: packageJson.version, + targetVersion, + updateId, + expiresAt: now + Duration.toMillis(HANDOFF_TIMEOUT), + }, + }), + ), + ), + ), + ), + ), ), ); @@ -354,7 +401,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { runtimePath: paths.entryPath, }); return { targetVersion, method: "boot-service" as const, updateId }; - }).pipe(Effect.onError(() => Ref.set(inFlight, false))); + }).pipe(Effect.onError(() => Ref.set(inFlight, undefined))); }); return ServerSelfUpdate.of({ @@ -363,9 +410,16 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { desktopAppUpdate.commit(requestId, onHandoffAccepted), capability, stage: (targetVersion) => - stageRuntime(targetVersion).pipe( - Effect.map(({ orchestrationProtocol }) => ({ orchestrationProtocol })), - ), + Effect.gen(function* () { + if (capability !== "boot-service" || !isExactServiceVersion(targetVersion)) { + return yield* failWith("Only a managed service can stage an exact server version."); + } + yield* acquire; + return yield* stageRuntime(targetVersion).pipe( + Effect.map(({ orchestrationProtocol }) => ({ orchestrationProtocol })), + Effect.ensuring(Ref.set(inFlight, undefined)), + ); + }), }); }); diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts index dee00b0b203c3..832a73f08c971 100644 --- a/apps/server/src/cloud/serviceLauncherClient.test.ts +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -48,7 +48,7 @@ const makeClient = (host: FakeLauncherProcess, currentVersion: string) => Effect.provideService(HostProcess.Environment, host.env), ); -it.effect("waits for the launcher to durably commit the trial update ID", () => +it.effect("activates a patched runtime under a deployed protocol 3 launcher", () => Effect.gen(function* () { const pending = { id: "update-1", @@ -58,7 +58,7 @@ it.effect("waits for the launcher to durably commit the trial update ID", () => status: "pending" as const, }; const host = new FakeLauncherProcess({ - protocol: SERVICE_LAUNCHER_PROTOCOL, + protocol: 3, childVersion: "1.1.0", update: pending, }); @@ -78,10 +78,10 @@ it.effect("waits for the launcher to durably commit the trial update ID", () => }), ); -it.effect("returns the launcher-generated ID only after update acceptance", () => +it.effect("requests another self-update under a deployed protocol 3 launcher", () => Effect.gen(function* () { const host = new FakeLauncherProcess({ - protocol: SERVICE_LAUNCHER_PROTOCOL, + protocol: 3, childVersion: "1.0.0", }); const client = yield* makeClient(host, "1.0.0"); diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts index f8b6055ce5c36..f113c7d789b4a 100644 --- a/apps/server/src/cloud/servicePreflight.test.ts +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -2,7 +2,6 @@ import { expect, it } from "@effect/vitest"; import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; import { runServicePreflight } from "./servicePreflight.ts"; -import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; it.each([1, 2])("blocks legacy launcher protocol %i", (launcherProtocol) => { expect( @@ -19,17 +18,17 @@ it.each([1, 2])("blocks legacy launcher protocol %i", (launcherProtocol) => { }); }); -it("accepts the current launcher protocol", () => { +it("accepts the deployed protocol 3 launcher without requiring a service reinstall", () => { expect( runServicePreflight({ databasePath: "/missing/state.sqlite", - launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + launcherProtocol: 3, version: "1.2.3", }), ).toEqual({ status: "ready", version: "1.2.3", - launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + launcherProtocol: 3, orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION, }); }); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 2d32a996ee2c2..8bdab1026940f 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,7 +1,7 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; -// Protocol 3 requires the standalone executable layout. Bump when runtimePaths -// or the installed runtime tree changes incompatibly; launchers survive self-updates. +// Protocol 3 requires the standalone executable layout. Post-commit health +// observation is launcher-owned and does not change the runtime's IPC contract. export const SERVICE_LAUNCHER_PROTOCOL = 3 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_STATE_FILE = "service-state.json"; @@ -29,6 +29,8 @@ export interface ServiceState { readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; readonly activeVersion: string; readonly update?: ServiceUpdateRecord; + readonly launcherVersion?: string; + readonly rollback?: { readonly update: PendingServiceUpdate; readonly healthyAfter: number }; } /** Context is copied from launcher-owned state when a child is spawned. */ @@ -150,7 +152,26 @@ export function compareExactServiceVersions(left: string, right: string): number export function decodeServiceState(value: unknown): ServiceState | undefined { if (!isRecord(value)) return undefined; const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update); + const rollbackUpdate = isRecord(value.rollback) + ? decodeServiceUpdate(value.rollback.update) + : undefined; + const rollback = + isRecord(value.rollback) && + rollbackUpdate?.status === "pending" && + typeof value.rollback.healthyAfter === "number" && + Number.isFinite(value.rollback.healthyAfter) + ? { update: rollbackUpdate, healthyAfter: value.rollback.healthyAfter } + : undefined; if ( + (value.launcherVersion !== undefined && + (typeof value.launcherVersion !== "string" || + !isExactServiceVersion(value.launcherVersion))) || + (value.rollback !== undefined && + (rollback === undefined || + update?.status !== "committed" || + rollback.update.id !== update.id || + rollback.update.targetVersion !== value.activeVersion || + rollback.update.fromVersion !== update.fromVersion)) || value.protocol !== SERVICE_LAUNCHER_PROTOCOL || typeof value.activeVersion !== "string" || !isExactServiceVersion(value.activeVersion) || @@ -168,6 +189,10 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: value.activeVersion, ...(update === undefined ? {} : { update }), + ...(typeof value.launcherVersion === "string" + ? { launcherVersion: value.launcherVersion } + : {}), + ...(rollback === undefined ? {} : { rollback }), }; } @@ -183,7 +208,11 @@ export function parseServiceState(value: string): ServiceState | undefined { export function serviceStateHasPendingUpdate(value: string): boolean { try { const parsed: unknown = JSON.parse(value); - return isRecord(parsed) && isRecord(parsed.update) && parsed.update.status === "pending"; + return ( + isRecord(parsed) && + isRecord(parsed.update) && + (parsed.update.status === "pending" || parsed.rollback !== undefined) + ); } catch { return false; } diff --git a/apps/server/src/serverLifecycleEvents.test.ts b/apps/server/src/serverLifecycleEvents.test.ts index 28ba41125b57f..ea0807f5277d4 100644 --- a/apps/server/src/serverLifecycleEvents.test.ts +++ b/apps/server/src/serverLifecycleEvents.test.ts @@ -2,6 +2,8 @@ import { EnvironmentId } from "@t3tools/contracts"; import { assert, it } from "@effect/vitest"; import { assertTrue } from "@effect/vitest/utils"; import * as Effect from "effect/Effect"; +import * as TestClock from "effect/testing/TestClock"; +import * as Stream from "effect/Stream"; import * as Option from "effect/Option"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; @@ -74,3 +76,32 @@ it.effect( assert.equal(migration?.payload.status, "complete"); }).pipe(Effect.provide(ServerLifecycleEvents.layer)), ); + +it.effect("announces restarts only to capable clients and expires abandoned handoffs", () => + Effect.gen(function* () { + const lifecycle = yield* ServerLifecycleEvents.ServerLifecycleEvents; + yield* lifecycle.publish({ + version: 1, + type: "updateRestart", + payload: { + fromVersion: "1.0.0", + targetVersion: "1.1.0", + updateId: "background", + expiresAt: 300_000, + }, + }); + const events = Stream.fromIterable((yield* lifecycle.snapshot).events); + assert.equal( + (yield* ServerLifecycleEvents.eventsForClient(events, { updateRestarts: true }).pipe( + Stream.runCollect, + )).length, + 1, + ); + assert.equal( + (yield* ServerLifecycleEvents.eventsForClient(events, {}).pipe(Stream.runCollect)).length, + 0, + ); + yield* TestClock.adjust("5 minutes"); + assert.equal((yield* lifecycle.snapshot).events.length, 0); + }).pipe(Effect.provide(ServerLifecycleEvents.layer)), +); diff --git a/apps/server/src/serverLifecycleEvents.ts b/apps/server/src/serverLifecycleEvents.ts index 7d394e4dc9134..c0eaa2efe614e 100644 --- a/apps/server/src/serverLifecycleEvents.ts +++ b/apps/server/src/serverLifecycleEvents.ts @@ -1,4 +1,5 @@ import type { ServerLifecycleStreamEvent } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -6,7 +7,17 @@ import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +/** Older clients cannot decode the added restart event. */ +export const eventsForClient = ( + events: Stream.Stream, + input: { readonly updateRestarts?: boolean }, +) => + events.pipe( + Stream.filter((event) => event.type !== "updateRestart" || input.updateRestarts === true), + ); + type LifecycleEventInput = + | Omit, "sequence"> | Omit, "sequence"> | Omit, "sequence"> | Omit, "sequence">; @@ -46,7 +57,19 @@ const make = Effect.gen(function* () { ]; return [nextEvent, { sequence: nextSequence, events: nextEvents }] as const; }).pipe(Effect.tap((event) => PubSub.publish(pubsub, event))), - snapshot: Ref.get(state), + snapshot: Effect.gen(function* () { + const current = yield* Ref.get(state); + const now = yield* Clock.currentTimeMillis; + return { + ...current, + events: current.events.filter( + (event) => + event.type !== "updateRestart" || + event.payload.expiresAt === undefined || + event.payload.expiresAt > now, + ), + }; + }), get stream() { return Stream.fromPubSub(pubsub); }, diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index df2ab9e9cd061..d52c5c492341c 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -4,7 +4,12 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; -import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; +import { + Launcher, + pruneServiceRuntimes, + readServiceState, + writeServiceState, +} from "./serviceLauncher.ts"; import { compareExactServiceVersions, decodeServiceState, @@ -33,6 +38,13 @@ it("orders exact semantic versions without treating build metadata as precedence assert.equal(compareExactServiceVersions("2.0.0+one", "2.0.0+two"), 0); }); +it("reads deployed protocol 3 state without optional health fields", () => { + assert.deepEqual(decodeServiceState({ protocol: 3, activeVersion: "1.0.0" }), { + protocol: 3, + activeVersion: "1.0.0", + }); +}); + it("rejects contradictory service state", () => { assert.isUndefined( decodeServiceState({ @@ -189,22 +201,27 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("commits only after the trial reports prepared", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, "before trial"); - const encodedDatabasePath = JSON.stringify(databasePath); - const childSource = ` + it.effect( + "rolls back and restores the database when a prepared candidate crashes after commit", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { process.send({ type: "prepared", updateId: context.update.id }); process.on("message", (message) => { - if (message.type === "committed") process.exit(0); + if (message.type === "committed") { + require("node:fs").writeFileSync(${encodedDatabasePath}, "post-commit migration"); + process.exit(42); + } }); } else if (context.update === undefined) { process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); @@ -213,33 +230,146 @@ if (context.update?.status === "pending") { process.exit(0); } `; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), ); - } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), - ), - ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.1.0"); - assert.equal(state.update?.status, "committed"); - }), + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal( + state.update?.status === "rolled-back" ? state.update.reason : undefined, + "candidate-exited-after-commit:42", + ); + assert.equal(yield* fs.readFileString(databasePath), "before trial"); + assert.isUndefined(state.rollback); + }), + ); + + it.effect( + "recovers a committed candidate crash using the retained backup after launcher restart", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-health-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const dbPath = path.join(root, "database"); + yield* fs.writeFileString(dbPath, "migrated database"); + const backup = path.join(root, "runtime", "db-backup", "health-update"); + yield* fs.makeDirectory(backup, { recursive: true }); + yield* fs.writeFileString(path.join(backup, "database"), "original database"); + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + "process.exit(42);\n", + ); + } + const update = { + id: "health-update", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath, + status: "pending" as const, + }; + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + launcherVersion: "1.0.0", + update: { ...update, status: "committed" }, + rollback: { update, healthyAfter: 60_000 }, + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.promise(() => launcher.run().catch(() => undefined)); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal(yield* fs.readFileString(dbPath), "original database"); + assert.isFalse(yield* fs.exists(backup)); + }), + ); + + it.effect( + "prunes staged runtimes while preserving launcher, active, rollback and current staging", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-prune-" }); + const versions = path.join(root, "runtime", "versions"); + for (const version of ["1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0"]) { + yield* writeFakeRuntime(fs, path, path.join(versions, version), "process.exit(0);\n"); + } + yield* fs.makeDirectory(path.join(versions, ".staging-in-progress")); + const update = { + id: "health-update", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + dbPath: path.join(root, "database"), + status: "pending" as const, + }; + const statePath = path.join(root, "runtime", "service-state.json"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.0", + launcherVersion: "1.0.0", + update: { ...update, status: "committed" }, + rollback: { update, healthyAfter: 60_000 }, + }), + ); + yield* Effect.promise(() => pruneServiceRuntimes(root, new Set(["1.4.0"]))); + assert.deepEqual((yield* fs.readDirectory(versions)).sort(), [ + ".staging-in-progress", + "1.0.0", + "1.1.0", + "1.2.0", + "1.4.0", + ]); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.0", + launcherVersion: "1.0.0", + update: { ...update, status: "committed" }, + }), + ); + yield* Effect.promise(() => pruneServiceRuntimes(root)); + assert.deepEqual((yield* fs.readDirectory(versions)).sort(), [ + ".staging-in-progress", + "1.0.0", + "1.2.0", + ]); + }), ); it.effect("rolls back a trial that reports the wrong update ID", () => diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index cfdd3d75b57ee..3f0501be42dd8 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -1,5 +1,5 @@ // @effect-diagnostics nodeBuiltinImport:off -// @effect-diagnostics globalTimers:off +// @effect-diagnostics globalTimers:off globalDate:off // The launcher supervises the server child for the boot service and must keep // working across server versions, so it stays on Node built-ins with no Effect // runtime: it is the one part of the executable that cannot depend on the @@ -33,6 +33,7 @@ import { const HANDOFF_DELAY_MS = 2_000; const PREPARED_TIMEOUT_MS = 120_000; const TERMINATE_GRACE_MS = 5_000; +const HEALTH_PERIOD_MS = 5 * 60_000; type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -226,6 +227,33 @@ async function runtimeExists(baseDir: string, version: string): Promise } } +/** Removes obsolete completed runtimes while preserving every executable needed for recovery. */ +export async function pruneServiceRuntimes( + baseDir: string, + keep: ReadonlySet = new Set(), +): Promise { + const state = await readServiceState(NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE)); + // Older launchers do not identify the runtime their service unit still names. + if (state.launcherVersion === undefined) return; + const protectedVersions = new Set([state.launcherVersion, state.activeVersion, ...keep]); + if (state.update?.status === "pending") { + protectedVersions.add(state.update.fromVersion); + protectedVersions.add(state.update.targetVersion); + } + if (state.rollback !== undefined) protectedVersions.add(state.rollback.update.fromVersion); + const versionsDir = NodePath.join(baseDir, "runtime", "versions"); + for (const entry of await NodeFSP.readdir(versionsDir, { withFileTypes: true })) { + if ( + !entry.isDirectory() || + !isExactServiceVersion(entry.name) || + protectedVersions.has(entry.name) + ) + continue; + if (!(await runtimeExists(baseDir, entry.name))) continue; + await NodeFSP.rm(NodePath.join(versionsDir, entry.name), { recursive: true, force: true }); + } +} + function terminalUpdate(input: { readonly pending: PendingServiceUpdate; readonly status: S; @@ -379,6 +407,32 @@ export class Launcher { if (awaitedVersion?.trim() === this.#state.activeVersion) { await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined); } + const executableDir = NodePath.dirname(await NodeFSP.realpath(process.execPath)); + const versionsDir = NodePath.join(this.#baseDir, "runtime", "versions"); + const launcherVersion = + NodePath.dirname(executableDir) === versionsDir + ? NodePath.basename(executableDir) + : (this.#state.launcherVersion ?? this.#state.activeVersion); + this.#state = { ...this.#state, launcherVersion }; + await writeServiceState(this.#statePath, this.#state); + const rollback = this.#state.rollback; + if (rollback !== undefined) { + if (await databaseRestorePending(this.#baseDir, rollback.update)) { + await this.#returnToPrevious(rollback.update, "rolled-back", "rollback-interrupted"); + return; + } + // Restarting the launcher must not turn an unproven release into a healthy one. + const healthyAfter = Date.now() + HEALTH_PERIOD_MS; + this.#state = { ...this.#state, rollback: { ...rollback, healthyAfter } }; + await writeServiceState(this.#statePath, this.#state); + try { + await this.#startChild(this.#state.activeVersion, "active", this.#state.update); + this.#watchHealth(healthyAfter); + } catch { + await this.#returnToPrevious(rollback.update, "rolled-back", "candidate-start-failed"); + } + return; + } const update = this.#state.update; if (update?.status !== "pending") { if (update !== undefined) { @@ -489,7 +543,7 @@ export class Launcher { await reject("The requesting server is not the selected active version."); return; } - if (this.#state.update?.status === "pending") { + if (this.#state.update?.status === "pending" || this.#state.rollback !== undefined) { await reject("Another server update is already pending."); return; } @@ -551,18 +605,43 @@ export class Launcher { } this.#clearTimer(); const committed = terminalUpdate({ pending, status: "committed" }); + const healthyAfter = Date.now() + HEALTH_PERIOD_MS; const next: ServiceState = { ...this.#state, activeVersion: pending.targetVersion, update: committed, + rollback: { update: pending, healthyAfter }, }; await writeServiceState(this.#statePath, next); this.#state = next; child.role = "active"; - await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); + this.#watchHealth(healthyAfter); await sendMessage(child.process, { type: "committed", updateId: committed.id }); } + #watchHealth(healthyAfter: number): void { + this.#timer = setTimeout( + () => + this.#enqueue(async () => { + const rollback = this.#state.rollback; + if ( + rollback === undefined || + this.#stopping || + this.#child === null || + this.#child.process.exitCode !== null || + this.#child.process.signalCode !== null + ) + return; + const { rollback: _rollback, ...next } = this.#state; + await writeServiceState(this.#statePath, next); + this.#state = next; + await discardDatabaseBackup(this.#baseDir, rollback.update.id).catch(() => undefined); + await pruneServiceRuntimes(this.#baseDir).catch(() => undefined); + }), + Math.max(0, healthyAfter - Date.now()), + ); + } + async #handlePreparedTimeout(child: ManagedChild): Promise { const pending = this.#state.update; if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") { @@ -594,6 +673,14 @@ export class Launcher { } this.#clearTimer(); + if (this.#state.rollback !== undefined) { + await this.#returnToPrevious( + this.#state.rollback.update, + "rolled-back", + `candidate-exited-after-commit:${String(code ?? signal ?? "unknown")}`, + ); + return; + } const pending = this.#state.update; if (pending?.status === "pending") { await this.#startTrial(pending); @@ -614,8 +701,9 @@ export class Launcher { } await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); + const { rollback: _rollback, ...state } = this.#state; const next: ServiceState = { - ...this.#state, + ...state, activeVersion: pending.fromVersion, update: outcome, }; @@ -623,6 +711,7 @@ export class Launcher { this.#state = next; await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); await this.#startChild(next.activeVersion, "active", outcome); + await pruneServiceRuntimes(this.#baseDir).catch(() => undefined); } } diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index daa617acc655c..7b1c710639a19 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -24,7 +24,7 @@ const RECHECK = Duration.seconds(30); /** T3 releases are checked this often; the desktop app polls its own feed. */ const RELEASE_CHECK_INTERVAL = Duration.hours(1); /** How long a handed-off install waits for the launcher to stop this server. */ -const HANDOFF_TIMEOUT = Duration.minutes(5); +const HANDOFF_TIMEOUT = ServerSelfUpdate.HANDOFF_TIMEOUT; /** * Updates this environment in the background when `automaticUpdates` is on. @@ -35,12 +35,12 @@ const HANDOFF_TIMEOUT = Duration.minutes(5); * cannot establish that every other backend the app hosts is idle. * - Foreground and `npx` servers cannot replace themselves and are skipped. */ -export const layer = Layer.effectDiscard( +export const makeUpdatePass = () => Effect.gen(function* () { // A dev server must not relaunch the desktop app or service it runs beside. - if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return; + if ((yield* ServerConfig.ServerConfig).devUrl !== undefined) return { run: Effect.void }; const selfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; - if (selfUpdate.capability !== "boot-service") return; + if (selfUpdate.capability !== "boot-service") return { run: Effect.void }; const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; const updateWindow = yield* UpdateWindow.UpdateWindow; const serverSettings = yield* ServerSettings.ServerSettingsService; @@ -63,11 +63,12 @@ export const layer = Layer.effectDiscard( if (!(yield* automaticUpdatesEnabled)) return; const nowMs = yield* Clock.currentTimeMillis; // One staged release at a time, so a closed window cannot pile up downloads. - if (nowMs >= nextReleaseCheckAt && pendingTarget === undefined) { - nextReleaseCheckAt = nowMs + Duration.toMillis(RELEASE_CHECK_INTERVAL); + if (nowMs >= nextReleaseCheckAt) { + pendingTarget = undefined; const targetVersion = yield* resolveNewestVersion(cliReleaseChannelOf(currentVersion)).pipe( Effect.provideService(HttpClient.HttpClient, httpClient), ); + nextReleaseCheckAt = nowMs + Duration.toMillis(RELEASE_CHECK_INTERVAL); if ( compareExactServiceVersions(targetVersion, currentVersion) > 0 && !skippedTargets.has(targetVersion) @@ -106,6 +107,16 @@ export const layer = Layer.effectDiscard( Effect.gen(function* () { // The setting may have been turned off while waiting. if (!(yield* automaticUpdatesEnabled)) return; + // Revalidate the feed at the handoff too: an hours-old staged release + // may have been withdrawn or superseded while the window was closed. + const newest = yield* resolveNewestVersion(cliReleaseChannelOf(currentVersion)).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + ); + if (newest !== targetVersion) { + pendingTarget = undefined; + nextReleaseCheckAt = 0; + return; + } pendingTarget = undefined; skippedTargets.add(targetVersion); yield* Effect.logInfo("Installing a background server update", { targetVersion }); @@ -138,9 +149,15 @@ export const layer = Layer.effectDiscard( Effect.withSpan("updates.ServerAutoUpdater.bootService"), ); + return { run: updateBootService }; + }); + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const { run: updatePass } = yield* makeUpdatePass(); yield* forkParked( Effect.sleep(FIRST_PASS_DELAY).pipe( - Effect.andThen(updateBootService.pipe(Effect.repeat(Schedule.spaced(RECHECK)))), + Effect.andThen(updatePass.pipe(Effect.repeat(Schedule.spaced(RECHECK)))), ), ); }), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index d8d81d7e65b6b..d3e43c42ef418 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -3051,7 +3051,7 @@ const layerWsRpc = ( ); }), ), - [WS_METHODS.subscribeServerLifecycle]: (_input) => + [WS_METHODS.subscribeServerLifecycle]: (input) => Stream.unwrap( Effect.gen(function* () { const liveBuffer = yield* Queue.unbounded(); @@ -3068,7 +3068,10 @@ const layerWsRpc = ( const liveEvents = Stream.fromQueue(liveBuffer).pipe( Stream.filter((event) => event.sequence > snapshot.sequence), ); - return Stream.concat(rpcInitialItems(snapshotEvents), liveEvents); + return ServerLifecycleEvents.eventsForClient( + Stream.concat(rpcInitialItems(snapshotEvents), liveEvents), + input, + ); }), ), [WS_METHODS.subscribeAuthAccess]: (_input) => diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index 330df548ba0f7..955a7661d8a99 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -43,6 +43,7 @@ import { resolveServerWelcomeState, resolveServerUpdateProgressResult, serverUpdateStateForProgressEvent, + serverUpdateStateForAnnouncedRestart, serverUpdateStateForServerVersion, validateServerUpdateReadyEvent, waitForNextEnvironmentReconnect, @@ -88,6 +89,39 @@ function session(client: WsRpcProtocolClient): RpcSession { }; } +it("treats an announced background restart as resuming and clears it on ready, including rollback", () => { + const active = {} as RpcSession; + const stale = {} as RpcSession; + const state = { currentSession: active, welcomeSession: active, welcome: null }; + const payload = { fromVersion: "1.0.0", targetVersion: "1.1.0", updateId: "background-update" }; + const announced = applyServerWelcomeEvent(state, active, { type: "updateRestart", payload }); + expect(announced.restart).toEqual(payload); + const failed = { + status: "failed" as const, + stage: "installing" as const, + fromVersion: "1.0.0", + targetVersion: "1.0.1", + message: "old manual update failed", + }; + expect(serverUpdateStateForAnnouncedRestart(failed, announced.restart)).toEqual({ + status: "running", + stage: "resuming", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + }); + const manual = { + status: "running" as const, + stage: "installing" as const, + fromVersion: "1.0.0", + targetVersion: "1.1.0", + }; + expect(serverUpdateStateForAnnouncedRestart(manual, announced.restart)).toBe(manual); + expect(applyServerWelcomeEvent(state, stale, { type: "updateRestart", payload })).toBe(state); + expect( + applyServerWelcomeEvent(announced, active, { type: "ready", payload: {} }).restart, + ).toBeUndefined(); +}); + describe("update restart reconnect nudges", () => { it.effect("retries a desktop commit that was lost before delivery", () => Effect.gen(function* () { @@ -530,6 +564,70 @@ describe("server state projection", () => { expect(resolveServerWelcomeState(afterBufferedOldWelcome)).toBeNull(); }); + it.effect( + "observes a background restart from the lifecycle stream and nudges reconnect until ready", + () => + Effect.gen(function* () { + const events = yield* Queue.unbounded<{ + readonly type: "updateRestart" | "ready"; + readonly payload: unknown; + }>(); + const subscribed = yield* Deferred.make(); + const client = { + [WS_METHODS.subscribeServerLifecycle]: () => + Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.fromQueue(events)), + ), + } as unknown as WsRpcProtocolClient; + const currentSession = session(client); + const connectionState = yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE); + const retried = yield* Deferred.make(); + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({ + target: TARGET, + state: connectionState, + session: yield* SubscriptionRef.make(Option.some(currentSession)), + prepared: yield* SubscriptionRef.make(Option.none()), + connect: Effect.void, + disconnect: Effect.void, + retryNow: Deferred.succeed(retried, undefined).pipe(Effect.asVoid), + }); + yield* Effect.scoped( + Effect.gen(function* () { + const state = yield* makeEnvironmentServerWelcomeState().pipe( + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), + ); + yield* Deferred.await(subscribed); + const announced = yield* SubscriptionRef.changes(state).pipe( + Stream.filter((value) => value.restart !== undefined), + Stream.runHead, + Effect.forkChild, + ); + yield* Queue.offer(events, { + type: "updateRestart", + payload: { fromVersion: "1.0.0", targetVersion: "1.1.0", updateId: "background" }, + }); + expect(Option.getOrThrow(yield* Fiber.join(announced)).restart?.updateId).toBe( + "background", + ); + yield* SubscriptionRef.set(connectionState, { + ...AVAILABLE_CONNECTION_STATE, + phase: "backoff" as const, + }); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(retried); + const resumed = yield* SubscriptionRef.changes(state).pipe( + Stream.filter((value) => value.restart === undefined), + Stream.runHead, + Effect.forkChild, + ); + yield* Queue.offer(events, { type: "ready", payload: {} }); + expect(Option.getOrThrow(yield* Fiber.join(resumed)).restart).toBeUndefined(); + }), + ); + }), + ); + it.effect("checks the authoritative session before accepting a buffered welcome", () => Effect.gen(function* () { const firstEvents = yield* Queue.unbounded<{ diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 1d49fd71f70c4..b419c0310c849 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -4,6 +4,7 @@ import { type ServerConfigStreamEvent, type ServerLifecycleWelcomePayload, type ServerLifecycleStreamReadyEvent, + type ServerLifecycleStreamUpdateRestartEvent, type ServerSelfUpdateProgressEvent, type ServerSelfUpdateResult, WS_METHODS, @@ -308,6 +309,20 @@ export function serverUpdateStateForServerVersion( : IDLE_SERVER_UPDATE_STATE; } +export function serverUpdateStateForAnnouncedRestart( + state: ServerUpdateState, + restart: ServerLifecycleStreamUpdateRestartEvent["payload"] | undefined, +): ServerUpdateState { + return restart !== undefined && state.status !== "running" + ? { + status: "running", + stage: "resuming", + fromVersion: restart.fromVersion, + targetVersion: restart.targetVersion, + } + : state; +} + export function serverUpdateFailureMessage(error: unknown): string { return error instanceof Error ? error.message : "Server update failed."; } @@ -484,10 +499,19 @@ export function applyServerWelcomeEvent( current: EnvironmentServerWelcomeState, session: RpcSession, event: { - readonly type: "welcome" | "ready" | "legacyThreadMigration"; + readonly type: "welcome" | "ready" | "legacyThreadMigration" | "updateRestart"; readonly payload: unknown; }, ): EnvironmentServerWelcomeState { + if (current.currentSession !== session) return current; + if (event.type === "updateRestart") { + return { + ...current, + restart: event.payload as ServerLifecycleStreamUpdateRestartEvent["payload"], + }; + } + if (event.type === "ready") + return current.restart === undefined ? current : { ...current, restart: undefined }; return event.type === "welcome" && current.currentSession === session ? { ...current, @@ -501,6 +525,7 @@ export interface EnvironmentServerWelcomeState { readonly currentSession: RpcSession | null; readonly welcomeSession: RpcSession | null; readonly welcome: ServerLifecycleWelcomePayload | null; + readonly restart?: ServerLifecycleStreamUpdateRestartEvent["payload"] | undefined; } export function resolveServerWelcomeState( @@ -537,6 +562,7 @@ export const makeEnvironmentServerWelcomeState = Effect.fn("EnvironmentServerWel ); }); + let restartObserver: Fiber.Fiber | undefined; yield* SubscriptionRef.changes(supervisor.session).pipe( Stream.runForEach(() => updateWithCurrentSession((current, currentSession) => ({ @@ -560,18 +586,36 @@ export const makeEnvironmentServerWelcomeState = Effect.fn("EnvironmentServerWel } : { ...current, currentSession }, ); - return {}; + return { updateRestarts: true }; }), ).pipe( Stream.runForEach(([session, event]) => updateWithCurrentSession((current, currentSession) => - applyServerWelcomeEvent( - { - ...current, - currentSession, - }, - session, - event, + applyServerWelcomeEvent({ ...current, currentSession }, session, event), + ).pipe( + Effect.andThen( + Effect.gen(function* () { + if ((yield* SubscriptionRef.get(state)).currentSession !== session) return; + if (event.type === "ready" && restartObserver !== undefined) { + yield* Fiber.interrupt(restartObserver); + restartObserver = undefined; + } + if (event.type !== "updateRestart") return; + if (restartObserver !== undefined) yield* Fiber.interrupt(restartObserver); + restartObserver = yield* nudgeReconnectDuringUpdateRestart({ + stateChanges: SubscriptionRef.changes(supervisor.state), + retryNow: supervisor.retryNow, + }).pipe( + Effect.andThen( + SubscriptionRef.update(state, (current) => + current.restart?.updateId === event.payload.updateId + ? { ...current, restart: undefined } + : current, + ), + ), + Effect.forkScoped, + ); + }), ), ), ), @@ -587,9 +631,7 @@ function serverWelcomeStateChanges(environmentId: EnvironmentId) { environmentId, Stream.unwrap( makeEnvironmentServerWelcomeState().pipe( - Effect.map((state) => - SubscriptionRef.changes(state).pipe(Stream.map(resolveServerWelcomeState)), - ), + Effect.map((state) => SubscriptionRef.changes(state)), ), ), ); @@ -674,7 +716,13 @@ export function createServerEnvironmentAtoms( const updateStateValueAtom = Atom.family((environmentId: EnvironmentId) => Atom.make((get) => serverUpdateStateForServerVersion( - get(serverUpdateStateAtom(environmentId)), + (() => { + const manual = get(serverUpdateStateAtom(environmentId)); + const lifecycle = Option.getOrNull( + AsyncResult.value(get(welcomeStateFamily(environmentId))), + ); + return serverUpdateStateForAnnouncedRestart(manual, lifecycle?.restart); + })(), get(configValueAtom(environmentId))?.environment.serverVersion ?? null, ), ).pipe(Atom.withLabel(`environment-data:server:update-state-value:${environmentId}`)), @@ -963,11 +1011,14 @@ export function createServerEnvironmentAtoms( ); const welcomeFamily = Atom.family((environmentId: EnvironmentId) => Atom.make((get) => { - const result = get(welcomeStateFamily(environmentId)); + const result = AsyncResult.map(get(welcomeStateFamily(environmentId)), (value) => + value === null ? null : resolveServerWelcomeState(value), + ); if (result._tag !== "Success") return result; - return result.value === null + const welcome = result.value; + return welcome === null ? AsyncResult.initial(result.waiting) - : AsyncResult.success(result.value, result); + : AsyncResult.success(welcome, result); }).pipe(Atom.withLabel(`environment-data:server:welcome:${environmentId}`)), ); const welcome = (target: { diff --git a/packages/contracts/src/rpc.test.ts b/packages/contracts/src/rpc.test.ts index 7b2832e896ecb..385cc865400f2 100644 --- a/packages/contracts/src/rpc.test.ts +++ b/packages/contracts/src/rpc.test.ts @@ -5,6 +5,8 @@ import * as Schema from "effect/Schema"; import { ORCHESTRATION_V2_WS_METHODS } from "./orchestrationV2.ts"; import { WsRpcGroup, WsSubscribeServerConfigRpc } from "./rpc.ts"; +const decodeLegacyLifecycleSubscription = Schema.decodeExit(Schema.Struct({})); + /** * The client always sends `environmentThemes`, including to servers built * before the field existed, whose payload schema was an empty struct. What @@ -68,3 +70,15 @@ describe("WebSocket RPC contracts", () => { ).toBe(true); }); }); + +describe("subscribeServerLifecycle restart compatibility", () => { + it("keeps legacy subscriptions valid and carries the explicit restart opt-in", () => { + const rpc = WsRpcGroup.requests.get("subscribeServerLifecycle"); + if (rpc === undefined) throw new Error("lifecycle RPC is missing"); + const decode = Schema.decodeUnknownSync(rpc.payloadSchema); + expect(decode({})).toEqual({}); + expect(decode({ updateRestarts: true })).toEqual({ updateRestarts: true }); + expect(() => decode({ updateRestarts: "yes" })).toThrow(); + expect(Exit.isSuccess(decodeLegacyLifecycleSubscription({ updateRestarts: true }))).toBe(true); + }); +}); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 05c124a51d56e..bd8701547ee56 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -1737,7 +1737,7 @@ export const WsSubscribeServerConfigRpc = Rpc.make(WS_METHODS.subscribeServerCon }); const WsSubscribeServerLifecycleRpc = Rpc.make(WS_METHODS.subscribeServerLifecycle, { - payload: Schema.Struct({}), + payload: Schema.Struct({ updateRestarts: Schema.optionalKey(Schema.Boolean) }), success: ServerLifecycleStreamEvent, error: EnvironmentAuthorizationError, stream: true, diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 6c4c0faf6a93d..edecb6ff616c9 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -925,10 +925,25 @@ export const ServerLifecycleStreamLegacyThreadMigrationEvent = Schema.Struct({ export type ServerLifecycleStreamLegacyThreadMigrationEvent = typeof ServerLifecycleStreamLegacyThreadMigrationEvent.Type; +export const ServerLifecycleStreamUpdateRestartEvent = Schema.Struct({ + version: Schema.Literal(1), + sequence: NonNegativeInt, + type: Schema.Literal("updateRestart"), + payload: Schema.Struct({ + fromVersion: TrimmedNonEmptyString, + targetVersion: TrimmedNonEmptyString, + updateId: TrimmedNonEmptyString, + expiresAt: Schema.optionalKey(NonNegativeInt), + }), +}); +export type ServerLifecycleStreamUpdateRestartEvent = + typeof ServerLifecycleStreamUpdateRestartEvent.Type; + export const ServerLifecycleStreamEvent = Schema.Union([ ServerLifecycleStreamWelcomeEvent, ServerLifecycleStreamReadyEvent, ServerLifecycleStreamLegacyThreadMigrationEvent, + ServerLifecycleStreamUpdateRestartEvent, ]); export type ServerLifecycleStreamEvent = typeof ServerLifecycleStreamEvent.Type; diff --git a/packages/shared/src/cliRelease.test.ts b/packages/shared/src/cliRelease.test.ts index c92421db5ec20..b2ebc1607097b 100644 --- a/packages/shared/src/cliRelease.test.ts +++ b/packages/shared/src/cliRelease.test.ts @@ -66,6 +66,7 @@ describe("cliRelease", () => { expect(cliReleaseChannelOf("1.2.3")).toBe("stable"); expect(cliReleaseChannelOf("1.2.3-nightly.20260911.4")).toBe("nightly"); expect(cliReleaseChannelOf("1.2.3-preview.20260911.4")).toBe("preview"); + expect(cliReleaseChannelOf("1.2.3-nightly.20260911.4+build.1")).toBe("nightly"); // A prerelease that is not one of our trains is not silently a nightly. expect(cliReleaseChannelOf("1.2.3-rc.1")).toBe("stable"); }); @@ -85,6 +86,35 @@ describe("cliRelease", () => { expect(newestCliReleaseVersion([{ tag_name: "v1.2.3" }], "preview")).toBeUndefined(); }); + it("excludes prereleases from stable and accepts nightly build metadata", () => { + const releases = [ + { tag_name: "v2.0.0-rc.1" }, + { tag_name: "v1.9.0", prerelease: true }, + { tag_name: "v1.8.0-nightly.20260911.4+build.1", prerelease: true }, + { tag_name: "v1.7.0+build.1" }, + ]; + expect(newestCliReleaseVersion(releases, "stable")).toBe("1.7.0+build.1"); + expect(newestCliReleaseVersion(releases, "nightly")).toBe("1.8.0-nightly.20260911.4+build.1"); + }); + + it("requires authenticated download origins except for loopback development", () => { + for (const origin of [ + "http://mirror.example", + "http://localhost.evil.test", + "ftp://127.0.0.1", + ]) { + expect(() => cliReleaseDownloadBaseUrl("1.0.0", origin)).toThrow(); + } + for (const origin of [ + "https://mirror.example", + "http://localhost:4321", + "http://127.0.0.2:4321", + "http://[::1]:4321", + ]) { + expect(cliReleaseDownloadBaseUrl("1.0.0", origin)).toBe(`${origin}/v1.0.0`); + } + }); + it("pages through the release index at the largest page GitHub allows", () => { expect(cliReleaseIndexPageUrl(1)).toBe( "https://api.github.com/repos/pingdotgg/t3code/releases?per_page=100&page=1", diff --git a/packages/shared/src/cliRelease.ts b/packages/shared/src/cliRelease.ts index 28f0d530bb297..4faf07127bbbd 100644 --- a/packages/shared/src/cliRelease.ts +++ b/packages/shared/src/cliRelease.ts @@ -61,7 +61,21 @@ export function cliReleaseDownloadBaseUrl( version: string, baseUrl: string | undefined = CLI_RELEASE_DEFAULT_BASE_URL, ): string { - return `${(baseUrl?.trim() || CLI_RELEASE_DEFAULT_BASE_URL).replace(/\/+$/, "")}/v${version}`; + const origin = validateCliReleaseDownloadUrl(baseUrl?.trim() || CLI_RELEASE_DEFAULT_BASE_URL); + return `${origin.href.replace(/\/+$/, "")}/v${version}`; +} + +/** Checks both configured mirrors and the final URL after HTTP redirects. */ +export function validateCliReleaseDownloadUrl(url: string): URL { + const origin = new URL(url); + const loopback = + origin.hostname === "localhost" || + origin.hostname === "[::1]" || + /^127\.\d+\.\d+\.\d+$/.test(origin.hostname); + if (origin.protocol !== "https:" && !(origin.protocol === "http:" && loopback)) { + throw new Error("T3 release downloads require HTTPS (HTTP is allowed only on loopback)."); + } + return origin; } /** @@ -88,7 +102,7 @@ export const CLI_RELEASE_CHANNELS: ReadonlyArray = [ /** The release train a version was published on, derived from its prerelease tag. */ export function cliReleaseChannelOf(version: string): CliReleaseChannel { - const channel = /^[^-+]+-(nightly|preview)\.\d{8}\.\d+$/.exec(version)?.[1]; + const channel = /^[^-+]+-(nightly|preview)\.\d{8}\.\d+(?:\+[0-9A-Za-z.-]+)?$/.exec(version)?.[1]; return channel === "nightly" || channel === "preview" ? channel : "stable"; } @@ -111,13 +125,18 @@ export function newestCliReleaseVersion( releases: ReadonlyArray<{ readonly tag_name: string; readonly draft?: boolean | undefined; + readonly prerelease?: boolean | undefined; }>, channel: CliReleaseChannel, ): string | undefined { for (const release of releases) { if (release.draft) continue; - const version = /^v(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)$/.exec(release.tag_name)?.[1]; + const version = /^v(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/.exec( + release.tag_name, + )?.[1]; if (version === undefined) continue; + if (channel === "stable" && (release.prerelease || version.split("+", 1)[0]?.includes("-"))) + continue; if (cliReleaseChannelOf(version) === channel) return version; } return undefined; From a708fcd9c1df4964a821685a06c9d109ab214b7a Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:06:54 +0000 Subject: [PATCH 086/149] fix(server): a background server restart waits for idle terminals and holds new terminal input --- .../providerMaintenanceRunner.test.ts | 72 +++++- apps/server/src/terminal/Manager.test.ts | 235 ++++++++++++++++- apps/server/src/terminal/Manager.ts | 241 +++++++++++++++--- apps/server/src/updates/UpdateWindow.ts | 49 ++-- 4 files changed, 528 insertions(+), 69 deletions(-) diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index ae07283f3d63a..7b91e1ff685fd 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -6,6 +6,9 @@ import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionMana import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as Option from "effect/Option"; import * as ThreadContracts from "@t3tools/contracts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import * as CodexInstallation from "./CodexInstallation.ts"; +import * as AntigravityInstallation from "./AntigravityInstallation.ts"; import { describe, it, assert } from "@effect/vitest"; import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; @@ -1947,18 +1950,48 @@ it.effect("manual updates wait for admission and verify under the same permit", it.effect.each([ { name: "open provider window", + restart: false, + busy: false, + installing: false, interaction: false, expected: true, }, + { + name: "open restart window", + restart: true, + busy: false, + installing: false, + interaction: false, + expected: true, + }, + { + name: "busy terminal", + restart: true, + busy: true, + installing: false, + interaction: false, + expected: false, + }, + { + name: "managed first install", + restart: true, + busy: false, + installing: true, + interaction: false, + expected: false, + }, { name: "interaction arriving during blocker reads", + restart: false, + busy: false, + installing: false, interaction: true, expected: false, }, ])("rechecks the update window for $name", (scenario) => Effect.gen(function* () { let quietReads = 0; - let installs = 0; + let handoffs = 0; const context = yield* Layer.build( UpdateWindow.layer.pipe( Layer.provide( @@ -1980,6 +2013,38 @@ it.effect.each([ getSettings: Effect.succeed(ThreadContracts.DEFAULT_SERVER_SETTINGS), }), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(CodexInstallation.CodexInstallation)({ + managedDirectory: "/managed/codex", + state: Effect.succeed({ + driver: ProviderDriverKind.make("codex"), + phase: scenario.installing ? "verifying" : "idle", + operationId: null, + downloadedBytes: 0, + totalBytes: null, + version: null, + installedVersion: null, + canRemove: false, + message: null, + }), + }), + Layer.mock(AntigravityInstallation.AntigravityInstallation)({ + managedDirectory: "/managed/antigravity", + state: Effect.succeed({ + driver: ProviderDriverKind.make("antigravity"), + phase: "idle", + operationId: null, + downloadedBytes: 0, + totalBytes: null, + version: null, + installedVersion: null, + canRemove: false, + message: null, + }), + }), + Layer.mock(TerminalManager.TerminalManager)({ + withIdleTerminals: (effect) => + scenario.busy ? Effect.succeed(Option.none()) : Effect.asSome(effect), + }), ), ), ), @@ -1987,10 +2052,11 @@ it.effect.each([ const window = yield* Effect.service(UpdateWindow.UpdateWindow).pipe(Effect.provide(context)); const result = yield* window.runIfOpen( Effect.sync(() => { - installs++; + handoffs++; }), + { restartsServer: scenario.restart }, ); assert.strictEqual(Option.isSome(result), scenario.expected); - assert.strictEqual(installs, scenario.expected ? 1 : 0); + assert.strictEqual(handoffs, scenario.expected ? 1 : 0); }), ); diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index d8639c8015bc5..aa4bf6f0fbf46 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -1282,8 +1282,8 @@ it.layer( const { manager, ptyAdapter } = yield* createManager(5, { processTable: Effect.succeed([ { pid: 9000, ppid: 1, name: "zsh" }, - // An async prompt worker: a copy of the shell with no children. - { pid: 100, ppid: 9000, name: "zsh" }, + // A known async prompt worker with no children. + { pid: 100, ppid: 9000, name: "gitstatusd" }, { pid: 9001, ppid: 1, name: "zsh" }, { pid: 200, ppid: 9001, name: "node" }, { pid: 9002, ppid: 1, name: "zsh" }, @@ -1298,6 +1298,8 @@ it.layer( yield* manager.open(openInput({ terminalId: "subshell" })); yield* manager.open(openInput({ threadId: "thread-2" })); + for (const process of ptyAdapter.processes) process.emitData("\u001b]133;A\u0007"); + yield* Effect.yieldNow; yield* manager.closeIdle({ threadId: "thread-1" }); expect(ptyAdapter.processes.map((process) => process.killed)).toEqual([ @@ -1412,7 +1414,11 @@ it.layer( fs .writeFileString(filePath, contents, options) .pipe( - Effect.tap(() => (contents.length > 0 ? Queue.offer(writes, contents) : Effect.void)), + Effect.tap(() => + filePath.endsWith(".log") && contents.length > 0 + ? Queue.offer(writes, contents) + : Effect.void, + ), ), }); const { manager, ptyAdapter } = yield* createManager(5, { processKillGraceMs: 0 }).pipe( @@ -1938,7 +1944,7 @@ it.layer( expect(ptyAdapter.spawnInputs[0]).toEqual( expect.objectContaining({ shell: "pwsh.exe", - args: ["-NoLogo"], + args: expect.arrayContaining(["-NoLogo", "-NoExit", "-Command"]), }), ); }), @@ -2032,8 +2038,16 @@ it.layer( "pwsh.exe", "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", ]); - expect(ptyAdapter.spawnInputs[1]?.args).toEqual(["-NoLogo"]); - expect(ptyAdapter.spawnInputs[2]?.args).toEqual(["-NoLogo"]); + expect(ptyAdapter.spawnInputs[1]?.args?.slice(0, 3)).toEqual([ + "-NoLogo", + "-NoExit", + "-Command", + ]); + expect(ptyAdapter.spawnInputs[2]?.args?.slice(0, 3)).toEqual([ + "-NoLogo", + "-NoExit", + "-Command", + ]); }), ); @@ -2959,4 +2973,213 @@ it.layer( expect(process.killSignals).toContain("SIGKILL"); }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("keeps a childless exec at the terminal root pid", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + processTable: Effect.succeed([{ pid: 9000, ppid: 1, name: "vim" }]), + }); + yield* manager.open(openInput()); + yield* manager.closeIdle({ threadId: "thread-1" }); + expect(ptyAdapter.processes[0]!.killed).toBe(false); + }), + ); + + it.effect("checks fresh terminal work instead of a stale idle poll", () => + Effect.gen(function* () { + let busy = false; + let checks = 0; + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "/bin/bash", + subprocessPollIntervalMs: 60_000, + subprocessInspector: () => + Effect.sync(() => { + checks++; + return { + hasRunningSubprocess: busy, + childCommand: busy ? "sleep" : null, + processIds: [], + }; + }), + }); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("\u001b]133;A\u0007"); + yield* waitFor(Effect.map(getEvents, (events) => events.some((e) => e.type === "output"))); + expect(yield* manager.hasBusyTerminals).toBe(false); + const before = checks; + busy = true; + expect(yield* manager.hasBusyTerminals).toBe(true); + expect(checks).toBeGreaterThan(before); + }), + ); + + it.effect("tracks PowerShell in-process work until its next prompt", () => + Effect.gen(function* () { + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "pwsh.exe", + processTable: Effect.succeed([{ pid: 9000, ppid: 1, name: "pwsh.exe" }]), + }).pipe(Effect.provide(layerWithHostPlatform("win32"))); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + process.emitData("\u001b]133;"); + process.emitData("A\u0007"); + yield* waitFor( + Effect.map(getEvents, (events) => events.filter((e) => e.type === "output").length === 2), + ); + expect(yield* manager.hasBusyTerminals).toBe(false); + yield* manager.write({ + threadId: "thread-1", + terminalId: DEFAULT_TERMINAL_ID, + data: "\u001b[1;1R", + }); + expect(yield* manager.hasBusyTerminals).toBe(false); + yield* manager.write({ + threadId: "thread-1", + terminalId: DEFAULT_TERMINAL_ID, + data: "Start-Sleep 60\r", + }); + expect(yield* manager.hasBusyTerminals).toBe(true); + process.emitData("\u001b]133;A\u0007"); + yield* waitFor( + Effect.map(getEvents, (events) => events.filter((e) => e.type === "output").length === 3), + ); + expect(yield* manager.hasBusyTerminals).toBe(false); + }), + ); + + it.effect("ignores a childless gitstatusd prompt helper", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + processTable: Effect.succeed([ + { pid: 9000, ppid: 1, name: "zsh" }, + { pid: 100, ppid: 9000, name: "gitstatusd-linux-x86_64" }, + ]), + }); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("\u001b]133;A\u0007"); + yield* Effect.yieldNow; + yield* manager.closeIdle({ threadId: "thread-1" }); + expect(ptyAdapter.processes[0]!.killed).toBe(true); + }), + ); + + it.effect("does not replay a previous prompt marker after new input", () => + Effect.gen(function* () { + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "/bin/bash", + subprocessInspector: () => + Effect.succeed({ hasRunningSubprocess: false, childCommand: null, processIds: [] }), + }); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + process.emitData("\u001b]133;A\u0007"); + yield* waitFor(Effect.map(getEvents, (events) => events.some((e) => e.type === "output"))); + yield* manager.write({ + threadId: "thread-1", + terminalId: DEFAULT_TERMINAL_ID, + data: "while :; do :; done\r", + }); + process.emitData("command echo"); + yield* waitFor( + Effect.map(getEvents, (events) => events.filter((e) => e.type === "output").length === 2), + ); + expect(yield* manager.hasBusyTerminals).toBe(true); + }), + ); + + it.effect.each(["sh", "dash", "fish", "ksh", "cmd"])( + "tracks input and prompts in %s terminals", + (shell) => + Effect.gen(function* () { + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => shell, + processTable: Effect.succeed([{ pid: 9000, ppid: 1, name: shell }]), + }); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + process.emitData("\u001b]133;A\u0007"); + yield* waitFor(Effect.map(getEvents, (events) => events.some((e) => e.type === "output"))); + expect(yield* manager.hasBusyTerminals).toBe(false); + yield* manager.write({ + threadId: "thread-1", + terminalId: DEFAULT_TERMINAL_ID, + data: "read\r", + }); + expect(yield* manager.hasBusyTerminals).toBe(true); + process.emitData("\u001b]133;A\u0007"); + yield* waitFor( + Effect.map(getEvents, (events) => events.filter((e) => e.type === "output").length === 2), + ); + expect(yield* manager.hasBusyTerminals).toBe(false); + }), + ); + + it.effect("keeps the restart window closed when the process table omits a live PTY", () => + Effect.gen(function* () { + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "/bin/bash", + processTable: Effect.succeed([]), + }); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("\u001b]133;A\u0007"); + yield* waitFor(Effect.map(getEvents, (events) => events.some((e) => e.type === "output"))); + expect(yield* manager.hasBusyTerminals).toBe(true); + expect(Option.isNone(yield* manager.withIdleTerminals(Effect.die("must not restart")))).toBe( + true, + ); + }), + ); + + it.effect("blocks restart for childless background shell jobs at an idle prompt", () => + Effect.gen(function* () { + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "/bin/bash", + processTable: Effect.succeed([ + { pid: 9000, ppid: 1, name: "bash" }, + { pid: 9001, ppid: 9000, name: "bash" }, + ]), + }); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("\u001b]133;A\u0007"); + yield* waitFor(Effect.map(getEvents, (events) => events.some((e) => e.type === "output"))); + expect(yield* manager.hasBusyTerminals).toBe(true); + yield* manager.closeIdle({ threadId: "thread-1" }); + expect(ptyAdapter.processes[0]!.killed).toBe(false); + }), + ); + + it.effect("holds new terminal launches and input through the restart handoff", () => + Effect.gen(function* () { + const { manager, ptyAdapter, getEvents } = yield* createManager(5, { + shellResolver: () => "/bin/bash", + subprocessInspector: () => + Effect.succeed({ hasRunningSubprocess: false, childCommand: null, processIds: [] }), + }); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("\u001b]133;A\u0007"); + yield* waitFor(Effect.map(getEvents, (events) => events.some((e) => e.type === "output"))); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const handoff = yield* manager + .withIdleTerminals( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const write = yield* manager + .write({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID, data: "make\r" }) + .pipe(Effect.forkScoped); + const open = yield* manager.open(openInput({ terminalId: "new" })).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(write.pollUnsafe() === undefined).toBe(true); + expect(open.pollUnsafe() === undefined).toBe(true); + expect(ptyAdapter.processes).toHaveLength(1); + expect(ptyAdapter.processes[0]!.writes).toEqual([]); + yield* Deferred.succeed(release, undefined); + expect(Option.isSome(yield* Fiber.join(handoff))).toBe(true); + yield* Fiber.join(write); + yield* Fiber.join(open); + expect(ptyAdapter.processes).toHaveLength(2); + expect(ptyAdapter.processes[0]!.writes).toEqual(["make\r"]); + }), + ); }); diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index fbc8614f4cda2..b6664239cce22 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -61,6 +61,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as SynchronizedRef from "effect/SynchronizedRef"; import * as ServerConfig from "../config.ts"; @@ -220,9 +221,6 @@ export class TerminalManager extends Context.Service< readonly terminalId?: string; }) => Effect.Effect; - /** True while a terminal starts, runs a command, or its last subprocess poll failed. */ - readonly hasBusyTerminals: Effect.Effect; - /** * Subscribe to terminal runtime events with a direct callback. * @@ -232,6 +230,13 @@ export class TerminalManager extends Context.Service< listener: (event: TerminalEvent) => Effect.Effect, ) => Effect.Effect<() => void>; + /** Fresh activity check, including root-pid commands and failed reads. */ + readonly hasBusyTerminals: Effect.Effect; + /** Holds terminal launches and input until an idle restart handoff finishes. */ + readonly withIdleTerminals: ( + effect: Effect.Effect, + ) => Effect.Effect, E, R>; + /** * Subscribe to lightweight terminal metadata with an initial full snapshot. * @@ -252,6 +257,7 @@ interface TerminalSubprocessInspectResult { interface TerminalSubprocessInspector { ( terminalPid: number, + includeShellChildren?: boolean, ): Effect.Effect; } @@ -311,6 +317,9 @@ interface TerminalSessionState { /** Normalized child command name when `hasRunningSubprocess`; cleared when idle. */ childCommandLabel: string | null; runtimeEnv: Record | null; + tracksPrompt: boolean; + commandPending: boolean; + promptTail: string; } interface PersistHistoryRequest { @@ -667,6 +676,19 @@ function isRetryableShellSpawnError(error: PtyAdapter.PtySpawnError): boolean { ); } +const PROMPT_READY = "\u001b]133;A\u0007"; +const SHELL_COMMANDS = new Set([ + "bash", + "zsh", + "sh", + "dash", + "fish", + "ksh", + "pwsh", + "powershell", + "cmd", +]); + interface TerminalProcessTableSnapshot { readonly childrenByParent: ReadonlyMap>; readonly commandById: ReadonlyMap; @@ -718,19 +740,25 @@ function deriveSubprocessInspectResult( snapshot: TerminalProcessTableSnapshot, terminalPid: number, platform: NodeJS.Platform, + includeShellChildren = false, ): TerminalSubprocessInspectResult { const commandName = (pid: number) => normalizeChildCommandName(snapshot.commandById.get(pid) ?? "", platform); const shellName = commandName(terminalPid); - // Async prompt themes fork the shell into a helper that waits with no - // children of its own. That copy is not a command the user started. + if (shellName === null && !snapshot.childrenByParent.has(terminalPid)) { + return { hasRunningSubprocess: true, childCommand: null, processIds: [terminalPid] }; + } + // Polling ignores shell copies used by async prompts. Restart/close checks + // also count childless shell jobs, which can execute builtins without children. + const rootIsCommand = shellName !== null && !SHELL_COMMANDS.has(shellName); const childPid = (snapshot.childrenByParent.get(terminalPid) ?? []).find( (pid) => shellName === null || - commandName(pid) !== shellName || + ((includeShellChildren || commandName(pid) !== shellName) && + !/^gitstatusd(?:-|$)/.test(commandName(pid) ?? "")) || (snapshot.childrenByParent.get(pid)?.length ?? 0) > 0, ); - if (childPid === undefined) { + if (!rootIsCommand && childPid === undefined) { return { hasRunningSubprocess: false, childCommand: null, processIds: [] }; } const processIds = new Set([terminalPid]); @@ -744,7 +772,7 @@ function deriveSubprocessInspectResult( pending.push(pid); } } - const normalized = commandName(childPid); + const normalized = commandName(rootIsCommand ? terminalPid : childPid!); return { hasRunningSubprocess: true, childCommand: normalized ? truncateTerminalWireLabel(normalized) : null, @@ -1500,6 +1528,27 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func // `options.env` is the test seam. const baseEnv = options.env ?? process.env; const shellResolver = options.shellResolver ?? (() => defaultShellResolver(platform, baseEnv)); + const restartPermit = yield* Semaphore.make(1); + const bashInitPath = path.join(options.logsDir, "shell-init.bash"); + const bashInit = `if [[ -f ~/.bashrc ]]; then source ~/.bashrc; fi +__t3_prompt_ready() { printf '\x1b]133;A\x07'; } +if [[ $(declare -p PROMPT_COMMAND 2>/dev/null) == 'declare -a '* ]]; then + PROMPT_COMMAND+=(__t3_prompt_ready) +else + PROMPT_COMMAND="\${PROMPT_COMMAND:+$PROMPT_COMMAND$'\n'}__t3_prompt_ready" +fi +`; + const powerShellInit = + "$global:__t3_original_prompt = $function:prompt; function global:prompt { [Console]::Write([char]27 + ']133;A' + [char]7); & $global:__t3_original_prompt }"; + let bashInitWritten = false; + const zshInitDir = path.join(options.logsDir, "zsh-init"); + let zshInitWritten = false; + const posixInitPath = path.join(options.logsDir, "shell-init.sh"); + let posixInitWritten = false; + const zshSource = (name: string) => `ZDOTDIR=$T3_TERMINAL_ZDOTDIR +if [[ -f $ZDOTDIR/${name} ]]; then source $ZDOTDIR/${name}; fi +`; + const processRunner = yield* ProcessRunner.ProcessRunner; const resolveLaunchInputEnvironment = Effect.fn("terminal.resolveLaunchInputEnvironment")( function* ( @@ -1572,8 +1621,15 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func readonly inspector: TerminalSubprocessInspector; readonly snapshotSucceeded: boolean; } => ({ - inspector: (terminalPid) => - Effect.succeed(deriveSubprocessInspectResult(snapshot, terminalPid, platform)), + inspector: (terminalPid, includeShellChildren) => + Effect.succeed( + deriveSubprocessInspectResult( + snapshot, + terminalPid, + platform, + includeShellChildren, + ), + ), snapshotSucceeded, }), ); @@ -2172,7 +2228,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func index = 0, lastError: PtyAdapter.PtySpawnError | null = null, ): Effect.fn.Return< - { process: PtyAdapter.PtyProcess; shellLabel: string }, + { process: PtyAdapter.PtyProcess; shellLabel: string; tracksPrompt: boolean }, PtyAdapter.PtySpawnError > { if (index >= shellCandidates.length) { @@ -2194,9 +2250,68 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); } + const shellName = normalizeChildCommandName(candidate.shell, platform); + let args = candidate.args ?? []; + const tracksPrompt = shellName !== null && SHELL_COMMANDS.has(shellName); + if (shellName === "bash") { + if (!bashInitWritten) { + yield* fileSystem.makeDirectory(options.logsDir, { recursive: true }).pipe(Effect.orDie); + yield* fileSystem.writeFileString(bashInitPath, bashInit).pipe(Effect.orDie); + bashInitWritten = true; + } + args = ["--rcfile", bashInitPath, ...args]; + } else if (shellName === "zsh") { + if (!zshInitWritten) { + yield* fileSystem.makeDirectory(zshInitDir, { recursive: true }).pipe(Effect.orDie); + for (const name of [".zshenv", ".zprofile"]) { + yield* fileSystem + .writeFileString( + path.join(zshInitDir, name), + zshSource(name) + "ZDOTDIR=$T3_TERMINAL_INIT_DIR\n", + ) + .pipe(Effect.orDie); + } + yield* fileSystem + .writeFileString( + path.join(zshInitDir, ".zshrc"), + zshSource(".zshrc") + + "__t3_prompt_ready() { printf '\\x1b]133;A\\x07'; }\nprecmd_functions+=(__t3_prompt_ready)\n", + ) + .pipe(Effect.orDie); + zshInitWritten = true; + } + spawnEnv = { + ...spawnEnv, + T3_TERMINAL_ZDOTDIR: spawnEnv.ZDOTDIR ?? spawnEnv.HOME ?? "", + T3_TERMINAL_INIT_DIR: zshInitDir, + ZDOTDIR: zshInitDir, + }; + } else if (shellName === "pwsh" || shellName === "powershell") { + args = [...args, "-NoExit", "-Command", powerShellInit]; + } else if (shellName === "fish") { + args = [ + ...args, + "--init-command", + "functions -c fish_prompt __t3_original_fish_prompt; function fish_prompt; printf '\\x1b]133;A\\x07'; __t3_original_fish_prompt; end", + ]; + } else if (shellName === "cmd") { + spawnEnv = { ...spawnEnv, PROMPT: PROMPT_READY + (spawnEnv.PROMPT ?? "$P$G") }; + } else if (shellName === "sh" || shellName === "dash" || shellName === "ksh") { + if (!posixInitWritten) { + yield* fileSystem.makeDirectory(options.logsDir, { recursive: true }).pipe(Effect.orDie); + yield* fileSystem + .writeFileString( + posixInitPath, + '[ -z "$T3_TERMINAL_ENV" ] || [ ! -f "$T3_TERMINAL_ENV" ] || . "$T3_TERMINAL_ENV"\nPS1=\'$(printf "\\033]133;A\\007")\'"$PS1"\n', + ) + .pipe(Effect.orDie); + posixInitWritten = true; + } + spawnEnv = { ...spawnEnv, T3_TERMINAL_ENV: spawnEnv.ENV ?? "", ENV: posixInitPath }; + } const launch = yield* agentScope.wrap({ command: candidate.shell, - args: candidate.args ?? [], + args, name: "terminal", env: spawnEnv, }); @@ -2215,6 +2330,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func return { process: attempt.success, shellLabel: formatShellCandidate(candidate), + tracksPrompt, }; } @@ -2307,6 +2423,9 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func } const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session); ptyProcess = spawnResult.process; + session.tracksPrompt = spawnResult.tracksPrompt; + session.commandPending = true; + session.promptTail = ""; startedShell = spawnResult.shellLabel; const processPid = ptyProcess.pid; @@ -2322,6 +2441,10 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func session.status = "running"; // onExit may replay an exit immediately; accept it before subscribing. session.unsubscribeData = spawnResult.process.onData((data) => { + const promptText = session.promptTail + data; + if (session.tracksPrompt && promptText.includes(PROMPT_READY)) + session.commandPending = false; + session.promptTail = promptText.slice(-(PROMPT_READY.length - 1)); if (!enqueueProcessEvent(session, processPid, { type: "output", data })) { return; } @@ -2439,8 +2562,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func } }); - // A failed inspection leaves cached activity stale, so busy checks distrust it. - let lastSubprocessPollSucceeded = true; const pollSubprocessActivity = Effect.fn("terminal.pollSubprocessActivity")(function* () { const state = yield* readManagerState; const runningSessions = [...state.sessions.values()].filter( @@ -2469,7 +2590,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); if (Option.isNone(inspectorOption)) { - lastSubprocessPollSucceeded = false; return false; } @@ -2492,7 +2612,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); if (Option.isNone(inspectResult)) { - lastSubprocessPollSucceeded = false; return; } @@ -2539,8 +2658,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func } }); - // `snapshotSucceeded` only drives backoff: the ps fallback still reads a fresh table. - lastSubprocessPollSucceeded = true; yield* Effect.forEach(runningSessions, checkSubprocessActivity, { concurrency: "unbounded", discard: true, @@ -2637,6 +2754,9 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func updatedAt: yield* nowIso, eventSequence: 0, inputCount: 0, + tracksPrompt: false, + commandPending: true, + promptTail: "", cols, rows, process: null, @@ -2740,9 +2860,11 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); const open: TerminalManager["Service"]["open"] = (input) => - withThreadLock( - input.threadId, - resolveLaunchInputEnvironment(input).pipe(Effect.flatMap(openLocked)), + restartPermit.withPermit( + withThreadLock( + input.threadId, + resolveLaunchInputEnvironment(input).pipe(Effect.flatMap(openLocked)), + ), ); const openOrAttachForStream = (input: TerminalAttachInput) => @@ -2889,7 +3011,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }; const attachStream: TerminalManager["Service"]["attachStream"] = (input, listener) => - streamSession(input, openOrAttachForStream(input), listener); + streamSession(input, restartPermit.withPermit(openOrAttachForStream(input)), listener); const observeStream: TerminalManager["Service"]["observeStream"] = (input, listener) => streamSession( @@ -2995,6 +3117,13 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }); } session.inputCount += 1; + // Cursor/device replies come from the terminal emulator, not the user. + const terminalReply = + input.data.startsWith("\u001b[") && /^\[(?:\d+;\d+R|[?>=][\d;]*c)$/.test(input.data.slice(1)); + if (input.data.length > 0 && !terminalReply) { + session.commandPending = true; + session.promptTail = ""; + } yield* Effect.try({ try: () => process.write(input.data), catch: (cause) => @@ -3005,7 +3134,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func cause, }), }); - }); + }, restartPermit.withPermit); const resizeLocked = Effect.fn("terminal.resize")(function* (input: TerminalResizeInput) { const session = yield* getSession(input.threadId, input.terminalId); @@ -3077,6 +3206,9 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func updatedAt: yield* nowIso, eventSequence: 0, inputCount: 0, + tracksPrompt: false, + commandPending: true, + promptTail: "", cols, rows, process: null, @@ -3127,13 +3259,15 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }); const restart: TerminalManager["Service"]["restart"] = (input) => - withThreadLock( - input.threadId, - resolveLaunchInputEnvironment(input).pipe( - Effect.flatMap((resolved) => - withWorkspaceLease( - path.resolve(resolved.worktreePath ?? resolved.cwd), - restartResolved(resolved), + restartPermit.withPermit( + withThreadLock( + input.threadId, + resolveLaunchInputEnvironment(input).pipe( + Effect.flatMap((resolved) => + withWorkspaceLease( + path.resolve(resolved.worktreePath ?? resolved.cwd), + restartResolved(resolved), + ), ), ), ), @@ -3186,9 +3320,11 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* Effect.forEach( running, (session) => - inspector(session.pid).pipe( + inspector(session.pid, true).pipe( Effect.flatMap((result) => result.hasRunningSubprocess || + !session.tracksPrompt || + session.commandPending || activityMark(session) !== marks.get(session.terminalId) ? Effect.void : closeSession(input.threadId, session.terminalId, false), @@ -3207,16 +3343,38 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ), ); - const hasBusyTerminals: TerminalManager["Service"]["hasBusyTerminals"] = readManagerState.pipe( - Effect.map((state) => - [...state.sessions.values()].some( - (session) => - session.status === "starting" || - (session.status === "running" && - (session.hasRunningSubprocess || !lastSubprocessPollSucceeded)), + const hasBusyTerminals: TerminalManager["Service"]["hasBusyTerminals"] = Effect.gen(function* () { + const before = yield* readManagerState; + const sessions = [...before.sessions.values()]; + if (sessions.some((session) => session.status === "starting")) return true; + const running = sessions.filter((session) => session.status === "running"); + if (running.length === 0) return false; + const activityMark = (session: TerminalSessionState) => + session.eventSequence + session.inputCount; + const marks = new Map(running.map((session) => [session, activityMark(session)])); + const { inspector, snapshotSucceeded } = yield* acquireSubprocessInspector; + if (!snapshotSucceeded) return true; + for (const session of running) { + if (session.pid === null || !session.tracksPrompt || session.commandPending) return true; + if ((yield* inspector(session.pid, true)).hasRunningSubprocess) return true; + } + const after = yield* readManagerState; + return [...after.sessions.values()].some( + (session) => + session.status === "starting" || + (session.status === "running" && + (marks.get(session) !== activityMark(session) || + session.commandPending || + session.pendingProcessEventIndex < session.pendingProcessEvents.length)), + ); + }).pipe(Effect.catch(() => Effect.succeed(true))); + + const withIdleTerminals: TerminalManager["Service"]["withIdleTerminals"] = (effect) => + restartPermit.withPermit( + hasBusyTerminals.pipe( + Effect.flatMap((busy) => (busy ? Effect.succeed(Option.none()) : Effect.asSome(effect))), ), - ), - ); + ); return TerminalManager.of({ open, @@ -3228,9 +3386,10 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func restart, close, closeIdle, - hasBusyTerminals, subscribe, subscribeMetadata, + hasBusyTerminals, + withIdleTerminals, }); }); diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index a3575198a1f34..07d6ae93636b2 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -58,10 +58,16 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const providers = yield* ProviderRegistry.ProviderRegistry; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; - const terminals = yield* TerminalManager.TerminalManager; + const terminals = yield* Effect.serviceOption(TerminalManager.TerminalManager); const managedInstalls = [ - yield* CodexInstallation.CodexInstallation, - yield* AntigravityInstallation.AntigravityInstallation, + Option.map( + yield* Effect.serviceOption(CodexInstallation.CodexInstallation), + (service) => service.state, + ), + Option.map( + yield* Effect.serviceOption(AntigravityInstallation.AntigravityInstallation), + (service) => service.state, + ), ]; /** Returns what keeps the window closed; empty means open. */ @@ -91,11 +97,11 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { provider.updateState?.status === "queued" || provider.updateState?.status === "running", ); if (providerUpdating) return ["provider-update"]; - if (restartsServer && (yield* terminals.hasBusyTerminals)) return ["terminal-work"]; // A first install downloads inside this server, so a restart would discard it. if (restartsServer) { for (const installation of managedInstalls) { - const { phase } = yield* installation.state; + if (Option.isNone(installation)) return ["provider-install"]; + const { phase } = yield* installation.value; if (phase === "downloading" || phase === "extracting" || phase === "verifying") { return ["provider-install"]; } @@ -169,20 +175,25 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (install, options) => admission.withPermit( blockers(options?.restartsServer === true).pipe( - Effect.flatMap((found) => - found.length > 0 - ? Effect.logDebug("Background update waiting for the update window", { - blockers: found, - }).pipe(Effect.as(Option.none())) - : // Interaction may have arrived during the blocker reads. - backgroundPolicy.sinceLastClientInteraction.pipe( - Effect.flatMap((quiet) => - Duration.isLessThan(quiet, QUIET_PERIOD) - ? Effect.succeed(Option.none()) - : Effect.asSome(install), - ), - ), - ), + Effect.flatMap((found) => { + if (found.length > 0) + return Effect.logDebug("Background update waiting for the update window", { + blockers: found, + }).pipe(Effect.as(Option.none())); + const stillQuiet = backgroundPolicy.sinceLastClientInteraction.pipe( + Effect.flatMap((quiet) => + Duration.isLessThan(quiet, QUIET_PERIOD) + ? Effect.succeed(Option.none()) + : Effect.asSome(install), + ), + ); + if (options?.restartsServer !== true) return stillQuiet; + return Option.match(terminals, { + onNone: () => Effect.succeed(Option.none()), + onSome: (manager) => + manager.withIdleTerminals(stillQuiet).pipe(Effect.map(Option.flatten)), + }); + }), ), ); From bf11dafd07a7dc08460ffadbe60b84afc90ab60d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:07:53 +0000 Subject: [PATCH 087/149] fix(server): open tabs keep loading the previous web build's chunks after an update --- apps/server/src/http.test.ts | 63 +++++++++++++++++++++++ apps/server/src/http.ts | 97 ++++++++++++++++++++++++++++++------ 2 files changed, 145 insertions(+), 15 deletions(-) diff --git a/apps/server/src/http.test.ts b/apps/server/src/http.test.ts index dc4ff0283d2e7..84d409b917816 100644 --- a/apps/server/src/http.test.ts +++ b/apps/server/src/http.test.ts @@ -144,6 +144,69 @@ it.layer( Layer.provideMerge(NodeServices.layer), ), )("static HTTP responses", (it) => { + it.effect("serves the previous build's lazy assets after restart and prunes older builds", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const staticDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-static-generations-" }); + yield* fs.makeDirectory(path.join(staticDir, "assets")); + yield* fs.makeDirectory(path.join(staticDir, ".vite")); + const builds = ["settings-aaaaaaaa.js", "settings-bbbbbbbb.js", "settings-cccccccc.js"]; + for (const [index, file] of builds.entries()) { + const asset = `assets/${file}`; + yield* fs.writeFileString(path.join(staticDir, asset), `export const build = ${index};`); + yield* fs.writeFileString( + path.join(staticDir, "index.html"), + `build ${index}`, + ); + yield* fs.writeFileString( + path.join(staticDir, ".vite", "manifest.json"), + JSON.stringify({ settings: { file: asset } }), + ); + const request = yield* makeStaticRequest(staticDir); + const current = yield* request(`/${asset}`); + expect(yield* current.text).toBe(`export const build = ${index};`); + if (index > 0) { + const previous = yield* request(`/assets/${builds[index - 1]}`); + expect(previous.status).toBe(200); + expect(previous.headers["content-type"]).toContain("javascript"); + expect(previous.headers["cache-control"]).toBe("public, max-age=31536000, immutable"); + expect(yield* previous.text).toBe(`export const build = ${index - 1};`); + const restarted = yield* makeStaticRequest(staticDir); + const stillPrevious = yield* restarted(`/assets/${builds[index - 1]}`); + expect(stillPrevious.status).toBe(200); + expect(yield* stillPrevious.text).toBe(`export const build = ${index - 1};`); + } + if (index === 2) { + const expired = yield* request(`/assets/${builds[0]}`); + expect(expired.status).toBe(404); + expect(yield* expired.text).toBe("Not Found"); + } + yield* fs.remove(path.join(staticDir, asset)); + } + }), + ); + + it.effect("returns 404 for missing assets while preserving SPA route fallback", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const staticDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-static-missing-" }); + yield* fs.writeFileString(path.join(staticDir, "index.html"), "app"); + const request = yield* makeStaticRequest(staticDir); + for (const resource of ["/assets/settings-deadbeef.js", "/assets/styles-deadbeef.css"]) { + const response = yield* request(resource); + expect(response.status).toBe(404); + expect(response.headers["content-type"]).not.toContain("text/html"); + expect(yield* response.text).toBe("Not Found"); + } + for (const route of ["/settings/providers", "/projects/github.com%2Fvercel%2Fnext.js"]) { + const response = yield* request(route); + expect(response.status).toBe(200); + expect(yield* response.text).toBe("app"); + } + }), + ); it.effect("revalidates non-HTML files and returns changed contents", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index de4ce26cc915d..450f92fcc4fab 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -498,6 +498,19 @@ const decodeBuildManifest = Schema.decodeUnknownEffect( ), ); +const isHashedBuildAsset = (asset: string) => /^assets\/[^/]+-[\w-]{8}\.[^/]+$/.test(asset); +const BuildAssetCache = Schema.fromJsonString( + Schema.Struct({ + current: Schema.Array(Schema.String), + previous: Schema.Array(Schema.String), + }), +); +const decodeBuildAssetCache = Schema.decodeUnknownEffect(BuildAssetCache); +const encodeBuildAssetCache = Schema.encodeEffect(BuildAssetCache); +const emptyBuildAssetCache = Schema.decodeUnknownSync(BuildAssetCache)( + '{"current":[],"previous":[]}', +); + const loadImmutableBuildAssets = Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; const staticDir = @@ -505,19 +518,59 @@ const loadImmutableBuildAssets = Effect.gen(function* () { if (!staticDir) return new Set(); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; - return yield* fileSystem.readFileString(path.join(staticDir, ".vite", "manifest.json")).pipe( - Effect.flatMap(decodeBuildManifest), - Effect.map( - (manifest) => - new Set( - Object.values(manifest).flatMap((entry) => [ - entry.file, - ...(entry.css ?? []), - ...(entry.assets ?? []), - ]), - ), + const current = yield* fileSystem + .readFileString(path.join(staticDir, ".vite", "manifest.json")) + .pipe( + Effect.flatMap(decodeBuildManifest), + Effect.map( + (manifest) => + new Set( + Object.values(manifest).flatMap((entry) => [ + entry.file, + ...(entry.css ?? []), + ...(entry.assets ?? []), + ]), + ), + ), + Effect.orElseSucceed(() => new Set()), + ); + if (config.devUrl || current.size === 0) return current; + + // Open tabs still import the previous build's lazy chunks after a service restart. + // Keep two generations on disk; never accumulate every installed build. + return yield* Effect.gen(function* () { + const cacheDir = path.join(config.providerStatusCacheDir, "web-assets"); + const manifestPath = path.join(cacheDir, "current.json"); + const saved = yield* fileSystem.readFileString(manifestPath).pipe( + Effect.flatMap(decodeBuildAssetCache), + Effect.orElseSucceed(() => emptyBuildAssetCache), + ); + const currentAssets = [...current].filter(isHashedBuildAsset); + const sameBuild = + currentAssets.length === saved.current.length && + currentAssets.every((asset) => saved.current.includes(asset)); + const previous = sameBuild ? saved.previous : saved.current; + const retained = new Set([...currentAssets, ...previous].filter(isHashedBuildAsset)); + const filesDir = path.join(cacheDir, "assets"); + yield* fileSystem.makeDirectory(filesDir, { recursive: true }); + for (const asset of current) { + if (!isHashedBuildAsset(asset)) continue; + yield* fileSystem.copyFile(path.join(staticDir, asset), path.join(cacheDir, asset)); + } + for (const file of yield* fileSystem.readDirectory(filesDir)) { + if (!retained.has(`assets/${file}`)) yield* fileSystem.remove(path.join(filesDir, file)); + } + yield* fileSystem.writeFileString( + manifestPath, + yield* encodeBuildAssetCache({ current: currentAssets, previous }), + ); + return new Set([...current, ...retained]); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not retain previous web build assets", { cause }).pipe( + Effect.as(current), + ), ), - Effect.orElseSucceed(() => new Set()), ); }); @@ -605,7 +658,23 @@ const handleStaticAndDevRequest = Effect.fn("handleStaticAndDevRequest")( } let opened = yield* openStaticFile(filePath); + if ( + !opened && + isHashedBuildAsset(staticRelativePath) && + immutableBuildAssets.has(staticRelativePath) + ) { + opened = yield* openStaticFile( + path.join(config.providerStatusCacheDir, "web-assets", staticRelativePath), + ); + } if (!opened) { + // A missing build chunk must fail its import, not load index.html as a script. + // Other dotted paths stay SPA routes: project keys can end in `next.js`. + if (/^assets[/\\]/.test(staticRelativePath)) + return HttpServerResponse.text("Not Found", { + status: 404, + headers: { "Cache-Control": "no-cache" }, + }); filePath = path.resolve(staticRoot, "index.html"); opened = yield* openStaticFile(filePath); if (!opened) { @@ -619,9 +688,7 @@ const handleStaticAndDevRequest = Effect.fn("handleStaticAndDevRequest")( // A hash-like name is not enough: custom static files can use the same naming pattern. const relativePath = path.relative(staticRoot, filePath).replaceAll("\\", "/"); const immutable = - !isHtml && - /^assets\/.+-[\w-]{8}\.[^/]+$/.test(relativePath) && - immutableBuildAssets.has(relativePath); + !isHtml && isHashedBuildAsset(relativePath) && immutableBuildAssets.has(relativePath); const headers: Record = { "Cache-Control": immutable ? "public, max-age=31536000, immutable" : "no-cache", }; From 7645122b6b20b810e5b1cd7b4bbb53e610f3b7bf Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:09:40 +0000 Subject: [PATCH 088/149] docs: a newer T3 Code release replaces one waiting for the update window --- docs/user/updating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/updating.md b/docs/user/updating.md index 63039cdfda224..40928de3a2222 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -84,7 +84,7 @@ hold updates back while they run. Then it installs: checks** is also on; - the newest T3 Code release on its channel, when it runs as a background service (`t3 service install`). It downloads first and restarts only inside that window. - While one release waits for the window, newer releases wait for it to install. + If a newer release is published while one waits, the newer one replaces it. Automatic server restarts also wait for integrated terminal commands to finish. From 678253595f022e5e452f3ef61a69738f1f2dbddd Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:11:01 +0000 Subject: [PATCH 089/149] fix(server): an interrupted provider install finishes its writes before it is stopped --- .../src/backend/DesktopBackendManager.test.ts | 2 +- .../src/backend/DesktopBackendManager.ts | 3 +- .../providerMaintenanceRunner.test.ts | 52 +++++++++++++++++++ .../src/provider/providerMaintenanceRunner.ts | 15 +++++- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index 24303aa66997c..e3e9ba62f317c 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -264,7 +264,7 @@ describe("DesktopBackendManager", () => { assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output"); assert.equal( Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)), - 2_000, + 5_000, ); assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 6b38dadcbdf85..81d84e7745336 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -65,7 +65,8 @@ const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5; const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1); const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100); const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1); -const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(2); +// Provider installers drain for up to three seconds during server shutdown. +const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5); const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5); const BACKEND_READINESS_PATH = "/.well-known/t3/environment"; const { logWarning: logBackendProcessWarning } = diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 7b91e1ff685fd..6819a8567b216 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -307,6 +307,58 @@ describe("providerMaintenanceRunner", () => { ), ), ); + + it.effect.each([true, false])( + "drains an interrupted installer before killing it (finishes: %s)", + (finishes) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const draining = yield* Deferred.make(); + let killed = false; + let exitReads = 0; + const spawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => ({ + ...mockHandle({}), + exitCode: Effect.suspend(() => { + exitReads++; + return Deferred.succeed(exitReads === 1 ? started : draining, undefined).pipe( + Effect.andThen(Deferred.await(finished)), + ); + }), + kill: () => + Effect.sync(() => { + killed = true; + }).pipe(Effect.andThen(Deferred.succeed(draining, undefined)), Effect.asVoid), + })), + ), + ); + const { registry } = yield* makeRegistry(baseNativeCliProvider); + yield* Effect.gen(function* () { + const runner = yield* makeTestRunner(registry); + const install = yield* runner.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkChild); + yield* Deferred.await(started); + const interrupted = yield* Fiber.interrupt(install).pipe(Effect.forkChild); + yield* Deferred.await(draining); + assert.isFalse(killed); + if (finishes) yield* Deferred.succeed(finished, ChildProcessSpawner.ExitCode(0)); + else yield* TestClock.adjust(Duration.seconds(3)); + yield* Fiber.join(interrupted); + assert.equal(killed, !finishes); + }).pipe(Effect.provide(spawner)); + }).pipe( + Effect.provide( + Layer.mergeAll( + TestClock.layer(), + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + ), + ), + ), + ); + it.effect("runs the allowlisted provider update command and records success", () => { const calls: Array<{ command: string; args: ReadonlyArray }> = []; return Effect.gen(function* () { diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 5193fc0b91e7f..b3a8b0fc62ac7 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -41,7 +41,8 @@ import { collectUint8StreamText } from "@t3tools/provider-core/server/collectStr const isServerProviderUpdateError = Schema.is(ServerProviderUpdateError); const UPDATE_TIMEOUT_MS = 5 * 60_000; -const UPDATE_FORCE_KILL_AFTER = Duration.seconds(5); +const UPDATE_FORCE_KILL_AFTER = Duration.seconds(1); +const UPDATE_SHUTDOWN_DRAIN = Duration.seconds(3); const UPDATE_OUTPUT_MAX_BYTES = 10_000; // Every progress publish resends the provider list to each client, so the // installer's output is sampled rather than forwarded line by line. @@ -130,7 +131,17 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR ), ); yield* Effect.addFinalizer(() => - child.kill({ forceKillAfter: UPDATE_FORCE_KILL_AFTER }).pipe(Effect.ignore), + // Give an interrupted installer time to finish its current writes before + // terminating it, within the desktop backend's five-second quit budget. + child.exitCode.pipe( + Effect.timeoutOption(UPDATE_SHUTDOWN_DRAIN), + Effect.orElseSucceed(() => Option.none()), + Effect.flatMap((exit) => + Option.isSome(exit) + ? Effect.void + : child.kill({ forceKillAfter: UPDATE_FORCE_KILL_AFTER }).pipe(Effect.ignore), + ), + ), ); // Holds the newest output line not yet reported; the sampler takes it. From 5267011ae77ac0e2b39d6e89dc7db707f19d0c9b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:11:01 +0000 Subject: [PATCH 090/149] fix(web): About shows the desktop version and the sidebar shows update progress right away --- .../settings/SettingsPanels.restore.test.tsx | 18 ++++++++++++++++- .../components/settings/SettingsPanels.tsx | 7 +++++-- .../sidebar/SidebarUpdatePill.test.tsx | 17 ++++++++++++++++ .../components/sidebar/SidebarUpdatePill.tsx | 20 +++++++++++-------- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/apps/web/src/components/settings/SettingsPanels.restore.test.tsx b/apps/web/src/components/settings/SettingsPanels.restore.test.tsx index 88c9cb1776e34..3fcb4abdbea47 100644 --- a/apps/web/src/components/settings/SettingsPanels.restore.test.tsx +++ b/apps/web/src/components/settings/SettingsPanels.restore.test.tsx @@ -1,10 +1,13 @@ import { DEFAULT_UNIFIED_SETTINGS, type UnifiedSettings } from "@t3tools/contracts"; +import { renderToString } from "react-dom/server"; +import { APP_VERSION } from "../../branding"; import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { reactHookHarness as hooks } from "../../test/reactHookHarness"; const state = vi.hoisted(() => ({ settings: null as UnifiedSettings | null, + desktopVersion: null as string | null, update: vi.fn(), confirm: vi.fn(), })); @@ -51,16 +54,29 @@ vi.mock("../../localApi", async (importOriginal) => ({ readLocalApi: () => ({ dialogs: { confirm: state.confirm } }), })); -import { useSettingsRestore } from "./SettingsPanels"; +vi.mock("../../state/desktopUpdate", () => ({ + useDesktopUpdateState: () => + state.desktopVersion === null ? null : { currentVersion: state.desktopVersion }, +})); + +import { AboutVersionTitle, useSettingsRestore } from "./SettingsPanels"; beforeEach(() => { hooks.reset(); vi.clearAllMocks(); + state.desktopVersion = null; state.settings = { ...DEFAULT_UNIFIED_SETTINGS }; state.confirm.mockResolvedValue(true); }); describe("restoring V2 settings", () => { + it("shows the desktop version rather than the web bundle version", () => { + state.desktopVersion = "9.8.7"; + expect(renderToString()).toContain("9.8.7"); + expect(renderToString()).not.toContain(APP_VERSION); + state.desktopVersion = null; + expect(renderToString()).toContain(APP_VERSION); + }); it.each([ ["persistComposerContextStrip", "Composer context"], ["autoResumeLimitedThreads", "Auto-resume limited threads"], diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 880588a24efd8..6e2053d19092b 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -269,11 +269,14 @@ function backgroundActivityProfileSettings(profile: BackgroundActivityProfile) { }; } -function AboutVersionTitle() { +export function AboutVersionTitle() { + const updateState = useDesktopUpdateState(); return ( Version - {APP_VERSION} + + {updateState?.currentVersion ?? APP_VERSION} + ); } diff --git a/apps/web/src/components/sidebar/SidebarUpdatePill.test.tsx b/apps/web/src/components/sidebar/SidebarUpdatePill.test.tsx index d3c6f368a54df..e6cc3993f5023 100644 --- a/apps/web/src/components/sidebar/SidebarUpdatePill.test.tsx +++ b/apps/web/src/components/sidebar/SidebarUpdatePill.test.tsx @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vite-plus/test"; import { handleSidebarUpdateReleaseNotesPopoverOpenChange, + resolveSidebarUpdatePresentation, openSidebarUpdateReleaseNotesPopoverOnForwardTab, shouldUseSidebarUpdateReleaseNotesPopover, } from "./SidebarUpdatePill"; @@ -27,6 +28,22 @@ const nightlyState: DesktopUpdateState = { }; describe("sidebar update release notes popover", () => { + it.each([ + ["download", false, "available"], + ["none", true, "downloading"], + ["install", false, "downloaded"], + ] as const)( + "shows %s immediately while the check animation is latched", + (action, isDownloading, iconStatus) => { + expect( + resolveSidebarUpdatePresentation({ action, isDownloading, showCheckIcon: true }), + ).toEqual({ + iconStatus, + showUpdateDetails: true, + showUpdateIconState: true, + }); + }, + ); it("uses the popover only for visible nightly release notes", () => { expect(shouldUseSidebarUpdateReleaseNotesPopover(true, nightlyState)).toBe(true); expect(shouldUseSidebarUpdateReleaseNotesPopover(false, nightlyState)).toBe(false); diff --git a/apps/web/src/components/sidebar/SidebarUpdatePill.tsx b/apps/web/src/components/sidebar/SidebarUpdatePill.tsx index a1c82d3d51f38..a595a85ffaa0b 100644 --- a/apps/web/src/components/sidebar/SidebarUpdatePill.tsx +++ b/apps/web/src/components/sidebar/SidebarUpdatePill.tsx @@ -61,7 +61,7 @@ export function openSidebarUpdateReleaseNotesPopoverOnForwardTab( flushSync(() => handle.open(triggerId)); } -function resolveSidebarUpdatePresentation({ +export function resolveSidebarUpdatePresentation({ action, isDownloading, showCheckIcon, @@ -71,7 +71,8 @@ function resolveSidebarUpdatePresentation({ readonly showCheckIcon: boolean; }) { const showUpdateDetails = action !== "none" || isDownloading; - const iconStatus = showCheckIcon + const showChecking = showCheckIcon && !showUpdateDetails; + const iconStatus = showChecking ? "checking" : action === "install" ? "downloaded" @@ -84,7 +85,7 @@ function resolveSidebarUpdatePresentation({ return { iconStatus, showUpdateDetails, - showUpdateIconState: showUpdateDetails && !showCheckIcon, + showUpdateIconState: showUpdateDetails && !showChecking, } as const; } @@ -133,11 +134,14 @@ function SidebarUpdateControl() { const action = state ? resolveDesktopUpdateButtonAction(state) : "none"; const isDownloading = state?.status === "downloading"; - const showCheckIcon = shouldShowDesktopUpdateCheckIcon({ - isAnimationLatched: isCheckAnimationLatched, - isChecking: state?.status === "checking", - prefersReducedMotion, - }); + const showCheckIcon = + action === "none" && + !isDownloading && + shouldShowDesktopUpdateCheckIcon({ + isAnimationLatched: isCheckAnimationLatched, + isChecking: state?.status === "checking", + prefersReducedMotion, + }); const { iconStatus, showUpdateDetails, showUpdateIconState } = resolveSidebarUpdatePresentation({ action, isDownloading, From 821364bbb175337d667232a2b8dae8c4a3227e02 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:11:01 +0000 Subject: [PATCH 091/149] fix(desktop): a downloaded update installs once at a real quit and a failed download is not retried every poll --- apps/desktop/src/app/DesktopLifecycle.test.ts | 138 +++++++++------ apps/desktop/src/app/DesktopLifecycle.ts | 40 ++++- .../src/electron/ElectronUpdater.test.ts | 38 ++++ apps/desktop/src/electron/ElectronUpdater.ts | 59 +++++-- .../src/updates/DesktopUpdates.test.ts | 165 ++++++++++++++---- apps/desktop/src/updates/DesktopUpdates.ts | 49 ++---- .../desktop/src/updates/updatesTestHarness.ts | 2 +- .../src/window/DesktopApplicationMenu.test.ts | 1 + docs/user/updating.md | 2 - 9 files changed, 357 insertions(+), 137 deletions(-) diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.ts index 5caa18a53aabf..70152f26e93ab 100644 --- a/apps/desktop/src/app/DesktopLifecycle.test.ts +++ b/apps/desktop/src/app/DesktopLifecycle.test.ts @@ -13,6 +13,7 @@ import * as DesktopEnvironment from "./DesktopEnvironment.ts"; import * as DesktopLifecycle from "./DesktopLifecycle.ts"; import * as DesktopShutdown from "./DesktopShutdown.ts"; import * as DesktopState from "./DesktopState.ts"; +import * as DesktopUpdates from "../updates/DesktopUpdates.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; function layerElectronApp( @@ -115,6 +116,11 @@ describe("DesktopLifecycle", () => { const layer = DesktopLifecycle.layer.pipe( Layer.provideMerge(layerElectronApp(appListeners)), Layer.provideMerge(layerElectronTheme), + Layer.provideMerge( + Layer.succeed(DesktopUpdates.DesktopUpdates, { + installOnQuit: Effect.succeed(false), + } as DesktopUpdates.DesktopUpdates["Service"]), + ), Layer.provideMerge( layerElectronWindow( Effect.sync(() => { @@ -157,67 +163,88 @@ describe("DesktopLifecycle", () => { }, ); - it.effect("destroys windows before waiting for backend shutdown", () => - Effect.gen(function* () { - const appListeners = new Map void>(); - const shutdownRequested = yield* Deferred.make(); - const allowShutdown = yield* Deferred.make(); - const quitRequested = yield* Deferred.make(); - const events: string[] = []; + it.effect.each([false, true])( + "destroys windows before waiting for backend shutdown (install: %s)", + (installOnQuit) => + Effect.gen(function* () { + const appListeners = new Map void>(); + const shutdownRequested = yield* Deferred.make(); + const allowShutdown = yield* Deferred.make(); + const quitRequested = yield* Deferred.make(); + const events: string[] = []; - const quit = Effect.sync(() => { - events.push("quit"); - }).pipe(Effect.andThen(Deferred.succeed(quitRequested, undefined)), Effect.asVoid); - const destroyAll = Effect.sync(() => { - events.push("destroy"); - }); - const flushMainWindowBounds = Effect.sync(() => { - events.push("flush"); - }); + const quit = Effect.sync(() => { + events.push("quit"); + }).pipe(Effect.andThen(Deferred.succeed(quitRequested, undefined)), Effect.asVoid); + const destroyAll = Effect.sync(() => { + events.push("destroy"); + }); + const flushMainWindowBounds = Effect.sync(() => { + events.push("flush"); + }); - const layerDesktopShutdown = Layer.succeed(DesktopShutdown.DesktopShutdown, { - request: Effect.sync(() => { - events.push("request"); - }).pipe(Effect.andThen(Deferred.succeed(shutdownRequested, undefined)), Effect.asVoid), - awaitRequest: Deferred.await(shutdownRequested), - markComplete: Deferred.succeed(allowShutdown, undefined).pipe(Effect.asVoid), - awaitComplete: Deferred.await(allowShutdown), - isComplete: Deferred.isDone(allowShutdown), - }); + const layerDesktopShutdown = Layer.succeed(DesktopShutdown.DesktopShutdown, { + request: Effect.sync(() => { + events.push("request"); + }).pipe(Effect.andThen(Deferred.succeed(shutdownRequested, undefined)), Effect.asVoid), + awaitRequest: Deferred.await(shutdownRequested), + markComplete: Deferred.succeed(allowShutdown, undefined).pipe(Effect.asVoid), + awaitComplete: Deferred.await(allowShutdown), + isComplete: Deferred.isDone(allowShutdown), + }); - const layerEnvironment = Layer.succeed(DesktopEnvironment.DesktopEnvironment, { - platform: "darwin", - isDevelopment: false, - } as DesktopEnvironment.DesktopEnvironment["Service"]); + const layerEnvironment = Layer.succeed(DesktopEnvironment.DesktopEnvironment, { + platform: "darwin", + isDevelopment: false, + } as DesktopEnvironment.DesktopEnvironment["Service"]); - const layer = DesktopLifecycle.layer.pipe( - Layer.provideMerge(layerElectronApp(appListeners, quit)), - Layer.provideMerge(layerElectronTheme), - Layer.provideMerge(layerElectronWindow(destroyAll)), - Layer.provideMerge(layerDesktopWindow({ flushMainWindowBounds })), - Layer.provideMerge(layerEnvironment), - Layer.provideMerge(layerDesktopShutdown), - Layer.provideMerge(DesktopState.layer), - ); + const layer = DesktopLifecycle.layer.pipe( + Layer.provideMerge(layerElectronApp(appListeners, quit)), + Layer.provideMerge(layerElectronTheme), + Layer.provideMerge( + Layer.succeed(DesktopUpdates.DesktopUpdates, { + installOnQuit: Effect.sync(() => { + events.push("installOnQuit"); + return installOnQuit; + }).pipe( + Effect.tap(() => + installOnQuit ? Deferred.succeed(quitRequested, undefined) : Effect.void, + ), + ), + } as DesktopUpdates.DesktopUpdates["Service"]), + ), + Layer.provideMerge(layerElectronWindow(destroyAll)), + Layer.provideMerge(layerDesktopWindow({ flushMainWindowBounds })), + Layer.provideMerge(layerEnvironment), + Layer.provideMerge(layerDesktopShutdown), + Layer.provideMerge(DesktopState.layer), + ); - yield* Effect.scoped( - Effect.gen(function* () { - const lifecycle = yield* DesktopLifecycle.DesktopLifecycle; - yield* lifecycle.register; + yield* Effect.scoped( + Effect.gen(function* () { + const lifecycle = yield* DesktopLifecycle.DesktopLifecycle; + yield* lifecycle.register; - const event = { preventDefault: () => undefined } as Electron.Event; - appListeners.get("before-quit")?.(event); + const event = { preventDefault: () => undefined } as Electron.Event; + appListeners.get("before-quit")?.(event); - yield* Deferred.await(shutdownRequested); - const eventsBeforeCleanup = [...events]; - yield* Deferred.succeed(allowShutdown, undefined); - yield* Deferred.await(quitRequested); + appListeners.get("before-quit")?.(event); + yield* Deferred.await(shutdownRequested); + const eventsBeforeCleanup = [...events]; + yield* Deferred.succeed(allowShutdown, undefined); + yield* Deferred.await(quitRequested); - assert.deepEqual(eventsBeforeCleanup, ["flush", "destroy", "request"]); - assert.deepEqual(events, ["flush", "destroy", "request", "quit"]); - }), - ).pipe(Effect.provide(layer)); - }), + assert.deepEqual(eventsBeforeCleanup, ["flush", "destroy", "request"]); + assert.deepEqual(events, [ + "flush", + "destroy", + "request", + "installOnQuit", + ...(installOnQuit ? [] : ["quit"]), + ]); + }), + ).pipe(Effect.provide(layer)); + }), ); it.effect("ignores app activation while quitting", () => @@ -234,6 +261,11 @@ describe("DesktopLifecycle", () => { const layer = DesktopLifecycle.layer.pipe( Layer.provideMerge(layerElectronApp(appListeners)), Layer.provideMerge(layerElectronTheme), + Layer.provideMerge( + Layer.succeed(DesktopUpdates.DesktopUpdates, { + installOnQuit: Effect.succeed(false), + } as DesktopUpdates.DesktopUpdates["Service"]), + ), Layer.provideMerge(layerElectronWindow()), Layer.provideMerge(layerDesktopWindow({ activate })), Layer.provideMerge(layerEnvironment), diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.ts index 3b1ca6c820484..00d35da9ccbb4 100644 --- a/apps/desktop/src/app/DesktopLifecycle.ts +++ b/apps/desktop/src/app/DesktopLifecycle.ts @@ -13,6 +13,7 @@ import * as DesktopShutdown from "./DesktopShutdown.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronTheme from "../electron/ElectronTheme.ts"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; +import * as DesktopUpdates from "../updates/DesktopUpdates.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as DesktopState from "./DesktopState.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; @@ -39,10 +40,11 @@ export type DesktopLifecycleRuntimeServices = type DesktopLifecycleRegistrationServices = | DesktopLifecycleRuntimeServices - | ElectronWindow.ElectronWindow; + | ElectronWindow.ElectronWindow + | DesktopUpdates.DesktopUpdates; /** - * @effect-expect-leaking DesktopEnvironment | DesktopShutdown | DesktopState | DesktopWindow | ElectronApp | ElectronTheme | ElectronUpdater | ElectronWindow + * @effect-expect-leaking DesktopEnvironment | DesktopShutdown | DesktopState | DesktopWindow | ElectronApp | ElectronTheme | ElectronUpdater | ElectronWindow | DesktopUpdates */ export class DesktopLifecycle extends Context.Service< DesktopLifecycle, @@ -138,7 +140,8 @@ function handleBeforeQuit( void runEffect( Effect.gen(function* () { const electronApp = yield* ElectronApp.ElectronApp; - yield* electronApp.quit; + const updates = yield* DesktopUpdates.DesktopUpdates; + if (!(yield* updates.installOnQuit)) yield* electronApp.quit; }).pipe(Effect.withSpan("desktop.lifecycle.quitAfterShutdown")), ); }); @@ -146,6 +149,7 @@ function handleBeforeQuit( function quitFromSignal( signal: "SIGINT" | "SIGTERM", + markQuitAllowed: () => void, runEffect: ( effect: Effect.Effect, ) => Promise, @@ -159,7 +163,9 @@ function quitFromSignal( if (wasQuitting) return; yield* logLifecycleInfo("process signal received", { signal }); yield* requestDesktopShutdownAndWait(); - yield* electronApp.quit; + markQuitAllowed(); + const updates = yield* DesktopUpdates.DesktopUpdates; + if (!(yield* updates.installOnQuit)) yield* electronApp.quit; }).pipe(Effect.withSpan("desktop.lifecycle.processSignal")), ); } @@ -181,8 +187,8 @@ export const make = DesktopLifecycle.of({ return; } // app.exit still emits "quit", where electron-updater would install a - // downloaded update under the relaunched app on Windows and Linux. Keep - // it for a real quit. A macOS update Squirrel already staged still applies. + // downloaded update under the relaunched app on Windows and Linux. + // macOS staging is deferred until a real quit by DesktopUpdates. yield* electronUpdater.setAutoInstallOnAppQuit(false); yield* electronApp.relaunch({ execPath: process.execPath, @@ -207,6 +213,7 @@ export const make = DesktopLifecycle.of({ const context = yield* Effect.context(); const runEffect = Effect.runPromiseWith(context); let quitAllowed = false; + let quitRequested = false; let updaterQuitAllowed = false; yield* electronTheme.onUpdated(() => { void runEffect( @@ -233,6 +240,11 @@ export const make = DesktopLifecycle.of({ ); }); yield* electronApp.on("before-quit", (event: Electron.Event) => { + if (quitRequested && !quitAllowed && !updaterQuitAllowed) { + event.preventDefault(); + return; + } + quitRequested = true; handleBeforeQuit( event, runEffect, @@ -265,10 +277,22 @@ export const make = DesktopLifecycle.of({ if (environment.platform !== "win32") { yield* addScopedListener(process, "SIGINT", () => { - quitFromSignal("SIGINT", runEffect); + quitFromSignal( + "SIGINT", + () => { + quitAllowed = true; + }, + runEffect, + ); }); yield* addScopedListener(process, "SIGTERM", () => { - quitFromSignal("SIGTERM", runEffect); + quitFromSignal( + "SIGTERM", + () => { + quitAllowed = true; + }, + runEffect, + ); }); } }).pipe(Effect.withSpan("desktop.lifecycle.register")), diff --git a/apps/desktop/src/electron/ElectronUpdater.test.ts b/apps/desktop/src/electron/ElectronUpdater.test.ts index 1e005d26fdf5d..43c7fdfbae682 100644 --- a/apps/desktop/src/electron/ElectronUpdater.test.ts +++ b/apps/desktop/src/electron/ElectronUpdater.test.ts @@ -1,5 +1,9 @@ import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Duration from "effect/Duration"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import { beforeEach, vi } from "vite-plus/test"; const { autoUpdaterMock } = vi.hoisted(() => ({ @@ -8,6 +12,7 @@ const { autoUpdaterMock } = vi.hoisted(() => ({ allowPrerelease: false, autoDownload: true, autoInstallOnAppQuit: true, + autoRunAppAfterInstall: true, channel: "latest", disableDifferentialDownload: false, fullChangelog: false, @@ -20,6 +25,12 @@ const { autoUpdaterMock } = vi.hoisted(() => ({ }, })); +vi.mock("electron", async () => { + const { EventEmitter } = await import("node:events"); + return { autoUpdater: new EventEmitter() }; +}); +import * as Electron from "electron"; + vi.mock("electron-updater", () => ({ autoUpdater: autoUpdaterMock, })); @@ -45,6 +56,32 @@ describe("ElectronUpdater", () => { autoUpdaterMock.setFeedURL.mockClear(); }); + it.effect.each(["downloaded", "error", "timeout"])( + "finishes a macOS real quit after native %s", + (outcome) => + Effect.gen(function* () { + autoUpdaterMock.quitAndInstall.mockImplementationOnce(() => { + if (outcome === "downloaded") Electron.autoUpdater.emit("update-downloaded"); + if (outcome === "error") Electron.autoUpdater.emit("error", new Error("staging failed")); + }); + const updater = yield* ElectronUpdater.ElectronUpdater; + const quit = yield* updater + .quitAndInstall({ isSilent: true, isForceRunAfter: false }) + .pipe(Effect.exit, Effect.forkChild); + yield* Effect.yieldNow; + if (outcome === "timeout") yield* TestClock.adjust(Duration.seconds(5)); + const result = yield* Fiber.join(quit); + assert.equal(result._tag, outcome === "downloaded" ? "Success" : "Failure"); + assert.isFalse(autoUpdaterMock.autoRunAppAfterInstall); + assert.equal(Electron.autoUpdater.listenerCount("error"), 0); + assert.equal(Electron.autoUpdater.listenerCount("update-downloaded"), 0); + }).pipe( + Effect.provide(ElectronUpdater.layer), + Effect.provideService(HostProcess.Platform, "darwin"), + Effect.provide(TestClock.layer()), + ), + ); + it.effect("scopes updater event listeners", () => Effect.gen(function* () { const listener = vi.fn(); @@ -134,6 +171,7 @@ describe("ElectronUpdater", () => { ); assert.notInclude(error.message, cause.message); assert.deepEqual(autoUpdaterMock.quitAndInstall.mock.calls, [[true, false]]); + assert.isFalse(autoUpdaterMock.autoRunAppAfterInstall); }).pipe(Effect.provide(ElectronUpdater.layer)), ); }); diff --git a/apps/desktop/src/electron/ElectronUpdater.ts b/apps/desktop/src/electron/ElectronUpdater.ts index e42ebc1003ab4..959a305d9d352 100644 --- a/apps/desktop/src/electron/ElectronUpdater.ts +++ b/apps/desktop/src/electron/ElectronUpdater.ts @@ -1,4 +1,8 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Electron from "electron"; import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Option from "effect/Option"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -138,18 +142,53 @@ export const make = ElectronUpdater.of({ }).pipe(Effect.asVoid); }), quitAndInstall: ({ isSilent, isForceRunAfter }) => - Effect.suspend(() => { + Effect.gen(function* () { const channel = autoUpdater.channel; - return Effect.try({ - try: () => autoUpdater.quitAndInstall(isSilent, isForceRunAfter), - catch: (cause) => - new ElectronUpdaterQuitAndInstallError({ - channel, - isSilent, - isForceRunAfter, - cause, + const makeError = (cause: unknown) => + new ElectronUpdaterQuitAndInstallError({ + channel, + isSilent, + isForceRunAfter, + cause, + }); + const install = () => { + // MacUpdater ignores the positional flags and reads this property. + autoUpdater.autoRunAppAfterInstall = isForceRunAfter; + autoUpdater.quitAndInstall(isSilent, isForceRunAfter); + }; + if ((yield* HostProcess.Platform) !== "darwin" || isForceRunAfter) { + return yield* Effect.try({ try: install, catch: makeError }); + } + // A real quit must still exit if Squirrel cannot stage the archive. + const staged = yield* Effect.tryPromise({ + try: (signal) => + new Promise((resolve, reject) => { + const native = Electron.autoUpdater; + const cleanup = () => { + native.removeListener("update-downloaded", downloaded); + native.removeListener("error", failed); + signal.removeEventListener("abort", cleanup); + }; + const downloaded = () => { + cleanup(); + resolve(); + }; + const failed = (cause: unknown) => { + cleanup(); + reject(cause); + }; + native.once("update-downloaded", downloaded); + native.once("error", failed); + signal.addEventListener("abort", cleanup, { once: true }); + try { + install(); + } catch (cause) { + failed(cause); + } }), - }); + catch: makeError, + }).pipe(Effect.timeoutOption(Duration.seconds(5))); + if (Option.isNone(staged)) return yield* makeError(new Error("Squirrel staging timed out.")); }), on: (eventName, listener) => { const eventTarget = autoUpdater as unknown as { diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index e4e5a3b6b054d..b82c8410d86de 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -106,6 +106,87 @@ describe("DesktopUpdates", () => { }), ); + it.effect("leaves RPM installs disabled", () => { + const harness = makeHarness({ platform: "linux", packageType: "rpm" }); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + assert.equal((yield* updates.getState).status, "disabled"); + assert.isFalse((yield* updates.check("manual")).checked); + assert.isFalse(yield* updates.installOnQuit); + assert.equal(harness.downloadCount(), 0); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + + it.effect( + "recovers background checks after being offline without downloading during errors", + () => { + let offline = true; + const harness = makeHarness({ + checkForUpdates: Effect.suspend(() => + offline + ? Effect.fail( + new ElectronUpdater.ElectronUpdaterCheckForUpdatesError({ + channel: "latest", + cause: new Error("offline"), + }), + ) + : Effect.sync(() => harness.emit("update-available", { version: "1.2.4" })), + ), + }); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* TestClock.adjust(Duration.seconds(15)); + assert.equal((yield* updates.getState).status, "error"); + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.checkCount(), 2); + assert.equal(harness.downloadCount(), 0); + offline = false; + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.checkCount(), 3); + assert.equal(harness.downloadCount(), 1); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }, + ); + + it.effect("refuses channel changes and installs while quitting during a download", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const harness = makeHarness({ + downloadUpdate: Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finish)), + ), + }); + yield* Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + const state = yield* DesktopState.DesktopState; + yield* updates.configure; + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + const download = yield* updates.download.pipe(Effect.forkScoped); + yield* Deferred.await(started); + const error = yield* updates.setChannel("nightly").pipe(Effect.flip); + assert.instanceOf(error, DesktopUpdates.DesktopUpdateActionInProgressError); + yield* Ref.set(state.quitting, true); + assert.isFalse(yield* updates.installOnQuit); + assert.isFalse((yield* updates.install).accepted); + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.checkCount(), 0); + assert.equal(harness.quitAndInstalls(), 0); + yield* Deferred.succeed(finish, undefined); + yield* Fiber.join(download); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }), + ); + it.effect("subscribe delivers the latest state plus subsequent changes", () => { const harness = makeHarness(); @@ -852,40 +933,60 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect( - "keeps a staged macOS download's channel even when a newer release is available", - () => { - const harness = makeHarness(); - - return Effect.scoped( - Effect.gen(function* () { - const settings = yield* DesktopAppSettings.DesktopAppSettings; - const updates = yield* DesktopUpdates.DesktopUpdates; - yield* updates.configure; - harness.emit("update-downloaded", { version: "1.2.4" }); - yield* flushCallbacks; - harness.emit("update-available", { version: "1.2.5" }); - yield* flushCallbacks; - const beforeChannelChange = yield* updates.getState; - assert.isNull(beforeChannelChange.downloadedVersion); + it.effect("defers Squirrel staging until a real quit and allows channel changes", () => { + const harness = makeHarness(); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + assert.isFalse(harness.autoInstallOnAppQuit()); + yield* flushCallbacks; + assert.equal((yield* updates.getState).downloadedVersion, "1.2.4"); + assert.equal(harness.quitAndInstalls(), 0); + assert.isTrue(yield* updates.installOnQuit); + assert.equal(harness.quitAndInstalls(), 1); + assert.equal((yield* updates.setChannel("nightly")).channel, "nightly"); + assert.isFalse(yield* updates.installOnQuit); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); - const exit = yield* updates.setChannel("nightly").pipe(Effect.exit); - assert.equal(exit._tag, "Failure"); - if (exit._tag === "Failure") { - const error = Cause.squash(exit.cause); - assert.instanceOf(error, DesktopUpdates.DesktopUpdateChannelStagedError); - assert.include(error.message, "Install the downloaded update"); - } - assert.deepEqual(yield* updates.getState, beforeChannelChange); - assert.equal((yield* settings.get).updateChannel, "latest"); - assert.isTrue(harness.autoInstallOnAppQuit()); - assert.equal(harness.checkCount(), 0); - assert.deepEqual(yield* updates.setChannel("latest"), beforeChannelChange); - assert.isTrue((yield* updates.check("manual")).checked); + it.effect("does not repeat a rejected background download on later polls", () => { + let offeredVersion = "1.2.4"; + const harness = makeHarness({ + platform: "win32", + checkForUpdates: Effect.sync(() => + harness.emit("update-available", { version: offeredVersion }), + ), + downloadUpdate: Effect.fail( + new ElectronUpdater.ElectronUpdaterDownloadUpdateError({ + channel: "latest", + cause: new Error("offline"), }), - ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); - }, - ); + ), + }); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* TestClock.adjust(Duration.seconds(15)); + assert.equal(harness.downloadCount(), 1); + assert.equal((yield* updates.getState).errorContext, "download"); + assert.isFalse(harness.autoInstallOnAppQuit()); + for (let poll = 0; poll < 3; poll++) { + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.checkCount(), poll + 2); + assert.equal(harness.downloadCount(), 1); + } + assert.isTrue((yield* updates.download).accepted); + assert.equal(harness.downloadCount(), 2); + offeredVersion = "1.2.5"; + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.downloadCount(), 3); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); it.effect("releases the channel and install on quit when macOS staging fails", () => { const harness = makeHarness({ diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 5b083a2a22640..6d15a3f563785 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -101,15 +101,6 @@ export class DesktopUpdateChannelPersistenceError extends Schema.TaggedError()( - "DesktopUpdateChannelStagedError", - { requestedChannel: DesktopUpdateChannelSchema }, -) { - override get message(): string { - return "Install the downloaded update by quitting and reopening the app before changing channels on macOS."; - } -} - export class DesktopUpdatePollerError extends Schema.TaggedError()( "DesktopUpdatePollerError", { @@ -163,7 +154,6 @@ export type DesktopUpdateConfigureError = never; export const DesktopUpdateSetChannelError = Schema.Union([ DesktopUpdateActionInProgressError, DesktopUpdateChannelPersistenceError, - DesktopUpdateChannelStagedError, ]); export type DesktopUpdateSetChannelError = typeof DesktopUpdateSetChannelError.Type; @@ -194,6 +184,8 @@ export class DesktopUpdates extends Context.Service< readonly check: (reason: string) => Effect.Effect; readonly download: Effect.Effect; readonly install: Effect.Effect; + /** Called after backend shutdown on a real quit, never a settings relaunch. */ + readonly installOnQuit: Effect.Effect; readonly installPrepared: ( expectedVersion: string, ) => Effect.Effect; @@ -300,8 +292,6 @@ export const make = Effect.gen(function* () { const updaterConfiguredRef = yield* Ref.make(false); const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1); const failedBackgroundDownloadRef = yield* Ref.make(null); - // Native macOS staging cannot be cancelled once Squirrel has the download. - const nativeUpdateStagedRef = yield* Ref.make(false); const updateStateRef = yield* Ref.make( createInitialDesktopUpdateState( environment.appVersion, @@ -481,17 +471,7 @@ export const make = Effect.gen(function* () { return { accepted: false, completed: false }; } - // A failed download leaves nothing to install on quit: electron-updater drops - // the previous installer, and on macOS update-downloaded fires before Squirrel - // stages the download, so a later failure means nothing was staged. - const stagedBefore = yield* Ref.get(nativeUpdateStagedRef); - const releaseFailedDownload = Effect.all( - [ - Ref.set(nativeUpdateStagedRef, stagedBefore), - electronUpdater.setAutoInstallOnAppQuit(false), - ], - { discard: true }, - ); + const releaseFailedDownload = electronUpdater.setAutoInstallOnAppQuit(false); return yield* Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); @@ -932,11 +912,10 @@ export const make = Effect.gen(function* () { }); return; } - if (environment.platform === "darwin") { - yield* Ref.set(nativeUpdateStagedRef, true); - } + // Squirrel applies a staged update on every process exit, including a + // settings relaunch. Defer staging until the lifecycle requests a real quit. // Installing a .deb asks for a password, which must not appear on quit. - if (!isDebPackage) { + if (environment.platform !== "darwin" && !isDebPackage) { yield* electronUpdater.setAutoInstallOnAppQuit(true); } yield* setState(reduceDesktopUpdateStateOnDownloadComplete(state, info.version)); @@ -1051,10 +1030,6 @@ export const make = Effect.gen(function* () { return state; } - if (yield* Ref.get(nativeUpdateStagedRef)) { - return yield* new DesktopUpdateChannelStagedError({ requestedChannel: nextChannel }); - } - yield* desktopSettings .setUpdateChannel(nextChannel) .pipe( @@ -1102,6 +1077,18 @@ export const make = Effect.gen(function* () { state: yield* Ref.get(updateStateRef), }; }).pipe(Effect.withSpan("desktop.updates.download")), + installOnQuit: Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + if (environment.platform !== "darwin" || state.downloadedVersion === null) return false; + yield* electronUpdater.quitAndInstall({ isSilent: true, isForceRunAfter: false }); + return true; + }).pipe( + Effect.catchCause((cause) => + logUpdaterError("Could not install the downloaded update on quit.", { cause }).pipe( + Effect.as(false), + ), + ), + ), install: installWithExpectedVersion().pipe( Effect.map(({ accepted, completed, state }) => ({ accepted, completed, state })), ), diff --git a/apps/desktop/src/updates/updatesTestHarness.ts b/apps/desktop/src/updates/updatesTestHarness.ts index d843552a065ec..46c6f2e125fe3 100644 --- a/apps/desktop/src/updates/updatesTestHarness.ts +++ b/apps/desktop/src/updates/updatesTestHarness.ts @@ -29,7 +29,7 @@ export interface UpdatesHarnessOptions { readonly beforeSetUpdateChannel?: Effect.Effect; readonly setUpdateChannelError?: DesktopAppSettings.DesktopSettingsWriteError; readonly setDisableDifferentialDownload?: Effect.Effect; - readonly downloadUpdate?: Effect.Effect; + readonly downloadUpdate?: Effect.Effect; readonly quitAndInstall?: Effect.Effect; readonly stopBackend?: Effect.Effect; readonly startBackend?: Effect.Effect; diff --git a/apps/desktop/src/window/DesktopApplicationMenu.test.ts b/apps/desktop/src/window/DesktopApplicationMenu.test.ts index f111dac25db8d..50bea71ba329a 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.test.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.test.ts @@ -58,6 +58,7 @@ const layerElectronDialog = Layer.succeed(ElectronDialog.ElectronDialog, { const layerDesktopUpdates = Layer.succeed(DesktopUpdates.DesktopUpdates, { getState: Effect.die("unexpected getState"), + installOnQuit: Effect.succeed(false), isActionActive: Effect.succeed(false), isInstallActive: Effect.succeed(false), subscribe: Effect.die("unexpected subscribe"), diff --git a/docs/user/updating.md b/docs/user/updating.md index 54eabaa4f1760..38ab22cc9fa7d 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -92,8 +92,6 @@ The desktop app downloads its updates in the background and installs them when you quit, whatever **Update automatically** is set to. Linux `.deb` updates ask for your password, so they wait for you to choose **Install**. The app stays open while you work, including when it hosts multiple backends. -On macOS, quit and reopen the app to install a downloaded update before switching -update channels. Servers started from a terminal or with `npx` are never replaced automatically. A failed provider update is retried after 6 hours, when a newer version is From 00b917029ae04053374b305a66370109521a4038 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:32:09 +0000 Subject: [PATCH 092/149] fix(server): steering lost in a restart is prompted instead of folded into a surviving OpenCode turn --- .../ProviderTurnStartService.test.ts | 51 +++++++++++++++++++ .../ProviderTurnStartService.ts | 7 ++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index d6c12fab7ee15..7cbba3cead61f 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -171,6 +171,8 @@ function makeLocalCommandHarness(input: { readonly writeFailure?: unknown; /** Loads the thread and starts the run, then fails every later state read. */ readonly failReadsAfterRunning?: boolean; + /** Makes the run a restart continuation of an interrupted source run. */ + readonly restartSource?: { readonly pendingSteering: boolean }; }) { const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z"); const threadId = ThreadId.make("thread-native-account-command"); @@ -184,6 +186,9 @@ function makeLocalCommandHarness(input: { const newInstanceId = ProviderInstanceId.make("codex-personal"); const checkpointScopeId = CheckpointScopeId.make("scope-native-account-command"); const messageId = MessageId.make("message-native-account-command"); + const sourceRunId = RunId.make("restart-source-run"); + const sourceMessageId = MessageId.make("restart-source-message"); + const steeringMessageId = MessageId.make("restart-source-steering"); const run: OrchestrationV2ThreadProjection["runs"][number] = { id: runId, threadId, @@ -200,6 +205,7 @@ function makeLocalCommandHarness(input: { completedAt: null, checkpointId: null, contextHandoffId: null, + ...(input.restartSource === undefined ? {} : { restartContinuationOfRunId: sourceRunId }), }; const providerThread: OrchestrationV2ThreadProjection["providerThreads"][number] = { id: providerThreadId, @@ -252,6 +258,22 @@ function makeLocalCommandHarness(input: { }, ] : []), + ...(input.restartSource === undefined + ? [] + : [ + { + ...run, + id: sourceRunId, + ordinal: 1, + status: "cancelled" as const, + userMessageId: sourceMessageId, + activeAttemptId: RunAttemptId.make("restart-source-attempt"), + restartContinuationOfRunId: undefined, + ...(input.restartSource.pendingSteering + ? { restartPendingMessageIds: [steeringMessageId] } + : {}), + }, + ]), run, ], attempts: [ @@ -313,6 +335,12 @@ function makeLocalCommandHarness(input: { id: MessageId.make(`previous-message-${index}`), text, })), + ...(input.restartSource === undefined + ? [] + : [ + { ...message, id: sourceMessageId, runId: sourceRunId, text: "Original prompt" }, + { ...message, id: steeringMessageId, runId: sourceRunId, text: "Steering input" }, + ]), message, ], checkpointScopes: [ @@ -513,6 +541,7 @@ function makeLocalCommandHarness(input: { (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0), ), }), + getThreadRecords: () => Effect.succeed(projection), getTurnStartHistory: () => Effect.fail( new ProjectionStore.ProjectionStoreReadError({ @@ -756,6 +785,28 @@ effectIt.effect("does not mistake a failed state read for a superseded run", () }), ); +effectIt.effect.each([false, true])( + "gives the adapter the restart source only without undelivered steering (steering: %s)", + (pendingSteering) => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "Continue", + failReadsAfterRunning: true, + restartSource: { pendingSteering }, + }); + + yield* harness.start; + + const started = harness.startRootRun.mock.calls[0]?.[0]; + expect(started).toBeDefined(); + // A source makes OpenCode follow or settle a surviving turn instead of + // prompting, which would drop the steering text. + expect(started?.restartContinuationSource?.message.text).toBe( + pendingSteering ? undefined : "Original prompt", + ); + }), +); + effectIt.effect("does not overwrite a run interrupted while its thread loads", () => Effect.gen(function* () { const harness = makeLocalCommandHarness({ diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 79f268c2d1789..dcde1a00726e0 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -50,6 +50,7 @@ import { isRestartNoteContinuation, pendingRestartCancelledBackgroundWork, restartCancelledBackgroundWorkNote, + restartPendingMessageIds, } from "./RestartBackgroundNote.ts"; export class ProviderTurnStartError extends Schema.TaggedError()( @@ -1231,8 +1232,12 @@ export const layer: Layer.Layer< !noteContinuation ? session : makeDeliverySession(session, startWithHandoffs); + // Undelivered steering must be prompted. A surviving native turn that is + // followed or settled from its source would drop it. const restartSourceRun = projection.runs.find( - (source) => source.id === run.restartContinuationOfRunId, + (source) => + source.id === run.restartContinuationOfRunId && + restartPendingMessageIds(source, projection.runs, projection.providerTurns).length === 0, ); const restartSourceTurn = projection.providerTurns.findLast( (turn) => turn.runAttemptId === restartSourceRun?.activeAttemptId, From 1629953fdd74ddf8cfc957b798a6189fc8109654 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:32:10 +0000 Subject: [PATCH 093/149] test(server): cover the webhook queue migrations and a turn without a native id --- .../src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts | 2 +- .../src/persistence/Migrations/055_OrchestrationV2.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 470aa46869b76..517eac9b82e62 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -1009,7 +1009,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { if (execution === "joined") { const childTurns = collected.flatMap((event) => event.type === "provider_turn.updated" && - event.providerTurn.nativeTurnRef?.nativeId.startsWith(CHILD) + event.providerTurn.nativeTurnRef?.nativeId?.startsWith(CHILD) ? [event.providerTurn.ordinal] : [], ); diff --git a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts index 74704d8ed17c9..ba3d035ab3348 100644 --- a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts +++ b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts @@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => { Effect.sync(() => { assert.deepStrictEqual( migrationEntries.map(([id]) => id), - Array.from({ length: 60 }, (_, index) => index + 1), + Array.from({ length: 62 }, (_, index) => index + 1), ); }), ); From 170ea4875ea42ad9a92f17312623a6bd346cb04c Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:37:59 +0000 Subject: [PATCH 094/149] fix(server): a service restarted by t3 update keeps external OpenCode turns --- apps/server/src/cloud/bootService.test.ts | 9 ++++++++- apps/server/src/cloud/bootService.ts | 13 ++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 957c25ae0ec34..cd6ae90ef9fdb 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -639,13 +639,20 @@ it.layer(NodeServices.layer)("boot service install", (it) => { it.effect("restarts an installed service when repair fails", () => Effect.gen(function* () { - const { service, commands, control } = yield* makeHarness(); + const { service, commands, control, fs, statePath } = yield* makeHarness(); yield* service.install(); + expect(control.restartingAtStop).toBeUndefined(); commands.length = 0; control.failCommand = "systemctl --user daemon-reload"; const error = yield* service.install().pipe(Effect.flip); expect(error._tag).toBe("BootServiceCommandError"); + // `t3 update` restarts a running service this way; it is a restart, not a stop. + expect(control.restartingAtStop).toBe(true); + const path = yield* Path.Path; + expect( + yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_MARKER_FILE)), + ).toBe(false); expect( commands.filter( (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 4a914fd6826d1..337037e141cff 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -829,8 +829,18 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { // written, from the same read the downgrade check uses; the stop that // normally serialises against the launcher is skipped on purpose. const start = options?.start !== false; + // A restarting install replaces the running server, like `restart`, so + // its external provider turns are kept rather than aborted. + const restartMarkerPath = path.join(input.baseDir, "runtime", SERVICE_RESTART_MARKER_FILE); + const removeRestartMarker = + installed && start + ? fs.remove(restartMarkerPath, { force: true }).pipe(Effect.ignore) + : Effect.void; if (installed && start) { - yield* runSteps(manager.stop); + yield* writeDurably(restartMarkerPath, "").pipe( + Effect.mapError((cause) => new BootServiceInstallError({ cause })), + ); + yield* runSteps(manager.stop).pipe(Effect.onError(() => removeRestartMarker)); } yield* Effect.gen(function* () { @@ -902,6 +912,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { Effect.tapError(() => installed && start ? runSteps(manager.restart).pipe(Effect.ignore) : Effect.void, ), + Effect.ensuring(removeRestartMarker), ); return plan; }); From 5fb1d545a733f99712126e8a8831e79c192dc7e9 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:37:59 +0000 Subject: [PATCH 095/149] fix(server): migrate-dev-db drops queued webhook deliveries with other pending work --- apps/server/scripts/migrate-dev-db.test.ts | 13 +++++++++++-- apps/server/scripts/migrate-dev-db.ts | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/server/scripts/migrate-dev-db.test.ts b/apps/server/scripts/migrate-dev-db.test.ts index 07d3dc61c7913..b91e3f2737cb8 100644 --- a/apps/server/scripts/migrate-dev-db.test.ts +++ b/apps/server/scripts/migrate-dev-db.test.ts @@ -86,6 +86,9 @@ const createFixtureSource = Effect.fn("createMigrateDevDbFixtureSource")(functio created_at, updated_at, last_run_status, run_count) VALUES ('task-1', 'Nightly', 'Run it', 1, '{}', 'project-kept', '{}', '{}', 'full-access', 'default', 'user', 'user', '2026-08-01', '2026-08-01', 'never', 0)`; + yield* sql`INSERT INTO scheduled_task_webhook_queue + (delivery_id, task_id, task_created_at, prompt) + VALUES ('delivery-1', 'task-1', '2026-08-01', 'Run it')`; yield* sql`INSERT INTO auth_sessions (session_id, subject, scopes, method, issued_at, expires_at) VALUES ('session-1', 'user', '[]', 'pairing', '2026-08-01', '2027-08-01')`; }), @@ -118,10 +121,16 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => { SELECT stream_id FROM orchestration_events ORDER BY stream_id`; const sessions = yield* sql<{ provider_session_id: string }>` SELECT provider_session_id FROM orchestration_v2_projection_provider_sessions`; - const [leftovers] = yield* sql<{ auth: number; tasks: number; transfers: number }>` + const [leftovers] = yield* sql<{ + auth: number; + tasks: number; + webhooks: number; + transfers: number; + }>` SELECT (SELECT COUNT(*) FROM auth_sessions) AS auth, (SELECT COUNT(*) FROM scheduled_tasks) AS tasks, + (SELECT COUNT(*) FROM scheduled_task_webhook_queue) AS webhooks, (SELECT COUNT(*) FROM orchestration_v2_projection_context_transfers) AS transfers`; return { threads, events, sessions, leftovers }; }), @@ -138,7 +147,7 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => { kept.sessions.map((row) => row.provider_session_id), ["session-shared"], ); - assert.deepStrictEqual(kept.leftovers, { auth: 0, tasks: 0, transfers: 0 }); + assert.deepStrictEqual(kept.leftovers, { auth: 0, tasks: 0, webhooks: 0, transfers: 0 }); }), ); diff --git a/apps/server/scripts/migrate-dev-db.ts b/apps/server/scripts/migrate-dev-db.ts index 4cfdf62333460..9c61ee4a4e6fd 100644 --- a/apps/server/scripts/migrate-dev-db.ts +++ b/apps/server/scripts/migrate-dev-db.ts @@ -250,6 +250,7 @@ const RECOVERY_KINDS: ReadonlyArray = [ const CLEARED_TABLES: ReadonlyArray = [ // Pending work the dev server would otherwise pick up and run. "scheduled_tasks", + "scheduled_task_webhook_queue", "orchestration_v2_effect_outbox", "orchestration_v2_thread_launch_workflows", "orchestration_command_receipts", From 38703ecba4232e1e38802674a3a806e354ac5afb Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:03:32 +0000 Subject: [PATCH 096/149] fix(server): background provider installs no longer stall or admit CLIs mid-install Live snapshot watchers released CLI admission only when their stream ended, so exclusive installs never started. Watchers now gate each finite probe, and cached maintenance resolution takes admission in the caller's context so installer verification can reenter. Queued installs recheck settings and the update window, host admission also covers probes from other servers, waiting readers stay interruptible, lock failures clear stuck update state, and SDK package links no longer prove manager ownership. --- .../provider/ProviderMaintenanceHostLock.ts | 97 ++++-- .../providerMaintenanceCommandCoordinator.ts | 10 +- .../providerMaintenanceRunner.test.ts | 276 +++++++++++++++++- .../src/provider/providerMaintenanceRunner.ts | 60 ++-- .../server/src/updates/ProviderAutoUpdater.ts | 23 +- apps/server/src/updates/UpdateWindow.ts | 13 +- .../src/server/maintenanceResolver.test.ts | 75 +++++ .../src/server/maintenanceResolver.ts | 25 +- .../src/server/managedProvider.test.ts | 27 ++ .../src/server/managedProvider.ts | 2 +- 10 files changed, 548 insertions(+), 60 deletions(-) diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index 1d6dacf23c38e..eb8d03a07de2b 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -10,6 +10,12 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Deferred from "effect/Deferred"; import * as Semaphore from "effect/Semaphore"; +import * as Maintenance from "@t3tools/provider-core/server/maintenance"; + +const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } | null>( + "t3/provider/ProviderMaintenanceHostLock/HeldWrite", + { defaultValue: () => null }, +); // Linux exposes a process start identity, so recycled PIDs cannot pin a dead reader. const processIdentity = (pid: number) => { @@ -24,6 +30,7 @@ const processIdentity = (pid: number) => { * an installer closes admission before waiting for active turns to drain. */ export const make = (directory: string) => { + const owner = {}; const readers = NodePath.join(directory, "readers"); const prepare = Effect.sync(() => NodeFS.mkdirSync(readers, { recursive: true })); const acquireGate = Effect.uninterruptibleMask((restore) => @@ -60,34 +67,44 @@ export const make = (directory: string) => { const readPermit = Semaphore.makeUnsafe(1); let readCount = 0; let readFile: string | undefined; + const registerRead = Effect.sync(() => { + const file = NodePath.join(readers, `${process.pid}-${NodeCrypto.randomUUID()}`); + NodeFS.writeFileSync(file, processIdentity(process.pid) ?? "", { flag: "wx" }); + return file; + }); // A provider can launch helper CLIs while its turn holds a read lease. // Joining that lease must not wait behind an installer draining this turn. - const acquireRead = Effect.uninterruptible( - readPermit.withPermits(1)( - Effect.gen(function* () { - if (readCount === 0) { - readFile = yield* withGate( - Effect.sync(() => { - const file = NodePath.join(readers, `${process.pid}-${NodeCrypto.randomUUID()}`); - NodeFS.writeFileSync(file, processIdentity(process.pid) ?? "", { flag: "wx" }); - return file; - }), - ); - } - readCount++; - let released = false; - return readPermit.withPermits(1)( - Effect.sync(() => { - if (released) return; - released = true; - if (--readCount === 0 && readFile !== undefined) { - NodeFS.rmSync(readFile, { force: true }); - readFile = undefined; - } - }), - ); - }), - ), + const acquireRead = Effect.flatMap(HeldWrite, (held) => + held?.owner === owner && held.active + ? // A scoped verification can outlive the install. Its own reader remains + // visible to the next writer without admitting unrelated launches here. + registerRead.pipe( + Effect.map((file) => Effect.sync(() => NodeFS.rmSync(file, { force: true }))), + ) + : Effect.acquireUseRelease( + Effect.interruptible(readPermit.take(1)), + () => + Effect.uninterruptible( + Effect.gen(function* () { + if (readCount === 0) { + readFile = yield* withGate(registerRead); + } + readCount++; + let released = false; + return readPermit.withPermits(1)( + Effect.sync(() => { + if (released) return; + released = true; + if (--readCount === 0 && readFile !== undefined) { + NodeFS.rmSync(readFile, { force: true }); + readFile = undefined; + } + }), + ); + }), + ), + () => Effect.asVoid(readPermit.release(1)), + ), ); const withRead = (effect: Effect.Effect) => Effect.acquireUseRelease( @@ -134,7 +151,14 @@ export const make = (directory: string) => { if (onBusy) yield* onBusy; while (!(yield* drained)) yield* Effect.sleep("100 millis"); } - return yield* effect; + return yield* Effect.acquireUseRelease( + Effect.sync(() => ({ owner, active: true })), + (token) => effect.pipe(Effect.provideService(HeldWrite, token)), + (token) => + Effect.sync(() => { + token.active = false; + }), + ); }), ); return { directory, acquireRead, withRead, withWrite }; @@ -165,4 +189,21 @@ export class ProviderMaintenanceHostLock extends Context.Reference make(userDirectory())); +export const layer = Layer.effectContext( + Effect.sync(() => { + const host = make(userDirectory()); + const admission = Maintenance.ProviderMaintenanceAdmission.defaultValue(); + return Context.make(ProviderMaintenanceHostLock, host).pipe( + Context.add(Maintenance.ProviderMaintenanceAdmission, { + // Taking the host lease first prevents a probe nested in an update + // window from waiting behind a writer that needs local admission. + withPermit: (effect) => host.withRead(admission.withPermit(effect)), + withLaunchPermit: (effect) => host.withRead(admission.withLaunchPermit(effect)), + withScopedLaunchPermit: (effect) => + Effect.acquireRelease(host.acquireRead, (release) => release, { + interruptible: true, + }).pipe(Effect.andThen(admission.withScopedLaunchPermit(effect))), + }), + ); + }), +); diff --git a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts index 663ebe281f08b..e609bb569ac7c 100644 --- a/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts +++ b/apps/server/src/provider/providerMaintenanceCommandCoordinator.ts @@ -1,6 +1,8 @@ import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as Effect from "effect/Effect"; +import * as Cause from "effect/Cause"; +import * as Exit from "effect/Exit"; import * as Ref from "effect/Ref"; import * as HostLock from "./ProviderMaintenanceHostLock.ts"; @@ -13,7 +15,7 @@ export interface ProviderMaintenanceCommandCoordinatorShape { readonly onQueued?: Effect.Effect; /** Runs once if turns on this or another server must finish first. */ readonly onWaitingForTurns?: Effect.Effect; - readonly onInterrupted?: Effect.Effect; + readonly onFailure?: (cause: Cause.Cause) => Effect.Effect; readonly run: Effect.Effect; }) => Effect.Effect; } @@ -49,7 +51,7 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( lockKey, onQueued, onWaitingForTurns, - onInterrupted, + onFailure, run, }) => Effect.gen(function* () { @@ -64,7 +66,9 @@ export const makeProviderMaintenanceCommandCoordinator = Effect.fn( Effect.andThen( locks.withLock(lockKey, hostLock.withWrite(admission.withPermit(run), onWaitingForTurns)), ), - Effect.onInterrupt(() => onInterrupted ?? Effect.void), + Effect.onExit((exit) => + Exit.isFailure(exit) ? (onFailure?.(exit.cause) ?? Effect.void) : Effect.void, + ), Effect.ensuring(releaseTarget(targetKey)), ); }); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index ae07283f3d63a..249073a734e6e 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -1549,6 +1549,136 @@ it.effect( }, ); +it.effect.each(["interaction", "opt-out"] as const)( + "postpones a queued background update after $0 and retries at the next pass", + (reason) => + Effect.gen(function* () { + const queued = yield* Deferred.make(); + const release = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const installed = yield* Deferred.make(); + let open = true; + let automaticUpdates = true; + let launches = 0; + const { registry, providersRef } = yield* makeRegistry({ + ...baseProvider, + version: "1.0.0", + versionAdvisory: { + status: "behind_latest", + currentVersion: "1.0.0", + latestVersion: "2.0.0", + canUpdate: true, + canInstallVersion: true, + updateCommand: "npm install", + checkedAt: null, + message: null, + }, + }); + const runner = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: () => + Effect.succeed( + makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "@openai/codex@latest"], + updateLockKey: "npm-global:/owned", + installationKey: "codex:/owned", + }), + ), + setProviderMaintenanceActionState: (input) => + registry + .setProviderMaintenanceActionState(input) + .pipe( + Effect.tap(() => + input.state?.status === "unchanged" || input.state?.status === "succeeded" + ? Deferred.succeed(finished, undefined) + : Effect.void, + ), + ), + refreshInstance: () => + Ref.updateAndGet(providersRef, (providers) => + providers.map((provider) => ({ + ...provider, + version: launches > 0 ? "2.0.0" : "1.0.0", + })), + ), + }).pipe( + Effect.provideService(HostLock.ProviderMaintenanceHostLock, { + directory: "", + acquireRead: Effect.succeed(Effect.void), + withRead: (effect) => effect, + withWrite: (effect) => + Deferred.succeed(queued, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(effect), + ), + }), + Effect.provide( + layerMockSpawner(() => { + launches++; + return { + exitCode: Deferred.succeed(installed, undefined).pipe( + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + }; + }), + ), + ); + yield* Layer.build( + ProviderAutoUpdater.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ServerConfig.ServerConfig, { + devUrl: undefined, + } as ServerConfig.ServerConfig["Service"]), + Layer.succeed(ProviderRegistry.ProviderRegistry, { + ...registry, + getProviderMaintenanceCapabilitiesForInstance: () => + Effect.succeed( + makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "@openai/codex@latest"], + updateLockKey: "npm-global:/owned", + installationKey: "codex:/owned", + }), + ), + }), + Layer.succeed(ProviderMaintenanceRunner.ProviderMaintenanceRunner, runner), + Layer.succeed(UpdateWindow.UpdateWindow, { + runIfOpen: (effect) => + open ? Effect.asSome(effect) : Effect.succeed(Option.none()), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.sync(() => ({ ...DEFAULT_SERVER_SETTINGS, automaticUpdates })), + }), + ), + ), + ), + ); + yield* TestClock.adjust("2 minutes"); + yield* Deferred.await(queued); + if (reason === "interaction") open = false; + else automaticUpdates = false; + yield* Deferred.succeed(release, undefined); + yield* Deferred.await(finished); + assert.strictEqual(launches, 0); + open = true; + automaticUpdates = true; + yield* TestClock.adjust("10 minutes"); + yield* Deferred.await(installed); + assert.strictEqual(launches, 1); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(layerNonWindowsPlatform, layerLatestVersionHttpClient("2.0.0")), + ), + ), +); + describe("provider maintenance host leases", () => { const coordinatedWrite = (directory: string, run: Effect.Effect) => Effect.gen(function* () { @@ -1603,6 +1733,40 @@ describe("provider maintenance host leases", () => { return Effect.sync(() => watcher.close()); }); + it.live("server admission keeps probes behind another server's install", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cache = yield* fs.makeTempDirectoryScoped(); + const previous = process.env.XDG_CACHE_HOME; + process.env.XDG_CACHE_HOME = cache; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (previous === undefined) delete process.env.XDG_CACHE_HOME; + else process.env.XDG_CACHE_HOME = previous; + }), + ); + const context = yield* Layer.build(HostLock.layer); + const host = yield* HostLock.ProviderMaintenanceHostLock.pipe(Effect.provide(context)); + const admission = yield* Maintenance.ProviderMaintenanceAdmission.pipe( + Effect.provide(context), + ); + const holder = yield* childLease(host.directory, "withWrite"); + yield* Effect.promise(() => holder.held); + const entered = yield* Deferred.make(); + const probe = yield* admission + .withLaunchPermit(Deferred.succeed(entered, undefined)) + .pipe(Effect.forkChild, Effect.provide(context)); + yield* Effect.yieldNow; + assert.isFalse(yield* Deferred.isDone(entered)); + holder.child.stdin.end("release"); + yield* Effect.promise(() => holder.exited); + yield* Fiber.join(probe); + assert.isTrue(yield* Deferred.isDone(entered)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("holds another server's install through a turn and admits nested CLI calls", () => Effect.scoped( Effect.gen(function* () { @@ -1686,6 +1850,83 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live("cancels a second host reader before the writer releases", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const host = HostLock.make(yield* fs.makeTempDirectoryScoped()); + const writer = yield* childLease(host.directory, "withWrite"); + yield* Effect.promise(() => writer.held); + const first = yield* Effect.forkChild(host.withRead(Effect.never), { + startImmediately: true, + }); + const second = yield* Effect.forkChild(host.withRead(Effect.never), { + startImmediately: true, + }); + const cancel = yield* Effect.forkChild(Fiber.interrupt(second), { startImmediately: true }); + yield* Effect.yieldNow; + const canceledBeforeRelease = second.pollUnsafe() !== undefined; + writer.child.stdin.end("release"); + yield* Effect.promise(() => writer.exited); + yield* Fiber.interrupt(first); + yield* Fiber.join(cancel); + assert.isTrue(canceledBeforeRelease); + assert.equal(NodeFS.readdirSync(NodePath.join(host.directory, "readers")).length, 0); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("clears queued update state when the host cache is not writable", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32" || process.getuid?.() === 0) return; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + NodeFS.chmodSync(directory, 0o500); + yield* Effect.addFinalizer(() => Effect.sync(() => NodeFS.chmodSync(directory, 0o700))); + const { registry } = yield* makeRegistry(baseNativeCliProvider); + const runner = yield* makeTestRunner(registry).pipe( + Effect.provideService(HostLock.ProviderMaintenanceHostLock, HostLock.make(directory)), + ); + const result = yield* runner.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(result)); + assert.equal((yield* registry.getProviders)[0]?.updateState?.status, "failed"); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + layerNonWindowsPlatform, + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner(() => { + throw new Error("must not install"); + }), + ), + ), + ), + ); + + it.live("retains a verification reader after its owning install releases the gate", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const host = HostLock.make(yield* fs.makeTempDirectoryScoped()); + const release = yield* host.withWrite(host.acquireRead); + const waiting = yield* Deferred.make(); + const installed = yield* Deferred.make(); + const writer = yield* host + .withWrite(Deferred.succeed(installed, undefined), Deferred.succeed(waiting, undefined)) + .pipe(Effect.forkScoped); + yield* Deferred.await(waiting); + assert.isFalse(yield* Deferred.isDone(installed)); + yield* release; + yield* Fiber.join(writer); + assert.isTrue(yield* Deferred.isDone(installed)); + assert.equal(NodeFS.readdirSync(NodePath.join(host.directory, "readers")).length, 0); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("drains host leases before an install closes CLI admission", () => Effect.scoped( Effect.gen(function* () { @@ -1949,11 +2190,25 @@ it.effect.each([ name: "open provider window", interaction: false, expected: true, + updatedInstance: null, }, { name: "interaction arriving during blocker reads", interaction: true, expected: false, + updatedInstance: null, + }, + { + name: "its own queued update", + interaction: false, + expected: true, + updatedInstance: CODEX_INSTANCE_ID, + }, + { + name: "another queued update", + interaction: false, + expected: false, + updatedInstance: ProviderInstanceId.make("other"), }, ])("rechecks the update window for $name", (scenario) => Effect.gen(function* () { @@ -1979,7 +2234,25 @@ it.effect.each([ Layer.mock(ServerSettings.ServerSettingsService)({ getSettings: Effect.succeed(ThreadContracts.DEFAULT_SERVER_SETTINGS), }), - Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed( + scenario.updatedInstance === null + ? [] + : [ + { + ...baseProvider, + instanceId: scenario.updatedInstance, + updateState: { + status: "queued", + startedAt: null, + finishedAt: null, + message: null, + output: null, + }, + }, + ], + ), + }), ), ), ), @@ -1989,6 +2262,7 @@ it.effect.each([ Effect.sync(() => { installs++; }), + CODEX_INSTANCE_ID, ); assert.strictEqual(Option.isSome(result), scenario.expected); assert.strictEqual(installs, scenario.expected ? 1 : 0); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 5193fc0b91e7f..90049bd9e622d 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -71,6 +71,8 @@ export interface ProviderMaintenanceRunnerShape { readonly quiet?: boolean | undefined; /** Background installs require proven ownership and pin the checked release. */ readonly origin?: ServerProviderUpdateState["origin"]; + /** Rechecks background eligibility after queued turns and installs have drained. */ + readonly beforeInstall?: Effect.Effect; }, ) => Effect.Effect; } @@ -594,6 +596,21 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { instanceId, fresh.update.installationKey, ); + if ( + background && + typeof target !== "string" && + target.beforeInstall && + !(yield* target.beforeInstall) + ) { + return yield* finish( + makeUpdateState({ + status: "unchanged", + startedAt: null, + finishedAt: yield* nowIso, + message: "Background update postponed until the update window opens.", + }), + ); + } yield* setRunningMessage(`Running ${describeCommand(command)}`); const result = yield* runMaintenanceCommand( command, @@ -698,25 +715,30 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { message: "Waiting for running turns to finish.", }), ).pipe(Effect.asVoid), - onInterrupted: Effect.gen(function* () { - const current = (yield* providerRegistry.getProviders).find( - (candidate) => candidate.instanceId === instanceId, - ); - // A state left queued or running would block background updates until restart. - const status = current?.updateState?.status; - if (status !== "queued" && status !== "running") return; - yield* setUpdateState( - makeUpdateState({ - status: "failed", - startedAt: current?.updateState?.startedAt ?? null, - finishedAt: yield* nowIso, - message: - status === "queued" - ? "Update canceled before installation started." - : "Update canceled.", - }), - ); - }), + onFailure: (cause) => + Effect.gen(function* () { + const current = (yield* providerRegistry.getProviders).find( + (candidate) => candidate.instanceId === instanceId, + ); + // A state left queued or running would block background updates until restart. + const status = current?.updateState?.status; + if (status !== "queued" && status !== "running") return; + const failure = Cause.squash(cause); + yield* setUpdateState( + makeUpdateState({ + status: "failed", + startedAt: current?.updateState?.startedAt ?? null, + finishedAt: yield* nowIso, + message: !Cause.hasInterrupts(cause) + ? failure instanceof Error + ? failure.message + : "Could not acquire the provider installation lock." + : status === "queued" + ? "Update canceled before installation started." + : "Update canceled.", + }), + ); + }), run: runProviderUpdate(), }) .pipe( diff --git a/apps/server/src/updates/ProviderAutoUpdater.ts b/apps/server/src/updates/ProviderAutoUpdater.ts index 7bb90eb36f76c..9b0732ec5d8a6 100644 --- a/apps/server/src/updates/ProviderAutoUpdater.ts +++ b/apps/server/src/updates/ProviderAutoUpdater.ts @@ -85,7 +85,7 @@ export const layer = Layer.effectDiscard( fromVersion: current.version, targetVersion, }); - return { provider: current.driver, targetVersion }; + return { provider: current.driver, targetVersion, attemptKey }; }); // Rechecked per install: the previous one may have run for minutes. const picked = yield* updateWindow.runIfOpen(pick); @@ -93,8 +93,27 @@ export const layer = Layer.effectDiscard( if (picked.value === undefined) continue; // Installed after the window releases admission: the runner first drains host // leases, and a session holding one may still need admission to launch its CLI. + const { attemptKey, ...target } = picked.value; + const beforeInstall = Effect.gen(function* () { + const allowed = yield* Effect.gen(function* () { + const settings = yield* serverSettings.getSettings; + return ( + settings.automaticUpdates && + settings.enableProviderUpdateChecks && + Option.isSome(yield* updateWindow.runIfOpen(Effect.void, instanceId)) + ); + }).pipe(Effect.catchCause(() => Effect.succeed(false))); + if (!allowed) attempts.delete(attemptKey); + return allowed; + }); yield* runner - .updateProvider({ ...picked.value, instanceId, quiet: true, origin: "background" }) + .updateProvider({ + ...target, + instanceId, + quiet: true, + origin: "background", + beforeInstall, + }) .pipe( Effect.catchCause((cause) => Effect.logWarning("Background provider update failed", { instanceId, cause }), diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index f9e3566354571..4a8fd2e85128a 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as SqlClient from "effect/sql/SqlClient"; +import type { ProviderInstanceId } from "@t3tools/contracts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; @@ -41,6 +42,7 @@ export class UpdateWindow extends Context.Service< */ readonly runIfOpen: ( install: Effect.Effect, + updatingInstanceId?: ProviderInstanceId, ) => Effect.Effect, E, R>; } >()("t3/updates/UpdateWindow") {} @@ -55,7 +57,9 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; /** Returns what keeps the window closed; empty means open. */ - const blockers = Effect.fn("updates.UpdateWindow.blockers")(function* () { + const blockers = Effect.fn("updates.UpdateWindow.blockers")(function* ( + updatingInstanceId?: ProviderInstanceId, + ) { const now = yield* DateTime.now; const horizon = DateTime.addDuration(now, UPDATE_LOOKAHEAD); @@ -78,7 +82,8 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { // Includes installs someone started by hand. const providerUpdating = (yield* providers.getProviders).some( (provider) => - provider.updateState?.status === "queued" || provider.updateState?.status === "running", + provider.instanceId !== updatingInstanceId && + (provider.updateState?.status === "queued" || provider.updateState?.status === "running"), ); if (providerUpdating) return ["provider-update"]; @@ -146,9 +151,9 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { return []; }); - const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (install) => + const runIfOpen: UpdateWindow["Service"]["runIfOpen"] = (install, updatingInstanceId) => admission.withPermit( - blockers().pipe( + blockers(updatingInstanceId).pipe( Effect.flatMap((found) => found.length > 0 ? Effect.logDebug("Background update waiting for the update window", { diff --git a/packages/provider-core/src/server/maintenanceResolver.test.ts b/packages/provider-core/src/server/maintenanceResolver.test.ts index b74400aec42c0..9523374561c9a 100644 --- a/packages/provider-core/src/server/maintenanceResolver.test.ts +++ b/packages/provider-core/src/server/maintenanceResolver.test.ts @@ -16,6 +16,7 @@ import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as ProviderLatestVersions from "./ProviderLatestVersions.ts"; +import * as Maintenance from "./maintenance.ts"; import { createProviderVersionAdvisory, makeTargetedProviderUpdateAction, @@ -1013,6 +1014,48 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { { skip: windowsHost }, ); + it.effect.each(["bun", "pnpm", "yarn", "vp"] as const)( + "does not treat a foreign SDK binary under %s as an owned provider install", + (manager) => + Effect.gen(function* () { + const root = yield* makeTempDir("r2-04-sdk-ownership"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const home = NodePath.join( + root, + manager === "bun" ? ".bun" : manager === "vp" ? ".vite-plus" : manager, + ); + const visible = NodePath.join(home, "bin", "package-tool"); + const real = NodePath.join( + home, + manager === "yarn" ? "global" : "install", + "node_modules", + "@example", + "sdk", + "bin", + "package-tool", + ); + writeExecutable(real); + const result = yield* resolvePackageManagedProviderMaintenance( + { + provider: driver("packageTool"), + npmPackageName: "@example/package-tool", + nativeUpdate: null, + }, + { + binaryPath: visible, + resolvedCommandPath: visible, + realCommandPath: real, + env: { PATH: "", PNPM_HOME: manager === "pnpm" ? home : undefined }, + platform: "linux", + }, + ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)); + expect(result.update?.installationKey).toBeUndefined(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + it.effect.each([ "installs/npm-codex/1.0.0/lib/node_modules/@example/package-tool/bin/tool.js", "installs/package-tool/1.0.0/bin/tool", @@ -1410,3 +1453,35 @@ it.each(["2.0.0", "2.0.0-beta.1"])( } }, ); + +it.effect( + "maintenance resolution waits for install admission before inspecting an executable", + () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + let inspections = 0; + const install = yield* admission + .withPermit( + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const resolve = yield* makeCachedProviderMaintenanceResolution( + Effect.sync(() => { + inspections++; + return manualPackageTool; + }), + ); + const resolution = yield* resolve().pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(inspections).toBe(0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(install); + expect(yield* Fiber.join(resolution)).toEqual(manualPackageTool); + expect(inspections).toBe(1); + expect(yield* admission.withPermit(resolve({ fresh: true }))).toEqual(manualPackageTool); + expect(inspections).toBe(2); + }), +); diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index 24cca5259c2a4..b528255e1faa1 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -2,6 +2,7 @@ export type { ProviderMaintenanceCapabilities, ProviderMaintenanceCommandAction, } from "@t3tools/provider-core/server/maintenance"; +import { ProviderMaintenanceAdmission } from "./maintenance.ts"; import type { ProviderMaintenanceCapabilities, ProviderMaintenanceCommandAction, @@ -375,6 +376,19 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( } const commandPaths = [context.resolvedCommandPath, context.realCommandPath]; const packageName = definition.npmPackageName; + // A manager's bin directory can contain links into a different package or SDK. + // Only its own entry point, including pnpm's store indirection, proves ownership. + const packagePaths = normalizeCommandPath(context.realCommandPath) + .split("/node_modules/") + .slice(1); + if ( + packagePaths.some((value, index) => + index === packagePaths.length - 1 + ? !value.startsWith(`${packageName.toLowerCase()}/`) + : !value.startsWith(".pnpm/"), + ) + ) + return manual; const fileSystem = yield* FileSystem.FileSystem; const miseDataDir = context.env.MISE_DATA_DIR; @@ -416,7 +430,11 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( env: { ...context.env, ...nativeUpdate.env }, }) : manual; - if (nativeUpdate?.isCommandPath && nativeUpdate.isCommandPath(context.realCommandPath)) { + if ( + packagePaths.length === 0 && + nativeUpdate?.isCommandPath && + nativeUpdate.isCommandPath(context.realCommandPath) + ) { const nativeRoot = context.realCommandPath .replaceAll("\\", "/") .replace(/(\/packages\/standalone)\/.*$/i, "$1") @@ -751,9 +769,12 @@ export const makeCachedProviderMaintenanceResolution = Effect.fn( timeToLive: MAINTENANCE_CAPABILITIES_CACHE_TTL, lookup: () => resolve, }); + const admission = yield* ProviderMaintenanceAdmission; const cached = Cache.get(cache, undefined); return (options?: { readonly fresh?: boolean }) => - options?.fresh ? Cache.invalidate(cache, undefined).pipe(Effect.andThen(cached)) : cached; + admission.withLaunchPermit( + options?.fresh ? Cache.invalidate(cache, undefined).pipe(Effect.andThen(cached)) : cached, + ); }); function deriveVersionAdvisory(input: { diff --git a/packages/provider-core/src/server/managedProvider.test.ts b/packages/provider-core/src/server/managedProvider.test.ts index 1b832a14b8d82..e588715da8aa0 100644 --- a/packages/provider-core/src/server/managedProvider.test.ts +++ b/packages/provider-core/src/server/managedProvider.test.ts @@ -728,3 +728,30 @@ it.effect("snapshot refresh waits outside its mutex so updater verification cann assert.strictEqual(checks, 3); }).pipe(Effect.provide(layerAlwaysRunTest)), ); + +it.effect("live snapshot watchers release CLI admission after their finite probe", () => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const watching = yield* Deferred.make(); + const update = yield* Deferred.make(); + yield* makeManagedServerProvider({ + resolveMaintenance: () => Effect.succeed(maintenanceCapabilities), + getSettings: Effect.succeed({ enabled: true }), + streamSettings: Stream.empty, + haveSettingsChanged: () => false, + initialSnapshot: () => Effect.succeed(initialSnapshot), + checkProvider: Effect.succeed(refreshedSnapshot), + enrichSnapshot: () => + admission + .withLaunchPermit(Effect.void) + .pipe( + Effect.andThen(Deferred.succeed(watching, undefined)), + Effect.andThen(Effect.never), + ), + refreshOnInterval: false, + }); + yield* Deferred.await(watching); + yield* admission.withPermit(Deferred.succeed(update, undefined)).pipe(Effect.forkScoped); + yield* Deferred.await(update); + }).pipe(Effect.provide(layerAlwaysRunTest)), +); diff --git a/packages/provider-core/src/server/managedProvider.ts b/packages/provider-core/src/server/managedProvider.ts index 36b2121b15337..b5ace6e6dd2cd 100644 --- a/packages/provider-core/src/server/managedProvider.ts +++ b/packages/provider-core/src/server/managedProvider.ts @@ -121,7 +121,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")( getSnapshot: Ref.get(snapshotStateRef).pipe(Effect.map((state) => state.snapshot)), publishSnapshot: (nextSnapshot) => publishEnrichedSnapshot(generation, nextSnapshot), }) - .pipe(admission.withLaunchPermit, Effect.ignoreCause({ log: true }), Effect.forkIn(scope)); + .pipe(Effect.ignoreCause({ log: true }), Effect.forkIn(scope)); yield* Ref.set(enrichmentFiberRef, fiber); }); From 1e0c9e69d4396219518291733ecbf48f51a020d7 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:03:33 +0000 Subject: [PATCH 097/149] fix(server): older clients no longer show background provider installs as manual updates Current clients opt into provider update origins on the config subscription. The server strips background update state from snapshots and status events sent to clients that do not. Docs describe recognized-installation limits, the manual native fallback, and host-wide lease ordering. --- apps/server/src/ws.test.ts | 64 +++++++++++++++++++ apps/server/src/ws.ts | 33 +++++++++- docs/internals/providers.md | 9 ++- docs/user/updating.md | 4 +- .../client-runtime/src/rpc/session.test.ts | 1 + packages/client-runtime/src/rpc/session.ts | 11 ++-- packages/contracts/src/rpc.test.ts | 10 +++ packages/contracts/src/rpc.ts | 2 + 8 files changed, 126 insertions(+), 8 deletions(-) diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 0f735efd31c70..16b723284a31d 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -1,6 +1,7 @@ import { assert, it } from "@effect/vitest"; import { ORCHESTRATION_PROTOCOL_VERSION, + ServerProvider, type ServerConfig, type ServerConfigStreamEvent, } from "@t3tools/contracts"; @@ -14,6 +15,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; +import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { ChildProcessSpawner } from "effect/process"; @@ -24,6 +26,7 @@ import { resolveAvailableEditorsForConfig, shouldUseBoundedThreadSnapshot, withLateEditorConfig, + withProviderUpdateOrigins, } from "./ws.ts"; it("accepts only the current orchestration protocol before websocket RPC setup", () => { @@ -67,6 +70,67 @@ it.effect("does not block server config when editor discovery never resolves", ( const snapshotConfig = (fields: Partial) => ({ availableEditors: [], settings: {}, ...fields }) as unknown as ServerConfig; +const decodeServerProvider = Schema.decodeSync(ServerProvider); + +it.effect.each(["running", "succeeded", "failed"] as const)( + "keeps background %s quiet for legacy config subscribers while retaining manual updates", + (status) => + Effect.gen(function* () { + const providers = (["background", "manual"] as const).map((origin) => + decodeServerProvider({ + instanceId: origin, + driver: "codex", + enabled: true, + installed: true, + version: "1.2.0", + status: "ready", + auth: { status: "authenticated" }, + checkedAt: "2026-10-11T00:00:00.000Z", + models: [], + updateState: { + status, + origin, + startedAt: null, + finishedAt: null, + message: null, + output: null, + }, + }), + ); + const snapshot: ServerConfigStreamEvent = { + version: 1, + type: "snapshot", + config: snapshotConfig({ providers }), + }; + const live: ServerConfigStreamEvent = { + version: 1, + type: "providerStatuses", + payload: { providers }, + }; + for (const flag of [undefined, false, true]) { + const events = yield* withProviderUpdateOrigins( + Stream.make(snapshot, live), + flag === undefined ? {} : { providerUpdateOrigins: flag }, + ).pipe(Stream.runCollect); + assert.equal(events.length, 2); + for (const event of events) { + const visible = + event.type === "snapshot" + ? event.config.providers + : event.type === "providerStatuses" + ? event.payload.providers + : []; + assert.equal(visible[0]?.version, "1.2.0"); + assert.deepEqual( + visible[0]?.updateState, + flag === true ? providers[0]?.updateState : undefined, + ); + assert.deepEqual(visible[1], providers[1]); + } + } + }), +); + const settingsUpdated = (settings: object): ServerConfigStreamEvent => ({ version: 1, type: "settingsUpdated", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index df20cddd2f585..e9ce76cafbcf1 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -264,6 +264,37 @@ export const resolveAvailableEditorsForConfig = ( discovery: Effect.Effect, E, R>, ) => resolveDiscoveryForConfig(discovery, () => []); +/** Keep background install diagnostics out of clients that cannot distinguish their origin. */ +export const withProviderUpdateOrigins = ( + events: Stream.Stream, + input: { readonly providerUpdateOrigins?: boolean }, +) => { + if (input.providerUpdateOrigins === true) return events; + const providersForClient = (providers: ClientServerConfig["providers"]) => + providers.map((provider) => { + if (provider.updateState?.origin !== "background") return provider; + const { updateState: _updateState, ...legacyProvider } = provider; + return legacyProvider; + }); + return events.pipe( + Stream.map((event): ServerConfigStreamEvent => { + if (event.type === "snapshot") { + return { + ...event, + config: { ...event.config, providers: providersForClient(event.config.providers) }, + }; + } + if (event.type === "providerStatuses") { + return { + ...event, + payload: { providers: providersForClient(event.payload.providers) }, + }; + } + return event; + }), + ); +}; + const resolveFileManagerRevealKindForConfig = ( discovery: Effect.Effect, ) => resolveDiscoveryForConfig(discovery, () => undefined); @@ -3052,7 +3083,7 @@ const layerWsRpc = ( return Stream.concat( rpcInitialItems([{ version: 1 as const, type: "snapshot" as const, config }]), withLateEditorConfig(config, liveUpdates, externalLauncher), - ); + ).pipe((events) => withProviderUpdateOrigins(events, input)); }), ), [WS_METHODS.subscribeServerLifecycle]: (_input) => diff --git a/docs/internals/providers.md b/docs/internals/providers.md index 5f9642c2db27d..29cc4883ce5f0 100644 --- a/docs/internals/providers.md +++ b/docs/internals/providers.md @@ -86,8 +86,8 @@ and npm are proven by the real path (symlinks followed): a versioned keg or cask Native installer layouts and the global directories of pnpm, Bun, Yarn, and Vite+ may match on either the resolved path or its real target, since those installers place real files or their own symlinks there. Volta is proven by its `volta-shim` link plus the package's image directory. When -nothing is proven, the provider's own updater (`claude update`, `codex update`, `opencode upgrade`, -`pi update --self`, `grok update`) runs instead, because each one detects its installer itself; +nothing is proven, a manual update can use the provider's own updater (`claude update`, +`codex update`, `opencode upgrade`, `pi update --self`, `grok update`), which detects its installer itself; the runner's version check catches an updater that exits 0 without updating. Mise installs stay manual-only because their version is pinned in mise's config. npm updates pin `--prefix` because the `npm` on `PATH` can belong to a different Node than the one that owns the provider. Homebrew @@ -95,6 +95,11 @@ compares against `brew info` since casks trail npm by hours; native installs sha train, so the registry stays authoritative for them. See the [resolver](../../apps/server/src/provider/providerMaintenance.ts). +Background updates require a proven installation identity. Manual and background +installs drain host-wide provider leases before closing admission to CLI launches: +a session holding a lease may still need admission to finish launching. This order +also protects another environment on the same machine from a binary being replaced. + Ownership is cached per instance and re-read immediately before an update runs. The [runner](../../apps/server/src/provider/providerMaintenanceRunner.ts) refuses when the lock key changed since the advisory, and reports success only when the refreshed provider is still installed diff --git a/docs/user/updating.md b/docs/user/updating.md index bda372495cf43..4e21b6745bb4e 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -86,6 +86,8 @@ automatic updates stay off. ACP Registry agents, including Antigravity, and local ACP commands do not offer one-click updates and are not updated by this setting. +If T3 Code cannot identify a CLI's installation, its update button can still run +the CLI's own updater manually, but automatic updates leave it alone. **Settings → Providers** shows provider updates for the selected environment. **Update all** updates every outdated provider on every connected environment @@ -102,7 +104,7 @@ the machine instead. The same page lets you refresh provider status and update supported providers. These controls require a connected environment and permission to operate it. -Provider update checks and restart continuation preferences are in +Automatic updates, provider update checks, and restart continuation preferences are in **Settings → Maintenance**. If provider update checks are disabled, enable them there before refreshing to find newer versions. diff --git a/packages/client-runtime/src/rpc/session.test.ts b/packages/client-runtime/src/rpc/session.test.ts index 38e1b7f89d31e..aff0b0f330a0c 100644 --- a/packages/client-runtime/src/rpc/session.test.ts +++ b/packages/client-runtime/src/rpc/session.test.ts @@ -264,6 +264,7 @@ const completeInitialConfig = Effect.fn("TestRpcSessionFactory.completeInitialCo tag: WS_METHODS.subscribeServerConfig, payload, }); + expect(request).toMatchObject({ payload: { providerUpdateOrigins: true } }); socket.serverMessage( encodeJson({ _tag: "Chunk", diff --git a/packages/client-runtime/src/rpc/session.ts b/packages/client-runtime/src/rpc/session.ts index 9f83e7c6019dc..ab33e15d91d67 100644 --- a/packages/client-runtime/src/rpc/session.ts +++ b/packages/client-runtime/src/rpc/session.ts @@ -154,6 +154,7 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* ( ) { const webSocketConstructor = yield* Socket.WebSocketConstructor; const serverConfigInput: ServerConfigSubscriptionInput = { + providerUpdateOrigins: true, ...(options.environmentThemes === true ? { environmentThemes: true } : {}), ...(options.usageLimitSources === true ? { usageLimitSources: true } : {}), ...(options.usageLimitsCommand === true ? { usageLimitsCommand: true } : {}), @@ -361,16 +362,18 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* ( }), ), ); - const subscribeServerConfig = (input: ServerConfigSubscriptionInput) => - Stream.unwrap( + const subscribeServerConfig = (input: ServerConfigSubscriptionInput) => { + const payload = { ...input, providerUpdateOrigins: true }; + return Stream.unwrap( validatedInitialConfig.pipe( Effect.as( - Equal.equals(input, serverConfigInput) + Equal.equals(payload, serverConfigInput) ? serverConfigEvents - : protocolClient[WS_METHODS.subscribeServerConfig](input), + : protocolClient[WS_METHODS.subscribeServerConfig](payload), ), ), ); + }; const probe = initialConfig.pipe( Effect.flatMap((config) => (config.environment.capabilities.connectionProbe === true diff --git a/packages/contracts/src/rpc.test.ts b/packages/contracts/src/rpc.test.ts index 7b2832e896ecb..1904e1c48bc10 100644 --- a/packages/contracts/src/rpc.test.ts +++ b/packages/contracts/src/rpc.test.ts @@ -5,6 +5,9 @@ import * as Schema from "effect/Schema"; import { ORCHESTRATION_V2_WS_METHODS } from "./orchestrationV2.ts"; import { WsRpcGroup, WsSubscribeServerConfigRpc } from "./rpc.ts"; +const decodeLegacyConfigSubscription = Schema.decodeExit(Schema.Struct({})); +const decodeConfigSubscription = Schema.decodeSync(WsSubscribeServerConfigRpc.payloadSchema); + /** * The client always sends `environmentThemes`, including to servers built * before the field existed, whose payload schema was an empty struct. What @@ -29,6 +32,13 @@ describe("subscribeServerConfig payload compatibility", () => { const decoded = Schema.decodeSync(WsSubscribeServerConfigRpc.payloadSchema)({}); expect(decoded).toEqual({}); }); + + it("negotiates provider update origins without rejecting older clients or servers", () => { + const payload = { providerUpdateOrigins: true }; + expect(Exit.isSuccess(decodeLegacyConfigSubscription(payload))).toBe(true); + expect(decodeConfigSubscription(payload)).toEqual(payload); + expect(decodeConfigSubscription({})).toEqual({}); + }); }); describe("WebSocket RPC contracts", () => { diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 225a38176e152..8b50aec496a16 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -1730,6 +1730,8 @@ const WsSubscribeTerminalMetadataRpc = Rpc.make(WS_METHODS.subscribeTerminalMeta export const WsSubscribeServerConfigRpc = Rpc.make(WS_METHODS.subscribeServerConfig, { payload: Schema.Struct({ + /** Older clients show background updates as manual update notifications. */ + providerUpdateOrigins: Schema.optionalKey(Schema.Boolean), /** * Whether this client understands `environmentThemesUpdated` events. * Already-shipped clients decode the stream against the old event union From c665cdb757c45a4bcd7d26fffd870e1151f3513a Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:05:49 +0000 Subject: [PATCH 098/149] fix(server): background server updates survive redirects, clock jumps, and interrupted writes Release redirects can no longer downgrade from https to plain http, accepted handoff latches and launcher health use monotonic time, an update stops when the setting is turned off during the feed lookup, the static asset manifest is written atomically so previous-build assets survive an interrupted write, restarts of the same build reuse cached assets, and restart announcements expire by server time rather than the client clock. --- apps/server/src/cloud/pinnedRuntime.test.ts | 41 ++++- apps/server/src/cloud/pinnedRuntime.ts | 33 +++- apps/server/src/cloud/selfUpdate.test.ts | 159 +++++++++++------- apps/server/src/cloud/selfUpdate.ts | 4 +- apps/server/src/http.test.ts | 116 ++++++++----- apps/server/src/http.ts | 21 ++- apps/server/src/serverLifecycleEvents.test.ts | 19 ++- apps/server/src/serverLifecycleEvents.ts | 13 ++ apps/server/src/serviceLauncher.test.ts | 68 ++++++-- apps/server/src/serviceLauncher.ts | 8 +- apps/server/src/updates/ServerAutoUpdater.ts | 1 + .../client-runtime/src/state/server.test.ts | 21 ++- packages/client-runtime/src/state/server.ts | 59 ++++--- packages/contracts/src/server.ts | 1 + 14 files changed, 393 insertions(+), 171 deletions(-) diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts index 36bc26bc82e56..39a872264f1ee 100644 --- a/apps/server/src/cloud/pinnedRuntime.test.ts +++ b/apps/server/src/cloud/pinnedRuntime.test.ts @@ -5,7 +5,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Path from "effect/Path"; -import { HttpClient, HttpClientResponse } from "effect/http"; +import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as ProcessRunner from "../processRunner.ts"; @@ -63,6 +63,45 @@ const extractingRunner = (fs: FileSystem.FileSystem, path: Path.Path, commands: }); it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { + it.effect.each(["http://mirror.example/hop", "http://127.0.0.1:1234/hop"])( + "refuses an HTTPS redirect through %s", + (location) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const checksums = yield* validChecksums; + const requests: string[] = []; + const httpClient = yield* HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer)); + const result = yield* ensurePinnedRuntimeInstalled({ + baseDir: yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-redirect-hop-" }), + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient, + runner: extractingRunner(fs, path), + validate: () => Effect.void, + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, async (input, init) => { + const url = String(input); + requests.push(url); + const response = + init?.redirect === "manual" + ? new Response(null, { + status: 302, + headers: { location }, + }) + : new Response(url.endsWith("/SHA256SUMS") ? checksums : archiveBytes); + Object.defineProperty(response, "url", { value: "https://mirror.example/final" }); + return response; + }), + Effect.result, + ); + assert.equal(result._tag, "Failure"); + assert.equal(requests.length, 1); + }), + ); it.effect("refuses a plaintext release mirror before fetching checksums or executing it", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts index 488a4426198c6..7ddd5931ed08e 100644 --- a/apps/server/src/cloud/pinnedRuntime.ts +++ b/apps/server/src/cloud/pinnedRuntime.ts @@ -7,7 +7,7 @@ import * as Schema from "effect/Schema"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; import { CLI_RELEASE_CHECKSUMS_FILE, @@ -138,11 +138,35 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")( ) { // The install lock is held for the whole transaction, so a stalled download // must fail rather than block every other caller. - return yield* httpClient.execute(HttpClientRequest.get(url)).pipe( - Effect.flatMap(HttpClientResponse.filterStatusOk), + const requiresHttps = new URL(url).protocol === "https:"; + const validateDownload = (targetUrl: string) => { + const target = validateCliReleaseDownloadUrl(targetUrl); + if (requiresHttps && target.protocol !== "https:") { + throw new Error("HTTPS release downloads cannot redirect to HTTP."); + } + }; + return yield* Effect.gen(function* () { + let nextUrl = url; + for (let redirects = 0; ; redirects++) { + yield* Effect.try(() => validateDownload(nextUrl)); + const response = yield* HttpClient.withScope(httpClient) + .execute(HttpClientRequest.get(nextUrl)) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })); + yield* Effect.try(() => validateDownload(response.url || nextUrl)); + if (![301, 302, 303, 307, 308].includes(response.status) || !response.headers.location) { + return yield* HttpClientResponse.filterStatusOk(response); + } + if (redirects >= 10) + return yield* new PinnedRuntimeInstallError({ + step: "following too many release redirects", + }); + nextUrl = yield* Effect.try( + () => new URL(response.headers.location!, response.url || nextUrl).href, + ); + } + }).pipe( Effect.flatMap( Effect.fn(function* (response) { - yield* Effect.try(() => validateCliReleaseDownloadUrl(response.url || url)); if (onProgress === undefined) return new Uint8Array(yield* response.arrayBuffer); const length = Number(response.headers["content-length"]); const total = Number.isFinite(length) && length > 0 ? length : undefined; @@ -170,6 +194,7 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")( return bytes; }), ), + Effect.scoped, Effect.mapError((cause) => new PinnedRuntimeInstallError({ step, cause })), Effect.timeoutOrElse({ duration: PINNED_RUNTIME_INSTALL_TIMEOUT, diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 440c686849742..0004613dff223 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "@effect/vitest"; import { ServerSelfUpdateError, ThreadId } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -14,7 +15,7 @@ import * as ServerLifecycleEvents from "../serverLifecycleEvents.ts"; import * as ServerAutoUpdater from "../updates/ServerAutoUpdater.ts"; import * as UpdateWindow from "../updates/UpdateWindow.ts"; import * as ServerSettings from "../serverSettings.ts"; -import { ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; +import { DEFAULT_SERVER_SETTINGS, ORCHESTRATION_PROTOCOL_VERSION } from "@t3tools/contracts"; import * as Path from "effect/Path"; import { HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -175,75 +176,107 @@ it.layer(NodeServices.layer)("server self update", (it) => { }), ); - it.effect("rechecks a staged release after a closed window and refuses a withdrawn target", () => + it.effect("expires the handoff latch by elapsed time despite wall clock corrections", () => Effect.gen(function* () { - let latest = "1.1.0"; - let open = false; - const staged: string[] = []; - const installed: string[] = []; - const config = yield* ServerConfig.ServerConfig.pipe( - Effect.provide( - ServerConfig.layerTest(process.cwd(), { prefix: "t3-background-update-test-" }), - ), - ); - const { run: pass } = yield* ServerAutoUpdater.makeUpdatePass().pipe( - Effect.provide( - Layer.mergeAll( - ServerConfig.layer({ ...config, devUrl: undefined }), - ServerSettings.layerTest({ automaticUpdates: true }), - ), - ), - Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, { - managed: true, - lastOutcome: undefined, - requestUpdate: () => Effect.succeed("id"), - prepareTrial: Effect.undefined, - }), - Effect.provideService(ServerSelfUpdate.ServerSelfUpdate, { - capability: "boot-service", - commitDesktopUpdate: () => Effect.never, - stage: (target) => - Effect.sync(() => { - staged.push(target); - return { orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION }; - }), - update: ({ targetVersion }) => - Effect.sync(() => { - installed.push(targetVersion); - return { targetVersion, method: "boot-service" as const }; - }), - }), - Effect.provideService(UpdateWindow.UpdateWindow, { - runIfOpen: (effect) => - open ? Effect.map(effect, Option.some) : Effect.succeed(Option.none()), + const { selfUpdate } = yield* makeHarness(); + const clock = yield* Clock.Clock; + let wallTime = 0n; + const update = selfUpdate.update({ targetVersion: "1.1.0" }).pipe( + Effect.provideService(Clock.Clock, { + ...clock, + currentTimeNanos: Effect.sync(() => wallTime), + currentTimeNanosUnsafe: () => wallTime, }), - Effect.provideService( - HttpClient.HttpClient, - HttpClient.make((request) => - Effect.sync(() => - HttpClientResponse.fromWeb( - request, - new Response(JSON.stringify([{ tag_name: `v${latest}` }])), - ), - ), - ), - ), ); - yield* pass; - expect(staged).toEqual(["1.1.0"]); - latest = "1.2.0"; - yield* TestClock.adjust("1 hour"); - yield* pass; - expect(staged).toEqual(["1.1.0", "1.2.0"]); - latest = "1.0.0"; - open = true; - const handoff = yield* pass.pipe(Effect.forkChild); + yield* update; + wallTime = 600_000_000_000n; + expect((yield* update.pipe(Effect.flip)).reason).toBe( + "A server update is already in progress.", + ); + wallTime = -600_000_000_000n; yield* TestClock.adjust("5 minutes"); - yield* Fiber.join(handoff); - expect(installed).toEqual([]); + expect((yield* update).updateId).toBe("launcher-id"); }), ); + it.effect.each(["withdrawn", "disabled"] as const)( + "rechecks a staged release at handoff: %s", + (scenario) => + Effect.gen(function* () { + let latest = "1.1.0"; + let open = false; + let enabled = true; + const staged: string[] = []; + const installed: string[] = []; + const config = yield* ServerConfig.ServerConfig.pipe( + Effect.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-background-update-test-" }), + ), + ); + const { run: pass } = yield* ServerAutoUpdater.makeUpdatePass().pipe( + Effect.provide( + Layer.mergeAll( + ServerConfig.layer({ ...config, devUrl: undefined }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.sync(() => ({ + ...DEFAULT_SERVER_SETTINGS, + automaticUpdates: enabled, + })), + }), + ), + ), + Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, { + managed: true, + lastOutcome: undefined, + requestUpdate: () => Effect.succeed("id"), + prepareTrial: Effect.undefined, + }), + Effect.provideService(ServerSelfUpdate.ServerSelfUpdate, { + capability: "boot-service", + commitDesktopUpdate: () => Effect.never, + stage: (target) => + Effect.sync(() => { + staged.push(target); + return { orchestrationProtocol: ORCHESTRATION_PROTOCOL_VERSION }; + }), + update: ({ targetVersion }) => + Effect.sync(() => { + installed.push(targetVersion); + return { targetVersion, method: "boot-service" as const }; + }), + }), + Effect.provideService(UpdateWindow.UpdateWindow, { + runIfOpen: (effect) => + open ? Effect.map(effect, Option.some) : Effect.succeed(Option.none()), + }), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.sync(() => { + if (open && scenario === "disabled") enabled = false; + return HttpClientResponse.fromWeb( + request, + new Response(JSON.stringify([{ tag_name: `v${latest}` }])), + ); + }), + ), + ), + ); + yield* pass; + expect(staged).toEqual(["1.1.0"]); + latest = "1.2.0"; + yield* TestClock.adjust("1 hour"); + yield* pass; + expect(staged).toEqual(["1.1.0", "1.2.0"]); + if (scenario === "withdrawn") latest = "1.0.0"; + open = true; + const handoff = yield* pass.pipe(Effect.forkChild); + yield* TestClock.adjust("5 minutes"); + yield* Fiber.join(handoff); + expect(installed).toEqual([]); + }), + ); + it.effect("marks running threads at the boot-service handoff", () => Effect.gen(function* () { const events: string[] = []; diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 4f128af67bbe1..e297fa1aa2264 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -204,7 +204,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { const lifecycle = yield* ServerLifecycleEvents.ServerLifecycleEvents; const inFlight = yield* Ref.make<{ readonly acceptedAt?: bigint } | undefined>(undefined); const acquire = Effect.gen(function* () { - const now = yield* Clock.currentTimeNanos; + const now = yield* Clock.monotonicTimeNanos; const acquired = yield* Ref.modify(inFlight, (current) => { const expired = current?.acceptedAt !== undefined && @@ -371,7 +371,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { ), ), Effect.tap((updateId) => - Clock.currentTimeNanos.pipe( + Clock.monotonicTimeNanos.pipe( Effect.flatMap((acceptedAt) => Ref.set(inFlight, { acceptedAt })), Effect.andThen(onHandoffAccepted()), Effect.andThen( diff --git a/apps/server/src/http.test.ts b/apps/server/src/http.test.ts index 84d409b917816..98e99a69fd9c1 100644 --- a/apps/server/src/http.test.ts +++ b/apps/server/src/http.test.ts @@ -144,47 +144,83 @@ it.layer( Layer.provideMerge(NodeServices.layer), ), )("static HTTP responses", (it) => { - it.effect("serves the previous build's lazy assets after restart and prunes older builds", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const staticDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-static-generations-" }); - yield* fs.makeDirectory(path.join(staticDir, "assets")); - yield* fs.makeDirectory(path.join(staticDir, ".vite")); - const builds = ["settings-aaaaaaaa.js", "settings-bbbbbbbb.js", "settings-cccccccc.js"]; - for (const [index, file] of builds.entries()) { - const asset = `assets/${file}`; - yield* fs.writeFileString(path.join(staticDir, asset), `export const build = ${index};`); - yield* fs.writeFileString( - path.join(staticDir, "index.html"), - `build ${index}`, - ); - yield* fs.writeFileString( - path.join(staticDir, ".vite", "manifest.json"), - JSON.stringify({ settings: { file: asset } }), - ); - const request = yield* makeStaticRequest(staticDir); - const current = yield* request(`/${asset}`); - expect(yield* current.text).toBe(`export const build = ${index};`); - if (index > 0) { - const previous = yield* request(`/assets/${builds[index - 1]}`); - expect(previous.status).toBe(200); - expect(previous.headers["content-type"]).toContain("javascript"); - expect(previous.headers["cache-control"]).toBe("public, max-age=31536000, immutable"); - expect(yield* previous.text).toBe(`export const build = ${index - 1};`); - const restarted = yield* makeStaticRequest(staticDir); - const stillPrevious = yield* restarted(`/assets/${builds[index - 1]}`); - expect(stillPrevious.status).toBe(200); - expect(yield* stillPrevious.text).toBe(`export const build = ${index - 1};`); - } - if (index === 2) { - const expired = yield* request(`/assets/${builds[0]}`); - expect(expired.status).toBe(404); - expect(yield* expired.text).toBe("Not Found"); + it.effect.each([false, true])( + "retains two builds across restart (interrupted manifest write: %s)", + (interruptedWrite) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const staticDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-static-generations-" }); + yield* fs.makeDirectory(path.join(staticDir, "assets")); + yield* fs.makeDirectory(path.join(staticDir, ".vite")); + let copies = 0; + const cachedFs = { + ...fs, + copyFile: (...args: Parameters) => + Effect.sync(() => copies++).pipe(Effect.andThen(fs.copyFile(...args))), + }; + const builds = ["settings-aaaaaaaa.js", "settings-bbbbbbbb.js", "settings-cccccccc.js"]; + for (const [index, file] of builds.entries()) { + const asset = `assets/${file}`; + yield* fs.writeFileString(path.join(staticDir, asset), `export const build = ${index};`); + yield* fs.writeFileString( + path.join(staticDir, "index.html"), + `build ${index}`, + ); + yield* fs.writeFileString( + path.join(staticDir, ".vite", "manifest.json"), + JSON.stringify({ settings: { file: asset } }), + ); + if (interruptedWrite && index === 1) { + let writes = 0; + yield* makeStaticRequest(staticDir).pipe( + Effect.flatMap((request) => request(`/${asset}`)), + Effect.provideService(FileSystem.FileSystem, { + ...fs, + writeFileString: (file, contents, options) => + file.includes("/web-assets/") && contents.includes('"current"') + ? fs.writeFileString(file, "", options).pipe( + Effect.tap(() => + Effect.sync(() => { + writes++; + }), + ), + Effect.andThen(Effect.die("interrupted cache write")), + ) + : fs.writeFileString(file, contents, options), + }), + ); + expect(writes).toBe(1); + } + const request = yield* makeStaticRequest(staticDir).pipe( + Effect.provideService(FileSystem.FileSystem, cachedFs), + ); + const current = yield* request(`/${asset}`); + expect(yield* current.text).toBe(`export const build = ${index};`); + if (index > 0) { + const previous = yield* request(`/assets/${builds[index - 1]}`); + expect(previous.status).toBe(200); + expect(previous.headers["content-type"]).toContain("javascript"); + expect(previous.headers["cache-control"]).toBe("public, max-age=31536000, immutable"); + expect(yield* previous.text).toBe(`export const build = ${index - 1};`); + // A restart of the same build reuses the cached copies. + const beforeCopies = copies; + const restarted = yield* makeStaticRequest(staticDir).pipe( + Effect.provideService(FileSystem.FileSystem, cachedFs), + ); + expect(copies).toBe(beforeCopies); + const stillPrevious = yield* restarted(`/assets/${builds[index - 1]}`); + expect(stillPrevious.status).toBe(200); + expect(yield* stillPrevious.text).toBe(`export const build = ${index - 1};`); + } + if (index === 2) { + const expired = yield* request(`/assets/${builds[0]}`); + expect(expired.status).toBe(404); + expect(yield* expired.text).toBe("Not Found"); + } + yield* fs.remove(path.join(staticDir, asset)); } - yield* fs.remove(path.join(staticDir, asset)); - } - }), + }), ); it.effect("returns 404 for missing assets while preserving SPA route fallback", () => diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 450f92fcc4fab..84d870421b086 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -5,6 +5,7 @@ import { EnvironmentHttpApi, } from "@t3tools/contracts"; import { isDevProxiedPath } from "@t3tools/shared/devProxy"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import { decodeOtlpTraceRecords } from "@t3tools/shared/observability"; import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; @@ -546,24 +547,26 @@ const loadImmutableBuildAssets = Effect.gen(function* () { Effect.orElseSucceed(() => emptyBuildAssetCache), ); const currentAssets = [...current].filter(isHashedBuildAsset); + const savedCurrent = new Set(saved.current); const sameBuild = - currentAssets.length === saved.current.length && - currentAssets.every((asset) => saved.current.includes(asset)); + currentAssets.length === savedCurrent.size && + currentAssets.every((asset) => savedCurrent.has(asset)); const previous = sameBuild ? saved.previous : saved.current; const retained = new Set([...currentAssets, ...previous].filter(isHashedBuildAsset)); const filesDir = path.join(cacheDir, "assets"); yield* fileSystem.makeDirectory(filesDir, { recursive: true }); - for (const asset of current) { - if (!isHashedBuildAsset(asset)) continue; - yield* fileSystem.copyFile(path.join(staticDir, asset), path.join(cacheDir, asset)); + for (const asset of currentAssets) { + const cached = path.join(cacheDir, asset); + if (savedCurrent.has(asset) && (yield* fileSystem.exists(cached))) continue; + yield* fileSystem.copyFile(path.join(staticDir, asset), cached); } for (const file of yield* fileSystem.readDirectory(filesDir)) { if (!retained.has(`assets/${file}`)) yield* fileSystem.remove(path.join(filesDir, file)); } - yield* fileSystem.writeFileString( - manifestPath, - yield* encodeBuildAssetCache({ current: currentAssets, previous }), - ); + yield* writeFileStringAtomically({ + filePath: manifestPath, + contents: yield* encodeBuildAssetCache({ current: currentAssets, previous }), + }); return new Set([...current, ...retained]); }).pipe( Effect.catchCause((cause) => diff --git a/apps/server/src/serverLifecycleEvents.test.ts b/apps/server/src/serverLifecycleEvents.test.ts index ea0807f5277d4..c25fc9b84bfc4 100644 --- a/apps/server/src/serverLifecycleEvents.test.ts +++ b/apps/server/src/serverLifecycleEvents.test.ts @@ -91,17 +91,26 @@ it.effect("announces restarts only to capable clients and expires abandoned hand }, }); const events = Stream.fromIterable((yield* lifecycle.snapshot).events); + yield* TestClock.adjust("1 minute"); + const capable = yield* ServerLifecycleEvents.eventsForClient(events, { + updateRestarts: true, + }).pipe(Stream.runCollect); + assert.equal(capable.length, 1); assert.equal( - (yield* ServerLifecycleEvents.eventsForClient(events, { updateRestarts: true }).pipe( - Stream.runCollect, - )).length, - 1, + capable[0]?.type === "updateRestart" ? capable[0].payload.expiresInMs : undefined, + 240_000, ); assert.equal( (yield* ServerLifecycleEvents.eventsForClient(events, {}).pipe(Stream.runCollect)).length, 0, ); - yield* TestClock.adjust("5 minutes"); + yield* TestClock.adjust("4 minutes"); assert.equal((yield* lifecycle.snapshot).events.length, 0); + assert.equal( + (yield* ServerLifecycleEvents.eventsForClient(events, { updateRestarts: true }).pipe( + Stream.runCollect, + )).length, + 0, + ); }).pipe(Effect.provide(ServerLifecycleEvents.layer)), ); diff --git a/apps/server/src/serverLifecycleEvents.ts b/apps/server/src/serverLifecycleEvents.ts index c0eaa2efe614e..4c2a0a8c5eda7 100644 --- a/apps/server/src/serverLifecycleEvents.ts +++ b/apps/server/src/serverLifecycleEvents.ts @@ -14,6 +14,19 @@ export const eventsForClient = ( ) => events.pipe( Stream.filter((event) => event.type !== "updateRestart" || input.updateRestarts === true), + Stream.mapEffect((event) => { + if (event.type !== "updateRestart" || event.payload.expiresAt === undefined) { + return Effect.succeed(event); + } + const expiresAt = event.payload.expiresAt; + return Clock.currentTimeMillis.pipe( + Effect.map((now) => ({ + ...event, + payload: { ...event.payload, expiresInMs: Math.max(0, expiresAt - now) }, + })), + ); + }), + Stream.filter((event) => event.type !== "updateRestart" || event.payload.expiresInMs !== 0), ); type LifecycleEventInput = diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index d52c5c492341c..0fde07960c07a 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,8 +1,13 @@ +// @effect-diagnostics nodeBuiltinImport:off - Launcher recovery injects native filesystem failures. import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as NodeFSP from "node:fs/promises"; + +vi.mock("node:fs/promises", { spy: true }); import { Launcher, @@ -201,9 +206,9 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect( - "rolls back and restores the database when a prepared candidate crashes after commit", - () => + it.effect.each([false, true])( + "restores the database after a post-commit crash (wall clock jumps: %s)", + (clockJumps) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -220,7 +225,7 @@ if (context.update?.status === "pending") { process.on("message", (message) => { if (message.type === "committed") { require("node:fs").writeFileSync(${encodedDatabasePath}, "post-commit migration"); - process.exit(42); + setTimeout(() => process.exit(42), 100); } }); } else if (context.update === undefined) { @@ -249,6 +254,13 @@ if (context.update?.status === "pending") { root, yield* Effect.promise(() => readServiceState(statePath)), ); + if (clockJumps) { + let now = 0; + yield* Effect.acquireRelease( + Effect.sync(() => vi.spyOn(Date, "now").mockImplementation(() => (now += 600_000))), + (clock) => Effect.sync(() => clock.mockRestore()), + ); + } yield* Effect.promise(() => launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), @@ -268,9 +280,9 @@ if (context.update?.status === "pending") { }), ); - it.effect( - "recovers a committed candidate crash using the retained backup after launcher restart", - () => + it.effect.each(["committed", "pending", "restore-interrupted", "restore-disk-full"] as const)( + "recovers a retained database backup after %s", + (checkpoint) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -281,6 +293,12 @@ if (context.update?.status === "pending") { const backup = path.join(root, "runtime", "db-backup", "health-update"); yield* fs.makeDirectory(backup, { recursive: true }); yield* fs.writeFileString(path.join(backup, "database"), "original database"); + yield* fs.writeFileString(`${dbPath}-wal`, "trial wal"); + yield* fs.writeFileString(`${dbPath}-shm`, "trial shm"); + if (checkpoint === "restore-interrupted") { + yield* fs.writeFileString(path.join(backup, ".restore-pending"), ""); + yield* fs.writeFileString(dbPath, "partially restored database"); + } for (const version of ["1.0.0", "1.1.0"]) { yield* writeFakeRuntime( fs, @@ -299,21 +317,51 @@ if (context.update?.status === "pending") { yield* Effect.promise(() => writeServiceState(statePath, { protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.1.0", + activeVersion: checkpoint === "pending" ? "1.0.0" : "1.1.0", launcherVersion: "1.0.0", - update: { ...update, status: "committed" }, - rollback: { update, healthyAfter: 60_000 }, + ...(checkpoint === "pending" + ? { update } + : { + update: { ...update, status: "committed" }, + rollback: { update, healthyAfter: 60_000 }, + }), }), ); const launcher = new Launcher( root, yield* Effect.promise(() => readServiceState(statePath)), ); + const diskFull = + checkpoint === "restore-disk-full" + ? yield* Effect.acquireRelease( + Effect.sync(() => + vi + .spyOn(NodeFSP, "copyFile") + .mockRejectedValue(Object.assign(new Error("disk full"), { code: "ENOSPC" })), + ), + (copy) => Effect.sync(() => copy.mockRestore()), + ) + : undefined; yield* Effect.promise(() => launcher.run().catch(() => undefined)); + if (diskFull !== undefined) { + assert.equal( + (yield* Effect.promise(() => readServiceState(statePath))).activeVersion, + "1.1.0", + ); + assert.isTrue(yield* fs.exists(path.join(backup, ".restore-pending"))); + diskFull.mockRestore(); + const retry = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.promise(() => retry.run().catch(() => undefined)); + } const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.0.0"); assert.equal(state.update?.status, "rolled-back"); assert.equal(yield* fs.readFileString(dbPath), "original database"); + assert.isFalse(yield* fs.exists(`${dbPath}-wal`)); + assert.isFalse(yield* fs.exists(`${dbPath}-shm`)); assert.isFalse(yield* fs.exists(backup)); }), ); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 3f0501be42dd8..452368e98ce2b 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -427,7 +427,7 @@ export class Launcher { await writeServiceState(this.#statePath, this.#state); try { await this.#startChild(this.#state.activeVersion, "active", this.#state.update); - this.#watchHealth(healthyAfter); + this.#watchHealth(); } catch { await this.#returnToPrevious(rollback.update, "rolled-back", "candidate-start-failed"); } @@ -615,11 +615,11 @@ export class Launcher { await writeServiceState(this.#statePath, next); this.#state = next; child.role = "active"; - this.#watchHealth(healthyAfter); + this.#watchHealth(); await sendMessage(child.process, { type: "committed", updateId: committed.id }); } - #watchHealth(healthyAfter: number): void { + #watchHealth(): void { this.#timer = setTimeout( () => this.#enqueue(async () => { @@ -638,7 +638,7 @@ export class Launcher { await discardDatabaseBackup(this.#baseDir, rollback.update.id).catch(() => undefined); await pruneServiceRuntimes(this.#baseDir).catch(() => undefined); }), - Math.max(0, healthyAfter - Date.now()), + HEALTH_PERIOD_MS, ); } diff --git a/apps/server/src/updates/ServerAutoUpdater.ts b/apps/server/src/updates/ServerAutoUpdater.ts index 7b1c710639a19..b4c24008a738c 100644 --- a/apps/server/src/updates/ServerAutoUpdater.ts +++ b/apps/server/src/updates/ServerAutoUpdater.ts @@ -112,6 +112,7 @@ export const makeUpdatePass = () => const newest = yield* resolveNewestVersion(cliReleaseChannelOf(currentVersion)).pipe( Effect.provideService(HttpClient.HttpClient, httpClient), ); + if (!(yield* automaticUpdatesEnabled)) return; if (newest !== targetVersion) { pendingTarget = undefined; nextReleaseCheckAt = 0; diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index 955a7661d8a99..72bb028ac9b67 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -564,9 +564,9 @@ describe("server state projection", () => { expect(resolveServerWelcomeState(afterBufferedOldWelcome)).toBeNull(); }); - it.effect( - "observes a background restart from the lifecycle stream and nudges reconnect until ready", - () => + it.effect.each(["ready", "expiry", "expiry with server clock ahead"] as const)( + "observes a background restart and clears it on %s", + (completion) => Effect.gen(function* () { const events = yield* Queue.unbounded<{ readonly type: "updateRestart" | "ready"; @@ -605,7 +605,13 @@ describe("server state projection", () => { ); yield* Queue.offer(events, { type: "updateRestart", - payload: { fromVersion: "1.0.0", targetVersion: "1.1.0", updateId: "background" }, + payload: { + fromVersion: "1.0.0", + targetVersion: "1.1.0", + updateId: "background", + expiresAt: completion === "expiry with server clock ahead" ? 3_602_000 : 2_000, + expiresInMs: 2_000, + }, }); expect(Option.getOrThrow(yield* Fiber.join(announced)).restart?.updateId).toBe( "background", @@ -621,7 +627,12 @@ describe("server state projection", () => { Stream.runHead, Effect.forkChild, ); - yield* Queue.offer(events, { type: "ready", payload: {} }); + if (completion === "ready") yield* Queue.offer(events, { type: "ready", payload: {} }); + else { + yield* TestClock.adjust("1 second"); + expect((yield* SubscriptionRef.get(state)).restart).toBeUndefined(); + return; + } expect(Option.getOrThrow(yield* Fiber.join(resumed)).restart).toBeUndefined(); }), ); diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 411b8b45defb0..08f1e6f17f326 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -590,34 +590,37 @@ export const makeEnvironmentServerWelcomeState = Effect.fn("EnvironmentServerWel }), ).pipe( Stream.runForEach(([session, event]) => - updateWithCurrentSession((current, currentSession) => - applyServerWelcomeEvent({ ...current, currentSession }, session, event), - ).pipe( - Effect.andThen( - Effect.gen(function* () { - if ((yield* SubscriptionRef.get(state)).currentSession !== session) return; - if (event.type === "ready" && restartObserver !== undefined) { - yield* Fiber.interrupt(restartObserver); - restartObserver = undefined; - } - if (event.type !== "updateRestart") return; - if (restartObserver !== undefined) yield* Fiber.interrupt(restartObserver); - restartObserver = yield* nudgeReconnectDuringUpdateRestart({ - stateChanges: SubscriptionRef.changes(supervisor.state), - retryNow: supervisor.retryNow, - }).pipe( - Effect.andThen( - SubscriptionRef.update(state, (current) => - current.restart?.updateId === event.payload.updateId - ? { ...current, restart: undefined } - : current, - ), - ), - Effect.forkScoped, - ); - }), - ), - ), + Effect.gen(function* () { + const remaining = + event.type === "updateRestart" && event.payload.expiresInMs !== undefined + ? event.payload.expiresInMs + : Duration.toMillis(SERVER_UPDATE_RESUME_TIMEOUT); + if (remaining <= 0) return; + yield* updateWithCurrentSession((current, currentSession) => + applyServerWelcomeEvent({ ...current, currentSession }, session, event), + ); + if ((yield* SubscriptionRef.get(state)).currentSession !== session) return; + if (event.type === "ready" && restartObserver !== undefined) { + yield* Fiber.interrupt(restartObserver); + restartObserver = undefined; + } + if (event.type !== "updateRestart") return; + if (restartObserver !== undefined) yield* Fiber.interrupt(restartObserver); + restartObserver = yield* nudgeReconnectDuringUpdateRestart({ + stateChanges: SubscriptionRef.changes(supervisor.state), + retryNow: supervisor.retryNow, + }).pipe( + Effect.timeoutOption(Duration.millis(remaining)), + Effect.andThen( + SubscriptionRef.update(state, (current) => + current.restart?.updateId === event.payload.updateId + ? { ...current, restart: undefined } + : current, + ), + ), + Effect.forkScoped, + ); + }), ), Effect.forkScoped, ); diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 17799d281ab5c..52905dde2b853 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -955,6 +955,7 @@ export const ServerLifecycleStreamUpdateRestartEvent = Schema.Struct({ targetVersion: TrimmedNonEmptyString, updateId: TrimmedNonEmptyString, expiresAt: Schema.optionalKey(NonNegativeInt), + expiresInMs: Schema.optionalKey(NonNegativeInt), }), }); export type ServerLifecycleStreamUpdateRestartEvent = From efde7a7d3fa40ae66176f258aa0362bdfdc26a39 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:05:50 +0000 Subject: [PATCH 099/149] perf(server): busy-terminal checks skip the process scan when a prompt is already busy --- apps/server/src/terminal/Manager.test.ts | 21 +++++++++++++++++++++ apps/server/src/terminal/Manager.ts | 9 +++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index aa4bf6f0fbf46..bfe9537f271e2 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -3012,6 +3012,27 @@ it.layer( }), ); + it.effect("rejects a busy terminal without scanning the process table", () => + Effect.gen(function* () { + let scans = 0; + const { manager } = yield* createManager(5, { + shellResolver: () => "/bin/bash", + subprocessPollIntervalMs: 60_000, + processTable: Effect.sync(() => { + scans++; + return [{ pid: 9000, ppid: 1, name: "bash" }]; + }), + }); + yield* manager.open(openInput()); + const before = scans; + expect(yield* manager.hasBusyTerminals).toBe(true); + expect(Option.isNone(yield* manager.withIdleTerminals(Effect.die("must not restart")))).toBe( + true, + ); + expect(scans).toBe(before); + }), + ); + it.effect("tracks PowerShell in-process work until its next prompt", () => Effect.gen(function* () { const { manager, ptyAdapter, getEvents } = yield* createManager(5, { diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index b6664239cce22..f8b00080f0874 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -3349,14 +3349,19 @@ if [[ -f $ZDOTDIR/${name} ]]; then source $ZDOTDIR/${name}; fi if (sessions.some((session) => session.status === "starting")) return true; const running = sessions.filter((session) => session.status === "running"); if (running.length === 0) return false; + if ( + running.some( + (session) => session.pid === null || !session.tracksPrompt || session.commandPending, + ) + ) + return true; const activityMark = (session: TerminalSessionState) => session.eventSequence + session.inputCount; const marks = new Map(running.map((session) => [session, activityMark(session)])); const { inspector, snapshotSucceeded } = yield* acquireSubprocessInspector; if (!snapshotSucceeded) return true; for (const session of running) { - if (session.pid === null || !session.tracksPrompt || session.commandPending) return true; - if ((yield* inspector(session.pid, true)).hasRunningSubprocess) return true; + if ((yield* inspector(session.pid!, true)).hasRunningSubprocess) return true; } const after = yield* readManagerState; return [...after.sessions.values()].some( From 59f3900ed69495b306db2d0a331d5a201432dcaf Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:05:50 +0000 Subject: [PATCH 100/149] docs: describe the launcher health period and post-commit rollback --- docs/internals/server-updates.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 34fa5aa47b094..42d23be167e31 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -9,9 +9,10 @@ processes do not self-update. Exact-version installs keep restarts independent of npm cache eviction or a moving release tag. Installation and preflight happen in staging before publishing an -immutable runtime. Preflight checks the launcher protocol because a target that -needs new rollback guarantees cannot safely run under an older launcher. Upgrading -that launcher requires a local service update. +immutable runtime. Preflight checks the launcher's IPC protocol. The post-commit +health period is launcher-owned and remains compatible with protocol 3; older +launchers can run the new runtime without that extra rollback protection. +Upgrading the launcher requires a local service update. ## Commit boundary @@ -27,8 +28,9 @@ child release its gates, accept commands, and publish ready. Keep fallible start acquisitions before this boundary. A listener alone does not prove the runtime is ready to commit. -A failed or timed-out trial returns to the old version. After commit, the target -is authoritative and the service manager's ordinary restart policy applies. +A failed or timed-out trial returns to the old version. The launcher also rolls +back a committed target that exits during its health period. After that period, +the service manager's ordinary restart policy applies. ## Database rollback @@ -39,7 +41,9 @@ replacing it during a retry could capture changes from the failed trial. Rollback stops the trial before restoring. A durable restore marker makes an interrupted restore finish before either version boots. Keep the snapshot until -commit, or until both restoration and the terminal rollback state are durable. +the committed target survives its health period, or until both restoration and +the terminal rollback state are durable. +Post-commit rollback also discards database writes accepted during the health period. Attachments and other files outside SQLite are outside this rollback boundary. ## Client acknowledgement From f115cab9ad6ed1cc5cefec9b2493ba8d222a0adf Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:06:04 +0000 Subject: [PATCH 101/149] fix(desktop): failed downloads, repeated quits, and AppImage detection no longer misfire Polls no longer retry a manual download that failed, an old failed release is not retried when the feed flips back, a second quit during macOS staging does not start another install, the macOS quit callback is released after a staging timeout, and an inherited APPIMAGE variable no longer enables updates for rpm installs. --- apps/desktop/src/app/DesktopLifecycle.test.ts | 154 ++++++++++++++++-- apps/desktop/src/app/DesktopLifecycle.ts | 14 +- .../src/backend/DesktopBackendPool.test.ts | 41 +++++ .../src/electron/ElectronUpdater.test.ts | 44 ++++- apps/desktop/src/electron/ElectronUpdater.ts | 5 +- .../src/updates/DesktopUpdates.test.ts | 104 ++++++++++-- apps/desktop/src/updates/DesktopUpdates.ts | 31 ++-- 7 files changed, 345 insertions(+), 48 deletions(-) diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.ts index 70152f26e93ab..335b973e736eb 100644 --- a/apps/desktop/src/app/DesktopLifecycle.test.ts +++ b/apps/desktop/src/app/DesktopLifecycle.test.ts @@ -9,6 +9,7 @@ import type * as Electron from "electron"; import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronTheme from "../electron/ElectronTheme.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; +import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; import * as DesktopLifecycle from "./DesktopLifecycle.ts"; import * as DesktopShutdown from "./DesktopShutdown.ts"; @@ -103,6 +104,145 @@ function layerDesktopWindow( } describe("DesktopLifecycle", () => { + it.effect.each(["darwin", "win32", "linux"] satisfies ReadonlyArray)( + "disarms quit installation before a settings relaunch on %s", + (platform) => + Effect.gen(function* () { + const exited = yield* Deferred.make(); + const events: string[] = []; + const layer = DesktopLifecycle.layer.pipe( + Layer.provideMerge(layerElectronApp(new Map())), + Layer.provideMerge(layerElectronTheme), + Layer.provideMerge( + layerDesktopWindow({ + flushMainWindowBounds: Effect.sync(() => { + events.push("flush"); + }), + }), + ), + Layer.provideMerge( + Layer.succeed(DesktopEnvironment.DesktopEnvironment, { + platform, + isDevelopment: false, + } as DesktopEnvironment.DesktopEnvironment["Service"]), + ), + Layer.provideMerge( + Layer.mock(ElectronUpdater.ElectronUpdater)({ + setAutoInstallOnAppQuit: (value) => + Effect.sync(() => { + events.push(`autoInstall:${value}`); + }), + }), + ), + Layer.provideMerge(DesktopShutdown.layer), + Layer.provideMerge(DesktopState.layer), + ); + yield* Effect.gen(function* () { + const lifecycle = yield* DesktopLifecycle.DesktopLifecycle; + const app = yield* ElectronApp.ElectronApp; + const shutdown = yield* DesktopShutdown.DesktopShutdown; + yield* lifecycle.relaunch("settings").pipe( + Effect.provideService(ElectronApp.ElectronApp, { + ...app, + relaunch: () => + Effect.sync(() => { + events.push("relaunch"); + }), + exit: (code) => + Effect.sync(() => { + events.push(`exit:${code}`); + }).pipe(Effect.andThen(Deferred.succeed(exited, undefined)), Effect.asVoid), + }), + ); + yield* shutdown.awaitRequest; + assert.deepEqual(events, ["flush"]); + yield* shutdown.markComplete; + yield* Deferred.await(exited); + assert.deepEqual(events, ["flush", "autoInstall:false", "relaunch", "exit:0"]); + }).pipe(Effect.provide(layer)); + }), + ); + + it.effect.each(["quit", "SIGTERM"] as const)( + "blocks repeated quit events during %s staging", + (trigger) => + Effect.gen(function* () { + const appListeners = new Map void>(); + const stagingStarted = yield* Deferred.make(); + const finishStaging = yield* Deferred.make(); + const quitFinished = yield* Deferred.make(); + let installs = 0; + const attemptQuit = () => { + let prevented = false; + appListeners.get("before-quit")?.({ + preventDefault: () => { + prevented = true; + }, + }); + return prevented; + }; + const layer = DesktopLifecycle.layer.pipe( + Layer.provideMerge( + layerElectronApp( + appListeners, + Effect.sync(() => { + assert.isFalse(attemptQuit()); + }).pipe(Effect.andThen(Deferred.succeed(quitFinished, undefined)), Effect.asVoid), + ), + ), + Layer.provideMerge(layerElectronTheme), + Layer.provideMerge(layerElectronWindow()), + Layer.provideMerge(layerDesktopWindow()), + Layer.provideMerge( + Layer.succeed(DesktopEnvironment.DesktopEnvironment, { + platform: "darwin", + isDevelopment: false, + } as DesktopEnvironment.DesktopEnvironment["Service"]), + ), + Layer.provideMerge( + Layer.succeed(DesktopShutdown.DesktopShutdown, { + request: Effect.void, + awaitRequest: Effect.void, + markComplete: Effect.void, + awaitComplete: Effect.void, + isComplete: Effect.succeed(true), + }), + ), + Layer.provideMerge( + Layer.succeed(DesktopUpdates.DesktopUpdates, { + installOnQuit: Effect.sync(() => { + installs++; + }).pipe( + Effect.andThen(Deferred.succeed(stagingStarted, undefined)), + Effect.andThen(Deferred.await(finishStaging)), + // MacUpdater calls app.quit synchronously from its native event. + Effect.tap(() => Effect.sync(() => assert.isTrue(attemptQuit()))), + Effect.as(true), + ), + } as DesktopUpdates.DesktopUpdates["Service"]), + ), + Layer.provideMerge(DesktopState.layer), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const lifecycle = yield* DesktopLifecycle.DesktopLifecycle; + const previousSignals = process.listeners("SIGTERM"); + yield* lifecycle.register; + if (trigger === "quit") assert.isTrue(attemptQuit()); + else + process + .listeners("SIGTERM") + .find((listener) => !previousSignals.includes(listener))?.("SIGTERM"); + yield* Deferred.await(stagingStarted); + assert.isTrue(attemptQuit()); + yield* Deferred.succeed(finishStaging, undefined); + yield* Deferred.await(quitFinished); + assert.equal(installs, 1); + }), + ).pipe(Effect.provide(layer)); + }), + ); + it.effect.each(["darwin", "win32", "linux"] satisfies ReadonlyArray)( "lets the updater's quit event proceed on %s", (platform) => { @@ -206,11 +346,7 @@ describe("DesktopLifecycle", () => { installOnQuit: Effect.sync(() => { events.push("installOnQuit"); return installOnQuit; - }).pipe( - Effect.tap(() => - installOnQuit ? Deferred.succeed(quitRequested, undefined) : Effect.void, - ), - ), + }), } as DesktopUpdates.DesktopUpdates["Service"]), ), Layer.provideMerge(layerElectronWindow(destroyAll)), @@ -235,13 +371,7 @@ describe("DesktopLifecycle", () => { yield* Deferred.await(quitRequested); assert.deepEqual(eventsBeforeCleanup, ["flush", "destroy", "request"]); - assert.deepEqual(events, [ - "flush", - "destroy", - "request", - "installOnQuit", - ...(installOnQuit ? [] : ["quit"]), - ]); + assert.deepEqual(events, ["flush", "destroy", "request", "installOnQuit", "quit"]); }), ).pipe(Effect.provide(layer)); }), diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.ts index 00d35da9ccbb4..88a7127001840 100644 --- a/apps/desktop/src/app/DesktopLifecycle.ts +++ b/apps/desktop/src/app/DesktopLifecycle.ts @@ -136,12 +136,13 @@ function handleBeforeQuit( ); }).pipe(Effect.withSpan("desktop.lifecycle.beforeQuit")), ).finally(() => { - markQuitAllowed(); void runEffect( Effect.gen(function* () { const electronApp = yield* ElectronApp.ElectronApp; const updates = yield* DesktopUpdates.DesktopUpdates; - if (!(yield* updates.installOnQuit)) yield* electronApp.quit; + yield* updates.installOnQuit; + markQuitAllowed(); + yield* electronApp.quit; }).pipe(Effect.withSpan("desktop.lifecycle.quitAfterShutdown")), ); }); @@ -163,9 +164,10 @@ function quitFromSignal( if (wasQuitting) return; yield* logLifecycleInfo("process signal received", { signal }); yield* requestDesktopShutdownAndWait(); - markQuitAllowed(); const updates = yield* DesktopUpdates.DesktopUpdates; - if (!(yield* updates.installOnQuit)) yield* electronApp.quit; + yield* updates.installOnQuit; + markQuitAllowed(); + yield* electronApp.quit; }).pipe(Effect.withSpan("desktop.lifecycle.processSignal")), ); } @@ -277,6 +279,8 @@ export const make = DesktopLifecycle.of({ if (environment.platform !== "win32") { yield* addScopedListener(process, "SIGINT", () => { + if (quitRequested) return; + quitRequested = true; quitFromSignal( "SIGINT", () => { @@ -286,6 +290,8 @@ export const make = DesktopLifecycle.of({ ); }); yield* addScopedListener(process, "SIGTERM", () => { + if (quitRequested) return; + quitRequested = true; quitFromSignal( "SIGTERM", () => { diff --git a/apps/desktop/src/backend/DesktopBackendPool.test.ts b/apps/desktop/src/backend/DesktopBackendPool.test.ts index d8d7813e79721..23e797d2e3d08 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.test.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.test.ts @@ -1,5 +1,8 @@ import { assert, describe, it } from "@effect/vitest"; import * as Duration from "effect/Duration"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -10,6 +13,7 @@ import { HttpClient } from "effect/http"; import { ChildProcessSpawner } from "effect/process"; import * as DesktopObservability from "../app/DesktopObservability.ts"; +import { stopAllPoolInstances } from "../app/DesktopApp.ts"; import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublisher.ts"; import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts"; @@ -109,6 +113,43 @@ function layerPool(labelRef: Ref.Ref): Layer.Layer { + it.effect("stops Windows and WSL backends concurrently within the quit budget", () => + Effect.gen(function* () { + const startedWindows = yield* Deferred.make(); + const startedWsl = yield* Deferred.make(); + const stopped: string[] = []; + const instances = [ + [DesktopBackendPool.PRIMARY_INSTANCE_ID, startedWindows], + [DesktopBackendPool.BackendInstanceId("wsl:ubuntu"), startedWsl], + ] as const; + const layer = DesktopBackendPool.layerTest( + instances.map(([id, started]) => ({ + ...makeStubInstance(id, id), + stop: (options) => + Effect.sync(() => { + assert.equal(Duration.toMillis(options?.timeout ?? Duration.zero), 5_000); + }).pipe( + Effect.andThen(Deferred.succeed(started, undefined)), + Effect.andThen(Effect.sleep(Duration.seconds(4))), + Effect.andThen( + Effect.sync(() => { + stopped.push(id); + }), + ), + ), + })), + ); + yield* Effect.gen(function* () { + const stop = yield* stopAllPoolInstances().pipe(Effect.forkChild); + yield* Deferred.await(startedWindows); + yield* Deferred.await(startedWsl); + yield* TestClock.adjust(Duration.seconds(4)); + yield* Fiber.join(stop); + assert.equal(stopped.length, 2); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("layerTest exposes registered instances by id", () => Effect.gen(function* () { const pool = yield* DesktopBackendPool.DesktopBackendPool; diff --git a/apps/desktop/src/electron/ElectronUpdater.test.ts b/apps/desktop/src/electron/ElectronUpdater.test.ts index 43c7fdfbae682..8917e54540679 100644 --- a/apps/desktop/src/electron/ElectronUpdater.test.ts +++ b/apps/desktop/src/electron/ElectronUpdater.test.ts @@ -27,7 +27,7 @@ const { autoUpdaterMock } = vi.hoisted(() => ({ vi.mock("electron", async () => { const { EventEmitter } = await import("node:events"); - return { autoUpdater: new EventEmitter() }; + return { autoUpdater: Object.assign(new EventEmitter(), { checkForUpdates: vi.fn() }) }; }); import * as Electron from "electron"; @@ -39,6 +39,7 @@ import * as ElectronUpdater from "./ElectronUpdater.ts"; describe("ElectronUpdater", () => { beforeEach(() => { + vi.mocked(Electron.autoUpdater.checkForUpdates).mockReset(); autoUpdaterMock.allowDowngrade = false; autoUpdaterMock.allowPrerelease = false; autoUpdaterMock.autoDownload = true; @@ -51,18 +52,19 @@ describe("ElectronUpdater", () => { autoUpdaterMock.downloadUpdate.mockClear(); autoUpdaterMock.downloadUpdate.mockImplementation(() => Promise.resolve([])); autoUpdaterMock.on.mockClear(); - autoUpdaterMock.quitAndInstall.mockClear(); + autoUpdaterMock.quitAndInstall.mockReset(); autoUpdaterMock.removeListener.mockClear(); autoUpdaterMock.setFeedURL.mockClear(); }); - it.effect.each(["downloaded", "error", "timeout"])( + it.effect.each(["downloaded", "error", "timeout", "throw"])( "finishes a macOS real quit after native %s", (outcome) => Effect.gen(function* () { - autoUpdaterMock.quitAndInstall.mockImplementationOnce(() => { + vi.mocked(Electron.autoUpdater.checkForUpdates).mockImplementationOnce(() => { if (outcome === "downloaded") Electron.autoUpdater.emit("update-downloaded"); if (outcome === "error") Electron.autoUpdater.emit("error", new Error("staging failed")); + if (outcome === "throw") throw new Error("native check failed"); }); const updater = yield* ElectronUpdater.ElectronUpdater; const quit = yield* updater @@ -73,6 +75,7 @@ describe("ElectronUpdater", () => { const result = yield* Fiber.join(quit); assert.equal(result._tag, outcome === "downloaded" ? "Success" : "Failure"); assert.isFalse(autoUpdaterMock.autoRunAppAfterInstall); + assert.equal(autoUpdaterMock.quitAndInstall.mock.calls.length, 0); assert.equal(Electron.autoUpdater.listenerCount("error"), 0); assert.equal(Electron.autoUpdater.listenerCount("update-downloaded"), 0); }).pipe( @@ -82,6 +85,39 @@ describe("ElectronUpdater", () => { ), ); + it.effect("does not leave MacUpdater's delayed quit handler after staging times out", () => + Effect.gen(function* () { + let lateQuits = 0; + const lateQuit = () => { + lateQuits++; + }; + // MacUpdater 6.8.9 registers this persistent callback in quitAndInstall. + autoUpdaterMock.quitAndInstall.mockImplementationOnce(() => { + Electron.autoUpdater.on("update-downloaded", lateQuit); + }); + yield* Effect.gen(function* () { + const updater = yield* ElectronUpdater.ElectronUpdater; + const staging = yield* updater + .quitAndInstall({ isSilent: true, isForceRunAfter: false }) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust(Duration.seconds(5)); + assert.equal((yield* Fiber.join(staging))._tag, "Failure"); + Electron.autoUpdater.emit("update-downloaded"); + assert.equal(lateQuits, 0); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + Electron.autoUpdater.removeListener("update-downloaded", lateQuit); + }), + ), + ); + }).pipe( + Effect.provide(ElectronUpdater.layer), + Effect.provideService(HostProcess.Platform, "darwin"), + Effect.provide(TestClock.layer()), + ), + ); + it.effect("scopes updater event listeners", () => Effect.gen(function* () { const listener = vi.fn(); diff --git a/apps/desktop/src/electron/ElectronUpdater.ts b/apps/desktop/src/electron/ElectronUpdater.ts index 959a305d9d352..10402cf6482df 100644 --- a/apps/desktop/src/electron/ElectronUpdater.ts +++ b/apps/desktop/src/electron/ElectronUpdater.ts @@ -181,7 +181,10 @@ export const make = ElectronUpdater.of({ native.once("error", failed); signal.addEventListener("abort", cleanup, { once: true }); try { - install(); + // MacUpdater.quitAndInstall adds a persistent listener that quits + // immediately on staging. The lifecycle owns quit after this wait. + autoUpdater.autoRunAppAfterInstall = false; + native.checkForUpdates(); } catch (cause) { failed(cause); } diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index b82c8410d86de..9fe707a852031 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -21,6 +21,45 @@ import * as DesktopUpdates from "./DesktopUpdates.ts"; import { flushCallbacks, makeHarness } from "./updatesTestHarness.ts"; describe("DesktopUpdates", () => { + it.effect.each([true, false])( + "does not retry failed releases on polls (manual: %s)", + (manual) => { + let offeredVersion = "1.2.4"; + const harness = makeHarness({ + platform: "win32", + checkForUpdates: Effect.sync(() => + harness.emit("update-available", { version: offeredVersion }), + ), + downloadUpdate: Effect.fail( + new ElectronUpdater.ElectronUpdaterDownloadUpdateError({ + channel: "latest", + cause: new Error("offline"), + }), + ), + }); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + if (manual) { + yield* updates.check("manual"); + assert.isFalse((yield* updates.download).completed); + } + yield* TestClock.adjust(Duration.seconds(15)); + assert.equal(harness.downloadCount(), 1); + offeredVersion = "1.2.5"; + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.downloadCount(), 2); + offeredVersion = "1.2.4"; + yield* TestClock.adjust(Duration.minutes(4)); + assert.equal(harness.downloadCount(), 2); + assert.isTrue((yield* updates.download).accepted); + assert.equal(harness.downloadCount(), 3); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }, + ); + it("preserves complete causes for update poller and event failures", () => { const cause = Cause.combine( Cause.fail(new Error("updater failed")), @@ -106,19 +145,26 @@ describe("DesktopUpdates", () => { }), ); - it.effect("leaves RPM installs disabled", () => { - const harness = makeHarness({ platform: "linux", packageType: "rpm" }); - return Effect.scoped( - Effect.gen(function* () { - const updates = yield* DesktopUpdates.DesktopUpdates; - yield* updates.configure; - assert.equal((yield* updates.getState).status, "disabled"); - assert.isFalse((yield* updates.check("manual")).checked); - assert.isFalse(yield* updates.installOnQuit); - assert.equal(harness.downloadCount(), 0); - }), - ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); - }); + it.effect.each([false, true])( + "leaves RPM installs disabled (inherited APPIMAGE: %s)", + (inherited) => { + const harness = makeHarness({ + platform: "linux", + packageType: "rpm", + env: inherited ? { APPIMAGE: "/tmp/T3.AppImage" } : {}, + }); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + assert.equal((yield* updates.getState).status, "disabled"); + assert.isFalse((yield* updates.check("manual")).checked); + assert.isFalse(yield* updates.installOnQuit); + assert.equal(harness.downloadCount(), 0); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }, + ); it.effect( "recovers background checks after being offline without downloading during errors", @@ -353,6 +399,38 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); + it.effect.each([ + { platform: "win32" as const, env: {}, packageType: undefined, installOnQuit: true }, + { + platform: "linux" as const, + env: { APPIMAGE: "/tmp/T3.AppImage" }, + packageType: undefined, + installOnQuit: true, + }, + { platform: "linux" as const, env: {}, packageType: "deb", installOnQuit: false }, + ])("downloads on $platform ($packageType) and preserves its installer for quit", (options) => { + const harness = makeHarness({ + ...options, + checkForUpdates: Effect.sync(() => harness.emit("update-available", { version: "1.2.4" })), + downloadUpdate: Effect.sync(() => harness.emit("update-downloaded", { version: "1.2.4" })), + }); + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* TestClock.adjust(Duration.seconds(15)); + assert.equal(harness.downloadCount(), 1); + assert.equal((yield* updates.getState).downloadedVersion, "1.2.4"); + assert.equal(harness.autoInstallOnAppQuit(), options.installOnQuit); + yield* TestClock.adjust(Duration.minutes(12)); + assert.equal(harness.checkCount(), 1); + assert.equal(harness.downloadCount(), 1); + assert.isTrue((yield* updates.install).accepted); + assert.equal(harness.quitAndInstalls(), 1); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + it.effect("retries a failed background download only for a newer release", () => { let offeredVersion = "1.2.4"; let downloadSetups = 0; diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 6d15a3f563785..e0fe553bfc226 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -291,7 +291,7 @@ export const make = Effect.gen(function* () { const finishedUpdateActions = yield* PubSub.unbounded(); const updaterConfiguredRef = yield* Ref.make(false); const lastLoggedDownloadMilestoneRef = yield* Ref.make(-1); - const failedBackgroundDownloadRef = yield* Ref.make(null); + const failedDownloads = new Set(); const updateStateRef = yield* Ref.make( createInitialDesktopUpdateState( environment.appVersion, @@ -336,17 +336,18 @@ export const make = Effect.gen(function* () { ), ); - // The .deb carries electron-builder's resources/package-type marker. - // electron-updater reads the same file and installs updates with dpkg. - const isDebPackage = + // Use the same package identity as electron-updater. An inherited APPIMAGE + // variable must not enable updates for an RPM or pacman installation. + const linuxPackageType = environment.platform === "linux" && environment.isPackaged ? yield* fileSystem .readFileString(environment.path.join(environment.resourcesPath, "package-type")) .pipe( - Effect.map((packageType) => packageType.trim() === "deb"), - Effect.orElseSucceed(() => false), + Effect.map((packageType) => packageType.trim()), + Effect.orElseSucceed(() => null), ) - : false; + : null; + const isDebPackage = linuxPackageType === "deb"; const hasUpdateFeedConfig = Ref.get(appUpdateYmlConfigRef).pipe( Effect.map((appUpdateYmlConfig) => Option.isSome(appUpdateYmlConfig) || config.mockUpdates), @@ -359,7 +360,10 @@ export const make = Effect.gen(function* () { isDevelopment: environment.isDevelopment, isPackaged: environment.isPackaged, platform: environment.platform, - appImage: Option.getOrUndefined(config.appImagePath), + appImage: + linuxPackageType === "rpm" || linuxPackageType === "pacman" + ? undefined + : Option.getOrUndefined(config.appImagePath), isDebPackage, disabledByEnv: config.disableAutoUpdate, hasUpdateFeedConfig: hasFeedConfig, @@ -471,7 +475,9 @@ export const make = Effect.gen(function* () { return { accepted: false, completed: false }; } - const releaseFailedDownload = electronUpdater.setAutoInstallOnAppQuit(false); + const releaseFailedDownload = Effect.sync(() => { + if (state.availableVersion !== null) failedDownloads.add(state.availableVersion); + }).pipe(Effect.andThen(electronUpdater.setAutoInstallOnAppQuit(false))); return yield* Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); @@ -532,11 +538,8 @@ export const make = Effect.gen(function* () { if ((yield* Ref.get(updateStateRef)).downloadedVersion !== null) return; if (!(yield* checkForUpdates(reason)) || (yield* Ref.get(desktopState.quitting))) return; const version = (yield* Ref.get(updateStateRef)).availableVersion; - if (version === null || version === (yield* Ref.get(failedBackgroundDownloadRef))) return; - const result = yield* downloadAvailableUpdate; - if (result.accepted && !result.completed) { - yield* Ref.set(failedBackgroundDownloadRef, version); - } + if (version === null || failedDownloads.has(version)) return; + yield* downloadAvailableUpdate; }, ); From 2268203a4f1991fc6e4c74656c9d0ed055f7d641 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:07:46 +0000 Subject: [PATCH 102/149] fix(pi): steering accepted before a restart is no longer lost Pi acknowledged steering once it was queued in memory. The adapter now keeps the delivery pending until Pi has saved the user message, and fails unfinished deliveries on teardown so restart recovery replays them. --- .../provider-pi/src/server/adapter.test.ts | 99 ++++++++++++++++--- packages/provider-pi/src/server/adapter.ts | 47 +++++++++ 2 files changed, 133 insertions(+), 13 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index 05e4ba1c3444a..f872b5f02407d 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -21,6 +21,7 @@ import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as PlatformError from "effect/PlatformError"; @@ -4038,6 +4039,71 @@ describe("PiAdapterV2", () => { }).pipe(Effect.scoped, Effect.provide(layerTest)), ); + it.effect.each(["persisted", "transport closed"] as const)( + "keeps steering delivery pending until Pi's user message is %s", + (outcome) => + Effect.gen(function* () { + const fake = yield* makeFakePi; + const { runtime, takeEvent } = yield* openRuntime(fake); + const providerThread = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + yield* startTurn(runtime, providerThread); + yield* fake.takeRequest("prompt"); + const running = yield* takeEvent( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "running", + ); + assert.equal(running.type, "provider_turn.updated"); + if (running.type !== "provider_turn.updated") return; + yield* fake.emit({ type: "agent_start" }); + const steering = yield* runtime + .steerTurn({ + threadId: THREAD_ID, + runId: RunId.make("run:thread-pi-test:1"), + providerThread, + providerTurnId: running.providerTurn.id, + message: { + messageId: "message:thread-pi-test:durable-steer" as never, + text: "Keep this steering after a restart", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + }) + .pipe(Effect.forkChild); + const prompt = yield* fake.takeRequest("prompt"); + assert.equal(prompt["streamingBehavior"], "steer"); + yield* fake.emit({ type: "response", command: "prompt", success: true }); + yield* fake.emit({ + type: "tool_execution_start", + toolCallId: "queued-steering-tool", + toolName: "bash", + args: { command: "sleep 60" }, + }); + yield* takeEvent( + (event) => event.type === "turn_item.updated" && event.turnItem.status === "running", + ); + assert.isUndefined(steering.pollUnsafe()); + if (outcome === "transport closed") { + yield* fake.closeStdout; + assert.isTrue(Exit.isFailure(yield* Fiber.await(steering))); + } else { + yield* fake.emit({ + type: "message_end", + message: { + role: "user", + content: [{ type: "text", text: "Keep this steering after a restart" }], + }, + }); + yield* Fiber.join(steering); + yield* fake.takeRequest("get_state"); + } + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + it.effect("ignores an idle snapshot made stale by a steer", () => Effect.gen(function* () { const fake = yield* makeFakePi; @@ -4062,19 +4128,21 @@ describe("PiAdapterV2", () => { fake.deferNextState(); yield* fake.emit({ type: "agent_settled" }); yield* fake.takeRequest("get_state"); - yield* runtime.steerTurn({ - threadId: THREAD_ID, - runId: RunId.make("run:thread-pi-test:1"), - providerThread, - providerTurnId: providerTurnId!, - message: { - messageId: "message:thread-pi-test:late-steer" as never, - text: "Continue after settlement", - attachments: [], - createdBy: "user", - creationSource: "web", - }, - }); + const steering = yield* runtime + .steerTurn({ + threadId: THREAD_ID, + runId: RunId.make("run:thread-pi-test:1"), + providerThread, + providerTurnId: providerTurnId!, + message: { + messageId: "message:thread-pi-test:late-steer" as never, + text: "Continue after settlement", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + }) + .pipe(Effect.forkChild); yield* fake.takeRequest("prompt"); yield* fake.resolveDeferredState({ isStreaming: false, @@ -4083,6 +4151,11 @@ describe("PiAdapterV2", () => { }); yield* fake.emit({ type: "agent_start" }); + yield* fake.emit({ + type: "message_end", + message: { role: "user", content: [{ type: "text", text: "Continue after settlement" }] }, + }); + yield* Fiber.join(steering); yield* fake.emit({ type: "message_start", message: { role: "assistant" } }); yield* fake.emit({ type: "message_update", diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 5245fbfda5128..b7c68db21ea7c 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -588,6 +588,18 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( ProviderAdapter.ProviderAdapterV2Error | Cause.Done >(); const pendingPrompts = new Map(); + const pendingSteers = new Map< + string, + Array> + >(); + const failPendingSteers = Effect.fnUntraced(function* () { + for (const pending of pendingSteers.values()) { + for (const delivery of pending) { + yield* Deferred.fail(delivery, protocolError("Pi stopped before persisting steering")); + } + } + pendingSteers.clear(); + }); const sessionApprovals = new Set(); // Answering a dialog and terminalizing a turn both publish lifecycle // events. Pi can settle immediately after `extension_ui_response`, so @@ -614,6 +626,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( Effect.gen(function* () { closed = true; pendingWake = null; + yield* failPendingSteers(); if (rollbackBarrier !== null) yield* Deferred.fail( rollbackBarrier, @@ -1607,6 +1620,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( const turn = state.activeTurn; if (turn === null) return; state.activeTurn = null; + yield* failPendingSteers(); const interrupted = turn.interrupted || turn.nativeAborted; const completedAt = yield* DateTime.now; yield* completeOpenStreamItems(turn); @@ -1875,6 +1889,27 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( case "message_end": { if (turn === null) return; const message = event["message"]; + if (recordString(message, "role") === "user") { + const text = contentText(recordField(message, "content")); + const deliveries = pendingSteers.get(text); + const pending = deliveries?.[0]; + if (deliveries !== undefined && pending !== undefined) { + // Pi emits message_end before appending the entry. The RPC + // reply is a barrier after that append, unlike prompt's queue ack. + deliveries.shift(); + if (deliveries.length === 0) pendingSteers.delete(text); + yield* Deferred.complete( + pending, + request({ type: "get_state" }).pipe( + Effect.asVoid, + Effect.mapError((cause) => + protocolError("Pi could not confirm persisted steering", cause), + ), + ), + ); + } + return; + } if (recordString(message, "role") !== "assistant") return; yield* completeOpenStreamItems(turn); if (recordString(message, "stopReason") === "error" && turn.failure === null) { @@ -2787,6 +2822,10 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( steerInput.message.attachments, ) : null; + const persisted = + payload !== null && !payload.message.trimStart().startsWith("/") + ? yield* Deferred.make() + : null; // Prompt with streamingBehavior steer is atomic on Pi's side: it // queues during an active run and starts a new run if settlement // won the race. A direct `steer` sent after Pi became idle would @@ -2807,6 +2846,11 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( kind: "steer", }); } else if (payload !== null) { + if (persisted !== null) { + const deliveries = pendingSteers.get(payload.message) ?? []; + deliveries.push(persisted); + pendingSteers.set(payload.message, deliveries); + } yield* connection.send({ type: "prompt", message: payload.message, @@ -2821,6 +2865,9 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( turn.settleProbeGeneration += 1; }), ); + // Queued steering lives only in Pi's memory. Keep the outbox + // delivery unfinished until the native conversation contains it. + if (persisted !== null) yield* Deferred.await(persisted); }).pipe( Effect.mapError( (cause) => From aba8a23453beef6bb18ac6d2a921f90b0aeea89c Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:07:46 +0000 Subject: [PATCH 103/149] fix(server): restart continuation ignores ended runs and survives clock changes Saved steering no longer revives failed, interrupted, or rolled-back runs. Recovery keeps continuation ancestors and their native phases, orders runs by native turn ordinal and continuation links instead of wall-clock time, and replays pending steering in the order it was queued. --- .../Orchestrator.control-reads.test.ts | 24 ++ .../ProjectionRecovery.test.ts | 278 ++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 100 +++---- .../ProviderRuntimeRecoveryService.test.ts | 90 ++++++ .../ProviderRuntimeRecoveryService.ts | 11 +- .../RestartContinuation.test.ts | 145 +++++++++ .../orchestration-v2/RestartContinuation.ts | 14 +- 7 files changed, 605 insertions(+), 57 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts index f0907479a4409..dfd7e06dccf20 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts @@ -62,6 +62,9 @@ it.effect.each([ "completed", "failed", "interrupted", + "run-failed", + "run-interrupted", + "run-rolled_back", ] as const)("rechecks %s state when dispatching recorded restart intent", (state) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -136,6 +139,27 @@ it.effect.each([ threadId, }, ); + } else if ( + state === "run-failed" || + state === "run-interrupted" || + state === "run-rolled_back" + ) { + yield* projections.apply({ + id: EventId.make(`ended-run-restart-${state}`), + type: "run.updated", + threadId, + occurredAt: now, + payload: { + ...before.runs[0]!, + status: + state === "run-failed" + ? "failed" + : state === "run-interrupted" + ? "interrupted" + : "rolled_back", + restartPendingMessageIds: [source.userMessageId], + }, + }); } else if (state === "completed" || state === "failed" || state === "interrupted") { yield* projections.apply({ id: EventId.make(`settled-turn-restart-${state}`), diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index e2814be016724..1dc7514e8d535 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -13,6 +13,7 @@ import { ProviderSessionId, ProviderThreadId, ProviderTurnId, + RunAttemptId, RunId, RuntimeRequestId, ThreadId, @@ -36,6 +37,176 @@ import { restartContinuationRun } from "./RestartContinuation.ts"; const layerTest = Layer.mergeAll(ProjectionStore.layer, EffectOutbox.layer).pipe( Layer.provideMerge(SqlitePersistence.layerMemory), ); + +it.effect.each(["running", "completed", "failed", "interrupted"] as const)( + "retains continuation ancestors and the latest %s phase for pending steering", + (status) => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = yield* createThread(`steering-chain-${status}`); + const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`); + const sessionId = ProviderSessionId.make(`session:${threadId}`); + const nodeId = NodeId.make(`node:${threadId}`); + const sourceAttempt = RunAttemptId.make(`attempt:${threadId}:1`); + const currentAttempt = RunAttemptId.make(`attempt:${threadId}:2`); + const messageId = MessageId.make(`steering:${threadId}`); + const source = yield* createRun(threadId, "cancelled", { + providerThreadId, + activeAttemptId: sourceAttempt, + restartPendingMessageIds: [messageId], + completedAt: now, + }); + const current = yield* createRun(threadId, status === "running" ? "starting" : "completed", { + ordinal: 2, + providerThreadId, + activeAttemptId: currentAttempt, + restartContinuationOfRunId: source, + ...(status === "running" ? {} : { restartPendingMessageIds: [messageId] }), + }); + yield* projections.apply({ + id: EventId.make(`session:${threadId}`), + type: "provider-session.attached", + threadId, + occurredAt: now, + payload: { + id: sessionId, + driver, + providerInstanceId, + status: "stopped", + cwd: "/workspace", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + }); + yield* projections.apply({ + id: EventId.make(`provider-thread:${threadId}`), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: { + id: providerThreadId, + appThreadId: threadId, + ownerNodeId: null, + driver, + providerInstanceId, + providerSessionId: sessionId, + nativeThreadRef: { driver, nativeId: "native", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 2, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }); + for (const [runId, attemptId, ordinal] of [ + [source, sourceAttempt, 1], + [current, currentAttempt, 2], + ] as const) { + yield* projections.apply({ + id: EventId.make(`attempt:${attemptId}`), + type: "run-attempt.created", + threadId, + occurredAt: now, + payload: { + id: attemptId, + runId, + attemptOrdinal: 1, + rootNodeId: nodeId, + providerInstanceId, + providerThreadId, + providerTurnId: null, + reason: "initial", + status: "cancelled", + startedAt: now, + completedAt: now, + }, + }); + if (ordinal === 2 && status === "running") continue; + yield* projections.apply({ + id: EventId.make(`turn:${attemptId}`), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: ProviderTurnId.make(`turn:${attemptId}`), + providerThreadId, + nodeId, + runAttemptId: attemptId, + nativeTurnRef: null, + ordinal, + status: ordinal === 1 ? "cancelled" : status, + startedAt: now, + completedAt: now, + }, + }); + } + if (status !== "running") { + yield* projections.apply({ + id: EventId.make(`message:${threadId}`), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + id: messageId, + threadId, + runId: current, + nodeId: null, + role: "user", + text: "steer", + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + }, + }); + yield* (yield* EffectOutbox.EffectOutboxV2).enqueue([ + { + id: `effect:${threadId}`, + commandId: CommandId.make(`command:${threadId}`), + threadId, + request: { + type: "provider-turn.steer", + messageId, + providerSessionId: sessionId, + providerThreadId, + providerTurnId: ProviderTurnId.make(`turn:${currentAttempt}`), + }, + }, + ]); + } + const recovered = yield* projections.getRuntimeRecoveryProjection(threadId); + assert.deepEqual( + recovered.runs.map((run) => run.id), + [source, current], + ); + assert.deepEqual( + recovered.attempts.map((attempt) => attempt.id), + [sourceAttempt, currentAttempt], + ); + assert.lengthOf(recovered.providerTurns, status === "running" ? 1 : 2); + const withSteering = { + ...recovered, + runs: recovered.runs.map((run) => + run.id === current && status !== "running" + ? { ...run, restartPendingMessageIds: [messageId] } + : run, + ), + }; + assert.equal( + restartContinuationRun(withSteering, now)?.id, + status === "failed" || status === "interrupted" ? undefined : current, + ); + }).pipe(Effect.provide(layerTest)), +); const providerInstanceId = ProviderInstanceId.make("codex"); const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" }; const driver = ProviderDriverKind.make("codex"); @@ -758,6 +929,62 @@ it.effect("reads the run that owns a background roster, not a queued or resumed }); yield* withRoster(resumed); assert.deepEqual(yield* runIds(resumed), [resumedRun, continuationRun]); + // A backward clock and a later turn on another provider must not move + // the roster's owning run back to an earlier native turn. + const clock = yield* createThread("roster-backward-clock"); + const clockProviderThreadId = ProviderThreadId.make(`provider-thread:${clock}`); + const clockRuns: RunId[] = []; + for (const ordinal of [1, 2, 3]) { + const attemptId = RunAttemptId.make(`attempt:${clock}:${ordinal}`); + const runId = yield* createRun(clock, "completed", { + ordinal, + providerThreadId: ordinal === 3 ? null : clockProviderThreadId, + activeAttemptId: attemptId, + completedAt: DateTime.makeUnsafe( + ordinal === 2 ? "2026-10-03T09:00:00Z" : "2026-10-03T10:00:00Z", + ), + }); + clockRuns.push(runId); + if (ordinal === 3) continue; + yield* projections.apply({ + id: EventId.make(`event:${attemptId}`), + type: "run-attempt.created", + threadId: clock, + occurredAt: now, + payload: { + id: attemptId, + runId, + attemptOrdinal: 1, + rootNodeId: NodeId.make(`node:${clock}:${ordinal}`), + providerInstanceId, + providerThreadId: clockProviderThreadId, + providerTurnId: null, + reason: "initial", + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + yield* projections.apply({ + id: EventId.make(`event:turn:${attemptId}`), + type: "provider-turn.updated", + threadId: clock, + occurredAt: now, + payload: { + id: ProviderTurnId.make(`turn:${attemptId}`), + providerThreadId: clockProviderThreadId, + nodeId: NodeId.make(`node:${clock}:${ordinal}`), + runAttemptId: attemptId, + nativeTurnRef: null, + ordinal, + status: "completed", + startedAt: now, + completedAt: now, + }, + }); + } + yield* withRoster(clock); + assert.deepEqual(yield* runIds(clock), clockRuns.slice(1)); // Without a roster, settled history is not read. const quiet = yield* createThread("no-roster-before-queue"); yield* createRun(quiet, "completed"); @@ -930,3 +1157,54 @@ it.effect( ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("preserves steering submission order across equal and backward clocks", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = yield* createThread("steering-order"); + const runId = yield* createRun(threadId, "completed"); + const messageIds = ["z", "y", "a"].map((id) => MessageId.make(`message:steering-order:${id}`)); + for (const [index, messageId] of messageIds.entries()) { + const now = DateTime.makeUnsafe(index < 2 ? "2026-10-03T10:00:00Z" : "2026-10-03T09:00:00Z"); + yield* projections.apply({ + id: EventId.make(`event:${messageId}`), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + id: messageId, + threadId, + runId, + nodeId: null, + role: "user", + text: String(index), + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + }, + }); + yield* outbox.enqueue([ + { + id: `effect:${messageId}`, + commandId: CommandId.make(`command:${messageId}`), + threadId, + request: { + type: "provider-turn.steer", + messageId, + providerSessionId: ProviderSessionId.make("session:steering-order"), + providerThreadId: ProviderThreadId.make("provider-thread:steering-order"), + providerTurnId: ProviderTurnId.make("turn:steering-order"), + }, + }, + ]); + } + assert.deepEqual( + (yield* projections.getRuntimeRecoveryProjection(threadId)).pendingSteering, + messageIds.map((messageId) => ({ runId, messageId })), + ); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 790fabaf7a183..a73c982931b1c 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -4055,22 +4055,16 @@ export const layer: Layer.Layer = if (threadRows[0] === undefined) { return yield* new ProjectionStoreThreadNotFoundError({ threadId }); } - const [ - thread, - runRows, - attemptRows, - nodeRows, - subagentRows, - providerSessionRows, - providerThreadRows, - providerTurnRows, - runtimeRequestRows, - messageRows, - turnItemRows, - ] = yield* Effect.all([ - decodeThreadPayload(threadRows[0].payload_json), - sql` - SELECT payload_json FROM orchestration_v2_projection_runs AS run + const runRows = yield* sql` + WITH RECURSIVE native_run_ordinals AS ( + SELECT attempt.run_id, turn.provider_thread_id, MAX(turn.ordinal) AS ordinal + FROM orchestration_v2_projection_provider_turns AS turn + JOIN orchestration_v2_projection_run_attempts AS attempt + ON attempt.attempt_id = turn.run_attempt_id + WHERE turn.thread_id = ${threadId} + GROUP BY attempt.run_id, turn.provider_thread_id + ), recovery_runs(run_id, payload_json) AS ( + SELECT run_id, payload_json FROM orchestration_v2_projection_runs AS run WHERE run.thread_id = ${threadId} AND ( run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') @@ -4084,11 +4078,15 @@ export const layer: Layer.Layer = OR run.run_id IN ( SELECT ( SELECT ended.run_id FROM orchestration_v2_projection_runs AS ended + LEFT JOIN native_run_ordinals AS native + ON native.run_id = ended.run_id + AND native.provider_thread_id = roster.provider_thread_id WHERE ended.thread_id = ${threadId} AND ended.provider_thread_id = roster.provider_thread_id AND ended.status NOT IN ('queued', 'rolled_back') - ORDER BY ended.completed_at IS NULL DESC, ended.completed_at DESC, - ended.ordinal DESC + ORDER BY ended.completed_at IS NULL DESC, + native.ordinal DESC, + ended.completed_at DESC, ended.ordinal DESC LIMIT 1 ) FROM orchestration_v2_projection_provider_threads AS roster @@ -4111,23 +4109,43 @@ export const layer: Layer.Layer = ) OR run.run_id IN ( SELECT item.run_id FROM orchestration_v2_projection_turn_items AS item + INDEXED BY orchestration_v2_projection_turn_items_recovery_idx WHERE item.thread_id = ${threadId} AND item.type IN ('command_execution', 'dynamic_tool', 'subagent') AND item.status IN ('pending', 'running', 'waiting') AND item.run_id IS NOT NULL ) ) - ORDER BY run.ordinal ASC - `, + UNION + SELECT ancestor.run_id, ancestor.payload_json + FROM orchestration_v2_projection_runs AS ancestor + JOIN recovery_runs AS child + ON ancestor.run_id = json_extract(child.payload_json, '$.restartContinuationOfRunId') + WHERE ancestor.thread_id = ${threadId} + ) + SELECT payload_json FROM recovery_runs + ORDER BY json_extract(payload_json, '$.ordinal') ASC + `; + const runs = yield* decodeRows(decodeRunPayload, threadId)(runRows); + const recoveryRunIds = runs.map((run) => run.id); + const [ + thread, + attemptRows, + nodeRows, + subagentRows, + providerSessionRows, + providerThreadRows, + providerTurnRows, + runtimeRequestRows, + messageRows, + turnItemRows, + ] = yield* Effect.all([ + decodeThreadPayload(threadRows[0].payload_json), sql` SELECT attempt.payload_json FROM orchestration_v2_projection_run_attempts AS attempt WHERE attempt.thread_id = ${threadId} - AND attempt.run_id IN ( - SELECT run_id FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - ) + AND attempt.run_id IN ${sql.in(recoveryRunIds)} ORDER BY attempt.run_id ASC, attempt.attempt_ordinal ASC `, sql` @@ -4196,16 +4214,7 @@ export const layer: Layer.Layer = SELECT run.provider_thread_id FROM orchestration_v2_projection_runs AS run WHERE run.thread_id = ${threadId} - AND ( - run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - OR run.run_id IN ( - SELECT message.run_id FROM orchestration_v2_projection_messages AS message - JOIN orchestration_v2_effect_outbox AS effect - ON json_extract(effect.payload_json, '$.messageId') = message.message_id - WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' - AND effect.status IN ('pending', 'running') - ) - ) + AND run.run_id IN ${sql.in(recoveryRunIds)} ) ) ) @@ -4235,16 +4244,7 @@ export const layer: Layer.Layer = OR provider_thread.provider_thread_id IN ( SELECT run.provider_thread_id FROM orchestration_v2_projection_runs AS run WHERE run.thread_id = ${threadId} - AND ( - run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - OR run.run_id IN ( - SELECT message.run_id FROM orchestration_v2_projection_messages AS message - JOIN orchestration_v2_effect_outbox AS effect - ON json_extract(effect.payload_json, '$.messageId') = message.message_id - WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' - AND effect.status IN ('pending', 'running') - ) - ) + AND run.run_id IN ${sql.in(recoveryRunIds)} ) OR provider_thread.provider_thread_id IN ( SELECT item.provider_thread_id @@ -4279,11 +4279,7 @@ export const layer: Layer.Layer = OR provider_turn.run_attempt_id IN ( SELECT attempt_id FROM orchestration_v2_projection_run_attempts WHERE thread_id = ${threadId} - AND run_id IN ( - SELECT run_id FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - ) + AND run_id IN ${sql.in(recoveryRunIds)} ) ) ORDER BY provider_turn.provider_thread_id ASC, provider_turn.ordinal ASC @@ -4328,7 +4324,6 @@ export const layer: Layer.Layer = `, ]); const [ - runs, attempts, nodes, subagents, @@ -4339,7 +4334,6 @@ export const layer: Layer.Layer = messages, turnItems, ] = yield* Effect.all([ - decodeRows(decodeRunPayload, threadId)(runRows), decodeRows(decodeRunAttemptPayload, threadId)(attemptRows), decodeRows(decodeNodePayload, threadId)(nodeRows), decodeRows(decodeSubagentPayload, threadId)(subagentRows), @@ -4362,7 +4356,7 @@ export const layer: Layer.Layer = WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' AND effect.status IN ('pending', 'running') AND json_extract(message.payload_json, '$.delegatedCompletion') IS NULL - ORDER BY message.created_at ASC, message.message_id ASC + ORDER BY effect.rowid ASC `; return { pendingSteering: pendingSteering.map((row) => ({ diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 7de34a5d7fbea..b637aaf384192 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -14,6 +14,7 @@ import { TurnItemId, type OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; @@ -1619,3 +1620,92 @@ it.effect("notes native child cancellation on its parent", () => { assert.deepEqual(noted, [`subagent ${nativeSubagentId}`]); }).pipe(Effect.provide(layer)); }); + +it.effect.each(["startup", "shutdown"] as const)( + "records lost background work on the last native turn after a backward clock on %s", + (trigger) => + Effect.gen(function* () { + const threadId = ThreadId.make("thread:clock-background"); + const providerThreadId = ProviderThreadId.make("provider-thread:clock-background"); + const earlierId = RunId.make("run:clock-background:earlier"); + const latestId = RunId.make("run:clock-background:latest"); + const earlierAttempt = RunAttemptId.make("attempt:clock-background:earlier"); + const latestAttempt = RunAttemptId.make("attempt:clock-background:latest"); + const events: Parameters< + EventSink.EventSinkV2["Service"]["commitCommand"] + >[0]["events"][number][] = []; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [], + attempts: [], + nodes: [], + subagents: [], + messages: [], + providerThreads: [ + { + id: providerThreadId, + ownerNodeId: null, + status: "idle", + providerInstanceId: ProviderInstanceId.make("codex"), + pendingBackgroundTasks: [ + { kind: "command", taskId: "background", description: "watch build" }, + ], + }, + ], + runs: [ + { + id: earlierId, + ordinal: 1, + status: "completed", + providerThreadId, + activeAttemptId: earlierAttempt, + providerInstanceId: ProviderInstanceId.make("codex"), + completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00Z"), + }, + { + id: latestId, + ordinal: 2, + status: "completed", + providerThreadId, + activeAttemptId: latestAttempt, + providerInstanceId: ProviderInstanceId.make("codex"), + completedAt: DateTime.makeUnsafe("2026-10-03T09:00:00Z"), + }, + ], + providerTurns: [ + { providerThreadId, runAttemptId: earlierAttempt, ordinal: 1, status: "completed" }, + { providerThreadId, runAttemptId: latestAttempt, ordinal: 2, status: "completed" }, + ], + turnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + yield* ProviderRuntimeRecovery.make.pipe( + Effect.flatMap((recovery) => recovery.reconcile(trigger)), + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest(), + IdAllocator.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + events.push(...input.events); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + assert.deepEqual( + events.flatMap((event) => + event.type === "run.background-work-cancelled" ? [event.runId] : [], + ), + [latestId], + ); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 5c48032cddd8e..5741436a83270 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -1,4 +1,3 @@ -import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { CommandId, @@ -32,6 +31,7 @@ import { cancelledTurnItemWork, isWorktreeContinuationRun, mergeRestartCancelledBackgroundWork, + restartRunRanAfter, } from "./RestartBackgroundNote.ts"; export class ProviderRuntimeRecoveryError extends Schema.TaggedError()( @@ -184,7 +184,14 @@ function latestStartedRun( run.providerThreadId === providerThreadId && run.status !== "queued" && run.status !== "rolled_back" && - (latest === undefined || runRanAfter(run, latest)) + (latest === undefined || + restartRunRanAfter( + run, + latest, + projection.providerTurns, + projection.runs, + projection.attempts, + )) ? run : latest, undefined, diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 47cd1c0dbbddf..db6195a0a7db0 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -133,6 +133,92 @@ it("requires matching saved native state for an unfinished root run", () => { assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection, testNow)); }); +it("checks the thread, run, turn and request recovery decision table", () => { + const base = makeProjection(); + const holds = [ + [{}, false], + [{ archivedAt: testNow }, true], + [{ deletedAt: testNow }, true], + [{ settledOverride: "settled" }, true], + [{ settledOverride: "unsettled" }, false], + [{ snoozedUntil: DateTime.add(testNow, { minutes: 1 }) }, true], + [{ snoozedUntil: testNow }, false], + [{ snoozedUntil: DateTime.subtract(testNow, { minutes: 1 }) }, false], + ] as const; + const requests = [ + [undefined, false], + [{ kind: "user_input", status: "pending", responseCapability: { type: "live" } }, true], + [{ kind: "command", status: "pending", responseCapability: { type: "live" } }, true], + [{ kind: "user_input", status: "pending", responseCapability: { type: "message" } }, false], + [{ kind: "user_input", status: "resolved", responseCapability: { type: "live" } }, false], + [ + { kind: "user_input", status: "expired", responseCapability: { type: "not_resumable" } }, + true, + ], + [{ kind: "command", status: "cancelled", responseCapability: { type: "not_resumable" } }, true], + [{ kind: "dynamic_tool_call", status: "pending", responseCapability: { type: "live" } }, false], + [{ kind: "auth_refresh", status: "pending", responseCapability: { type: "live" } }, false], + ] as const; + for (const status of [ + "queued", + "preparing", + "starting", + "running", + "waiting", + "completed", + "failed", + "interrupted", + "cancelled", + "rolled_back", + ] as const) + for (const pending of [false, true]) + for (const [thread, held] of holds) + for (const turnStatus of [ + "running", + "completed", + "failed", + "interrupted", + "cancelled", + ] as const) + for (const [request, blocked] of requests) { + const eligible = status === "running" || pending; + const turnBlocked = + turnStatus === "failed" || + turnStatus === "interrupted" || + (turnStatus === "completed" && !pending); + const projection = { + ...base, + thread: { ...base.thread, ...thread }, + runs: [ + { + ...base.runs[0]!, + status, + ...(pending + ? { restartPendingMessageIds: [MessageId.make("message:matrix-steer")] } + : {}), + }, + ], + providerTurns: [{ ...base.providerTurns[0]!, status: turnStatus }], + runtimeRequests: + request === undefined + ? [] + : [{ ...request, providerTurnId: base.providerTurns[0]!.id }], + } as unknown as OrchestrationV2ThreadProjection; + const expected = + !["queued", "failed", "interrupted", "rolled_back"].includes(status) && + eligible && + !held && + !blocked && + !turnBlocked && + (pending || turnStatus === "running"); + assert.equal( + restartContinuationRun(projection, testNow) !== undefined, + expected, + JSON.stringify({ status, pending, thread, turnStatus, request }), + ); + } +}); + it.effect.each([ ["user_input", "pending", "live", false], ["user_input", "expired", "not_resumable", false], @@ -196,6 +282,27 @@ it.effect.each([ }), ); +it.effect.each(["failed", "interrupted", "rolled_back"] as const)( + "never revives a %s run with saved pending steering", + (status) => + Effect.gen(function* () { + const source = { + ...cutMidTurn({ + status, + restartPendingMessageIds: [MessageId.make("message:user")], + }), + messages: [ + { + id: MessageId.make("message:user"), + text: "Undelivered steering.", + attachments: [], + } as never, + ], + }; + assert.deepEqual(yield* continuationTexts(source), []); + }), +); + it.effect.each(["pending", "cancelled"] as const)( "rebinds OpenCode's %s question after restart", (status) => @@ -1971,3 +2078,41 @@ it.effect("recovers an unstarted continuation after a second restart with a back ); }), ); + +it("selects a later continuation after its provider settled and the clock moved backward", () => { + const base = makeProjection(); + const earlier = { + ...base.runs[0]!, + status: "completed" as const, + completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"), + }; + const current = { + ...earlier, + id: RunId.make("run:backward-pending-steer"), + ordinal: 2, + restartContinuationOfRunId: earlier.id, + restartPendingMessageIds: [MessageId.make("message:backward-pending-steer")], + activeAttemptId: RunAttemptId.make("attempt:backward-pending-steer"), + completedAt: DateTime.makeUnsafe("2026-10-03T09:00:00.000Z"), + }; + assert.equal( + restartContinuationRun( + { + ...base, + runs: [earlier, current], + providerTurns: [ + { ...base.providerTurns[0]!, ordinal: 1, status: "completed" }, + { + ...base.providerTurns[0]!, + id: ProviderTurnId.make("turn:backward-pending-steer"), + runAttemptId: current.activeAttemptId, + ordinal: 2, + status: "completed", + }, + ], + }, + testNow, + )?.id, + current.id, + ); +}); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 3bc6ab43abc54..122697e8e3332 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -1,4 +1,3 @@ -import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { CommandId, @@ -67,6 +66,9 @@ export function restartContinuationBlocked( // service counts (a run that completed or failed after the snooze) cannot // apply here: the source is the latest run and it was cut, not finished. if ( + source.status === "failed" || + source.status === "interrupted" || + source.status === "rolled_back" || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.thread.settledOverride === "settled" || @@ -130,7 +132,15 @@ export function restartContinuationRun( // Queued runs never started; recovery holds them behind the cut run. const run = projection.runs.reduce( (latest, candidate) => - candidate.status !== "queued" && (!latest || runRanAfter(candidate, latest)) + candidate.status !== "queued" && + (!latest || + restartRunRanAfter( + candidate, + latest, + projection.providerTurns, + projection.runs, + projection.attempts, + )) ? candidate : latest, undefined, From b7ff4b0c8468d44301a8c14a4548f61be3334063 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:07:46 +0000 Subject: [PATCH 104/149] fix(server): restarts keep external OpenCode work, unanswered questions, and Claude script CLIs in check OpenCode 2 validates the original admission before following a restarted turn, replays the original text and id when it was never admitted, keeps a lost question interrupted instead of continuing, lets external server turns survive a restart while a quit still stops them, and lets queued notes follow external work. Claude JS launchers run under the parent-death wrapper, and the webhook delivery log stays capped at 50 rows. --- .../Adapters/ClaudeAdapterV2.test.ts | 36 +++- .../Adapters/ClaudeAdapterV2.ts | 29 +++- .../Adapters/OpenCode2AdapterV2.test.ts | 156 +++++++++++++++--- .../scheduledTasks/ScheduledTaskService.ts | 1 - .../ScheduledTaskService.webhook.test.ts | 51 ++++-- .../src/server/v2/adapter.ts | 89 ++++++++-- 6 files changed, 290 insertions(+), 72 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 6dcfdd44269b3..6631428e2ce86 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -2044,9 +2044,9 @@ describe("ClaudeAdapterV2 MCP credential channel", () => { }); describe("ClaudeAdapterV2 native session identity", () => { - it.live( - "ends the native CLI when its hosting process crashes", - () => + it.live.each(["native", "javascript", "shell wrapper", "agent scope"] as const)( + "ends the %s CLI when its hosting process crashes", + (launchKind) => Effect.gen(function* () { const platform = yield* HostProcess.Platform; if (platform !== "linux") return; @@ -2072,22 +2072,46 @@ describe("ClaudeAdapterV2 native session identity", () => { ); const cliSource = [ "const fs = require('node:fs');", - `process.on('SIGTERM', () => { fs.writeFileSync(${JSON.stringify(stoppedPath)}, 'stopped'); process.exit(0); });`, - `fs.writeFileSync(${JSON.stringify(readyPath)}, String(process.pid));`, + "const publish = (file, text) => { fs.writeFileSync(file + '.tmp', text); fs.renameSync(file + '.tmp', file); };", + `process.on('SIGTERM', () => { publish(${JSON.stringify(stoppedPath)}, 'stopped'); process.exit(0); });`, + `publish(${JSON.stringify(readyPath)}, String(process.pid));`, "setInterval(() => {}, 1000);", ].join("\n"); + const scriptPath = NodePath.join(directory, "cli.js"); + NodeFS.writeFileSync(scriptPath, cliSource); + const shellPath = NodePath.join(directory, "cli.sh"); + const shellQuote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; + NodeFS.writeFileSync( + shellPath, + `#!/bin/sh\nexec ${shellQuote(executablePath)} ${shellQuote(scriptPath)} "$@"\n`, + { mode: 0o755 }, + ); + const binaryPath = + launchKind === "javascript" + ? scriptPath + : launchKind === "shell wrapper" + ? shellPath + : executablePath; + const executableArgs = + launchKind === "native" || launchKind === "agent scope" ? ["-e", cliSource, "--"] : []; const parentSource = [ `import * as Effect from ${JSON.stringify(import.meta.resolve("effect/Effect"))};`, `import * as NodeServices from ${JSON.stringify(import.meta.resolve("@effect/platform-node/NodeServices"))};`, `import * as Adapter from ${JSON.stringify(import.meta.resolve("./ClaudeAdapterV2.ts"))};`, `import * as Loggers from ${JSON.stringify(import.meta.resolve("@t3tools/provider-core/server/ProviderEventLoggers"))};`, + `import { AgentScope, agentScopeCommand } from ${JSON.stringify(import.meta.resolve("@t3tools/shared/AgentScope"))};`, "await Effect.runPromise(Effect.scoped(Effect.gen(function* () {", "const runner = yield* Adapter.ClaudeAgentSdkQueryRunner;", "yield* runner.open({ threadId: 'parent-death', providerSessionId: 'parent-death', options: {", - `pathToClaudeCodeExecutable: process.execPath, executableArgs: ['-e', ${JSON.stringify(cliSource)}, '--'],`, + `pathToClaudeCodeExecutable: ${JSON.stringify(binaryPath)}, executableArgs: ${JSON.stringify(executableArgs)},`, "sessionId: '12345678-1234-4123-8123-123456789012', model: 'sonnet', tools: [], permissionMode: 'default', env: process.env }});", "yield* Effect.never;", "}).pipe(Effect.provide(Adapter.layerQueryRunner), Effect.provide(NodeServices.layer),", + ...(launchKind === "agent scope" + ? [ + "Effect.provideService(AgentScope, { wrap: (input) => Effect.succeed(agentScopeCommand(input)), oomKilled: () => Effect.succeed(false), clear: () => Effect.void }),", + ] + : []), "Effect.provideService(Loggers.ProviderEventLoggers, Loggers.NoOpProviderEventLoggers))));", ].join("\n"); const ready = yield* waitForFile(readyPath).pipe(Effect.forkScoped); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6a12b8f94bef7..34532b94e6abb 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -747,9 +747,12 @@ export const layerQueryRunner: Layer.Layer< const { mcpServers, ...queryOptions } = input.options; // The SDK spawns a native binary as `path ...executableArgs ...sdkArgs`, // which lets the agent scope wrapper go in front of the CLI while the - // SDK keeps its own stderr capture and exit reporting. The SDK runs a - // script entry through node instead, so those launch unwrapped. + // SDK keeps its own stderr capture and exit reporting. A script entry + // needs its interpreter inside the parent-death wrapper too. const binaryPath = queryOptions.pathToClaudeCodeExecutable; + const isScript = + binaryPath !== undefined && + CLAUDE_SCRIPT_EXTENSIONS.some((ext) => binaryPath.endsWith(ext)); const parentDeathSignal = (yield* HostProcess.Platform) === "linux" ? yield* resolveCommandPath("setpriv", { @@ -761,27 +764,35 @@ export const layerQueryRunner: Layer.Layer< ) : Option.none(); const launch = - binaryPath === undefined || - CLAUDE_SCRIPT_EXTENSIONS.some((ext) => binaryPath.endsWith(ext)) + binaryPath === undefined || (isScript && Option.isNone(parentDeathSignal)) ? undefined : yield* agentScope.wrap({ // stdin EOF does not stop Claude's background work. A crashed // server must end its CLI before recovery resumes the transcript. command: Option.isSome(parentDeathSignal) ? parentDeathSignal.value : binaryPath, - args: Option.isSome(parentDeathSignal) - ? ["--pdeathsig", "TERM", "--", binaryPath] - : [], + args: [ + ...(Option.isSome(parentDeathSignal) + ? [ + "--pdeathsig", + "TERM", + "--", + isScript ? (queryOptions.executable ?? "node") : binaryPath, + ] + : []), + ...(queryOptions.executableArgs ?? []), + ...(isScript ? [binaryPath] : []), + ], name: "claude", threadId: input.threadId, env: queryOptions.env, }); const options = - launch === undefined || launch.args.length === 0 + launch === undefined ? queryOptions : { ...queryOptions, pathToClaudeCodeExecutable: launch.command, - executableArgs: [...launch.args, ...(queryOptions.executableArgs ?? [])], + executableArgs: [...launch.args], }; // The CLI is running once query() returns, so it is closed if anything // before the session is handed out fails or is interrupted. diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index efa79413895c2..34c39205d5b53 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -424,7 +424,7 @@ const history = { }; it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { - it.effect.each([false, true])( + it.effect.each([false, true, "note"])( "follows a surviving execution after a T3 restart (buffered=%s) without another prompt", (buffered) => Effect.gen(function* () { @@ -449,7 +449,18 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ); yield* runtime.startTurn({ ...turnInput(thread), - restartContinuationOfRunId: RunId.make("interrupted-run"), + ...(buffered === "note" + ? { + restartContinuationSource: { + nativeTurnRef: { + driver: OPENCODE_PROVIDER, + nativeId: history.data[0]!.id, + strength: "weak" as const, + }, + message: turnInput(thread).message, + }, + } + : { restartContinuationOfRunId: RunId.make("interrupted-run") }), }); const collected = yield* Fiber.join(events); assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); @@ -464,8 +475,17 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); - it.effect.each(["empty", "already settled", "failed"] as const)( - "prompts after a restart with %s history instead of following an earlier completion", + it.effect.each([ + "empty", + "already settled", + "failed", + "different admission", + "unrecorded admission", + "unknown admission", + "earlier admission", + "lost question", + ] as const)( + "reconciles a restart with %s history without following an earlier completion", (previous) => Effect.gen(function* () { const { runtime, thread } = yield* resumed([ @@ -474,7 +494,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), reply("message.list", { data: - previous === "empty" + previous === "empty" || previous === "lost question" ? [] : history.data .toReversed() @@ -485,9 +505,60 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ), cursor: {}, }), - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - event("session.execution.succeeded", { sessionID: SESSION }), + ...(previous === "different admission" || + previous === "unrecorded admission" || + previous === "earlier admission" + ? [ + out("message.list", { + sessionID: SESSION, + order: "desc", + limit: "50", + type: "user", + }), + reply("message.list", { + data: [history.data[0]!], + cursor: previous === "earlier admission" ? { next: "older" } : {}, + }), + ...(previous === "earlier admission" + ? [ + out("message.list", { + sessionID: SESSION, + cursor: "older", + limit: "50", + type: "user", + }), + reply("message.list", { + data: [{ ...history.data[0]!, id: "msg_not_admitted" }], + cursor: {}, + }), + ] + : []), + ] + : []), + ...(previous === "lost question" + ? [] + : [ + out("session.prompt", { + sessionID: SESSION, + text: + previous === "different admission" || + previous === "unrecorded admission" || + previous === "unknown admission" + ? "original request before the crash" + : previous === "earlier admission" + ? "hi" + : "", + ...(previous === "different admission" + ? { id: "msg_not_admitted" } + : previous === "unrecorded admission" + ? { id: `msg_t3_turn_${SESSION}:attempt:not-admitted` } + : previous === "earlier admission" + ? { id: `msg_t3_turn_${SESSION}:attempt:opencode2-adapter` } + : {}), + }), + promptAccepted, + event("session.execution.succeeded", { sessionID: SESSION }), + ]), ]); const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); yield* runtime.startTurn({ @@ -504,8 +575,38 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { : thread, ), restartContinuationOfRunId: RunId.make("interrupted-run"), + ...(previous === "different admission" || + previous === "unrecorded admission" || + previous === "unknown admission" || + previous === "earlier admission" || + previous === "lost question" + ? { + restartContinuationSource: { + attemptId: + previous === "unknown admission" + ? null + : RunAttemptId.make("attempt:not-admitted"), + nativeTurnRef: + previous === "unrecorded admission" || previous === "unknown admission" + ? null + : { + driver: OPENCODE_PROVIDER, + nativeId: "msg_not_admitted", + strength: "weak" as const, + }, + message: { + ...turnInput(thread).message, + text: "original request before the crash", + }, + awaitingUser: previous === "lost question", + }, + } + : {}), }); - assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + assert.equal( + (yield* Fiber.join(terminal))?.status, + previous === "lost question" ? "interrupted" : "completed", + ); }).pipe(Effect.scoped), ); @@ -721,21 +822,28 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); - it.effect("stops running turns on an external server when the session closes", () => - Effect.gen(function* () { - const scope = yield* Scope.make(); - const { runtime, thread } = yield* resumed( - [ - out("session.prompt", { sessionID: SESSION, text: "" }), - promptAccepted, - out("session.interrupt", { sessionID: SESSION }), - reply("session.interrupt", { interrupted: true }), - ], - { external: true }, - ).pipe(Scope.provide(scope)); - yield* runtime.startTurn(turnInput(thread)); - yield* Scope.close(scope, Exit.void); - }), + it.effect.each([false, true])( + "external shutdown preserves work only for restart=%s", + (restarting) => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const { runtime, thread } = yield* resumed( + [ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + ...(restarting + ? [] + : [ + out("session.interrupt", { sessionID: SESSION }), + reply("session.interrupt", { interrupted: true }), + ]), + ], + { external: true }, + ).pipe(Scope.provide(scope)); + yield* runtime.startTurn(turnInput(thread)); + yield* runtime.prepareForServerShutdown?.(restarting) ?? Effect.void; + yield* Scope.close(scope, Exit.void); + }), ); it.effect("sends nothing for a turn stopped while it reads the workspace's commands", () => diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 6f1425c672f1f..5c270a7252931 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1450,7 +1450,6 @@ export const layer = Layer.effect( yield* sql` DELETE FROM scheduled_task_webhook_deliveries WHERE task_id = ${input.taskId} - AND delivery_id NOT IN (SELECT delivery_id FROM scheduled_task_webhook_queue) AND delivery_id NOT IN ( SELECT delivery_id FROM scheduled_task_webhook_deliveries WHERE task_id = ${input.taskId} diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index a86121f351544..25736e1cb57d4 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -682,23 +682,40 @@ it.effect("a held request whose sender hung up mid-request still runs", () => ); it.effect("a held request runs once even after the log has trimmed it", () => - withService(({ service, launches }) => - Effect.gen(function* () { - const { task } = yield* service.upsert(yield* webhookTaskInput()); - const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" })); - assert.equal(first._tag, "accepted"); - yield* Queue.take(launches); - // Push the original row out of the 50-row delivery log. - const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); - yield* Effect.forEach(Array.from({ length: 55 }), () => - service.triggerWebhook(requestFor(paused.task)), - ); - yield* service.upsert(yield* webhookTaskInput()); - const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" })); - assert.equal(replay._tag, "accepted"); - assert.equal(yield* Queue.size(launches), 0); - }), - ), + Effect.gen(function* () { + const gate = yield* Deferred.make(); + yield* withService( + ({ service, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert(yield* webhookTaskInput()); + const first = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "kept" }), + ); + assert.equal(first._tag, "accepted"); + yield* Queue.take(launches); + // Push the original row out of the 50-row delivery log. + const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false })); + yield* Effect.forEach(Array.from({ length: 55 }), () => + service.triggerWebhook(requestFor(paused.task)), + ); + const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id }); + assert.lengthOf(deliveries, 50); + assert.isFalse( + deliveries.some( + (delivery) => delivery.id === (first._tag === "accepted" ? first.deliveryId : ""), + ), + ); + yield* service.upsert(yield* webhookTaskInput()); + const replay = yield* service.triggerWebhook( + requestFor(task, { relayDeliveryId: "kept" }), + ); + assert.equal(replay._tag, "accepted"); + assert.equal(yield* Queue.size(launches), 0); + yield* Deferred.succeed(gate, undefined); + }), + { gate }, + ); + }), ); it.effect("a rate-limited held request can run on a later pass", () => diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 91b1bc84a90a8..40c2579ed50db 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -3084,18 +3084,21 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr // Subscribed before any session or prompt call, so no event of theirs is missed. yield* follow(yield* connection.events).pipe(Effect.forkScoped); - // A server T3 did not start keeps running after T3 stops, so stop the turns - // it would otherwise finish unseen. A spawned server stops with its owner. + let preserveExternalTurns = false; + // External executions survive a restart so recovery can follow them. + // A regular quit still stops them. A spawned server stops with its owner. if (connection.external) { yield* Effect.addFinalizer(() => - Effect.forEach( - [...threads].filter(([, state]) => state.active !== undefined), - ([sessionId]) => - client.session - .interrupt({ sessionID: Session.ID.make(sessionId) }) - .pipe(Effect.timeout("1 second"), Effect.ignore({ log: true })), - { concurrency: 8, discard: true }, - ), + preserveExternalTurns + ? Effect.void + : Effect.forEach( + [...threads].filter(([, state]) => state.active !== undefined), + ([sessionId]) => + client.session + .interrupt({ sessionID: Session.ID.make(sessionId) }) + .pipe(Effect.timeout("1 second"), Effect.ignore({ log: true })), + { concurrency: 8, discard: true }, + ), ); } @@ -3524,6 +3527,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr state: ThreadState, turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, awaitingStart: boolean, + admissionId?: string, ) => Effect.gen(function* () { const startedAt = yield* DateTime.now; @@ -3539,7 +3543,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr nativeTurnRef: ref( isContinuation(turnInput) ? wakeTurnId(state.sessionId, turnInput.attemptId) - : turnPromptId(state.sessionId, turnInput.attemptId), + : (admissionId ?? turnPromptId(state.sessionId, turnInput.attemptId)), "weak", ), ordinal: turnInput.providerTurnOrdinal, @@ -3798,7 +3802,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr const text = turnInput.message.text.trim(); const bare = turnInput.message.attachments.length === 0; // The turn's own id, so fork and rollback cut before this turn's item. - const id = turnPromptId(sessionId, turnInput.attemptId); + const id = SessionMessage.ID.make( + promptOf(turn.providerTurn) ?? turnPromptId(sessionId, turnInput.attemptId), + ); if (bare && text === "/compact") { if (!sending()) return; return yield* client.session.compact({ sessionID, id }).pipe(Effect.asVoid); @@ -3854,6 +3860,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr // A background subagent keeps its session busy after its parent's turn, // and a held wake still needs its turn: idle release must wait for both. hasPendingBackgroundWork: Effect.sync(() => [...threads.values()].some(owesWork)), + prepareForServerShutdown: (restarting) => + Effect.sync(() => { + preserveExternalTurns = restarting; + }), hasPendingBackgroundWorkForThread: (providerThread) => Effect.sync(() => { const nativeId = providerThread.nativeThreadRef?.nativeId; @@ -4009,6 +4019,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr // Its checks and setup take the session's gate, so a rollback or // fork sees either no turn or this one installed; the prompt after // it does not hold the gate. + let admissionId: string | undefined; const started = yield* exclusive(turnInput.providerThread)( Effect.gen(function* () { const sessionId = yield* sessionIdOf(turnInput.providerThread); @@ -4027,7 +4038,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } // OpenCode already ran this turn on its own; it prompts nothing. if (isContinuation(turnInput)) return yield* runWake(state, turnInput); - if (turnInput.restartContinuationOfRunId !== undefined && !state.unsettled) { + const reconcileRestart = + turnInput.restartContinuationOfRunId !== undefined || + (connection.external && turnInput.restartContinuationSource !== undefined); + if (reconcileRestart && !state.unsettled) { const followed = yield* lock.withPermit( Effect.gen(function* () { const active = yield* client.session @@ -4044,12 +4058,57 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr ); const prompt = history.at(-1); const idle = history.find((message) => message.type === "idle"); + const source = turnInput.restartContinuationSource; + const recordedId = source?.nativeTurnRef?.nativeId; + const sourcePromptId = + recordedId?.startsWith("msg_") === true + ? recordedId + : source?.attemptId != null + ? turnPromptId(sessionId, source.attemptId) + : undefined; const finishedWhileDown = prompt?.type === "user" && + (sourcePromptId === undefined + ? source === undefined + : prompt.id === sourcePromptId) && prompt.id !== turnInput.providerThread.nativeConversationHeadRef?.nativeId && idle?.type === "idle" && idle.outcome === "succeeded"; - if (!(sessionId in active) && !finishedWhileDown) return false; + if (!(sessionId in active) && !finishedWhileDown) { + if (source?.awaitingUser === true) { + yield* beginTurn(state, turnInput, false); + yield* finishTurn(state, { status: "interrupted" }); + return true; + } + if ( + source !== undefined && + (sourcePromptId === undefined || prompt?.id !== sourcePromptId) + ) { + // The latest prompt may be a later one; search user + // history before retrying an original never admitted. + const admitted = + sourcePromptId !== undefined && + Option.isSome( + yield* OpenCode2Client.paginate( + { + sessionID: Session.ID.make(sessionId), + order: "desc" as const, + limit: 50, + type: "user" as const, + }, + client.message.list, + ).pipe( + Stream.filter((message) => message.id === sourcePromptId), + Stream.runHead, + ), + ); + if (!admitted) { + turnInput = { ...turnInput, message: source.message }; + admissionId = sourcePromptId; + } + } + return false; + } const turn = yield* beginTurn(state, turnInput, false); yield* takeRunningWake(state); if (state.active !== turn) return true; @@ -4096,7 +4155,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } } // Installs the turn; every path after it ends the turn with a terminal. - const begin = beginTurn(state, turnInput, afterUnsettled); + const begin = beginTurn(state, turnInput, afterUnsettled, admissionId); if (stillStopping) { yield* begin; return yield* finishTurn(state, { From 0db1be115cb7cbc410551b45bd817dfae5afe464 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:07:46 +0000 Subject: [PATCH 105/149] docs: describe continuation limits, waiting users, and accepted webhook durability --- docs/internals/server-updates.md | 10 ++++++---- docs/user/project-settings.md | 2 ++ docs/user/updating.md | 2 ++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 26550280fbae1..b865eb68e50f7 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -66,8 +66,8 @@ Restart continuation is an environment-owned preference, on by default. The [v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts) requires matching durable run, provider thread, session, and native resume identity. Queued runs never started, so recovery holds them and continues the run they wait -behind. A finished run qualifies only when the restart cancelled its background work; -its continuation tells the provider what will not report back. +behind. Cancelled background work is reported on the next user turn; it does not +wake a finished run. Undelivered steering can still qualify as unfinished input. Recovery retires effects tied to the lost process and records continuation intent in the durable outbox. That intent survives another restart before provider startup. @@ -80,8 +80,10 @@ The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinu rechecks the preference, archive state, provider selection, newer user work, a stop the user requested, and maintenance turns such as `/compact` before dispatching. Stable command and message IDs prevent duplicate submissions after an outbox retry. Codex -resumes without adding provider prompt text unless the turn lost background work; -other adapters receive the continuation message through their normal turn path. +first resumes without prompt text when its native thread has history; a fresh +thread or an active goal that rejects empty input receives the continuation prompt. +Recovered steering and background-work notes also require prompt text. Other +adapters receive the continuation message through their normal turn path. Delegated tasks (`delegate_task` child threads) are reconciled as their own threads, never as the parent's background work. The orchestrator settles child results and diff --git a/docs/user/project-settings.md b/docs/user/project-settings.md index c92226a9ee4f3..78d94ae57cb8f 100644 --- a/docs/user/project-settings.md +++ b/docs/user/project-settings.md @@ -89,6 +89,8 @@ one on. On desktop and web, pick **Deliveries** from a task's menu to see recent requests and the prompt each one produced. +Once the environment accepts a request, it keeps the pending delivery through +server restarts. Pausing or deleting the task skips deliveries still waiting to run. If the environment is offline, the sender gets an error and nothing runs; redeliver from the sender, such as GitHub's **Recent Deliveries**, once it is diff --git a/docs/user/updating.md b/docs/user/updating.md index 0de79162b28c2..da63204687acc 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -18,6 +18,8 @@ different value, use **Apply to all** in Settings after it connects. T3 Code must start again on that machine; the setting does not enable automatic startup. Terminal commands may still be interrupted, and threads without saved provider resume state need a new message. +It does not answer approvals, questions, or secret requests for you. After three +consecutive automatic continuations, send a new message to continue the work yourself. Updates from the previous orchestration system preserve conversation transcripts but cannot carry every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md) From a2a5a16e0dee14799176f92acfe4c61462dea9f4 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:26:07 +0000 Subject: [PATCH 106/149] fix(server): background updates recognize native pi, grok symlinks, CODEX_HOME, bun, volta and vite-plus installs --- .../src/provider/Drivers/CodexDriver.test.ts | 66 +++ .../src/provider/Drivers/CodexDriver.ts | 9 +- .../providerMaintenanceRunner.test.ts | 384 ++++++++++++++++++ .../src/server/maintenanceResolver.test.ts | 89 +++- .../src/server/maintenanceResolver.ts | 70 +++- .../provider-grok/src/server/driver.test.ts | 62 +++ packages/provider-grok/src/server/driver.ts | 58 ++- .../provider-pi/src/server/driver.test.ts | 84 ++++ packages/provider-pi/src/server/driver.ts | 55 ++- 9 files changed, 828 insertions(+), 49 deletions(-) diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index 7ac25a2e6bc13..ce5c4f2155d89 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -329,6 +329,72 @@ it.layer(layerTest)("CodexDriver", (it) => { }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), Effect.scoped), ); + it.effect.each(["default", "environment", "linked-environment", "configured"] as const)( + "owns the official standalone Codex release through its active alias (%s)", + (layout) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "r3-03-codex-" }); + let sharedHome = NodePath.join(root, layout === "default" ? ".codex" : "custom-home"); + const release = NodePath.join( + sharedHome, + "packages/standalone/releases/0.162.1-x86_64/bin/codex", + ); + yield* fs.makeDirectory(NodePath.dirname(release), { recursive: true }); + yield* fs.writeFileString(release, "#!/bin/sh\n"); + yield* fs.chmod(release, 0o755); + const current = NodePath.join(sharedHome, "packages/standalone/current"); + yield* fs.symlink("releases/0.162.1-x86_64", current); + if (layout === "linked-environment") { + const linked = NodePath.join(root, "linked-home"); + yield* fs.symlink(sharedHome, linked); + sharedHome = linked; + } + const alias = NodePath.join(root, ".local/bin/codex"); + yield* fs.makeDirectory(NodePath.dirname(alias), { recursive: true }); + yield* fs.symlink( + NodePath.join(sharedHome, "packages/standalone/current/bin/codex"), + alias, + ); + const environment = + layout === "default" + ? [] + : [ + { + name: "CODEX_HOME", + value: layout === "configured" ? NodePath.join(root, "other-home") : sharedHome, + sensitive: false, + }, + ]; + const resolve = Effect.fnUntraced(function* (binaryPath: string) { + const instance = yield* CodexDriver.create({ + instanceId: ProviderInstanceId.make("codex-official-layout"), + displayName: undefined, + enabled: false, + environment, + config: { + ...CodexDriver.defaultConfig(), + binaryPath, + ...(layout === "configured" ? { homePath: sharedHome } : {}), + }, + }).pipe(Effect.provideService(HostProcess.HomeDirectory, root)); + return yield* instance.snapshot.resolveMaintenance(); + }); + const active = yield* resolve(alias); + expect(active.update?.installationKey).toBeDefined(); + expect(active.update).toMatchObject({ + executable: alias, + args: ["update"], + env: { CODEX_HOME: sharedHome }, + }); + expect((yield* resolve(release)).update?.installationKey).toBeUndefined(); + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), + Effect.scoped, + ), + { skip: windowsHost }, + ); + for (const fixture of [ { name: "leaves mise npm-backend installations manual-only", diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index 8aa546d103e12..ef01dc85b6f96 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -184,7 +184,14 @@ export const CodexDriver: ProviderDriver { + let root = ""; + let updated = false; + const calls: Array<{ command: string; args: ReadonlyArray; env?: NodeJS.ProcessEnv }> = + []; + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + root = yield* fs.makeTempDirectoryScoped({ prefix: "r3-03-runner-" }); + const write = (file: string, source = '#!/bin/sh\nprintf "%s\\n" "$@"\n') => { + NodeFS.mkdirSync(NodePath.dirname(file), { recursive: true }); + NodeFS.writeFileSync(file, source); + NodeFS.chmodSync(file, 0o755); + }; + const link = (target: string, alias: string) => { + NodeFS.mkdirSync(NodePath.dirname(alias), { recursive: true }); + NodeFS.symlinkSync(NodePath.relative(NodePath.dirname(alias), target), alias); + }; + const bin = NodePath.join(root, ".local/bin"); + const cli = + method === "claude" + ? "claude" + : method === "opencode" + ? "opencode" + : method === "pi" + ? "pi" + : method === "grok" + ? "grok" + : method === "muse" + ? "muse" + : "codex"; + const alias = NodePath.join(bin, cli); + const kind = ProviderDriverKind.make(cli === "claude" ? "claudeAgent" : cli); + const packageName = + cli === "claude" + ? "@anthropic-ai/claude-code" + : cli === "opencode" + ? "opencode-ai" + : cli === "pi" + ? "@earendil-works/pi-coding-agent" + : cli === "grok" + ? "@xai-official/grok" + : "@openai/codex"; + let expectedExecutable = alias; + let expectedArgs: ReadonlyArray = [cli === "opencode" ? "upgrade" : "update"]; + let codexHome = NodePath.join(root, ".codex"); + const env = { HOME: root, PATH: bin, GROK_HOME: NodePath.join(root, ".grok") }; + if (method === "claude") { + const release = NodePath.join(root, ".local/share/claude/versions/2.1.293"); + write(release); + link(release, alias); + } else if (method.startsWith("codex")) { + if (method === "codex-custom") codexHome = NodePath.join(root, "custom-codex"); + const release = NodePath.join( + codexHome, + "packages/standalone/releases/0.162.1-x86_64/bin/codex", + ); + write(release); + const current = NodePath.join(codexHome, "packages/standalone/current"); + link(NodePath.dirname(NodePath.dirname(release)), current); + if (method === "codex-linked") { + const linkedHome = NodePath.join(root, "linked-codex"); + link(codexHome, linkedHome); + codexHome = linkedHome; + } + link(NodePath.join(codexHome, "packages/standalone/current/bin/codex"), alias); + } else if (method === "opencode") { + const native = NodePath.join(root, ".opencode/bin/opencode"); + write(native); + link(native, alias); + } else if (method === "pi") { + const agent = NodePath.join(root, ".pi/agent"); + const launcher = NodePath.join(agent, "bin/pi"); + const release = NodePath.join( + agent, + "install/releases/1.0.0/node_modules/@earendil-works/pi-coding-agent/dist/bundle/cli.js", + ); + write(release); + link(release, NodePath.join(agent, "install/releases/1.0.0/node_modules/.bin/pi")); + write(NodePath.join(agent, "install/current-version"), "1.0.0\n"); + write( + NodePath.join(agent, "install/managed-install.json"), + JSON.stringify({ + kind: "pi-managed-install", + schemaVersion: 1, + layout: "releases-v1", + entrypoint: { type: "symlink", path: alias }, + }), + ); + write( + launcher, + '#!/bin/sh\npi_bin_dir=${0%/*}\npi_agent_dir=${pi_bin_dir%/*}\npi_current_file=$pi_agent_dir/install/current-version\nIFS= read -r pi_current_version < "$pi_current_file"\npi_release_bin=$pi_agent_dir/install/releases/$pi_current_version/node_modules/.bin/pi\nPI_MANAGED_INSTALL_ROOT=$pi_agent_dir/install\nexport PI_MANAGED_INSTALL_ROOT\nexec "$pi_release_bin" "$@"\n', + ); + link(launcher, alias); + expectedArgs = ["update", "--self"]; + } else if (method === "grok") { + const download = NodePath.join(env.GROK_HOME, "downloads/grok-linux-x86_64"); + write(download); + link(download, NodePath.join(env.GROK_HOME, "bin/grok")); + link(NodePath.join(env.GROK_HOME, "bin/grok"), alias); + } else if (method === "muse") { + write( + alias, + '#!/bin/bash\n# muse-code/launcher-2\nMUSE_SYNC_UPDATE=${MUSE_SYNC_UPDATE:-0}\nexec "${0%/*}/muse-bin-2.0.0-R1" "$@"\n', + ); + write(NodePath.join(bin, "muse-bin-2.0.0-R1")); + expectedArgs = ["--version"]; + } else if (method === "volta") { + const shim = NodePath.join(root, ".volta/bin/volta-shim"); + write(shim); + NodeFS.mkdirSync(NodePath.join(root, ".volta/tools/image/packages/@openai/codex/1.0.0"), { + recursive: true, + }); + const voltaAlias = NodePath.join(root, ".volta/bin/codex"); + link(shim, voltaAlias); + link(voltaAlias, alias); + expectedExecutable = "volta"; + expectedArgs = ["install", "@openai/codex@2.0.0"]; + } else if (method.startsWith("vp-")) { + const data = NodePath.join( + root, + method === "vp-legacy" ? ".vite-plus" : ".local/share/vite-plus", + ); + const vp = NodePath.join(data, "1.1.0/bin/vp"); + write(vp); + link(NodePath.dirname(NodePath.dirname(vp)), NodePath.join(data, "current")); + write( + NodePath.join(data, "bins/codex.json"), + JSON.stringify({ + name: "codex", + package: packageName, + version: "1.0.0", + nodeVersion: "24.13.1", + source: "vp", + }), + ); + write( + NodePath.join(data, "packages/@openai/codex.json"), + JSON.stringify({ + name: packageName, + version: "1.0.0", + bins: ["codex"], + jsBins: ["codex"], + manager: "npm", + platform: { node: "24.13.1" }, + installedAt: "2026-10-11T00:00:00Z", + }), + ); + const vpAlias = NodePath.join(data, "bin/codex"); + link(NodePath.join(data, "current/bin/vp"), vpAlias); + link(vpAlias, alias); + expectedExecutable = "vp"; + expectedArgs = ["i", "-g", "@openai/codex@2.0.0"]; + } else if (method.startsWith("brew-")) { + const real = NodePath.join( + root, + method === "brew-cask" + ? "Caskroom/codex/0.162.1/bin/codex" + : "Cellar/opencode/2.0.25/bin/opencode", + ); + write(real); + link(real, alias); + write(NodePath.join(root, "bin/brew")); + expectedExecutable = NodePath.join(root, "bin/brew"); + expectedArgs = + method === "brew-cask" ? ["upgrade", "--cask", "codex"] : ["upgrade", "opencode"]; + } else { + const prefix = + method === "npm" + ? "npm/lib" + : method === "bun" + ? ".bun/install/global" + : method === "yarn" + ? ".config/yarn/global" + : method === "pnpm-5" + ? ".local/share/pnpm/global/5" + : ".local/share/pnpm/global/v11/store/codex/node_modules/.pnpm/@openai+codex@1.0.0"; + const real = NodePath.join(root, prefix, "node_modules/@openai/codex/bin/codex.js"); + write(real); + const ownerAlias = NodePath.join( + root, + method === "bun" + ? ".bun/bin/codex" + : method.startsWith("pnpm") + ? ".local/share/pnpm/bin/codex" + : ".local/bin/codex-link", + ); + if (method.startsWith("pnpm")) { + // pnpm writes a shell shim, not an npm-style executable symlink. + write( + ownerAlias, + `#!/bin/sh\nbasedir=$(dirname "$0")\nexec node "$basedir/${NodePath.relative(NodePath.dirname(ownerAlias), real)}" "$@"\n`, + ); + } else link(real, ownerAlias); + link(ownerAlias, alias); + expectedExecutable = method.startsWith("pnpm") ? "pnpm" : method; + expectedArgs = + method === "npm" + ? [ + "install", + "-g", + "--prefix", + NodePath.join(root, "npm"), + "--allow-scripts=@openai/codex", + "@openai/codex@2.0.0", + ] + : method.startsWith("pnpm") + ? ["add", "-g", "@openai/codex@2.0.0"] + : method === "bun" + ? ["i", "-g", "@openai/codex@2.0.0"] + : ["global", "add", "@openai/codex@2.0.0"]; + } + const resolver = makePackageManagedProviderMaintenanceResolver({ + provider: kind, + npmPackageName: packageName, + nativeUpdate: { + args: expectedArgs, + isCommandPath: (value) => + method === "claude" + ? value.includes("/.local/share/claude/versions/") + : method.startsWith("codex") + ? value.includes("/packages/standalone/") + : method === "opencode" + ? value.endsWith("/.opencode/bin/opencode") + : false, + ...(method.startsWith("codex") ? { env: { CODEX_HOME: codexHome } } : {}), + }, + }); + const instanceId = ProviderInstanceId.make("official-install"); + const instance = + method === "pi" + ? yield* PiDriver.create({ + instanceId, + displayName: undefined, + enabled: false, + environment: [], + config: { ...PiDriver.defaultConfig(), binaryPath: alias }, + }) + : method === "grok" + ? yield* GrokDriver.create({ + instanceId, + displayName: undefined, + enabled: false, + environment: [{ name: "GROK_HOME", value: env.GROK_HOME, sensitive: false }], + config: { ...GrokDriver.defaultConfig(), binaryPath: alias }, + }) + : method === "muse" + ? yield* MuseDriver.create({ + instanceId, + displayName: undefined, + enabled: false, + environment: [], + config: { ...MuseDriver.defaultConfig(), binaryPath: alias }, + }) + : null; + const resolve = (options?: { fresh?: boolean }) => + instance + ? instance.snapshot.resolveMaintenance(options) + : resolveProviderMaintenanceCapabilitiesEffect(resolver, { binaryPath: alias, env }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + assert.isDefined((yield* resolve()).update?.installationKey); + const { registry, providersRef } = yield* makeRegistry({ + ...baseProvider, + instanceId, + driver: kind, + version: "1.0.0", + }); + const runner = yield* makeTestRunner( + { + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (_id, _driver, options) => + resolve(options), + refreshInstance: () => + Ref.updateAndGet(providersRef, (providers) => + providers.map((provider) => (updated ? { ...provider, version: "2.0.0" } : provider)), + ), + }, + { + version: 1, + currentModels: {}, + compatibility: [ + { + driver: kind, + t3CodeRange: ">=0.0.42", + ranges: [{ range: ">=1.0.0", status: "supported" }], + }, + ], + }, + ); + const result = yield* runner.updateProvider({ + instanceId, + provider: kind, + origin: "background", + }); + assert.strictEqual(result.providers[0]?.updateState?.status, "succeeded"); + assert.strictEqual(calls.length, 1); + assert.strictEqual(calls[0]?.command, expectedExecutable); + assert.deepStrictEqual(calls[0]?.args, expectedArgs); + if (method.startsWith("codex")) assert.strictEqual(calls[0]?.env?.CODEX_HOME, codexHome); + if (method === "muse") assert.strictEqual(calls[0]?.env?.MUSE_SYNC_UPDATE, "1"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + NodeServices.layer, + TestProviderHost.layer({ runBackgroundWork: false }).pipe( + Layer.provide(NodeServices.layer), + ), + IdAllocator.layer, + McpProviderSessions.layer, + ProviderLatestVersions.layer, + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + layerMockSpawner((command, args, options) => { + if (args[0] === "--prefix") return { stdout: root }; + if (args[0] === "info") + return { + stdout: JSON.stringify({ + formulae: [{ versions: { stable: "2.0.0" } }], + casks: [{ version: "2.0.0" }], + }), + }; + calls.push({ command, args, ...(options.env ? { env: options.env } : {}) }); + updated = true; + return { stdout: "updated" }; + }), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + version: request.url.includes("meta.ai") ? "2.0.0-R1" : "2.0.0", + }), + ), + ), + ), + ), + ), + ), + Effect.scoped, + ); + }, + { skip: HostProcess.Platform.defaultValue() === "win32" }, +); + const CODEX_DRIVER = ProviderDriverKind.make("codex"); const NATIVE_CLI_DRIVER = ProviderDriverKind.make("nativeCli"); const OPENCODE_DRIVER = ProviderDriverKind.make("opencode"); diff --git a/packages/provider-core/src/server/maintenanceResolver.test.ts b/packages/provider-core/src/server/maintenanceResolver.test.ts index 9523374561c9a..3c63c8e4abef7 100644 --- a/packages/provider-core/src/server/maintenanceResolver.test.ts +++ b/packages/provider-core/src/server/maintenanceResolver.test.ts @@ -1157,6 +1157,75 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { { skip: windowsHost }, ); + it.effect.each(["legacy", "split", "custom"] as const)( + "owns official Vite+ package shims through their metadata (%s)", + (layout) => + Effect.gen(function* () { + const root = yield* makeTempDir("r3-03-vp"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const data = NodePath.join( + root, + layout === "legacy" + ? ".vite-plus" + : layout === "split" + ? ".local/share/vite-plus" + : "custom-data", + ); + const bin = NodePath.join(layout === "custom" ? root : data, "bin"); + const vp = NodePath.join(data, "1.1.0/bin/vp"); + writeExecutable(vp); + NodeFS.symlinkSync("1.1.0", NodePath.join(data, "current")); + NodeFS.mkdirSync(bin, { recursive: true }); + const alias = NodePath.join(bin, "package-tool"); + NodeFS.symlinkSync(NodePath.relative(bin, NodePath.join(data, "current/bin/vp")), alias); + NodeFS.mkdirSync(NodePath.join(data, "bins"), { recursive: true }); + NodeFS.writeFileSync( + NodePath.join(data, "bins/package-tool.json"), + JSON.stringify({ + name: "package-tool", + package: "@example/package-tool", + version: "1.0.0", + nodeVersion: "24.13.1", + source: "vp", + }), + ); + NodeFS.mkdirSync(NodePath.join(data, "packages/@example"), { recursive: true }); + const metadata = NodePath.join(data, "packages/@example/package-tool.json"); + const installed = { + name: "@example/package-tool", + version: "1.0.0", + installId: "", + platform: { node: "24.13.1" }, + bins: ["package-tool"], + jsBins: ["package-tool"], + manager: "npm", + installedAt: "2026-10-11T00:00:00Z", + }; + NodeFS.writeFileSync(metadata, JSON.stringify(installed)); + const resolve = () => + resolveProviderMaintenanceCapabilitiesEffect(packageToolUpdate, { + binaryPath: alias, + env: { PATH: bin }, + }); + const active = yield* resolve(); + expect(active.update).toMatchObject({ + executable: "vp", + args: ["i", "-g", "@example/package-tool"], + lockKey: "vite-plus-global", + }); + expect(active.update?.installationKey).toBeDefined(); + NodeFS.writeFileSync(metadata, JSON.stringify({ ...installed, versionSpec: "1.0.0" })); + expect((yield* resolve()).update?.installationKey).toBeUndefined(); + NodeFS.writeFileSync(metadata, JSON.stringify({ ...installed, name: "@example/sdk" })); + expect((yield* resolve()).update).toBeNull(); + NodeFS.rmSync(metadata); + expect((yield* resolve()).update).toBeNull(); + }).pipe(Effect.scoped), + { skip: windowsHost }, + ); + it.effect.skipIf(windowsHost)( "native aliases share stable ownership but configured release paths stay pinned", () => @@ -1165,15 +1234,7 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { yield* Effect.addFinalizer(() => Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), ); - const release = NodePath.join( - root, - ".local", - "share", - "claude", - "versions", - "1.0.0", - "claude", - ); + const release = NodePath.join(root, ".local", "share", "claude", "versions", "1.0.0"); writeExecutable(release); const alias = NodePath.join(root, ".local", "bin", "claude"); NodeFS.mkdirSync(NodePath.dirname(alias), { recursive: true }); @@ -1193,15 +1254,7 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { const pinned = yield* resolve(release); expect(active.update?.installationKey).toBeDefined(); expect(pinned.update?.installationKey).toBeUndefined(); - const newer = NodePath.join( - root, - ".local", - "share", - "claude", - "versions", - "2.0.0", - "claude", - ); + const newer = NodePath.join(root, ".local", "share", "claude", "versions", "2.0.0"); writeExecutable(newer); NodeFS.unlinkSync(alias); NodeFS.symlinkSync(newer, alias); diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index b528255e1faa1..e014020d287a3 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -201,7 +201,10 @@ function isVitePlusGlobalCommandPath(commandPath: string): boolean { } function isBunGlobalCommandPath(commandPath: string): boolean { - return normalizeCommandPath(commandPath).includes("/.bun/bin/"); + const normalized = normalizeCommandPath(commandPath); + return ( + normalized.includes("/.bun/bin/") || normalized.includes("/.bun/install/global/node_modules/") + ); } function isYarnGlobalCommandPath(commandPath: string): boolean { @@ -465,6 +468,21 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( ) ? manual : native; + const vitePlus = yield* resolveVitePlusPackageInstall(context, packageName); + if (vitePlus) { + return makeProviderMaintenanceCapabilities({ + provider: definition.provider, + env: context.env, + platform: context.platform, + installationKey: keyFor(vitePlus), + packageName, + updateExecutable: "vp", + updateArgs: ["i", "-g", packageName], + updateLockKey: "vite-plus-global", + }); + } + // Vite+ package shims resolve to vp itself, not the installed package. + if (/\/vp(?:\.exe)?$/.test(normalizeCommandPath(context.realCommandPath))) return manual; if (commandPaths.some(isVitePlusGlobalCommandPath)) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, @@ -530,7 +548,7 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( provider: definition.provider, env: context.env, platform: context.platform, - installationKey: keyFor(context.resolvedCommandPath.replaceAll("\\", "/").split("/bin/")[0]!), + installationKey: keyFor(context.realCommandPath.replaceAll("\\", "/").split("/bin/")[0]!), packageName, updateExecutable: "volta", updateArgs: ["install", `${packageName}@latest`], @@ -619,6 +637,52 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( return fallback; }); +const VitePlusBinConfig = Schema.fromJsonString( + Schema.Struct({ + name: Schema.String, + package: Schema.String, + source: Schema.optional(Schema.Literal("vp")), + }), +); +const VitePlusPackageMetadata = Schema.fromJsonString( + Schema.Struct({ + name: Schema.String, + bins: Schema.Array(Schema.String), + versionSpec: Schema.optional(Schema.String), + }), +); + +/** Vite+ records the owner of a shim in /bins and /packages. */ +const resolveVitePlusPackageInstall = Effect.fn("resolveVitePlusPackageInstall")(function* ( + context: ProviderMaintenanceResolutionContext, + packageName: string, +) { + if (!/\/vp(?:\.exe)?$/.test(normalizeCommandPath(context.realCommandPath))) return null; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const data = path.dirname(path.dirname(path.dirname(context.realCommandPath))); + const name = path.basename(context.resolvedCommandPath).replace(/\.exe$/i, ""); + return yield* Effect.gen(function* () { + const config = yield* fs + .readFileString(path.join(data, "bins", `${name}.json`)) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(VitePlusBinConfig))); + const metadata = yield* fs + .readFileString(path.join(data, "packages", `${packageName}.json`)) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(VitePlusPackageMetadata))); + const active = yield* fs.realPath( + path.join(data, "current/bin", path.basename(context.realCommandPath)), + ); + return active === context.realCommandPath && + config.name === name && + config.package === packageName && + metadata.name === packageName && + metadata.bins.includes(name) && + (!metadata.versionSpec || metadata.versionSpec === "latest") + ? data + : null; + }).pipe(Effect.orElseSucceed(() => null)); +}); + /** A launcher script that runs the provider through mise (`exec mise x codex -- codex`). */ const isMiseWrapperScript = Effect.fn("isMiseWrapperScript")(function* ( context: ProviderMaintenanceResolutionContext, @@ -652,7 +716,7 @@ const isVoltaPackageInstall = Effect.fn("isVoltaPackageInstall")(function* ( ) { return false; } - const voltaHome = path.dirname(path.dirname(context.resolvedCommandPath)); + const voltaHome = path.dirname(path.dirname(context.realCommandPath)); const packageDir = path.join(voltaHome, "tools", "image", "packages", ...packageName.split("/")); const fileSystem = yield* FileSystem.FileSystem; return yield* fileSystem.exists(packageDir).pipe(Effect.orElseSucceed(() => false)); diff --git a/packages/provider-grok/src/server/driver.test.ts b/packages/provider-grok/src/server/driver.test.ts index 42f2ae4841f24..e663fb124b7c9 100644 --- a/packages/provider-grok/src/server/driver.test.ts +++ b/packages/provider-grok/src/server/driver.test.ts @@ -44,6 +44,68 @@ const noSpawner = ChildProcessSpawner.make(() => const windowsHost = HostProcess.Platform.defaultValue() === "win32"; it.layer(layerTest)("GrokDriver", (it) => { + it.effect.each(["default", "linked-home", "custom-bin"] as const)( + "owns the official Grok download symlinks (%s) but not a pinned download", + (layout) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "r3-03-grok-" }); + const home = path.join(root, ".grok"); + const download = path.join(home, "downloads/grok-linux-x86_64"); + const bin = layout === "custom-bin" ? path.join(root, "tools") : path.join(home, "bin"); + yield* fs.makeDirectory(path.dirname(download), { recursive: true }); + yield* fs.writeFileString(download, "#!/bin/sh\nprintf '%s\\n' \"$@\"\n"); + yield* fs.chmod(download, 0o755); + yield* fs.makeDirectory(bin, { recursive: true }); + yield* fs.symlink(path.relative(bin, download), path.join(bin, "grok")); + yield* fs.symlink(path.relative(bin, download), path.join(bin, "agent")); + let visibleHome = home; + if (layout === "linked-home") { + visibleHome = path.join(root, "linked-grok"); + yield* fs.symlink(home, visibleHome); + } + const alias = path.join(root, ".local/bin/grok"); + yield* fs.makeDirectory(path.dirname(alias), { recursive: true }); + yield* fs.symlink( + layout === "custom-bin" ? path.join(bin, "grok") : path.join(visibleHome, "bin/grok"), + alias, + ); + const resolve = Effect.fnUntraced(function* (binaryPath: string) { + const instance = yield* GrokDriver.create({ + instanceId: ProviderInstanceId.make("grok-native-layout"), + displayName: undefined, + enabled: false, + environment: [ + { name: "GROK_HOME", value: visibleHome, sensitive: false }, + ...(layout === "custom-bin" + ? [{ name: "GROK_BIN_DIR", value: bin, sensitive: false }] + : []), + ], + config: { ...GrokDriver.defaultConfig(), binaryPath }, + }); + return yield* instance.snapshot.resolveMaintenance(); + }); + const active = yield* resolve(alias); + expect(active.update).toMatchObject({ executable: alias, args: ["update"] }); + expect(active.update?.installationKey).toBeDefined(); + expect((yield* resolve(path.join(bin, "grok"))).update?.installationKey).toBe( + active.update?.installationKey, + ); + expect((yield* resolve(download)).update?.installationKey).toBeUndefined(); + const foreign = path.join(root, "pinned/grok"); + yield* fs.makeDirectory(path.dirname(foreign), { recursive: true }); + yield* fs.writeFileString(foreign, "#!/bin/sh\n"); + yield* fs.chmod(foreign, 0o755); + yield* fs.remove(alias); + yield* fs.symlink(foreign, alias); + expect((yield* resolve(alias)).update?.installationKey).toBeUndefined(); + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawner), + Effect.scoped, + ), + { skip: windowsHost }, + ); it.effect.skipIf(windowsHost)("updates through the configured executable's own updater", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/provider-grok/src/server/driver.ts b/packages/provider-grok/src/server/driver.ts index 588b57af5f54c..4e21d2d3899e2 100644 --- a/packages/provider-grok/src/server/driver.ts +++ b/packages/provider-grok/src/server/driver.ts @@ -50,27 +50,45 @@ const DRIVER_KIND = ProviderDriverKind.make("grok"); const GROK_NPM_PACKAGE = "@xai-official/grok"; // Native updates target GROK_HOME; package-manager installs stay with their owner. const UPDATE: ProviderMaintenanceCapabilitiesResolver = { - resolve: (context) => { - const home = ( - context?.env.GROK_HOME || (context?.env.HOME ? `${context.env.HOME}/.grok` : undefined) - ) - ?.trim() - .replace(/[\\/]+$/, ""); - return resolvePackageManagedProviderMaintenance( - { - provider: DRIVER_KIND, - npmPackageName: GROK_NPM_PACKAGE, - nativeUpdate: { - args: ["update"], - isCommandPath: (value) => - Boolean(home) && - normalizeCommandPath(value).replace(/\.exe$/, "") === - normalizeCommandPath(`${home}/bin/grok`), + resolve: (context) => + Effect.gen(function* () { + const home = ( + context?.env.GROK_HOME || (context?.env.HOME ? `${context.env.HOME}/.grok` : undefined) + ) + ?.trim() + .replace(/[\\/]+$/, ""); + const fs = yield* FileSystem.FileSystem; + const realHome = home + ? yield* fs.realPath(home).pipe(Effect.orElseSucceed(() => home)) + : undefined; + const bin = context?.env.GROK_BIN_DIR || (home ? `${home}/bin` : undefined); + const activePath = bin + ? yield* fs.realPath(`${bin}/grok`).pipe(Effect.orElseSucceed(() => undefined)) + : undefined; + return yield* resolvePackageManagedProviderMaintenance( + { + provider: DRIVER_KIND, + npmPackageName: GROK_NPM_PACKAGE, + nativeUpdate: { + args: ["update"], + isCommandPath: (value) => + Boolean(realHome) && + (normalizeCommandPath(value).replace(/\.exe$/, "") === + normalizeCommandPath(`${realHome}/bin/grok`) || + (value === activePath && + context?.resolvedCommandPath !== value && + !/[\\/]downloads[\\/]grok-/.test(context?.binaryPath ?? "") && + normalizeCommandPath(value).startsWith( + `${normalizeCommandPath(realHome!)}/downloads/`, + ) && + /\/grok-(?:linux|macos|windows)-(?:x86_64|aarch64)(?:\.exe)?$/.test( + normalizeCommandPath(value), + ))), + }, }, - }, - context, - ); - }, + context, + ); + }), }; export type GrokDriverEnv = diff --git a/packages/provider-pi/src/server/driver.test.ts b/packages/provider-pi/src/server/driver.test.ts index 84e6d86b0ddf7..88c020576bf27 100644 --- a/packages/provider-pi/src/server/driver.test.ts +++ b/packages/provider-pi/src/server/driver.test.ts @@ -1,9 +1,12 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { DEFAULT_SERVER_SETTINGS, ProviderInstanceId } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; @@ -127,6 +130,87 @@ const create = (config: Partial = {}, enabled = true) => }); it.layer(layerTest)("PiDriver workspace discovery", (it) => { + it.effect.each(["default", "custom", "linked"] as const)( + "owns the official managed Pi launcher (%s), never a pinned release or wrapper", + (layout) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "r3-03-pi-" }); + const agent = path.join(root, layout === "default" ? ".pi/agent" : "custom-agent"); + const install = path.join(agent, "install"); + const launcher = path.join(agent, "bin/pi"); + const release = path.join( + install, + "releases/1.0.0/node_modules/@earendil-works/pi-coding-agent/dist/bundle/cli.js", + ); + const releaseBin = path.join(install, "releases/1.0.0/node_modules/.bin/pi"); + yield* fs.makeDirectory(path.dirname(release), { recursive: true }); + yield* fs.writeFileString(release, "#!/bin/sh\nprintf '%s\\n' \"$@\"\n"); + yield* fs.chmod(release, 0o755); + yield* fs.makeDirectory(path.dirname(releaseBin), { recursive: true }); + yield* fs.symlink("../@earendil-works/pi-coding-agent/dist/bundle/cli.js", releaseBin); + yield* fs.makeDirectory(path.dirname(launcher), { recursive: true }); + yield* fs.writeFileString( + launcher, + [ + "#!/bin/sh", + 'pi_launcher="$0"', + 'while [ -L "$pi_launcher" ]; do pi_launcher=$(readlink "$pi_launcher"); done', + "pi_bin_dir=${pi_launcher%/*}", + "pi_agent_dir=${pi_bin_dir%/*}", + "pi_current_file=$pi_agent_dir/install/current-version", + 'IFS= read -r pi_current_version < "$pi_current_file"', + "pi_release_dir=$pi_agent_dir/install/releases/$pi_current_version", + "pi_release_bin=$pi_release_dir/node_modules/.bin/pi", + "PI_MANAGED_INSTALL_ROOT=$pi_agent_dir/install", + "export PI_MANAGED_INSTALL_ROOT", + 'exec "$pi_release_bin" "$@"', + "", + ].join("\n"), + ); + yield* fs.chmod(launcher, 0o755); + let visibleAgent = agent; + if (layout === "linked") { + visibleAgent = path.join(root, "linked-agent"); + yield* fs.symlink(agent, visibleAgent); + } + const alias = path.join(root, ".local/bin/pi"); + yield* fs.makeDirectory(path.dirname(alias), { recursive: true }); + yield* fs.symlink(path.join(visibleAgent, "bin/pi"), alias); + const marker = path.join(install, "managed-install.json"); + yield* fs.writeFileString( + marker, + JSON.stringify({ + kind: "pi-managed-install", + schemaVersion: 1, + layout: "releases-v1", + entrypoint: { type: "symlink", path: alias }, + }), + ); + yield* fs.writeFileString(path.join(install, "current-version"), "1.0.0\n"); + const resolve = Effect.fnUntraced(function* (binaryPath: string) { + const instance = yield* create({ binaryPath }, false); + return yield* instance.snapshot.resolveMaintenance(); + }); + const active = yield* resolve(alias); + assert.isDefined(active.update?.installationKey); + assert.equal(active.update?.executable, alias); + assert.deepEqual(active.update?.args, ["update", "--self"]); + assert.equal( + (yield* resolve(launcher)).update?.installationKey, + active.update?.installationKey, + ); + assert.isUndefined((yield* resolve(release)).update?.installationKey); + const wrapper = path.join(root, "custom-pi"); + yield* fs.writeFileString(wrapper, `#!/bin/sh\nexec "${launcher}" "$@"\n`); + yield* fs.chmod(wrapper, 0o755); + assert.isUndefined((yield* resolve(wrapper)).update?.installationKey); + yield* fs.writeFileString(marker, "{}"); + assert.isUndefined((yield* resolve(alias)).update?.installationKey); + }).pipe(Effect.scoped), + { skip: HostProcess.Platform.defaultValue() === "win32" }, + ); it.effect("keeps each workspace's skills and commands separate from the machine catalog", () => Effect.gen(function* () { const { spawner, launches } = yield* makePiSpawner(); diff --git a/packages/provider-pi/src/server/driver.ts b/packages/provider-pi/src/server/driver.ts index e36909fa39902..219f34a512d1e 100644 --- a/packages/provider-pi/src/server/driver.ts +++ b/packages/provider-pi/src/server/driver.ts @@ -37,8 +37,9 @@ import type { ServerProviderDraft } from "@t3tools/provider-core/server/snapshot import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { makeCachedProviderMaintenanceResolution, - makePackageManagedProviderMaintenanceResolver, + resolvePackageManagedProviderMaintenance, resolveProviderMaintenanceCapabilitiesEffect, + type ProviderMaintenanceCapabilitiesResolver, } from "@t3tools/provider-core/server/maintenanceResolver"; import { haveProviderSnapshotSettingsChanged, @@ -49,12 +50,52 @@ import { const decodePiSettings = Schema.decodeSync(PiSettings); const DRIVER_KIND = ProviderDriverKind.make("pi"); -const UPDATE = makePackageManagedProviderMaintenanceResolver({ - provider: DRIVER_KIND, - npmPackageName: "@earendil-works/pi-coding-agent", - // Pi's updater covers its own installer and npm, pnpm, yarn, and bun globals. - nativeUpdate: { args: ["update", "--self"] }, -}); +const ManagedInstallMarker = Schema.fromJsonString( + Schema.Struct({ + kind: Schema.Literal("pi-managed-install"), + schemaVersion: Schema.Literal(1), + layout: Schema.Literal("releases-v1"), + }), +); +const UPDATE: ProviderMaintenanceCapabilitiesResolver = { + resolve: (context) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // The official installer writes a launcher beside install/managed-install.json. + // Release entry points are project-local node_modules and stay pinned. + const ownsLauncher = + context && + path.basename(context.realCommandPath) === "pi" && + path.basename(path.dirname(context.realCommandPath)) === "bin" + ? yield* Effect.gen(function* () { + const marker = path.join( + path.dirname(path.dirname(context.realCommandPath)), + "install/managed-install.json", + ); + if (Number((yield* fs.stat(context.realCommandPath)).size) > 16 * 1_024) return false; + if (Number((yield* fs.stat(marker)).size) > 16 * 1_024) return false; + yield* fs + .readFileString(marker) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(ManagedInstallMarker))); + const source = yield* fs.readFileString(context.realCommandPath); + return ( + source.includes("pi_current_file=$pi_agent_dir/install/current-version") && + source.includes("PI_MANAGED_INSTALL_ROOT=$pi_agent_dir/install") && + source.includes('exec "$pi_release_bin" "$@"') + ); + }).pipe(Effect.orElseSucceed(() => false)) + : false; + return yield* resolvePackageManagedProviderMaintenance( + { + provider: DRIVER_KIND, + npmPackageName: "@earendil-works/pi-coding-agent", + nativeUpdate: { args: ["update", "--self"], isCommandPath: () => Boolean(ownsLauncher) }, + }, + context, + ); + }), +}; export type PiDriverEnv = | PiAdapterV2DriverEnv From 7e66636ce54a726394b79010b73925893388b0cf Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:26:07 +0000 Subject: [PATCH 107/149] fix(mobile): writable project settings follow write destinations and continuous input reports activity once --- .../connection/background-activity-scopes.ts | 10 +- .../connection/background-activity.test.ts | 16 ++- .../SettingsServerControlsRouteScreen.tsx | 24 ++-- .../settings/settings-scoped-server.test.ts | 104 +++++++++++++++++- 4 files changed, 132 insertions(+), 22 deletions(-) diff --git a/apps/mobile/src/connection/background-activity-scopes.ts b/apps/mobile/src/connection/background-activity-scopes.ts index bd60d7ce669c1..5fe6c0f5272e3 100644 --- a/apps/mobile/src/connection/background-activity-scopes.ts +++ b/apps/mobile/src/connection/background-activity-scopes.ts @@ -4,14 +4,12 @@ import * as Effect from "effect/Effect"; const INTERACTION_WINDOW_MS = 30_000; let lastInteractionAt = Number.NEGATIVE_INFINITY; -let lastInteractionReportAt = Number.NEGATIVE_INFINITY; export function recordMobileInteraction(): void { - lastInteractionAt = performance.now(); - if (lastInteractionAt - lastInteractionReportAt >= 1_000) { - lastInteractionReportAt = lastInteractionAt; - notify(); - } + const now = performance.now(); + const wasRecent = mobileRecentlyInteracted(now); + lastInteractionAt = now; + if (!wasRecent) notify(); } export function mobileRecentlyInteracted(now = performance.now()): boolean { diff --git a/apps/mobile/src/connection/background-activity.test.ts b/apps/mobile/src/connection/background-activity.test.ts index aa790e5a1e547..cf3e37cfa99f0 100644 --- a/apps/mobile/src/connection/background-activity.test.ts +++ b/apps/mobile/src/connection/background-activity.test.ts @@ -101,21 +101,25 @@ it("expires real mobile interaction independently of foreground heartbeats", () expect(mobileRecentlyInteracted()).toBe(true); }); -it("reports continuous interaction without starving the reporter debounce", () => { - const start = performance.now() + 10_000; +it("reports renewed interaction once and lets heartbeats cover continuous use", () => { + const start = performance.now() + 40_000; const clock = vi.spyOn(performance, "now"); let reports = 0; const unsubscribe = onRetainedMobileBackgroundScopesChange(() => { reports++; }); try { - for (let elapsed = 0; elapsed < 10_000; elapsed += 100) { + for (let elapsed = 0; elapsed < 120_000; elapsed += 100) { clock.mockReturnValue(start + elapsed); recordMobileInteraction(); } - expect(reports).toBe(10); - expect(mobileRecentlyInteracted(start + 10_000)).toBe(true); - expect(mobileRecentlyInteracted(start + 40_000)).toBe(false); + expect(reports).toBe(1); + expect(mobileRecentlyInteracted(start + 120_000)).toBe(true); + expect(mobileRecentlyInteracted(start + 150_000)).toBe(false); + clock.mockReturnValue(start + 150_000); + recordMobileInteraction(); + expect(reports).toBe(2); + expect(mobileRecentlyInteracted(start + 150_000)).toBe(true); } finally { unsubscribe(); clock.mockRestore(); diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 7a0ff111af609..6e05f1f74bd6c 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -145,24 +145,30 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { const navigation = useNavigation(); const { selectedTargets, projectGroups, selectedProjectKey } = useSettingsEnvironmentFilter(); const selectedProject = projectGroups.find((group) => group.key === selectedProjectKey); + const projectSelected = selectedProjectKey !== null; + const targets = useMemo( + () => + resolveMobileSettingsTargets( + selectedTargets, + projectSelected ? (selectedProject?.members.map((member) => member.project) ?? []) : null, + ), + [selectedTargets, selectedProject, projectSelected], + ); const canWriteSettings = useAtomValue( useMemo( () => Atom.make( (get) => - selectedTargets.length > 0 && - selectedTargets.every((target) => - get(serverEnvironment.updateSettings.permissionAtom(target.environmentId)), + targets.length > 0 && + targets.every((target) => + get( + serverEnvironment.updateSettings.permissionAtom(target.environment.environmentId), + ), ), ), - [selectedTargets], + [targets], ), ); - const projectSelected = selectedProjectKey !== null; - const targets = resolveMobileSettingsTargets( - selectedTargets, - projectSelected ? (selectedProject?.members.map((member) => member.project) ?? []) : null, - ); const [pendingWrites, setPendingWrites] = useState(0); const writeInFlight = useRef(false); const [pendingTargets, setPendingTargets] = useState< diff --git a/apps/mobile/src/features/settings/settings-scoped-server.test.ts b/apps/mobile/src/features/settings/settings-scoped-server.test.ts index 1c2de9b662f10..5fd45f0354438 100644 --- a/apps/mobile/src/features/settings/settings-scoped-server.test.ts +++ b/apps/mobile/src/features/settings/settings-scoped-server.test.ts @@ -1,10 +1,69 @@ +// @vitest-environment jsdom import { DEFAULT_SERVER_SETTINGS, type EnvironmentId, type ProjectId, type ServerSettings, } from "@t3tools/contracts"; -import { describe, expect, it } from "vite-plus/test"; +import { act, createElement, type ReactNode } from "react"; +import { createRoot } from "react-dom/client"; +import { Atom, AtomRegistry } from "effect/reactivity"; +import { describe, expect, it, vi } from "vite-plus/test"; + +const screenState = vi.hoisted(() => ({ + targets: [] as SettingsTarget[], + projectMembers: [] as { project: { environmentId: EnvironmentId; id: ProjectId } }[], + writable: new Set(), + write: vi.fn(), +})); + +vi.mock("@react-navigation/native", () => ({ useNavigation: () => ({}) })); +vi.mock("react-native-safe-area-context", () => ({ useSafeAreaInsets: () => ({ bottom: 0 }) })); +vi.mock("react-native", () => ({ + Platform: { OS: "android" }, + View: ({ children }: { children: ReactNode }) => children, + Pressable: (props: { children: ReactNode; disabled?: boolean; onPress?: () => void }) => + createElement("button", { disabled: props.disabled, onClick: props.onPress }, props.children), +})); +vi.mock("../../components/AppText", () => ({ + AppText: ({ children }: { children: ReactNode }) => createElement("span", null, children), + AppTextInput: () => null, +})); +vi.mock("../../components/AppSymbol", () => ({ SymbolView: () => null })); +vi.mock("../../components/ThemedSwitch", () => ({ ThemedSwitch: () => null })); +vi.mock("../../components/ScreenScrollView", () => ({ + ScreenScrollView: ({ children }: { children: ReactNode }) => children, +})); +vi.mock("./components/SettingsScreen", () => ({ + SettingsScreen: ({ children }: { children: ReactNode }) => children, +})); +vi.mock("./components/SettingsEnvironmentFilterHeader", () => ({ + AndroidSettingsEnvironmentFilter: () => null, + SettingsEnvironmentFilterHeader: () => null, +})); +vi.mock("./components/SettingsRow", () => ({ SettingsRow: () => null })); +vi.mock("./settings-environment-filter", () => ({ + useSettingsEnvironmentFilter: () => ({ + selectedTargets: screenState.targets, + selectedProjectKey: "project", + projectGroups: [{ key: "project", label: "Project", members: screenState.projectMembers }], + }), +})); +vi.mock("../../state/session", () => ({ + readEnvironmentScope: (environmentId: EnvironmentId) => screenState.writable.has(environmentId), +})); +vi.mock("../../state/server", () => ({ + serverEnvironment: { + updateSettings: { + permissionAtom: (environmentId: EnvironmentId) => + Atom.make(screenState.writable.has(environmentId)), + }, + }, +})); +vi.mock("../../state/use-atom-command", () => ({ useAtomCommand: () => screenState.write })); +vi.mock("@effect/atom-react", () => ({ + useAtomValue: (atom: Atom.Atom) => AtomRegistry.make().get(atom), +})); import type { SettingsTarget } from "./settings-environment-filter"; import { @@ -13,6 +72,7 @@ import { resolveMobileSettingsTargets, uniformMobileSetting, } from "./settings-scoped-server"; +import { SettingsEnvironmentAgentBehaviorRouteScreen } from "./SettingsServerControlsRouteScreen"; const firstId = "first" as EnvironmentId; const secondId = "second" as EnvironmentId; @@ -66,6 +126,48 @@ function environment(environmentId: EnvironmentId, settings: ServerSettings): Se } describe("mobile project settings scope", () => { + it.each([false, true])( + "checks permissions only for project write destinations (read-only checkout: %s)", + async (readOnlyCheckout) => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + screenState.targets = [ + environment(firstId, DEFAULT_SERVER_SETTINGS), + environment(secondId, DEFAULT_SERVER_SETTINGS), + ]; + screenState.projectMembers = [ + { project: { environmentId: firstId, id: firstProject } }, + ...(readOnlyCheckout ? [{ project: { environmentId: secondId, id: secondProject } }] : []), + ]; + screenState.writable = new Set([firstId]); + screenState.write.mockReset().mockResolvedValue({ _tag: "Success" }); + const container = document.createElement("div"); + const root = createRoot(container); + try { + await act(async () => + root.render(createElement(SettingsEnvironmentAgentBehaviorRouteScreen)), + ); + const choice = Array.from(container.querySelectorAll("button")).find((button) => + button.textContent?.includes("After the turn"), + ); + expect(choice).toBeDefined(); + await act(async () => choice!.click()); + if (readOnlyCheckout) expect(screenState.write).not.toHaveBeenCalled(); + else + expect(screenState.write).toHaveBeenCalledExactlyOnceWith({ + environmentId: firstId, + input: { + patch: { + projectSettingsOverrides: { [firstProject]: { responseStreamingMode: "turn" } }, + }, + }, + }); + } finally { + await act(async () => root.unmount()); + vi.unstubAllGlobals(); + } + }, + ); + it("edits each checkout's own override without changing either environment default", () => { const firstSettings: ServerSettings = { ...DEFAULT_SERVER_SETTINGS, From fe27fa05ce36a1c32a5f7489f0928564d5849464 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:26:07 +0000 Subject: [PATCH 108/149] fix(server): a detached supervisor finishes provider installs after the server quits or crashes --- apps/server/src/bin.ts | 5 +- .../ProviderSessionManager.test.ts | 2 + .../src/provider/ProviderInstallSupervisor.ts | 345 ++++++++++++++ .../provider/ProviderMaintenanceHostLock.ts | 212 +++++++-- .../providerMaintenanceRunner.test.ts | 424 +++++++++++++++++- .../src/provider/providerMaintenanceRunner.ts | 156 +++++-- apps/server/src/server.ts | 3 +- 7 files changed, 1067 insertions(+), 80 deletions(-) create mode 100644 apps/server/src/provider/ProviderInstallSupervisor.ts diff --git a/apps/server/src/bin.ts b/apps/server/src/bin.ts index 063093874503d..fc0e63c277ed6 100644 --- a/apps/server/src/bin.ts +++ b/apps/server/src/bin.ts @@ -17,7 +17,10 @@ if ( }) ) { const command = process.argv[2]; - if (command === "acp-mcp-bridge" || command === "acp-mcp-call") { + if (command === "__provider-install" || command === "__provider-install-relay") { + const { runInstallSubcommand } = await import("./provider/ProviderInstallSupervisor.ts"); + await runInstallSubcommand(command, process.argv.slice(3)); + } else if (command === "acp-mcp-bridge" || command === "acp-mcp-call") { const { runAcpMcpCliFastPath } = await import("./mcp/AcpMcpStdioBridge.ts"); await runAcpMcpCliFastPath(command, process.argv.slice(3)); } else { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 527a2a3047b80..8e6a88624f480 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -58,6 +58,7 @@ const layerHostLock = Layer.succeed(HostLock.ProviderMaintenanceHostLock, { acquireRead: Effect.succeed(Effect.void), withRead: (effect) => effect, withWrite: (effect) => effect, + handoff: (effect) => effect, }); const layerTestDatabase = SqlitePersistence.layerMemory; const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( @@ -4744,6 +4745,7 @@ it.effect("ProviderSessionManagerV2 holds the host lease through terminal backgr directory: "", withRead: (effect: Effect.Effect) => effect, withWrite: (effect: Effect.Effect) => effect, + handoff: (effect: Effect.Effect) => effect, acquireRead: Ref.update(held, (value) => value + 1).pipe( Effect.as(Ref.update(held, (value) => value - 1)), ), diff --git a/apps/server/src/provider/ProviderInstallSupervisor.ts b/apps/server/src/provider/ProviderInstallSupervisor.ts new file mode 100644 index 0000000000000..7d483678f6475 --- /dev/null +++ b/apps/server/src/provider/ProviderInstallSupervisor.ts @@ -0,0 +1,345 @@ +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off preferSchemaOverJson:off - Detached runtime boundary; receipts survive their server. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeCrypto from "node:crypto"; +import * as Context from "effect/Context"; +import * as Clock from "effect/Clock"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { resolveSelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime"; +import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; +import * as ServerConfig from "../config.ts"; +import * as HostLock from "./ProviderMaintenanceHostLock.ts"; +import type { ProviderMaintenanceCommandResult } from "./providerMaintenanceRunner.ts"; + +const Result = Schema.Struct({ + stdout: Schema.String, + stderr: Schema.String, + exitCode: Schema.NullOr(Schema.Number), + timedOut: Schema.Boolean, + stdoutTruncated: Schema.Boolean, + stderrTruncated: Schema.Boolean, +}); +const Receipt = Schema.Struct({ + id: Schema.String, + provider: ProviderDriverKind, + instanceId: ProviderInstanceId, + instanceIds: Schema.Array(ProviderInstanceId), + origin: Schema.optional(Schema.Literals(["manual", "background"])), + targetVersion: Schema.optional(Schema.String), + exactVersion: Schema.optional(Schema.Boolean), + startedAt: Schema.String, + finishedAt: Schema.optional(Schema.String), + command: Schema.String, + args: Schema.Array(Schema.String), + env: Schema.optional(Schema.Record(Schema.String, Schema.UndefinedOr(Schema.String))), + parentPid: Schema.Number, + directory: Schema.String, + pid: Schema.optional(Schema.Number), + progress: Schema.optional(Schema.String), + result: Schema.optional(Result), +}); +export type InstallReceipt = typeof Receipt.Type; +const decodeReceipt = Schema.decodeSync(Schema.fromJsonString(Receipt)); +export const readReceipt = (file: string) => decodeReceipt(NodeFS.readFileSync(file, "utf8")); +const writeReceipt = (file: string, receipt: InstallReceipt) => { + const temp = `${file}.${process.pid}.tmp`; + NodeFS.writeFileSync(temp, JSON.stringify(receipt), { mode: 0o600 }); + NodeFS.renameSync(temp, file); +}; + +type InstallInput = Pick< + InstallReceipt, + | "provider" + | "instanceId" + | "instanceIds" + | "origin" + | "targetVersion" + | "exactVersion" + | "startedAt" + | "command" + | "args" + | "env" +> & { + readonly onProgress?: (line: string) => Effect.Effect; +}; + +export class ProviderInstallError extends Data.TaggedError("ProviderInstallError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} +const installError = (cause: unknown) => + new ProviderInstallError({ + message: cause instanceof Error ? cause.message : "Provider installer could not run.", + cause, + }); + +export class ProviderInstallSupervisor extends Context.Service< + ProviderInstallSupervisor, + { + readonly run: ( + input: InstallInput, + ) => Effect.Effect; + readonly receipts: Effect.Effect, ProviderInstallError>; + } +>()("t3/provider/ProviderInstallSupervisor") {} + +export const layer = Layer.effect( + ProviderInstallSupervisor, + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const host = yield* HostLock.ProviderMaintenanceHostLock; + const invocation = yield* resolveSelfInvocation(); + const root = NodePath.join(config.stateDir, "provider-updates"); + const run: ProviderInstallSupervisor["Service"]["run"] = Effect.fn( + "ProviderInstallSupervisor.run", + )(function* (input) { + const file = NodePath.join(root, `${encodeURIComponent(input.instanceId)}.json`); + const receipt: InstallReceipt = { + ...input, + id: NodeCrypto.randomUUID(), + parentPid: process.pid, + directory: host.directory, + }; + yield* Effect.try(() => { + NodeFS.mkdirSync(root, { recursive: true, mode: 0o700 }); + NodeFS.rmSync(`${file}.start`, { force: true }); + writeReceipt(file, receipt); + }); + yield* Effect.tryPromise( + () => + new Promise((resolve, reject) => { + // The relay exits before the supervisor starts the installer. On Windows, + // detached alone does not remove a process from taskkill /t's parent tree. + const relay = NodeChildProcess.spawn( + invocation.command, + [...selfInvocationArgs(invocation, ["__provider-install-relay", file])], + { + detached: true, + stdio: "ignore", + windowsHide: true, + env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, + }, + ); + relay.once("error", reject); + relay.once("spawn", () => { + relay.unref(); + resolve(); + }); + }), + ); + const readCurrent = Effect.try(() => { + const current = readReceipt(file); + if (current.id !== receipt.id) + throw new ProviderInstallError({ message: "Provider install receipt replaced." }); + return current; + }); + const deadline = (yield* Clock.currentTimeMillis) + 30_000; + while (true) { + const current = yield* readCurrent; + if (current.result) return current.result; + if (current.pid !== undefined) break; + if ((yield* Clock.currentTimeMillis) >= deadline) + return yield* Effect.fail( + new ProviderInstallError({ message: "Provider install supervisor did not start." }), + ); + yield* Effect.sleep("100 millis"); + } + return yield* host.handoff( + Effect.gen(function* () { + let started = false; + let progress: string | undefined; + while (true) { + const current = yield* readCurrent; + if (current.result) return current.result; + if (current.pid !== undefined && !started) { + // Only acknowledge after the durable lease is visible. A server dying + // before this write cannot authorize a late installer to start. + yield* Effect.try(() => + NodeFS.writeFileSync(`${file}.start`, receipt.id, { mode: 0o600 }), + ); + started = true; + } + if (current.progress && current.progress !== progress) { + progress = current.progress; + if (input.onProgress) yield* input.onProgress(progress); + } + if (!started && (yield* Clock.currentTimeMillis) >= deadline) + return yield* Effect.fail( + new ProviderInstallError({ message: "Provider install supervisor did not start." }), + ); + if (started && !(yield* Effect.sync(() => HostLock.installIsAlive(host.directory)))) { + return yield* Effect.fail( + new ProviderInstallError({ + message: "Provider install supervisor exited without a result.", + }), + ); + } + yield* Effect.sleep("1 second"); + } + }), + ); + }, Effect.mapError(installError)); + return ProviderInstallSupervisor.of({ + run, + receipts: Effect.try(() => + NodeFS.existsSync(root) + ? NodeFS.readdirSync(root) + .filter((file) => file.endsWith(".json")) + .map((file) => readReceipt(NodePath.join(root, file))) + : [], + ).pipe(Effect.mapError(installError)), + }); + }), +); + +/** The server holds the gate until the supervisor has registered its own install lease. */ +export async function runInstallSubcommand(command: string, args: ReadonlyArray) { + const file = args[0]; + if (!file) throw new Error("Missing provider install receipt"); + if (command === "__provider-install-relay") { + const invocation = await Effect.runPromise( + resolveSelfInvocation().pipe(Effect.provide(NodeServices.layer)), + ); + NodeFS.writeFileSync(`${file}.stdout`, "", { mode: 0o600 }); + NodeFS.writeFileSync(`${file}.stderr`, "", { mode: 0o600 }); + const stdout = NodeFS.openSync(`${file}.stdout`, "a", 0o600); + const stderr = NodeFS.openSync(`${file}.stderr`, "a", 0o600); + const child = NodeChildProcess.spawn( + invocation.command, + [...selfInvocationArgs(invocation, ["__provider-install", file, String(process.pid)])], + { + detached: true, + stdio: ["ignore", stdout, stderr], + windowsHide: true, + env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, + }, + ); + NodeFS.closeSync(stdout); + NodeFS.closeSync(stderr); + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + child.unref(); + return; + } + let receipt = readReceipt(file); + const relayPid = Number(args[1]); + // The live relay is the only link into the desktop/server's Windows tree. + while (HostLock.isProcessAlive(relayPid)) await new Promise((resolve) => setTimeout(resolve, 25)); + const lease = HostLock.installLeasePath(receipt.directory); + const candidate = `${lease}.${process.pid}`; + NodeFS.writeFileSync(candidate, String(process.pid), { flag: "wx", mode: 0o600 }); + try { + // A delayed relay must never overwrite an already running install's owner. + NodeFS.linkSync(candidate, lease); + } finally { + NodeFS.rmSync(candidate, { force: true }); + } + // Quit signals must never interrupt an install. SIGKILL still leaves the + // process group in the lease, which other servers inspect before reclaiming. + process.on("SIGTERM", () => {}); + process.on("SIGINT", () => {}); + const truncated = new Set(); + const output = (suffix: string) => { + const fd = NodeFS.openSync(file + suffix, "r+"); + try { + const size = NodeFS.fstatSync(fd).size; + const buffer = Buffer.alloc(Math.min(size, 10_000)); + NodeFS.readSync(fd, buffer, 0, buffer.length, Math.max(0, size - buffer.length)); + if (size > buffer.length) truncated.add(suffix); + if (size > 20_000) { + NodeFS.ftruncateSync(fd, 0); + NodeFS.writeSync(fd, buffer, 0, buffer.length, 0); + } + return { text: buffer.toString("utf8"), truncated: truncated.has(suffix) }; + } finally { + NodeFS.closeSync(fd); + } + }; + let heartbeat: NodeJS.Timeout | undefined; + let ownsGate = false; + const beat = (format: (line: string) => string | null) => { + NodeFS.utimesSync(lease, new Date(), new Date()); + NodeFS.utimesSync(NodePath.join(receipt.directory, "gate.lock"), new Date(), new Date()); + const text = output(".stderr").text || output(".stdout").text; + const progress = text + .split(/\r?\n|\r/) + .map(format) + .findLast((line) => line !== null); + if (progress) { + receipt = { ...receipt, progress }; + writeReceipt(file, receipt); + } + }; + try { + receipt = { ...receipt, pid: process.pid }; + writeReceipt(file, receipt); + const deadline = Date.now() + 30_000; + while (!NodeFS.existsSync(`${file}.start`)) { + if (!HostLock.isProcessAlive(receipt.parentPid) || Date.now() >= deadline) { + throw new Error("Server exited before authorizing provider installation."); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (NodeFS.readFileSync(`${file}.start`, "utf8") !== receipt.id) + throw new Error("Provider install receipt replaced"); + ownsGate = true; + NodeFS.rmSync(`${file}.start`, { force: true }); + const { runProviderMaintenanceCommandWithSpawner, toProgressLine } = + await import("./providerMaintenanceRunner.ts"); + heartbeat = setInterval(() => beat(toProgressLine), 1_000); + const result = await Effect.runPromise( + Effect.gen(function* () { + const { ChildProcessSpawner } = yield* Effect.promise(() => import("effect/process")); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* runProviderMaintenanceCommandWithSpawner({ + spawner, + command: receipt.command, + args: receipt.args, + ...(receipt.env ? { env: receipt.env } : {}), + inheritOutput: true, + }); + }).pipe(Effect.provide(NodeServices.layer)), + ); + const stdout = output(".stdout"); + const stderr = output(".stderr"); + writeReceipt(file, { + ...receipt, + finishedAt: new Date().toISOString(), + result: { + ...result, + stdout: stdout.text, + stderr: stderr.text, + stdoutTruncated: stdout.truncated, + stderrTruncated: stderr.truncated, + }, + }); + } catch (error) { + writeReceipt(file, { + ...receipt, + finishedAt: new Date().toISOString(), + result: { + stdout: "", + stderr: error instanceof Error ? error.message : String(error), + exitCode: null, + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }, + }); + } finally { + clearInterval(heartbeat); + if (ownsGate) + NodeFS.rmSync(NodePath.join(receipt.directory, "gate.lock"), { + recursive: true, + force: true, + }); + NodeFS.rmSync(lease, { force: true }); + } +} diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index eb8d03a07de2b..979b4a436eb47 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -1,11 +1,13 @@ // @effect-diagnostics nodeBuiltinImport:off - Host leases coordinate independent Node servers. // oxlint-disable t3code/no-global-process-runtime -- Leases belong to the OS user and process, not an environment. import * as NodeCrypto from "node:crypto"; +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import { lock } from "proper-lockfile"; import * as Context from "effect/Context"; +import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Deferred from "effect/Deferred"; @@ -17,6 +19,11 @@ const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } { defaultValue: () => null }, ); +const HeldGate = Context.Reference<{ + readonly handoff: Effect.Effect; + readonly resume: Effect.Effect; +} | null>("t3/provider/ProviderMaintenanceHostLock/HeldGate", { defaultValue: () => null }); + // Linux exposes a process start identity, so recycled PIDs cannot pin a dead reader. const processIdentity = (pid: number) => { if (NodeOS.platform() !== "linux") return null; @@ -24,6 +31,86 @@ const processIdentity = (pid: number) => { return fields[0] === "Z" ? "" : fields[19]!; }; +export const installLeasePath = (directory: string) => NodePath.join(directory, "install.json"); + +export const isProcessAlive = (pid: number) => { + try { + process.kill(pid, 0); + return processIdentity(pid) !== ""; + } catch (error) { + if ( + error instanceof Error && + "code" in error && + (error.code === "ESRCH" || error.code === "ENOENT") + ) + return false; + throw error; + } +}; + +/** A dead supervisor's installer still owns admission until its entire process group exits. */ +export const installIsAlive = (directory: string): boolean => { + const file = installLeasePath(directory); + let pid: number; + try { + pid = Number(NodeFS.readFileSync(file, "utf8")); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return false; + throw error; + } + if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid provider install lease"); + if (isProcessAlive(pid)) return true; + // Snapshot only after an owner dies. A failed scan keeps the lease closed. + const windows = NodeOS.platform() === "win32"; + const scan = windows + ? NodeChildProcess.spawnSync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + 'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)" }', + ], + { encoding: "utf8", timeout: 10_000, windowsHide: true }, + ) + : NodeChildProcess.spawnSync("ps", ["-ax", "-o", "pgid=", "-o", "stat="], { + encoding: "utf8", + timeout: 10_000, + }); + if (scan.error || scan.status !== 0) + throw new Error("Could not inspect provider installer processes"); + if (windows) { + const descendants = new Set([pid]); + const rows = scan.stdout + .trim() + .split(/\r?\n/) + .map((line) => line.split("|").map(Number)); + let changed = true; + while (changed) { + changed = false; + for (const [child, parent] of rows) { + if ( + parent !== undefined && + child !== undefined && + descendants.has(parent) && + !descendants.has(child) + ) { + descendants.add(child); + changed = true; + } + } + } + if (descendants.size > 1) return true; + } else if ( + scan.stdout.split(/\r?\n/).some((line) => { + const [group, state] = line.trim().split(/\s+/); + return Number(group) === pid && state !== undefined && !state.startsWith("Z"); + }) + ) + return true; + return false; +}; + /** * One user-wide reader/writer gate, deliberately covering every CLI: package * managers can replace several providers in one installation. Readers overlap; @@ -33,35 +120,106 @@ export const make = (directory: string) => { const owner = {}; const readers = NodePath.join(directory, "readers"); const prepare = Effect.sync(() => NodeFS.mkdirSync(readers, { recursive: true })); - const acquireGate = Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - yield* prepare; - const compromised = yield* Deferred.make(); - const services = yield* Effect.context(); - while (true) { - const result = yield* Effect.tryPromise(() => - lock(NodePath.join(directory, "gate"), { - realpath: false, - stale: 120_000, - update: 10_000, - onCompromised: (error) => - Effect.runSyncWith(services)(Deferred.die(compromised, error)), - }), - ).pipe(Effect.result); - if (result._tag === "Success") return { release: result.success, compromised }; - const cause = result.failure.cause; - if (!(cause instanceof Error) || !("code" in cause) || cause.code !== "ELOCKED") { - return yield* Effect.die(cause); + const acquireGate = (compromised: Deferred.Deferred) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* prepare; + const services = yield* Effect.context(); + while (true) { + while (yield* Effect.sync(() => installIsAlive(directory))) { + yield* Effect.interruptible(restore(Effect.sleep("1 second"))); + } + let preserve = false; + const result = yield* Effect.tryPromise(() => + lock(NodePath.join(directory, "gate"), { + realpath: false, + stale: 120_000, + update: 10_000, + fs: { + ...NodeFS, + rmdir: ( + path: NodeFS.PathLike, + callback: (error: NodeJS.ErrnoException | null) => void, + ) => { + if (preserve) callback(null); + else NodeFS.rmdir(path, callback); + }, + }, + onCompromised: (error) => + Effect.runSyncWith(services)(Deferred.die(compromised, error)), + }), + ).pipe(Effect.result); + if (result._tag === "Success") { + // An installer may have registered after our first check, before its + // server died and this gate's heartbeat expired. + if (yield* Effect.sync(() => installIsAlive(directory))) { + yield* Effect.promise(() => result.success()); + continue; + } + // Reclaim only under the gate, so a concurrent stale-owner scan + // cannot erase a newer supervisor's lease. + yield* Effect.sync(() => NodeFS.rmSync(installLeasePath(directory), { force: true })); + return { + release: result.success, + handoff: async () => { + preserve = true; + await result.success(); + }, + }; + } + const cause = result.failure.cause; + if (!(cause instanceof Error) || !("code" in cause) || cause.code !== "ELOCKED") { + return yield* Effect.die(cause); + } + yield* Effect.interruptible(restore(Effect.sleep("100 millis"))); } - yield* Effect.interruptible(restore(Effect.sleep("100 millis"))); - } - }), - ); + }), + ); const withGate = (effect: Effect.Effect) => Effect.acquireUseRelease( - acquireGate, - ({ compromised }) => Effect.raceFirst(effect, Deferred.await(compromised)), - ({ release }) => Effect.promise(() => release()), + Effect.gen(function* () { + const compromised = yield* Deferred.make(); + let current = yield* acquireGate(compromised); + let held = true; + const token = { + handoff: Effect.promise(async () => { + await current.handoff(); + held = false; + }), + resume: Effect.map(acquireGate(compromised), (gate) => { + current = gate; + held = true; + }), + }; + return { + token, + compromised, + release: () => (held ? current.release() : Promise.resolve()), + }; + }), + ({ token, compromised }) => + effect.pipe( + Effect.provideService(HeldGate, token), + Effect.raceFirst(Deferred.await(compromised)), + ), + ({ release }) => Effect.promise(release), + ); + // Stop this process's heartbeat and exit hook without opening the gate. The + // acknowledged supervisor maintains the same gate, then releases it on exit. + const handoff = (effect: Effect.Effect) => + Effect.flatMap(HeldGate, (gate) => + gate === null + ? Effect.die(new Error("Provider install requires the host write lease")) + : Effect.uninterruptibleMask((restore) => + gate.handoff.pipe( + Effect.andThen(restore(effect)), + Effect.onExit((exit) => + exit._tag === "Failure" && Cause.hasInterrupts(exit.cause) + ? Effect.void + : restore(gate.resume), + ), + ), + ), ); const readPermit = Semaphore.makeUnsafe(1); @@ -161,7 +319,7 @@ export const make = (directory: string) => { ); }), ); - return { directory, acquireRead, withRead, withWrite }; + return { directory, acquireRead, withRead, withWrite, handoff }; }; /** Every T3 server on the host shares the user's provider CLIs, so servers share one gate. */ diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 479eb0280f066..7b4a8b4de7041 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -15,6 +15,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as HostLock from "./ProviderMaintenanceHostLock.ts"; +import * as InstallSupervisor from "./ProviderInstallSupervisor.ts"; import { makeProviderMaintenanceCommandCoordinator } from "./providerMaintenanceCommandCoordinator.ts"; import { DEFAULT_SERVER_SETTINGS, @@ -25,6 +26,7 @@ import { } from "@t3tools/contracts"; import { ServerProviderUpdateError } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -457,6 +459,7 @@ const layerNonWindowsPlatform = Layer.merge( acquireRead: Effect.succeed(Effect.void), withRead: (effect) => effect, withWrite: (effect) => effect, + handoff: (effect) => effect, }), ); @@ -638,25 +641,54 @@ const makeTestRunner = ( ranges: [], })), }, + receipts = Effect.succeed>([]), + onInstall: ( + input: Parameters[0], + ) => Effect.Effect = () => Effect.void, ) => - Effect.service(ProviderMaintenanceRunner.ProviderMaintenanceRunner).pipe( - Effect.provide( - ProviderMaintenanceRunner.layer.pipe( - Layer.provide( - Layer.mergeAll( - Layer.succeed(ProviderRegistry.ProviderRegistry, registry), - Layer.succeed(ModelManifest.ModelManifest, { - current: Effect.succeed(manifest), - refresh: Effect.succeed(manifest), - forceRefresh: Effect.succeed(manifest), - refreshInBackground: Effect.void, + Layer.build( + ProviderMaintenanceRunner.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ProviderRegistry.ProviderRegistry, registry), + Layer.succeed(ModelManifest.ModelManifest, { + current: Effect.succeed(manifest), + refresh: Effect.succeed(manifest), + forceRefresh: Effect.succeed(manifest), + refreshInBackground: Effect.void, + }), + // Fresh per runner so a version cached by one test cannot leak into another. + ProviderLatestVersions.layer, + Layer.effect( + InstallSupervisor.ProviderInstallSupervisor, + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return InstallSupervisor.ProviderInstallSupervisor.of({ + run: (input) => + ProviderMaintenanceRunner.runProviderMaintenanceCommandWithSpawner({ + ...input, + spawner, + }).pipe( + Effect.tap(() => onInstall(input)), + Effect.mapError( + (cause) => + new InstallSupervisor.ProviderInstallError({ + message: cause.message, + cause, + }), + ), + ), + receipts, + }); }), - // Fresh per runner so a version cached by one test cannot leak into another. - ProviderLatestVersions.layer, ), ), ), ), + ).pipe( + Effect.map((context) => + Context.get(context, ProviderMaintenanceRunner.ProviderMaintenanceRunner), + ), ); describe("providerMaintenanceRunner", () => { @@ -688,6 +720,145 @@ describe("providerMaintenanceRunner", () => { ), ), ); + + it.effect("does not time out or kill an installer still writing after five minutes", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finished = yield* Deferred.make(); + let killed = false; + const spawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed({ + ...mockHandle({}), + exitCode: Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finished)), + ), + kill: () => + Effect.sync(() => { + killed = true; + }), + }), + ), + ); + yield* Effect.gen(function* () { + const { registry } = yield* makeRegistry(baseNativeCliProvider); + const runner = yield* makeTestRunner(registry); + const install = yield* runner.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkChild); + yield* Deferred.await(started); + yield* TestClock.adjust("6 minutes"); + assert.isUndefined(install.pollUnsafe()); + assert.isFalse(killed); + yield* Deferred.succeed(finished, ChildProcessSpawner.ExitCode(0)); + assert.equal((yield* Fiber.join(install)).providers[0]?.updateState?.status, "succeeded"); + }).pipe(Effect.provide(spawner)); + }).pipe( + Effect.provide( + Layer.mergeAll( + TestClock.layer(), + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + ), + ), + ), + ); + + it.effect.each([ + { origin: "background" as const, installedVersion: "1.0.0" }, + { origin: "manual" as const, installedVersion: "1.1.0" }, + ])( + "restores a detached $origin install result and refreshes its installed version on the next server", + ({ origin, installedVersion }) => + Effect.gen(function* () { + const { registry, providersRef } = yield* makeRegistry(); + const recovered = yield* Deferred.make(); + let receipt: InstallSupervisor.InstallReceipt = { + id: "install", + provider: CODEX_DRIVER, + instanceId: CODEX_INSTANCE_ID, + instanceIds: [CODEX_INSTANCE_ID], + origin, + targetVersion: "1.0.0", + exactVersion: true, + startedAt: "2026-04-10T00:00:00.000Z", + finishedAt: "2026-04-10T00:01:00.000Z", + command: "npm", + args: [], + parentPid: 1, + directory: "", + result: { + stdout: "installed", + stderr: "", + exitCode: 0, + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }, + }; + if (origin === "manual") { + const runner = yield* makeTestRunner( + { + ...registry, + getProviderMaintenanceCapabilitiesForInstance: () => + Effect.succeed( + makeProviderMaintenanceCapabilities({ + provider: CODEX_DRIVER, + packageName: "@openai/codex", + updateExecutable: "npm", + updateArgs: ["install", "-g", "@openai/codex@latest"], + updateLockKey: "npm-global:/owned", + }), + ), + }, + undefined, + undefined, + (input) => + Effect.sync(() => { + receipt = { ...receipt, ...input }; + }), + ); + yield* runner.updateProvider({ provider: CODEX_DRIVER, origin }); + } + yield* makeTestRunner( + { + ...registry, + refreshInstance: () => + Ref.updateAndGet(providersRef, (providers) => + providers.map((provider) => ({ ...provider, version: installedVersion })), + ), + setProviderMaintenanceActionState: (input) => + registry + .setProviderMaintenanceActionState(input) + .pipe( + Effect.tap(() => + input.state?.status === "succeeded" || + input.state?.status === "unchanged" || + input.state?.status === "failed" + ? Deferred.succeed(recovered, undefined) + : Effect.void, + ), + ), + }, + undefined, + Effect.succeed([receipt]), + ); + yield* Deferred.await(recovered); + const provider = (yield* registry.getProviders)[0]; + assert.equal(provider?.version, installedVersion); + assert.equal(provider?.updateState?.status, "succeeded"); + assert.equal(provider?.updateState?.origin, origin); + assert.equal(provider?.updateState?.output, "installed"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + layerMockSpawner(() => ({})), + ), + ), + ), + ); + it.effect("runs the allowlisted provider update command and records success", () => { const calls: Array<{ command: string; args: ReadonlyArray }> = []; return Effect.gen(function* () { @@ -1993,6 +2164,7 @@ it.effect.each(["interaction", "opt-out"] as const)( directory: "", acquireRead: Effect.succeed(Effect.void), withRead: (effect) => effect, + handoff: (effect) => effect, withWrite: (effect) => Deferred.succeed(queued, undefined).pipe( Effect.andThen(Deferred.await(release)), @@ -2419,6 +2591,232 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live.each(["unacknowledged", "existing owner"] as const)( + "a late supervisor preserves the current host gate (%s)", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const host = HostLock.make(directory); + const dead = NodeChildProcess.spawn(process.execPath, ["-e", ""], { stdio: "ignore" }); + yield* Effect.promise( + () => new Promise((resolve) => dead.once("exit", () => resolve())), + ); + assert.ok(dead.pid); + const file = NodePath.join(directory, "receipt.json"); + NodeFS.writeFileSync( + file, + JSON.stringify({ + id: "late", + provider: CODEX_DRIVER, + instanceId: CODEX_INSTANCE_ID, + instanceIds: [CODEX_INSTANCE_ID], + startedAt: "2026-04-10T00:00:00.000Z", + parentPid: dead.pid, + directory, + command: process.execPath, + args: ["-e", ""], + }), + ); + yield* host.withWrite( + Effect.gen(function* () { + if (mode === "existing owner") + NodeFS.writeFileSync(HostLock.installLeasePath(directory), String(process.pid)); + const child = NodeChildProcess.spawn( + process.execPath, + [ + NodePath.resolve(import.meta.dirname, "../bin.ts"), + "__provider-install", + file, + String(dead.pid), + ], + { stdio: "ignore" }, + ); + const code = yield* Effect.promise( + () => new Promise((resolve) => child.once("exit", resolve)), + ); + assert.equal(code, mode === "existing owner" ? 1 : 0); + if (mode === "existing owner") assert.isTrue(HostLock.installIsAlive(directory)); + // Independent admission must still reject the gate while its writer holds it. + yield* Effect.promise(async () => { + const { lock } = await import("proper-lockfile"); + try { + const release = await lock(NodePath.join(directory, "gate"), { realpath: false }); + await release(); + } catch (error) { + assert.instanceOf(error, Error); + assert.equal((error as NodeJS.ErrnoException).code, "ELOCKED"); + return; + } + assert.fail("Unacknowledged supervisor opened another server's gate"); + }); + NodeFS.rmSync(HostLock.installLeasePath(directory), { force: true }); + }), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + 30_000, + ); + + it.live.each(["SIGKILL", "SIGTERM", "supervisor"] as const)( + "keeps an orphaned installer exclusive until it exits (%s)", + (mode) => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const stateDir = NodePath.join(directory, "state"); + const started = NodePath.join(directory, "started"); + const finish = NodePath.join(directory, "finish"); + const done = NodePath.join(directory, "done"); + const readerPending = NodePath.join(directory, "reader-pending"); + const writing = NodePath.join(directory, "writing"); + const receiptFile = NodePath.join(stateDir, "provider-updates", "codex.json"); + const waitForFile = (file: string) => + Effect.callback((resume) => { + const watcher = NodeFS.watch(directory, () => { + if (NodeFS.existsSync(file)) { + watcher.close(); + resume(Effect.void); + } + }); + watcher.once("error", (error) => resume(Effect.die(error))); + if (NodeFS.existsSync(file)) { + watcher.close(); + resume(Effect.void); + } + return Effect.sync(() => watcher.close()); + }); + const installer = ` + const fs = require("node:fs"); + console.log("installer started"); + fs.writeFileSync(${JSON.stringify(started)}, String(process.pid)); + let chunks = 0; + const ticker = setInterval(() => { + if (!fs.existsSync(${JSON.stringify(readerPending)})) return; + console.log("installer writing chunk " + (++chunks)); + if (chunks === 2) fs.writeFileSync(${JSON.stringify(writing)}, "writing"); + }, 250); + const watcher = fs.watch(${JSON.stringify(directory)}, () => { + if (!fs.existsSync(${JSON.stringify(finish)})) return; + clearInterval(ticker); + console.log("installer still writing"); + fs.writeFileSync(${JSON.stringify(done)}, "installed"); + watcher.close(); + }); + `; + const parent = NodeChildProcess.spawn( + process.execPath, + [ + "--input-type=module", + "-e", + ` + import * as Effect from "effect/Effect"; + import * as Layer from "effect/Layer"; + import * as NodeServices from "@effect/platform-node/NodeServices"; + import * as HostProcess from "@t3tools/shared/HostProcess"; + import * as Supervisor from ${JSON.stringify(new URL("./ProviderInstallSupervisor.ts", import.meta.url).href)}; + import * as Host from ${JSON.stringify(new URL("./ProviderMaintenanceHostLock.ts", import.meta.url).href)}; + import * as Config from ${JSON.stringify(new URL("../config.ts", import.meta.url).href)}; + const abort = new AbortController(); + process.on("SIGTERM", () => abort.abort()); + const host = Host.make(${JSON.stringify(directory)}); + await Effect.runPromise(Effect.gen(function* () { + const supervisor = yield* Supervisor.ProviderInstallSupervisor; + yield* host.withWrite(supervisor.run({ + provider: "codex", instanceId: "codex", instanceIds: ["codex"], origin: "manual", + startedAt: new Date().toISOString(), command: process.execPath, + args: ["-e", ${JSON.stringify(installer)}], + })); + }).pipe(Effect.provide(Supervisor.layer), Effect.provide(Layer.mergeAll( + NodeServices.layer, + Layer.succeed(Config.ServerConfig, { stateDir: ${JSON.stringify(stateDir)} }), + Layer.succeed(Host.ProviderMaintenanceHostLock, host), + Layer.succeed(HostProcess.Arguments, [process.execPath, ${JSON.stringify(new URL("../bin.ts", import.meta.url).pathname)}]), + ))), { signal: abort.signal }).catch(() => {}); + `, + ], + { cwd: NodePath.resolve(import.meta.dirname, "../.."), stdio: "ignore" }, + ); + const exited = new Promise((resolve) => parent.once("exit", () => resolve())); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + const completed = NodeFS.existsSync(started) + ? new Promise((resolve) => { + const watcher = NodeFS.watch(directory, () => { + if (NodeFS.existsSync(done)) { + watcher.close(); + resolve(); + } + }); + if (NodeFS.existsSync(done)) { + watcher.close(); + resolve(); + } + }) + : Promise.resolve(); + NodeFS.writeFileSync(finish, "finish"); + if (parent.exitCode === null && parent.signalCode === null) parent.kill("SIGKILL"); + await exited; + await completed; + }), + ); + yield* waitForFile(started); + yield* Effect.callback((resume) => { + const check = () => { + if ( + NodeFS.existsSync(receiptFile) && + InstallSupervisor.readReceipt(receiptFile).progress + ) { + watcher.close(); + resume(Effect.void); + } + }; + const watcher = NodeFS.watch(NodePath.dirname(receiptFile), check); + watcher.once("error", (error) => resume(Effect.die(error))); + check(); + return Effect.sync(() => watcher.close()); + }); + const receipt = InstallSupervisor.readReceipt(receiptFile); + assert.isTrue(HostLock.installIsAlive(directory)); + if (mode === "supervisor") { + assert.ok(receipt.pid); + process.kill(receipt.pid, "SIGKILL"); + } + parent.kill(mode === "SIGTERM" ? "SIGTERM" : "SIGKILL"); + yield* Effect.promise(() => exited); + if (NodeFS.existsSync(NodePath.join(directory, "gate.lock"))) { + NodeFS.utimesSync(NodePath.join(directory, "gate.lock"), 0, 0); + } + assert.isTrue(HostLock.installIsAlive(directory)); + const admitted = yield* Deferred.make(); + const reader = yield* HostLock.make(directory) + .withRead( + Effect.sync(() => assert.equal(NodeFS.readFileSync(done, "utf8"), "installed")).pipe( + Effect.andThen(Deferred.succeed(admitted, undefined)), + ), + ) + .pipe(Effect.forkChild); + NodeFS.writeFileSync(readerPending, "pending"); + yield* waitForFile(writing); + assert.isFalse(yield* Deferred.isDone(admitted)); + NodeFS.writeFileSync(finish, "finish"); + yield* waitForFile(done); + yield* Fiber.join(reader); + if (mode !== "supervisor") { + assert.equal(InstallSupervisor.readReceipt(receiptFile).result?.exitCode, 0); + assert.match( + InstallSupervisor.readReceipt(receiptFile).result?.stdout ?? "", + /installer still writing/, + ); + } + }), + ).pipe(Effect.provide(NodeServices.layer)), + 30_000, + ); + it.live("reclaims a crashed server's turn and install leases", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 90049bd9e622d..27f1fdf054b03 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -16,7 +16,6 @@ import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; @@ -26,6 +25,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process"; import * as NodeUtil from "node:util"; import * as ModelManifest from "./ModelManifest.ts"; +import * as InstallSupervisor from "./ProviderInstallSupervisor.ts"; +import * as HostLock from "./ProviderMaintenanceHostLock.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; import { makeProviderMaintenanceCommandCoordinator } from "./providerMaintenanceCommandCoordinator.ts"; @@ -40,7 +41,6 @@ import type { ProviderMaintenanceCapabilities } from "@t3tools/provider-core/ser import { collectUint8StreamText } from "@t3tools/provider-core/server/collectStreamText"; const isServerProviderUpdateError = Schema.is(ServerProviderUpdateError); -const UPDATE_TIMEOUT_MS = 5 * 60_000; const UPDATE_FORCE_KILL_AFTER = Duration.seconds(5); const UPDATE_OUTPUT_MAX_BYTES = 10_000; // Every progress publish resends the provider list to each client, so the @@ -99,8 +99,9 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR readonly spawner: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly command: string; readonly args: ReadonlyArray; - readonly env?: NodeJS.ProcessEnv; - readonly onProgress?: (line: string) => Effect.Effect; + readonly env?: NodeJS.ProcessEnv | undefined; + readonly onProgress?: ((line: string) => Effect.Effect) | undefined; + readonly inheritOutput?: boolean; }) { const collectCommandResult = Effect.fn("ProviderMaintenanceRunner.collectCommandResult")( function* () { @@ -117,6 +118,8 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR .spawn( ChildProcess.make(resolved.command, resolved.args, { shell: resolved.shell, + detached: false, + ...(input.inheritOutput ? { stdout: "inherit", stderr: "inherit" } : {}), // An installer that ignores SIGTERM must not hold up the update forever. forceKillAfter: UPDATE_FORCE_KILL_AFTER, ...(input.env ? { env: input.env, extendEnv: true } : {}), @@ -199,27 +202,13 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR }, ); - return yield* collectCommandResult().pipe( - Effect.scoped, - Effect.timeoutOption(Duration.millis(UPDATE_TIMEOUT_MS)), - Effect.map((result) => - Option.match(result, { - onSome: (value) => value, - onNone: () => - ({ - stdout: "", - stderr: "", - exitCode: null, - timedOut: true, - stdoutTruncated: false, - stderrTruncated: false, - }) satisfies ProviderMaintenanceCommandResult, - }), - ), - ); + // An installer owns its lifetime. A timeout or quit cannot safely stop it mid-write. + return yield* collectCommandResult().pipe(Effect.scoped); }, ); +export { runProviderMaintenanceCommandWithSpawner }; + /** * Split a chunk of installer output into complete lines. `\r` ends a line too, * so a redrawn progress bar reports its latest frame. @@ -304,20 +293,10 @@ function makeUpdateState(input: { export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const manifestService = yield* ModelManifest.ModelManifest; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const supervisor = yield* InstallSupervisor.ProviderInstallSupervisor; + const host = yield* HostLock.ProviderMaintenanceHostLock; const httpClient = yield* HttpClient.HttpClient; const latestVersions = yield* ProviderLatestVersions.ProviderLatestVersions; - const runMaintenanceCommand = ( - update: ProviderMaintenanceCommandAction, - onProgress: (line: string) => Effect.Effect, - ) => - runProviderMaintenanceCommandWithSpawner({ - spawner, - command: update.executable, - args: update.args, - onProgress, - ...(update.env ? { env: update.env } : {}), - }); const commandCoordinator = yield* makeProviderMaintenanceCommandCoordinator({ makeAlreadyRunningError: () => new ServerProviderUpdateError({ @@ -612,10 +591,19 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { ); } yield* setRunningMessage(`Running ${describeCommand(command)}`); - const result = yield* runMaintenanceCommand( - command, - quiet ? () => Effect.void : setRunningMessage, - ); + const result = yield* supervisor.run({ + command: command.executable, + args: command.args, + ...(command.env ? { env: command.env } : {}), + provider, + instanceId, + instanceIds: affectedInstances, + origin, + startedAt, + ...(candidateVersion ? { targetVersion: candidateVersion } : {}), + exactVersion: targetVersion !== undefined || (background && pinnedCommand !== null), + onProgress: quiet ? () => Effect.void : setRunningMessage, + }); const finishedAt = yield* nowIso; if (result.timedOut || result.exitCode !== 0) { yield* refreshInstallationInstances(provider, affectedInstances); @@ -753,6 +741,98 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { ); }); + // A previous server may have quit while its detached installer kept running. + // Refresh only these receipts, once complete; normal discovery stays unchanged. + yield* supervisor.receipts.pipe( + Effect.flatMap((receipts) => + Effect.forEach( + receipts, + (receipt) => + Effect.gen(function* () { + yield* providerRegistry.setProviderMaintenanceActionState({ + instanceId: receipt.instanceId, + action: "update", + state: { + status: "running", + startedAt: receipt.startedAt, + finishedAt: null, + message: "Waiting for provider installation to finish.", + output: null, + ...(receipt.origin ? { origin: receipt.origin } : {}), + }, + }); + // Host admission also waits for an installer orphaned by its supervisor. + yield* host.withRead( + Effect.gen(function* () { + const current = (yield* supervisor.receipts).find((item) => item.id === receipt.id); + if (!current) return; + const providers = yield* refreshInstallationInstances( + current.provider, + current.instanceIds, + ).pipe(Effect.andThen(providerRegistry.getProviders)); + const capabilities = + yield* providerRegistry.getProviderMaintenanceCapabilitiesForInstance( + current.instanceId, + current.provider, + { fresh: true }, + ); + const installed = providers.filter((item) => + current.instanceIds.includes(item.instanceId), + ); + const verified = + installed.length > 0 && + installed.every( + (item) => + item.installed && + (!current.targetVersion || + (current.exactVersion === false + ? Boolean( + item.version && + (capabilities.compareVersions ?? compareSemverVersions)( + item.version, + current.targetVersion, + ) >= 0, + ) + : item.version?.replace(/^v/, "") === current.targetVersion)), + ); + yield* providerRegistry.setProviderMaintenanceActionState({ + instanceId: current.instanceId, + action: "update", + state: { + status: + current.result?.exitCode === 0 + ? verified + ? "succeeded" + : "unchanged" + : "failed", + startedAt: current.startedAt, + finishedAt: current.finishedAt ?? (yield* nowIso), + message: + current.result?.exitCode === 0 && verified + ? "Provider updated." + : current.result?.exitCode === 0 + ? "Update command completed, but T3 Code could not verify the provider version." + : current.result + ? failureMessage(current.result) + : "Provider install supervisor exited without a result.", + output: current.result ? commandOutput(current.result) : null, + ...(current.origin ? { origin: current.origin } : {}), + }, + }); + }), + ); + }), + { concurrency: "unbounded", discard: true }, + ), + ), + Effect.catchCause((cause) => + Effect.logWarning("Could not recover provider update receipts", { + cause: Cause.pretty(cause), + }), + ), + Effect.forkScoped, + ); + return ProviderMaintenanceRunner.of({ updateProvider, }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index af6fe123f8992..41a90ed1111a3 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -160,6 +160,7 @@ import * as CursorKeychain from "@t3tools/provider-cursor/server/CursorKeychain" import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsageAccounts"; import * as UsageService from "./usage/UsageService.ts"; import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts"; +import * as InstallSupervisor from "./provider/ProviderInstallSupervisor.ts"; import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; import * as ProviderAutoUpdater from "./updates/ProviderAutoUpdater.ts"; import * as ProviderMaintenanceHostLock from "./provider/ProviderMaintenanceHostLock.ts"; @@ -552,7 +553,7 @@ const layerProviderInstallationRefresh = Layer.effectDiscard( // behind the same per-installer lock. const layerProviderUpdates = ProviderAutoUpdater.layer.pipe( Layer.provide(UpdateWindow.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), - Layer.provideMerge(ProviderMaintenanceRunner.layer), + Layer.provideMerge(ProviderMaintenanceRunner.layer.pipe(Layer.provide(InstallSupervisor.layer))), ); const layerRuntimeCoreDependenciesBase = Layer.mergeAll( From 73928d1ee6f921a45740c2bf3fdaaf7760b3c79d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:29:03 +0000 Subject: [PATCH 109/149] fix(server): a crash loop after an update commit never restores the old database Only the runtime falls back, and only when the schema is unchanged. Otherwise the candidate keeps restarting and reports the failure. --- apps/server/src/cloud/selfUpdate.test.ts | 21 +- .../src/cloud/serviceLauncherClient.test.ts | 18 + apps/server/src/cloud/serviceProtocol.ts | 48 ++- apps/server/src/serviceLauncher.test.ts | 319 ++++++++++++++++-- apps/server/src/serviceLauncher.ts | 167 ++++++--- 5 files changed, 494 insertions(+), 79 deletions(-) diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 0004613dff223..2ebba6c0485af 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -199,7 +199,7 @@ it.layer(NodeServices.layer)("server self update", (it) => { }), ); - it.effect.each(["withdrawn", "disabled"] as const)( + it.effect.each(["withdrawn", "disabled", "rolled-back", "failed"] as const)( "rechecks a staged release at handoff: %s", (scenario) => Effect.gen(function* () { @@ -227,7 +227,16 @@ it.layer(NodeServices.layer)("server self update", (it) => { ), Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, { managed: true, - lastOutcome: undefined, + lastOutcome: + scenario === "rolled-back" || scenario === "failed" + ? { + id: "crash-loop", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: scenario, + reason: "candidate-crash-loop", + } + : undefined, requestUpdate: () => Effect.succeed("id"), prepareTrial: Effect.undefined, }), @@ -263,6 +272,14 @@ it.layer(NodeServices.layer)("server self update", (it) => { ), ); yield* pass; + if (scenario === "rolled-back" || scenario === "failed") { + expect(staged).toEqual([]); + latest = "1.2.0"; + yield* TestClock.adjust("1 hour"); + yield* pass; + expect(staged).toEqual(["1.2.0"]); + return; + } expect(staged).toEqual(["1.1.0"]); latest = "1.2.0"; yield* TestClock.adjust("1 hour"); diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts index 832a73f08c971..49566018f2962 100644 --- a/apps/server/src/cloud/serviceLauncherClient.test.ts +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -136,3 +136,21 @@ it.effect("rejects contradictory trial context instead of leaving activation clo expect(error.message).toBe("The service launcher supplied invalid startup context."); }), ); + +it.effect("exposes migrated crash-loop failure while keeping the candidate active", () => + Effect.gen(function* () { + const outcome = { + id: "crash-loop", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "failed" as const, + reason: "candidate-crash-loop: database schema changed; keeping candidate runtime", + }; + const host = new FakeLauncherProcess({ protocol: 3, childVersion: "1.1.0", update: outcome }); + const client = yield* makeClient(host, "1.1.0"); + expect(client.managed).toBe(true); + expect(client.lastOutcome).toEqual(outcome); + expect(yield* client.prepareTrial).toEqual(outcome); + expect(host.sent).toEqual([]); + }), +); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 8bdab1026940f..a112bbacfb6ec 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,7 +1,6 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; -// Protocol 3 requires the standalone executable layout. Post-commit health -// observation is launcher-owned and does not change the runtime's IPC contract. +// Protocol 3 requires the standalone executable layout. export const SERVICE_LAUNCHER_PROTOCOL = 3 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_STATE_FILE = "service-state.json"; @@ -30,6 +29,14 @@ export interface ServiceState { readonly activeVersion: string; readonly update?: ServiceUpdateRecord; readonly launcherVersion?: string; + /** Runtime-only recovery; its schema predates the trial, never accepted work. */ + readonly crashRecovery?: { + readonly update: PendingServiceUpdate; + readonly schema: string | null; + readonly committedAt: number; + readonly crashes: number; + }; + /** Legacy launchers retained post-commit backups; read them for recovery only. */ readonly rollback?: { readonly update: PendingServiceUpdate; readonly healthyAfter: number }; } @@ -162,7 +169,33 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { Number.isFinite(value.rollback.healthyAfter) ? { update: rollbackUpdate, healthyAfter: value.rollback.healthyAfter } : undefined; + const recoveryUpdate = isRecord(value.crashRecovery) + ? decodeServiceUpdate(value.crashRecovery.update) + : undefined; + const crashRecovery = + isRecord(value.crashRecovery) && + recoveryUpdate?.status === "pending" && + (value.crashRecovery.schema === null || typeof value.crashRecovery.schema === "string") && + typeof value.crashRecovery.committedAt === "number" && + Number.isFinite(value.crashRecovery.committedAt) && + typeof value.crashRecovery.crashes === "number" && + Number.isSafeInteger(value.crashRecovery.crashes) && + value.crashRecovery.crashes >= 0 + ? { + update: recoveryUpdate, + schema: value.crashRecovery.schema, + committedAt: value.crashRecovery.committedAt, + crashes: value.crashRecovery.crashes, + } + : undefined; if ( + (value.crashRecovery !== undefined && + (crashRecovery === undefined || + (update?.status !== "committed" && update?.status !== "failed") || + crashRecovery.update.id !== update.id || + crashRecovery.update.targetVersion !== value.activeVersion || + crashRecovery.update.targetVersion !== update.targetVersion || + crashRecovery.update.fromVersion !== update.fromVersion)) || (value.launcherVersion !== undefined && (typeof value.launcherVersion !== "string" || !isExactServiceVersion(value.launcherVersion))) || @@ -180,8 +213,10 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0) || (update?.status === "pending" && update.fromVersion !== value.activeVersion) || (update?.status === "committed" && update.targetVersion !== value.activeVersion) || - ((update?.status === "rolled-back" || update?.status === "failed") && - update.fromVersion !== value.activeVersion) + (update?.status === "rolled-back" && update.fromVersion !== value.activeVersion) || + (update?.status === "failed" && + update.fromVersion !== value.activeVersion && + (crashRecovery === undefined || update.targetVersion !== value.activeVersion)) ) { return undefined; } @@ -193,6 +228,7 @@ export function decodeServiceState(value: unknown): ServiceState | undefined { ? { launcherVersion: value.launcherVersion } : {}), ...(rollback === undefined ? {} : { rollback }), + ...(crashRecovery === undefined ? {} : { crashRecovery }), }; } @@ -250,7 +286,9 @@ export function decodeServiceLauncherContext(value: string): ServiceLauncherCont const update = parsed.update === undefined ? undefined : decodeServiceUpdate(parsed.update); if (parsed.update !== undefined && update === undefined) return undefined; const selectedVersion = - update?.status === "pending" || update?.status === "committed" + update?.status === "pending" || + update?.status === "committed" || + (update?.status === "failed" && parsed.childVersion === update.targetVersion) ? update.targetVersion : update === undefined ? parsed.childVersion diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 0fde07960c07a..5781fc0e3926a 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off - Launcher recovery injects native filesystem failures. +// @effect-diagnostics nodeBuiltinImport:off globalDateInEffect:off - Launcher recovery uses native filesystem failures and wall clocks. import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; @@ -6,6 +6,8 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as NodeFSP from "node:fs/promises"; +import * as NodeFS from "node:fs"; +import * as NodeSqlite from "node:sqlite"; vi.mock("node:fs/promises", { spy: true }); @@ -206,8 +208,250 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect.each([ + "unchanged", + "restart", + "clock-jump", + "before-prepared", + "after-commit", + "wal-migration", + "unreadable", + "untracked-ddl", + ] as const)( + "supervises committed crash loops with schema %s without restoring accepted work", + (schema) => + Effect.gen(function* () { + const compatible = + schema === "unchanged" || schema === "restart" || schema === "clock-jump"; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-crash-loop-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const dbPath = path.join(root, "database"); + const db = new NodeSqlite.DatabaseSync(dbPath); + db.exec( + "CREATE TABLE effect_sql_migrations (migration_id INTEGER PRIMARY KEY, name TEXT); INSERT INTO effect_sql_migrations VALUES (1, 'initial'); CREATE TABLE receipts (id TEXT PRIMARY KEY)", + ); + db.close(); + const pending = { + id: "crash-loop", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath, + status: "pending" as const, + }; + const migrate = + "db.exec(\"INSERT INTO effect_sql_migrations VALUES (2, 'candidate'); CREATE TABLE candidate_data (id TEXT)\");"; + const source = ` +const fs = require("node:fs"); +const { DatabaseSync } = require("node:sqlite"); +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +fs.appendFileSync(${JSON.stringify(path.join(root, "starts"))}, context.childVersion + "\\n"); +if (context.update.status === "rolled-back" || context.update.status === "failed") { + const ready = ${JSON.stringify(path.join(root, "ready"))}; + fs.writeFileSync(ready + ".staging", JSON.stringify(context)); + fs.renameSync(ready + ".staging", ready); + setInterval(() => {}, 1000); +} else if (context.update.status === "pending") { + const db = new DatabaseSync(${JSON.stringify(dbPath)}); + ${schema === "before-prepared" ? migrate : ""} + db.close(); + process.send({ type: "prepared", updateId: context.update.id }); + process.on("message", message => { + if (message.type !== "committed") return; + const db = new DatabaseSync(${JSON.stringify(dbPath)}); + ${schema === "wal-migration" ? 'db.exec("PRAGMA journal_mode=WAL");' : ""} + db.exec("INSERT INTO receipts VALUES ('accepted')"); + ${schema === "after-commit" || schema === "wal-migration" ? migrate : schema === "unreadable" ? 'db.exec("DROP TABLE effect_sql_migrations");' : schema === "untracked-ddl" ? 'db.exec("CREATE TABLE untracked (id TEXT)");' : ""} + ${schema === "wal-migration" ? "" : "db.close();"} + process.exit(42); + }); +} else { + process.exit(42); +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + source, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: pending, + }), + ); + const ready = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + NodeFS.watch(root, (_event, name) => { + if (name === "ready") ready.resolve(); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), + ); + if (schema === "clock-jump") { + let now = 0; + yield* Effect.acquireRelease( + Effect.sync(() => vi.spyOn(Date, "now").mockImplementation(() => (now += 600_000))), + (clock) => Effect.sync(() => clock.mockRestore()), + ); + } + if (schema === "restart") { + const crashed = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + NodeFS.watch(path.dirname(statePath), (_event, name) => { + if (name === "service-state.json") + void readServiceState(statePath).then( + (state) => { + if (state.crashRecovery?.crashes === 1) crashed.resolve(); + }, + () => {}, + ); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), + ); + const first = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.acquireRelease( + Effect.sync(() => { + void first.run().catch(crashed.reject); + }), + () => Effect.promise(() => first.stop("SIGTERM")), + ); + yield* Effect.promise(() => crashed.promise); + yield* Effect.promise(() => first.stop("SIGTERM")); + } + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.acquireRelease( + Effect.sync(() => { + void launcher.run().catch(ready.reject); + }), + () => Effect.promise(() => launcher.stop("SIGTERM")), + ); + yield* Effect.promise(() => ready.promise); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, compatible ? "1.0.0" : "1.1.0"); + assert.equal(state.update?.status, compatible ? "rolled-back" : "failed"); + assert.include( + state.update?.status === "pending" ? "" : (state.update?.reason ?? ""), + "crash-loop", + ); + const context = JSON.parse(yield* fs.readFileString(path.join(root, "ready"))); + assert.equal(context.childVersion, state.activeVersion); + assert.deepEqual(context.update, state.update); + assert.deepEqual((yield* fs.readFileString(path.join(root, "starts"))).trim().split("\n"), [ + "1.1.0", + "1.1.0", + "1.1.0", + state.activeVersion, + ]); + const current = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + try { + assert.deepEqual( + current + .prepare("SELECT id FROM receipts") + .all() + .map((row) => row.id), + ["accepted"], + ); + if ( + schema === "before-prepared" || + schema === "after-commit" || + schema === "wal-migration" + ) { + assert.equal( + current.prepare("SELECT MAX(migration_id) AS id FROM effect_sql_migrations").get() + ?.id, + 2, + ); + } + } finally { + current.close(); + } + assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", pending.id))); + }), + { timeout: 15_000 }, + ); + + it.effect.each(["expired", "backward-clock"] as const)( + "does not downgrade outside the bounded crash window: %s", + (clock) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-crash-window-" }); + const dbPath = path.join(root, "database"); + yield* fs.writeFileString(dbPath, "accepted work"); + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + "process.exit(42);\n", + ); + } + const pending = { + id: "window", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath, + status: "pending" as const, + }; + const statePath = path.join(root, "runtime", "service-state.json"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: pending.id, + fromVersion: pending.fromVersion, + targetVersion: pending.targetVersion, + status: "committed", + }, + crashRecovery: { + update: pending, + schema: "baseline", + crashes: 2, + committedAt: Date.now() + (clock === "expired" ? -600_000 : 600_000), + }, + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.promise(() => + launcher.run().then( + () => { + throw new Error("unexpected completion"); + }, + (error) => { + assert.include(String(error), "outside the update crash window"); + }, + ), + ); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update?.status, "committed"); + assert.isUndefined(state.crashRecovery); + assert.equal(yield* fs.readFileString(dbPath), "accepted work"); + }), + ); + it.effect.each([false, true])( - "restores the database after a post-commit crash (wall clock jumps: %s)", + "preserves accepted work after a post-commit crash (wall clock jumps: %s)", (clockJumps) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -224,7 +468,7 @@ if (context.update?.status === "pending") { process.send({ type: "prepared", updateId: context.update.id }); process.on("message", (message) => { if (message.type === "committed") { - require("node:fs").writeFileSync(${encodedDatabasePath}, "post-commit migration"); + require("node:fs").writeFileSync(${encodedDatabasePath}, "accepted command receipt"); setTimeout(() => process.exit(42), 100); } }); @@ -232,7 +476,8 @@ if (context.update?.status === "pending") { process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); setInterval(() => {}, 1_000); } else { - process.exit(0); + require("node:fs").writeFileSync(${JSON.stringify(path.join(root, "ready"))}, "ready"); + setInterval(() => {}, 1_000); } `; for (const version of ["1.0.0", "1.1.0"]) { @@ -261,21 +506,27 @@ if (context.update?.status === "pending") { (clock) => Effect.sync(() => clock.mockRestore()), ); } - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), + const ready = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + NodeFS.watch(root, (_event, name) => { + if (name === "ready") ready.resolve(); + }), ), + (watcher) => Effect.sync(() => watcher.close()), ); + yield* Effect.acquireRelease( + Effect.sync(() => { + void launcher.run().catch(ready.reject); + }), + () => Effect.promise(() => launcher.stop("SIGTERM")), + ); + yield* Effect.promise(() => ready.promise); const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.0.0"); - assert.equal(state.update?.status, "rolled-back"); - assert.equal( - state.update?.status === "rolled-back" ? state.update.reason : undefined, - "candidate-exited-after-commit:42", - ); - assert.equal(yield* fs.readFileString(databasePath), "before trial"); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update?.status, "committed"); + assert.equal(yield* fs.readFileString(databasePath), "accepted command receipt"); assert.isUndefined(state.rollback); }), ); @@ -295,7 +546,7 @@ if (context.update?.status === "pending") { yield* fs.writeFileString(path.join(backup, "database"), "original database"); yield* fs.writeFileString(`${dbPath}-wal`, "trial wal"); yield* fs.writeFileString(`${dbPath}-shm`, "trial shm"); - if (checkpoint === "restore-interrupted") { + if (checkpoint === "restore-interrupted" || checkpoint === "restore-disk-full") { yield* fs.writeFileString(path.join(backup, ".restore-pending"), ""); yield* fs.writeFileString(dbPath, "partially restored database"); } @@ -357,18 +608,27 @@ if (context.update?.status === "pending") { yield* Effect.promise(() => retry.run().catch(() => undefined)); } const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.0.0"); - assert.equal(state.update?.status, "rolled-back"); - assert.equal(yield* fs.readFileString(dbPath), "original database"); - assert.isFalse(yield* fs.exists(`${dbPath}-wal`)); - assert.isFalse(yield* fs.exists(`${dbPath}-shm`)); + if (checkpoint === "committed") { + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update?.status, "committed"); + assert.equal(yield* fs.readFileString(dbPath), "migrated database"); + assert.equal(yield* fs.readFileString(`${dbPath}-wal`), "trial wal"); + assert.equal(yield* fs.readFileString(`${dbPath}-shm`), "trial shm"); + assert.isUndefined(state.rollback); + } else { + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal(yield* fs.readFileString(dbPath), "original database"); + assert.isFalse(yield* fs.exists(`${dbPath}-wal`)); + assert.isFalse(yield* fs.exists(`${dbPath}-shm`)); + } assert.isFalse(yield* fs.exists(backup)); }), ); - it.effect( - "prunes staged runtimes while preserving launcher, active, rollback and current staging", - () => + it.effect.each(["legacy", "crash-recovery"] as const)( + "prunes staged runtimes while preserving launcher, active and %s recovery", + (recovery) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -392,7 +652,16 @@ if (context.update?.status === "pending") { activeVersion: "1.2.0", launcherVersion: "1.0.0", update: { ...update, status: "committed" }, - rollback: { update, healthyAfter: 60_000 }, + ...(recovery === "legacy" + ? { rollback: { update, healthyAfter: 60_000 } } + : { + crashRecovery: { + update, + schema: "baseline", + committedAt: Date.now(), + crashes: 1, + }, + }), }), ); yield* Effect.promise(() => pruneServiceRuntimes(root, new Set(["1.4.0"]))); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 452368e98ce2b..f1ee8fd5068d9 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -9,6 +9,7 @@ import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; import type { PendingServiceUpdate, @@ -33,7 +34,33 @@ import { const HANDOFF_DELAY_MS = 2_000; const PREPARED_TIMEOUT_MS = 120_000; const TERMINATE_GRACE_MS = 5_000; -const HEALTH_PERIOD_MS = 5 * 60_000; +const CRASH_WINDOW_MS = 5 * 60_000; +const CRASH_LIMIT = 3; + +// A read-only transaction includes WAL state. Compare both the migration ledger +// and actual schema: missing/corrupt history or untracked DDL must fail closed. +function databaseSchema(dbPath: string): string | null { + let db: NodeSqlite.DatabaseSync | undefined; + try { + db = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + db.exec("BEGIN"); + const ledger = db + .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id") + .all(); + if (ledger.length === 0) return null; + const schema = db + .prepare("SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name") + .all(); + const version = db.prepare("PRAGMA user_version").get(); + return NodeCrypto.createHash("sha256") + .update(JSON.stringify({ ledger, schema, version })) + .digest("hex"); + } catch { + return null; + } finally { + db?.close(); + } +} type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -241,6 +268,8 @@ export async function pruneServiceRuntimes( protectedVersions.add(state.update.targetVersion); } if (state.rollback !== undefined) protectedVersions.add(state.rollback.update.fromVersion); + if (state.crashRecovery !== undefined) + protectedVersions.add(state.crashRecovery.update.fromVersion); const versionsDir = NodePath.join(baseDir, "runtime", "versions"); for (const entry of await NodeFSP.readdir(versionsDir, { withFileTypes: true })) { if ( @@ -315,6 +344,7 @@ export class Launcher { #stopRequested = false; #stopping = false; #done = false; + #committedAtMonotonic: number | undefined; readonly #completion = Promise.withResolvers(); constructor(baseDir: string, state: ServiceState) { @@ -379,6 +409,7 @@ export class Launcher { // before this queued stop tears it down. That replacement owns the // pre-activation tunnel cleanup path and observes the marker above. this.#stopping = true; + this.#clearTimer(); const child = this.#child?.process; this.#child = null; if (child !== undefined) await terminateChild(child, signal); @@ -421,17 +452,11 @@ export class Launcher { await this.#returnToPrevious(rollback.update, "rolled-back", "rollback-interrupted"); return; } - // Restarting the launcher must not turn an unproven release into a healthy one. - const healthyAfter = Date.now() + HEALTH_PERIOD_MS; - this.#state = { ...this.#state, rollback: { ...rollback, healthyAfter } }; - await writeServiceState(this.#statePath, this.#state); - try { - await this.#startChild(this.#state.activeVersion, "active", this.#state.update); - this.#watchHealth(); - } catch { - await this.#returnToPrevious(rollback.update, "rolled-back", "candidate-start-failed"); - } - return; + // Older launchers retained a backup after activation. It may predate + // accepted work, so only an already-started restore can use it safely. + const { rollback: _rollback, ...next } = this.#state; + await writeServiceState(this.#statePath, next); + this.#state = next; } const update = this.#state.update; if (update?.status !== "pending") { @@ -571,7 +596,9 @@ export class Launcher { dbPath: message.dbPath, status: "pending", }; - const next: ServiceState = { ...this.#state, update: pending }; + const { crashRecovery: _recovery, ...state } = this.#state; + const next: ServiceState = { ...state, update: pending }; + this.#committedAtMonotonic = undefined; await writeServiceState(this.#statePath, next); this.#state = next; await sendMessage(child.process, { type: "update-accepted", updateId: pending.id }); @@ -605,41 +632,28 @@ export class Launcher { } this.#clearTimer(); const committed = terminalUpdate({ pending, status: "committed" }); - const healthyAfter = Date.now() + HEALTH_PERIOD_MS; const next: ServiceState = { ...this.#state, activeVersion: pending.targetVersion, update: committed, - rollback: { update: pending, healthyAfter }, + crashRecovery: { + update: pending, + schema: databaseSchema( + databaseBackupFile(databaseBackupDir(this.#baseDir, pending.id), ""), + ), + committedAt: Date.now(), + crashes: 0, + }, }; await writeServiceState(this.#statePath, next); this.#state = next; child.role = "active"; - this.#watchHealth(); + this.#committedAtMonotonic = performance.now(); + // Committed opens the runtime's provider and client gates. Never restore + // this snapshot afterward: it would erase receipts for accepted work. await sendMessage(child.process, { type: "committed", updateId: committed.id }); - } - - #watchHealth(): void { - this.#timer = setTimeout( - () => - this.#enqueue(async () => { - const rollback = this.#state.rollback; - if ( - rollback === undefined || - this.#stopping || - this.#child === null || - this.#child.process.exitCode !== null || - this.#child.process.signalCode !== null - ) - return; - const { rollback: _rollback, ...next } = this.#state; - await writeServiceState(this.#statePath, next); - this.#state = next; - await discardDatabaseBackup(this.#baseDir, rollback.update.id).catch(() => undefined); - await pruneServiceRuntimes(this.#baseDir).catch(() => undefined); - }), - HEALTH_PERIOD_MS, - ); + await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + await pruneServiceRuntimes(this.#baseDir).catch(() => undefined); } async #handlePreparedTimeout(child: ManagedChild): Promise { @@ -673,22 +687,81 @@ export class Launcher { } this.#clearTimer(); - if (this.#state.rollback !== undefined) { - await this.#returnToPrevious( - this.#state.rollback.update, - "rolled-back", - `candidate-exited-after-commit:${String(code ?? signal ?? "unknown")}`, - ); - return; - } const pending = this.#state.update; if (pending?.status === "pending") { await this.#startTrial(pending); return; } + if (this.#state.crashRecovery !== undefined) { + await this.#handleCommittedCrash(); + return; + } throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } + async #handleCommittedCrash(): Promise { + const recovery = this.#state.crashRecovery; + if (recovery === undefined) return; + const age = + this.#committedAtMonotonic === undefined + ? Date.now() - recovery.committedAt + : performance.now() - this.#committedAtMonotonic; + // A launcher restart uses durable time. A backward clock jump is not + // evidence of a bounded crash loop, so it cannot authorize a downgrade. + if (this.#state.update?.status !== "failed" && (age < 0 || age > CRASH_WINDOW_MS)) { + const { crashRecovery: _recovery, ...next } = this.#state; + await writeServiceState(this.#statePath, next); + this.#state = next; + await pruneServiceRuntimes(this.#baseDir).catch(() => undefined); + throw new Error("Active child exited outside the update crash window."); + } + const crashes = Math.min(recovery.crashes + 1, 6); + let next: ServiceState = { ...this.#state, crashRecovery: { ...recovery, crashes } }; + if (crashes >= CRASH_LIMIT && this.#state.update?.status !== "failed") { + const current = databaseSchema(recovery.update.dbPath); + if ( + recovery.schema !== null && + current === recovery.schema && + (await runtimeExists(this.#baseDir, recovery.update.fromVersion)) + ) { + const { crashRecovery: _recovery, ...state } = next; + next = { + ...state, + activeVersion: recovery.update.fromVersion, + update: terminalUpdate({ + pending: recovery.update, + status: "rolled-back", + reason: "candidate-crash-loop: previous runtime resumed with current database", + }), + }; + } else { + next = { + ...next, + update: terminalUpdate({ + pending: recovery.update, + status: "failed", + reason: + "candidate-crash-loop: database schema changed or compatibility could not be verified; keeping candidate runtime", + }), + }; + } + } + await writeServiceState(this.#statePath, next); + this.#state = next; + if (this.#stopRequested) return; + // Keep the launcher alive so systemd's start limit does not strand the + // service. The timer stays cancellable by explicit shutdown. + this.#timer = setTimeout( + () => { + this.#timer = undefined; + this.#enqueue(() => + this.#startChild(this.#state.activeVersion, "active", this.#state.update), + ); + }, + Math.min(1_000 * 2 ** (crashes - 1), 30_000), + ); + } + async #returnToPrevious( pending: PendingServiceUpdate, status: "rolled-back" | "failed", From 2e5508f8056e24fdf029fe0fb33a2291216b3add Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:29:30 +0000 Subject: [PATCH 110/149] docs: describe crash-loop recovery without a post-commit database restore --- docs/internals/server-updates.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 42d23be167e31..0d804acf9210d 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -9,10 +9,10 @@ processes do not self-update. Exact-version installs keep restarts independent of npm cache eviction or a moving release tag. Installation and preflight happen in staging before publishing an -immutable runtime. Preflight checks the launcher's IPC protocol. The post-commit -health period is launcher-owned and remains compatible with protocol 3; older -launchers can run the new runtime without that extra rollback protection. -Upgrading the launcher requires a local service update. +immutable runtime. Preflight checks the launcher's IPC protocol. Crash-loop +recovery after commit is launcher-owned and remains compatible with protocol 3; +older launchers can run the new runtime without it. Upgrading the launcher +requires a local service update. ## Commit boundary @@ -28,9 +28,12 @@ child release its gates, accept commands, and publish ready. Keep fallible start acquisitions before this boundary. A listener alone does not prove the runtime is ready to commit. -A failed or timed-out trial returns to the old version. The launcher also rolls -back a committed target that exits during its health period. After that period, -the service manager's ordinary restart policy applies. +A failed or timed-out trial returns to the old version. After commit, the database +is never restored: commit opens the gates, so a restore would erase accepted work. +If the committed target exits three times within five minutes, the launcher returns +to the previous runtime only when the migration ledger and schema still match the +pre-trial database. Otherwise it keeps restarting the target with backoff and +records the update as failed. ## Database rollback @@ -40,10 +43,8 @@ migrations. The snapshot is made once per update and survives launcher restarts; replacing it during a retry could capture changes from the failed trial. Rollback stops the trial before restoring. A durable restore marker makes an -interrupted restore finish before either version boots. Keep the snapshot until -the committed target survives its health period, or until both restoration and -the terminal rollback state are durable. -Post-commit rollback also discards database writes accepted during the health period. +interrupted restore finish before either version boots. Discard the snapshot at +commit, or once both restoration and the terminal rollback state are durable. Attachments and other files outside SQLite are outside this rollback boundary. ## Client acknowledgement From 4910b1803269582902b4576c40369d52da002c4a Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:29:48 +0000 Subject: [PATCH 111/149] docs: describe the detached provider install supervisor --- docs/internals/providers.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/internals/providers.md b/docs/internals/providers.md index 29cc4883ce5f0..d8d59281f16d1 100644 --- a/docs/internals/providers.md +++ b/docs/internals/providers.md @@ -100,6 +100,12 @@ installs drain host-wide provider leases before closing admission to CLI launche a session holding a lease may still need admission to finish launching. This order also protects another environment on the same machine from a binary being replaced. +Installers run under a detached +[supervisor](../../apps/server/src/provider/ProviderInstallSupervisor.ts) that owns the +host install lease and writes a result receipt, so quitting or crashing the server never +stops a half-written install. The next server reports the receipt's outcome. Admission +stays closed until the installer exits, even if the supervisor itself dies. + Ownership is cached per instance and re-read immediately before an update runs. The [runner](../../apps/server/src/provider/providerMaintenanceRunner.ts) refuses when the lock key changed since the advisory, and reports success only when the refreshed provider is still installed From e23f87ab794820de0f8ce7c438cbf0f891299d3f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:30:59 +0000 Subject: [PATCH 112/149] fix(desktop): quitting no longer waits on a provider install, which now finishes on its own --- .../src/backend/DesktopBackendManager.test.ts | 2 +- .../src/backend/DesktopBackendManager.ts | 4 +- .../providerMaintenanceRunner.test.ts | 51 ------------------- .../src/provider/providerMaintenanceRunner.ts | 16 ------ 4 files changed, 3 insertions(+), 70 deletions(-) diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index e3e9ba62f317c..24303aa66997c 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -264,7 +264,7 @@ describe("DesktopBackendManager", () => { assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output"); assert.equal( Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)), - 5_000, + 2_000, ); assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 81d84e7745336..fcd5d644ba336 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -65,8 +65,8 @@ const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5; const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1); const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100); const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1); -// Provider installers drain for up to three seconds during server shutdown. -const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5); +// Provider installers run independently; this grace period is only for the server. +const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(2); const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5); const BACKEND_READINESS_PATH = "/.well-known/t3/environment"; const { logWarning: logBackendProcessWarning } = diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index e50fe89bff45d..4f33579231bdc 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -724,57 +724,6 @@ describe("providerMaintenanceRunner", () => { ), ); - it.effect.each([true, false])( - "drains an interrupted installer before killing it (finishes: %s)", - (finishes) => - Effect.gen(function* () { - const started = yield* Deferred.make(); - const finished = yield* Deferred.make(); - const draining = yield* Deferred.make(); - let killed = false; - let exitReads = 0; - const spawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.sync(() => ({ - ...mockHandle({}), - exitCode: Effect.suspend(() => { - exitReads++; - return Deferred.succeed(exitReads === 1 ? started : draining, undefined).pipe( - Effect.andThen(Deferred.await(finished)), - ); - }), - kill: () => - Effect.sync(() => { - killed = true; - }).pipe(Effect.andThen(Deferred.succeed(draining, undefined)), Effect.asVoid), - })), - ), - ); - const { registry } = yield* makeRegistry(baseNativeCliProvider); - yield* Effect.gen(function* () { - const runner = yield* makeTestRunner(registry); - const install = yield* runner.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkChild); - yield* Deferred.await(started); - const interrupted = yield* Fiber.interrupt(install).pipe(Effect.forkChild); - yield* Deferred.await(draining); - assert.isFalse(killed); - if (finishes) yield* Deferred.succeed(finished, ChildProcessSpawner.ExitCode(0)); - else yield* TestClock.adjust(Duration.seconds(3)); - yield* Fiber.join(interrupted); - assert.equal(killed, !finishes); - }).pipe(Effect.provide(spawner)); - }).pipe( - Effect.provide( - Layer.mergeAll( - TestClock.layer(), - layerNonWindowsPlatform, - layerLatestVersionHttpClient("1.0.0"), - ), - ), - ), - ); - it.effect("does not time out or kill an installer still writing after five minutes", () => Effect.gen(function* () { const started = yield* Deferred.make(); diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index cc7ac0d299ecd..e0796a013b6e7 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -16,7 +16,6 @@ import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; @@ -43,7 +42,6 @@ import { collectUint8StreamText } from "@t3tools/provider-core/server/collectStr const isServerProviderUpdateError = Schema.is(ServerProviderUpdateError); const UPDATE_FORCE_KILL_AFTER = Duration.seconds(1); -const UPDATE_SHUTDOWN_DRAIN = Duration.seconds(3); const UPDATE_OUTPUT_MAX_BYTES = 10_000; // Every progress publish resends the provider list to each client, so the // installer's output is sampled rather than forwarded line by line. @@ -136,20 +134,6 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR }), ), ); - yield* Effect.addFinalizer(() => - // Give an interrupted installer time to finish its current writes before - // terminating it, within the desktop backend's five-second quit budget. - child.exitCode.pipe( - Effect.timeoutOption(UPDATE_SHUTDOWN_DRAIN), - Effect.orElseSucceed(() => Option.none()), - Effect.flatMap((exit) => - Option.isSome(exit) - ? Effect.void - : child.kill({ forceKillAfter: UPDATE_FORCE_KILL_AFTER }).pipe(Effect.ignore), - ), - ), - ); - // Holds the newest output line not yet reported; the sampler takes it. const pendingProgress = yield* Ref.make(null); const onProgress = input.onProgress; From fe27f9e2023ef3cf3124dcebadb530745434011b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:34:02 +0000 Subject: [PATCH 113/149] fix(server): providers and terminals orphaned by a crash are stopped before interrupted work continues --- .../Adapters/ClaudeAdapterV2.ts | 24 +- .../Adapters/CodexAdapterV2.ts | 3 + .../src/process/ProviderProcessLedger.ts | 360 ++++++++++++++++++ apps/server/src/process/agentScope.test.ts | 250 ++++++++++++ apps/server/src/process/agentScope.ts | 12 +- apps/server/src/provider/CodexProvider.ts | 4 + apps/server/src/terminal/NodePtyAdapter.ts | 5 + .../textGeneration/ClaudeTextGeneration.ts | 4 + .../src/textGeneration/CodexTextGeneration.ts | 4 + .../src/server/AcpSessionRuntime.ts | 3 + packages/provider-acp/src/server/adapter.ts | 1 + .../src/server/clientTerminals.ts | 3 + .../src/server/OpenCodeRuntime.ts | 2 + packages/provider-pi/src/server/adapter.ts | 6 +- packages/provider-pi/src/server/rpc.ts | 4 + .../provider-pi/src/server/textGeneration.ts | 7 +- packages/shared/src/AgentScope.ts | 2 + 17 files changed, 690 insertions(+), 4 deletions(-) create mode 100644 apps/server/src/process/ProviderProcessLedger.ts diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 34532b94e6abb..3a58d7d2e737c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- The SDK exposes a native spawn callback. +import * as NodeChildProcess from "node:child_process"; import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts"; import { dynamicToolTitle, @@ -753,8 +755,9 @@ export const layerQueryRunner: Layer.Layer< const isScript = binaryPath !== undefined && CLAUDE_SCRIPT_EXTENSIONS.some((ext) => binaryPath.endsWith(ext)); + const platform = yield* HostProcess.Platform; const parentDeathSignal = - (yield* HostProcess.Platform) === "linux" + platform === "linux" ? yield* resolveCommandPath("setpriv", { env: queryOptions.env ?? (yield* HostProcess.Environment), }).pipe( @@ -794,6 +797,23 @@ export const layerQueryRunner: Layer.Layer< pathToClaudeCodeExecutable: launch.command, executableArgs: [...launch.args], }; + const spawnedPids: Array = []; + if (agentScope.trackProcess && !options.spawnClaudeCodeProcess) { + options.spawnClaudeCodeProcess = ({ command, args, cwd, env, signal }) => { + const child = NodeChildProcess.spawn(command, args, { + cwd, + env, + signal, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"], + detached: platform !== "win32", + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (data: string) => queryOptions.stderr?.(data)); + if (child.pid !== undefined) spawnedPids.push(child.pid); + return child; + }; + } // The CLI is running once query() returns, so it is closed if anything // before the session is handed out fails or is interrupted. const queryRuntime = yield* Effect.uninterruptibleMask((restore) => @@ -804,6 +824,8 @@ export const layerQueryRunner: Layer.Layer< Effect.tap((queryRuntime) => restore( Effect.gen(function* () { + for (const pid of spawnedPids) + yield* agentScope.trackProcess?.(pid) ?? Effect.void; if (mcpServers !== undefined) { // A rejected request or a server that failed to connect // leaves the turn without those tools, as `--mcp-config` diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 85b289e1b2ecd..28713b7862905 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1412,6 +1412,8 @@ const layerCodexAppServerClientFactoryCommand = ( }), ), ); + const agentScope = yield* AgentScope; + yield* agentScope.trackProcess?.(Number(handle.pid)) ?? Effect.void; const context = yield* Layer.build(CodexClient.layerChildProcess(handle, options)); return yield* Effect.service(CodexClient.CodexAppServerClient).pipe( Effect.provide(context), @@ -1533,6 +1535,7 @@ export const layerAppServerClientFactory: Layer.Layer< }), ), ); + yield* agentScope.trackProcess?.(Number(handle.pid)) ?? Effect.void; const protocolLogger = makeCodexAppServerProtocolLogger({ nativeEventLogger, threadId: input.threadId, diff --git a/apps/server/src/process/ProviderProcessLedger.ts b/apps/server/src/process/ProviderProcessLedger.ts new file mode 100644 index 0000000000000..7344c13045865 --- /dev/null +++ b/apps/server/src/process/ProviderProcessLedger.ts @@ -0,0 +1,360 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Process ownership survives server crashes. +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; + +import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +const execFile = NodeUtil.promisify(NodeChildProcess.execFile); + +const Identity = Schema.Struct({ + pid: Schema.Int.check(Schema.isGreaterThan(0)), + startTime: Schema.NonEmptyString, +}); +const OwnedProcess = Schema.Struct({ ...Identity.fields, pgid: Schema.Int }); +const Entry = Schema.Struct({ + version: Schema.Literal(1), + instanceId: Schema.NonEmptyString, + hostname: Schema.String, + stateDir: Schema.String, + owner: Identity, + processes: Schema.Array(OwnedProcess), +}); +type Identity = typeof Identity.Type; +type Entry = typeof Entry.Type; +interface ObservedProcess extends Identity { + readonly ppid: number; + readonly pgid: number; + readonly zombie: boolean; +} + +export interface ProcessController { + readonly snapshot: () => ReadonlyArray | Promise>; + readonly identity?: (pid: number) => ObservedProcess | undefined; + readonly signal: (pid: number, signal: NodeJS.Signals) => void; + readonly exists: (pid: number) => boolean; +} + +const same = (a: Identity, b: Identity) => a.pid === b.pid && a.startTime === b.startTime; +const decodeEntry = Schema.decodeUnknownSync(Schema.fromJsonString(Entry)); +const decodeProcessTable = Schema.decodeUnknownSync( + Schema.Array( + Schema.Struct({ + ...Identity.fields, + ppid: Schema.Int, + pgid: Schema.Int, + zombie: Schema.Boolean, + }), + ), +); + +/** Uses the ACP Linux identities; macOS ps and Windows CIM supply the same creation identity. */ +function controllerFor( + platform: NodeJS.Platform, + environment: NodeJS.ProcessEnv, +): ProcessController { + const linux = + platform === "linux" ? AcpSessionRuntime.makePosixProcessTreeController(platform) : undefined; + // Start ticks repeat after reboot. Including boot id keeps power-loss records safe. + const boot = linux ? NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() : ""; + return { + ...(linux + ? { + identity: (pid: number) => { + const p = linux.identity(pid); + return p + ? { + pid, + ppid: p.ppid, + pgid: p.pgid, + startTime: `${boot}:${p.startTime}`, + zombie: AcpSessionRuntime.posixProcessIsZombie(p), + } + : undefined; + }, + } + : {}), + snapshot: async () => { + if (linux) + return linux.snapshot().map((p) => ({ + pid: p.pid, + ppid: p.ppid, + pgid: p.pgid, + startTime: `${boot}:${p.startTime}`, + zombie: AcpSessionRuntime.posixProcessIsZombie(p), + })); + if (platform === "darwin") { + const { stdout: output } = await execFile( + "/bin/ps", + ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], + { + encoding: "utf8", + timeout: 5_000, + maxBuffer: 8 * 1024 * 1024, + env: { ...environment, LC_ALL: "C", TZ: "UTC" }, + }, + ); + return output.split("\n").flatMap((line) => { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); + return match + ? [ + { + pid: Number(match[1]), + ppid: Number(match[2]), + pgid: Number(match[3]), + zombie: match[4]!.startsWith("Z"), + startTime: match[5]!, + }, + ] + : []; + }); + } + if (platform === "win32") { + const output = NodeChildProcess.execFileSync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference='Stop'; @(Get-CimInstance Win32_Process | Where-Object { $_.CreationDate -and $_.ProcessId -gt 0 } | ForEach-Object { @{pid=[int]$_.ProcessId; ppid=[int]$_.ParentProcessId; pgid=0; startTime=$_.CreationDate.ToUniversalTime().Ticks.ToString(); zombie=$false} }) | ConvertTo-Json -Compress", + ], + { encoding: "utf8", timeout: 10_000, windowsHide: true, maxBuffer: 8 * 1024 * 1024 }, + ); + return decodeProcessTable(JSON.parse(output)); + } + throw new Error(`Provider process ownership is unsupported on ${platform}`); + }, + signal: (pid, signal) => { + process.kill(pid, signal); + }, + exists: (pid) => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException).code !== "ESRCH"; + } + }, + }; +} + +function creationTime(identity: Identity): bigint | undefined { + const ticks = identity.startTime.split(":").at(-1)!; + if (/^\d+$/.test(ticks)) return BigInt(ticks); + const parsed = Date.parse(`${identity.startTime} UTC`); + return Number.isFinite(parsed) ? BigInt(parsed) : undefined; +} + +/** Admits descendants only through a live, matching identity, or its still-owned private process group. */ +function discover(entry: Entry, table: ReadonlyArray): Entry { + const found = new Map(entry.processes.map((p) => [`${p.pid}:${p.startTime}`, p])); + const live = table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); + const groups = new Set( + live + .filter( + (p) => + p.pgid > 0 && entry.processes.some((owned) => owned.pgid > 0 && owned.pgid === p.pgid), + ) + .map((p) => p.pgid), + ); + const parents = new Map(live.map((p) => [p.pid, p])); + let changed = true; + while (changed) { + changed = false; + for (const p of table) { + if (p.zombie || found.has(`${p.pid}:${p.startTime}`)) continue; + const parent = parents.get(p.ppid); + const parentBirth = parent ? creationTime(parent) : undefined; + const childBirth = creationTime(p); + const childOfParent = + parent !== undefined && + (parentBirth === undefined || childBirth === undefined || childBirth >= parentBirth); + if (!childOfParent && !groups.has(p.pgid)) continue; + found.set(`${p.pid}:${p.startTime}`, { + pid: p.pid, + startTime: p.startTime, + pgid: groups.has(p.pgid) ? p.pgid : 0, + }); + parents.set(p.pid, p); + changed = true; + } + } + return { ...entry, processes: [...found.values()] }; +} + +/** One instance's local children only. External servers, launchers and install supervisors never register. */ +export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { + readonly stateDir: string; + readonly ownerPid?: number; + readonly controller?: ProcessController; +}) { + const platform = yield* HostProcess.Platform; + const environment = yield* HostProcess.Environment; + const controller = input.controller ?? controllerFor(platform, environment); + const directory = NodePath.join(input.stateDir, "provider-processes"); + NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const hostname = yield* HostProcess.Hostname; + const ownerPid = input.ownerPid ?? process.pid; + const owner = (yield* Effect.promise(async () => controller.snapshot())).find( + (p) => p.pid === ownerPid && !p.zombie, + ); + if (!owner) throw new Error("Could not identify the provider process owner"); + const instanceId = NodeCrypto.randomUUID(); + const tracked = new Map(); + const write = (file: string, entry: Entry) => { + const temporary = `${file}.tmp`; + NodeFS.writeFileSync(temporary, JSON.stringify(entry), { mode: 0o600 }); + NodeFS.renameSync(temporary, file); + }; + const survivors = (entry: Entry, table: ReadonlyArray) => + table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); + const refresh = async () => { + if (tracked.size === 0) return; + const table = await controller.snapshot(); + for (const [file, entry] of tracked) { + const discovered = discover(entry, table); + const next = { + ...discovered, + processes: discovered.processes.filter( + (p) => + table.some((observed) => same(p, observed) && !observed.zombie) || + (!table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid)), + ), + }; + if (next.processes.length === 0) { + NodeFS.rmSync(file, { force: true }); + tracked.delete(file); + } else if (JSON.stringify(next.processes) !== JSON.stringify(entry.processes)) { + write(file, next); + tracked.set(file, next); + } + } + }; + const signal = async (p: Identity, signal: NodeJS.Signals) => { + // Refresh immediately before each signal, including escalation. Never signal by group id or taskkill /t. + const observed = controller.identity + ? controller.identity(p.pid) + : (await controller.snapshot()).find((candidate) => candidate.pid === p.pid); + if (!observed || observed.zombie || !same(p, observed)) return; + try { + controller.signal(p.pid, signal); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "ESRCH") throw cause; + } + }; + const reap = (file: string, entry: Entry) => + Effect.gen(function* () { + let table = yield* Effect.promise(async () => controller.snapshot()); + const liveOwner = table.find((p) => same(entry.owner, p)); + if (liveOwner && !liveOwner.zombie) return; + // A restricted/failed process query cannot establish that an owner died. + if ( + !liveOwner && + !table.some((p) => p.pid === entry.owner.pid) && + controller.exists(entry.owner.pid) + ) { + throw new Error("Could not verify the recorded provider owner"); + } + let next = discover(entry, table); + if (platform !== "win32") { + // Freeze to a fixed point. A child may fork while its parent is being stopped. + const stopped = new Set(); + for (let attempt = 0; attempt < 100; attempt++) { + const pending = survivors(next, table).filter( + (p) => !stopped.has(`${p.pid}:${p.startTime}`), + ); + if (pending.length === 0) break; + for (const p of pending) { + yield* Effect.promise(() => signal(p, "SIGSTOP")); + stopped.add(`${p.pid}:${p.startTime}`); + } + table = yield* Effect.promise(async () => controller.snapshot()); + next = discover(next, table); + if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); + } + } + // Persist newly discovered children before signalling: recovery can itself be killed. + write(file, next); + for (const p of survivors( + next, + yield* Effect.promise(async () => controller.snapshot()), + ).toReversed()) + yield* Effect.promise(() => signal(p, "SIGKILL")); + for (let attempt = 0; attempt < 100; attempt++) { + table = yield* Effect.promise(async () => controller.snapshot()); + if ( + next.processes.some( + (p) => !table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid), + ) + ) { + throw new Error("Could not verify a recorded provider process"); + } + if (survivors(next, table).length === 0) { + NodeFS.rmSync(file, { force: true }); + return; + } + yield* Effect.sleep("20 millis"); + } + throw new Error("Recorded provider processes did not stop"); + }); + // Await every reap before exposing the spawn hook, and therefore before restart recovery launches work. + for (const name of NodeFS.readdirSync(directory)) { + if (!/^[\da-f-]+\.json$/.test(name)) continue; + const file = NodePath.join(directory, name); + const entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); + if (entry.hostname !== hostname || entry.stateDir !== input.stateDir) { + NodeFS.rmSync(file, { force: true }); + continue; + } + yield* reap(file, entry); + } + const track = (pid: number) => + Effect.promise(async () => { + const table = await controller.snapshot(); + const root = + table.find((p) => p.pid === pid && !p.zombie) ?? + table.find((p) => p.pgid === pid && !p.zombie); + if (!root) return; + // Process groups shared with the server are never a source of ownership. + const entry = discover( + { + version: 1, + instanceId, + hostname, + stateDir: input.stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes: [ + { pid: root.pid, startTime: root.startTime, pgid: root.pgid === pid ? pid : 0 }, + ], + }, + table, + ); + const file = NodePath.join(directory, `${instanceId}-${pid}.json`); + try { + write(file, entry); + tracked.set(file, entry); + } catch (cause) { + for (const child of entry.processes.toReversed()) await signal(child, "SIGKILL"); + throw cause; + } + }); + yield* Effect.forever( + Effect.sleep("1 second").pipe( + Effect.andThen( + Effect.tryPromise(refresh).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not record provider descendants", { cause }), + ), + ), + ), + ), + ).pipe(Effect.forkScoped); + // Leave surviving entries for the next startup. A graceful provider stop makes refresh remove them. + yield* Effect.addFinalizer(() => Effect.tryPromise(refresh).pipe(Effect.ignore)); + return { track }; +}); diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 537bcc646748b..a1d7a15ca26ba 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -10,6 +10,7 @@ import * as Effect from "effect/Effect"; import { ChildProcessSpawner } from "effect/process"; import * as ProcessRunner from "../processRunner.ts"; +import * as ProviderProcessLedger from "./ProviderProcessLedger.ts"; import { agentSliceMemoryLimits, classifyScope, make, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; @@ -143,3 +144,252 @@ describe("AgentScope service", () => { ), ); }); + +// The controller models creation identities, including PID reuse and reparenting after a crash. +describe("provider process restart ownership", () => { + const scenario = ( + body: (input: { + directory: string; + processes: Map< + number, + { pid: number; ppid: number; pgid: number; startTime: string; zombie: boolean } + >; + signals: Array; + controller: ProviderProcessLedger.ProcessController; + }) => Effect.Effect, + ) => + Effect.gen(function* () { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-ledger-")); + const processes = new Map([ + [10, { pid: 10, ppid: 1, pgid: 10, startTime: "owner", zombie: false }], + [20, { pid: 20, ppid: 10, pgid: 20, startTime: "provider", zombie: false }], + [30, { pid: 30, ppid: 20, pgid: 20, startTime: "child", zombie: false }], + [90, { pid: 90, ppid: 1, pgid: 90, startTime: "next-owner", zombie: false }], + ]); + const signals: Array = []; + const controller: ProviderProcessLedger.ProcessController = { + snapshot: () => [...processes.values()], + exists: (pid) => processes.has(pid), + signal: (pid, signal) => { + signals.push(`${pid}:${signal}`); + if (signal === "SIGKILL") processes.delete(pid); + }, + }; + try { + yield* body({ directory, processes, signals, controller }); + } finally { + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); + const record = (directory: string, controller: ProviderProcessLedger.ProcessController) => + ProviderProcessLedger.make({ stateDir: directory, ownerPid: 10, controller }).pipe( + Effect.tap((ledger) => ledger.track(20)), + Effect.scoped, + ); + const restart = (directory: string, controller: ProviderProcessLedger.ProcessController) => + ProviderProcessLedger.make({ stateDir: directory, ownerPid: 90, controller }).pipe( + Effect.scoped, + ); + + it.effect("reaps an orphan provider and its forked children before exposing the spawn hook", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + yield* restart(directory, controller); + expect(processes.has(20)).toBe(false); + expect(processes.has(30)).toBe(false); + expect(signals.filter((s) => s.endsWith("SIGKILL"))).toEqual(["30:SIGKILL", "20:SIGKILL"]); + expect(NodeFS.readdirSync(NodePath.join(directory, "provider-processes"))).toEqual([]); + }), + ), + ); + it.effect("leaves every child of a live owning server alone", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + yield* restart(directory, controller); + expect(signals).toEqual([]); + expect(processes.has(20)).toBe(true); + expect(NodeFS.readdirSync(NodePath.join(directory, "provider-processes"))).toHaveLength(1); + }), + ), + ); + it.effect("does not signal a reused provider pid or admit its new descendants", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + processes.delete(30); + processes.set(20, { ...processes.get(20)!, startTime: "reused" }); + processes.set(40, { + pid: 40, + ppid: 20, + pgid: 20, + startTime: "unrelated-child", + zombie: false, + }); + yield* restart(directory, controller); + expect(signals).toEqual([]); + expect(processes.has(40)).toBe(true); + }), + ), + ); + it.effect("a reused owner pid cannot pin an orphan", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.set(10, { ...processes.get(10)!, startTime: "reused-owner" }); + yield* restart(directory, controller); + expect(signals).not.toContain("10:SIGKILL"); + expect(processes.has(20)).toBe(false); + }), + ), + ); + it.effect("retains ownership when a shell wrapper exits and its child is reparented", () => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + processes.delete(20); + processes.set(30, { ...processes.get(30)!, ppid: 1 }); + processes.set(40, { pid: 40, ppid: 1, pgid: 20, startTime: "later-fork", zombie: false }); + yield* restart(directory, controller); + expect(processes.has(30)).toBe(false); + expect(processes.has(40)).toBe(false); + }), + ), + ); + it.effect("records a private group even if the wrapper exits before registration", () => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + processes.delete(20); + processes.set(30, { ...processes.get(30)!, ppid: 1 }); + yield* record(directory, controller); + processes.delete(10); + yield* restart(directory, controller); + expect(processes.has(30)).toBe(false); + }), + ), + ); + it.effect( + "never owns unrelated siblings, an external server, launcher or install supervisor", + () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + processes.set(50, { pid: 50, ppid: 10, pgid: 10, startTime: "installer", zombie: false }); + processes.set(60, { pid: 60, ppid: 1, pgid: 60, startTime: "external", zombie: false }); + yield* record(directory, controller); + processes.delete(10); + yield* restart(directory, controller); + expect(processes.has(50)).toBe(true); + expect(processes.has(60)).toBe(true); + expect(processes.has(90)).toBe(true); + expect(signals.some((s) => /^(50|60|90):/.test(s))).toBe(false); + }), + ), + ); + it.effect("rejects copied state without signalling the original environment", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + const copied = NodePath.join(directory, "copy"); + NodeFS.cpSync( + NodePath.join(directory, "provider-processes"), + NodePath.join(copied, "provider-processes"), + { recursive: true }, + ); + yield* restart(copied, controller); + expect(signals).toEqual([]); + expect(processes.has(20)).toBe(true); + }), + ), + ); + it.effect("does not treat an unreadable living owner as dead", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + const restricted = { + ...controller, + snapshot: () => [...processes.values()].filter((p) => p.pid !== 10), + }; + const result = yield* restart(directory, restricted).pipe(Effect.exit); + expect(result._tag).toBe("Failure"); + expect(signals).toEqual([]); + }), + ), + ); + it.effect("windows cleanup checks each identity without taskkill tree selection", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + for (const [pid, p] of processes) processes.set(pid, { ...p, pgid: 0 }); + yield* record(directory, controller); + processes.delete(10); + yield* restart(directory, controller).pipe( + Effect.provideService(HostProcess.Platform, "win32"), + ); + expect(signals).toEqual(["30:SIGKILL", "20:SIGKILL"]); + }), + ), + ); + it.effect("rejects Windows stale parent ids from children born before their reused parent", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + for (const [pid, p] of processes) + processes.set(pid, { ...p, pgid: 0, startTime: String(pid * 100) }); + processes.set(40, { pid: 40, ppid: 20, pgid: 0, startTime: "100", zombie: false }); + yield* record(directory, controller); + processes.delete(10); + yield* restart(directory, controller).pipe( + Effect.provideService(HostProcess.Platform, "win32"), + ); + expect(processes.has(40)).toBe(true); + expect(signals.some((s) => s.startsWith("40:"))).toBe(false); + }), + ), + ); + it.effect("freezes children forked during cleanup before killing their parents", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + const forking = { + ...controller, + signal: (pid: number, signal: NodeJS.Signals) => { + controller.signal(pid, signal); + if (pid === 20 && signal === "SIGSTOP") + processes.set(40, { + pid: 40, + ppid: 20, + pgid: 40, + startTime: "racing-child", + zombie: false, + }); + }, + }; + yield* restart(directory, forking); + expect(signals.indexOf("40:SIGSTOP")).toBeLessThan(signals.indexOf("20:SIGKILL")); + expect(processes.has(40)).toBe(false); + }), + ), + ); + it.effect("stops the new tree if its durable ownership record cannot be written", () => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + const ledger = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 10, + controller, + }).pipe(Effect.scoped); + const records = NodePath.join(directory, "provider-processes"); + NodeFS.rmSync(records, { recursive: true }); + NodeFS.writeFileSync(records, "unwritable records directory"); + const result = yield* ledger.track(20).pipe(Effect.exit); + expect(result._tag).toBe("Failure"); + expect(processes.has(20)).toBe(false); + expect(processes.has(30)).toBe(false); + }), + ), + ); +}); diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 511ed6446fd32..b306a12eb68c1 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -16,6 +16,8 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as ProcessRunner from "../processRunner.ts"; +import * as ServerConfig from "../config.ts"; +import * as ProviderProcessLedger from "./ProviderProcessLedger.ts"; /** * Agents and terminals share this slice. It sits in app.slice next to the @@ -315,4 +317,12 @@ export const make = Effect.gen(function* () { } satisfies AgentScopeShape; }); -export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer)); +export const layer = Layer.effect( + AgentScope, + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const ledger = yield* ProviderProcessLedger.make({ stateDir: config.stateDir }); + const scope = yield* make; + return { ...scope, trackProcess: ledger.track }; + }), +).pipe(Layer.provide(ProcessRunner.layer)); diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index e9816055b7608..dbd73f858eece 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -1,3 +1,4 @@ +import { AgentScope } from "@t3tools/shared/AgentScope"; import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -411,6 +412,9 @@ export const withCodexAppServerClient = Effect.fn("withCodexAppServerClient")( }), ), ); + const processScope = yield* AgentScope; + yield* processScope.trackProcess?.(Number(child.pid)) ?? Effect.void; + const clientContext = yield* Layer.build(CodexClient.layerChildProcess(child)); const client = yield* Effect.service(CodexClient.CodexAppServerClient).pipe( Effect.provide(clientContext), diff --git a/apps/server/src/terminal/NodePtyAdapter.ts b/apps/server/src/terminal/NodePtyAdapter.ts index e865c3020a80d..151fb561389b7 100644 --- a/apps/server/src/terminal/NodePtyAdapter.ts +++ b/apps/server/src/terminal/NodePtyAdapter.ts @@ -8,6 +8,7 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { AgentScope } from "@t3tools/shared/AgentScope"; import * as PtyAdapter from "@t3tools/shared/PtyAdapter"; @@ -296,6 +297,10 @@ export const make = Effect.fn("NodePtyAdapter.make")(function* () { ), ); } + const agentScope = yield* AgentScope; + yield* (agentScope.trackProcess?.(ptyProcess.process.pid) ?? Effect.void).pipe( + Effect.onError(() => Effect.sync(() => ptyProcess.process.kill())), + ); return ptyProcess.process; }), }); diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.ts index 25695f7fdefae..08ecdbd60f326 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.ts @@ -1,3 +1,4 @@ +import { AgentScope } from "@t3tools/shared/AgentScope"; /** * ClaudeTextGeneration – Text generation layer using the Claude CLI. * @@ -65,6 +66,7 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu modelCatalog: Effect.Effect = Effect.succeed(BUNDLED_CLAUDE_MODEL_CATALOG), ) { const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const processScope = yield* AgentScope; const fileSystem = yield* FileSystem.FileSystem; const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, environment); const scopedModelCatalog = modelCatalog.pipe( @@ -215,6 +217,8 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu ), ); + yield* processScope.trackProcess?.(Number(child.pid)) ?? Effect.void; + const [stdout, stderr, exitCode] = yield* Effect.all( [ readStreamAsString(operation, child.stdout), diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 3902b0c74d78e..b4f7f31e0500b 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -1,3 +1,4 @@ +import { AgentScope } from "@t3tools/shared/AgentScope"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; @@ -47,6 +48,7 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const processScope = yield* AgentScope; const serverConfig = yield* Effect.service(ServerConfig.ServerConfig); const resolvedEnvironment = environment ?? process.env; @@ -238,6 +240,8 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func ), ); + yield* processScope.trackProcess?.(Number(child.pid)) ?? Effect.void; + const [stdout, stderr, exitCode] = yield* Effect.all( [ readStreamAsString(operation, child.stdout), diff --git a/packages/provider-acp/src/server/AcpSessionRuntime.ts b/packages/provider-acp/src/server/AcpSessionRuntime.ts index 1295509d41e52..349877325a0de 100644 --- a/packages/provider-acp/src/server/AcpSessionRuntime.ts +++ b/packages/provider-acp/src/server/AcpSessionRuntime.ts @@ -1616,6 +1616,9 @@ export const make = ( ), ); + const agentScope = yield* AgentScope; + yield* agentScope.trackProcess?.(Number(child.pid)) ?? Effect.void; + const posixOwnershipLedger = new Map(); const posixOwnershipFrontier = new Map(); const posixOwnershipChildQueues = new Map>(); diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 49b1e2d924663..dc23b17b7d7a0 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -1612,6 +1612,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }, }).pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner), + Effect.provideService(AgentScope, agentScope), ); if (clientTerminals !== undefined) { yield* Scope.addFinalizer(sessionScope, clientTerminals.disposeAll); diff --git a/packages/provider-acp/src/server/clientTerminals.ts b/packages/provider-acp/src/server/clientTerminals.ts index ce6b3d0436ae5..115ec5a321ef0 100644 --- a/packages/provider-acp/src/server/clientTerminals.ts +++ b/packages/provider-acp/src/server/clientTerminals.ts @@ -1,3 +1,4 @@ +import { AgentScope } from "@t3tools/shared/AgentScope"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -165,6 +166,7 @@ export const makeAcpClientTerminals = Effect.fn("makeAcpClientTerminals")(functi options: AcpClientTerminalsOptions, ): Effect.fn.Return { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const agentScope = yield* AgentScope; const creationPermit = yield* Semaphore.make(1); const terminals = new Map(); let nextTerminalNumber = 1; @@ -253,6 +255,7 @@ export const makeAcpClientTerminals = Effect.fn("makeAcpClientTerminals")(functi ) .pipe( Effect.provideService(Scope.Scope, terminalScope), + Effect.tap((child) => agentScope.trackProcess?.(Number(child.pid)) ?? Effect.void), Effect.mapError((cause) => terminalRequestError( `Could not start terminal command '${request.command}'.`, diff --git a/packages/provider-opencode/src/server/OpenCodeRuntime.ts b/packages/provider-opencode/src/server/OpenCodeRuntime.ts index c2feb6043dab3..fd2844a1c72ef 100644 --- a/packages/provider-opencode/src/server/OpenCodeRuntime.ts +++ b/packages/provider-opencode/src/server/OpenCodeRuntime.ts @@ -748,6 +748,8 @@ const makeOpenCodeRuntime = Effect.gen(function* () { ), ); + yield* agentScope.trackProcess?.(Number(child.pid)) ?? Effect.void; + const killOpenCodeProcessGroup = (signal: NodeJS.Signals) => hostPlatform === "win32" ? child.kill({ killSignal: signal, forceKillAfter: "1 second" }).pipe(Effect.asVoid) diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index b7c68db21ea7c..39335e944babb 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -561,6 +561,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( env: launch.env, }).pipe( Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(AgentScope, agentScope), Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterOpenSessionError({ @@ -3263,7 +3264,10 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( } return file; }), - ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + ).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(AgentScope, agentScope), + ); const now = yield* DateTime.now; return yield* registerThread( { diff --git a/packages/provider-pi/src/server/rpc.ts b/packages/provider-pi/src/server/rpc.ts index 3f67291330ae1..07c8704bb233b 100644 --- a/packages/provider-pi/src/server/rpc.ts +++ b/packages/provider-pi/src/server/rpc.ts @@ -1,3 +1,4 @@ +import { AgentScope } from "@t3tools/shared/AgentScope"; /** * PiRpc — stdio JSONL transport for the Pi coding agent's RPC mode. * @@ -241,6 +242,9 @@ export const makePiRpcConnection = Effect.fnUntraced(function* (options: PiRpcSp ) .pipe(Effect.mapError((cause) => new PiRpcError({ operation: "spawn", cause }))); + const processScope = yield* AgentScope; + yield* processScope.trackProcess?.(Number(child.pid)) ?? Effect.void; + let childExited = false; let diagnosingStdoutClose = false; diff --git a/packages/provider-pi/src/server/textGeneration.ts b/packages/provider-pi/src/server/textGeneration.ts index e8ac19a7e13ce..52a679f0023a0 100644 --- a/packages/provider-pi/src/server/textGeneration.ts +++ b/packages/provider-pi/src/server/textGeneration.ts @@ -1,3 +1,4 @@ +import { AgentScope } from "@t3tools/shared/AgentScope"; /** * PiTextGeneration — commit messages, PR content, branch names, and thread * titles generated through an ephemeral `pi --mode rpc --no-session` process. @@ -26,6 +27,7 @@ export const makePiTextGeneration = Effect.fn("makePiTextGeneration")(function* environment: NodeJS.ProcessEnv = process.env, ) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const agentScope = yield* AgentScope; const runPiJson: TextGenerationOperations.Runner = (request) => { const { operation, cwd, prompt, modelSelection } = request; @@ -56,7 +58,10 @@ export const makePiTextGeneration = Effect.fn("makePiTextGeneration")(function* args: launch.args, cwd, env: launch.env, - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(AgentScope, agentScope), + ); if (modelSelection.model !== "default") { // `customModels` accepts arbitrary strings, so an unusable slug is diff --git a/packages/shared/src/AgentScope.ts b/packages/shared/src/AgentScope.ts index 403cac7c4a74f..00501a660cc75 100644 --- a/packages/shared/src/AgentScope.ts +++ b/packages/shared/src/AgentScope.ts @@ -38,6 +38,8 @@ export interface AgentScopeShape { * it, even when the next session runs without one (Cursor, OpenCode). */ readonly clear: (threadId: string) => Effect.Effect; + /** Records a local provider or terminal before it can accept work. */ + readonly trackProcess?: (pid: number) => Effect.Effect; } export interface AgentScopeCommand { From 4f6f4c222f60668a3d2f0d38dd2bb419d41c0fa7 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:34:02 +0000 Subject: [PATCH 114/149] fix(server): a prompt accepted during a restart handoff resumes once after the restart --- .../orchestration-v2/RestartBackgroundNote.ts | 5 +-- .../RestartContinuation.test.ts | 33 +++++++++++++++++++ .../orchestration-v2/RestartContinuation.ts | 26 +++++++++++---- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index ce4574199c00d..b881343976f19 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -240,7 +240,7 @@ export function restartContinuationNote( export const isWorktreeContinuationRun = (run: Pick) => /^message:mcp:[^:]+:worktree-continuation:/.test(run.userMessageId); -/** A handoff or replacement prompt that never reached the provider. */ +/** An initial, handoff, or replacement prompt that never reached the provider. */ export function restartPromptSource( source: OrchestrationV2Run, runs: ReadonlyArray, @@ -256,7 +256,8 @@ export function restartPromptSource( isWorktreeContinuationRun(current) || attempts.some( (attempt) => - attempt.id === current?.activeAttemptId && attempt.reason === "steering_restart", + attempt.id === current?.activeAttemptId && + (attempt.reason === "initial" || attempt.reason === "steering_restart"), ) ) return current; diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index db6195a0a7db0..b4102721d75cb 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -416,6 +416,25 @@ it("recovers an admitted continuation after another crash before provider start" assert.equal(restartContinuationRun(starting, testNow)?.id, runId); }); +it("recovers an accepted initial prompt before admission creates its native session", () => { + const projection = makeProjection(); + const starting = { + ...projection, + runs: [{ ...projection.runs[0]!, status: "starting" as const }], + attempts: [{ id: attemptId, runId, reason: "initial" }], + providerThreads: [{ ...projection.providerThreads[0]!, nativeThreadRef: null }], + providerSessions: [], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection; + assert.equal(restartContinuationRun(starting, testNow)?.id, runId); + assert.isUndefined( + restartContinuationRun( + { ...starting, runs: [{ ...starting.runs[0]!, status: "queued" }] }, + testNow, + ), + ); +}); + it("recovers an accepted steering restart before the replacement turn starts", () => { const projection = makeProjection(); const starting = { @@ -924,6 +943,20 @@ const cutMidTurn = (extra: Record = {}) => { } as unknown as OrchestrationV2ThreadProjection; }; +it.effect("delivers the original prompt when a restart cuts initial provider admission", () => + Effect.gen(function* () { + const projection = { + ...cutMidTurn(), + attempts: [{ id: attemptId, runId, reason: "initial" }], + providerTurns: [], + messages: [ + { id: MessageId.make("message:user"), text: "Do the accepted work.", attachments: [] }, + ], + } as unknown as OrchestrationV2ThreadProjection; + assert.deepEqual(yield* continuationTexts(projection), [projection.messages[0]!.text]); + }), +); + it.effect( "delivers the replacement prompt when a steering restart never reached the provider", () => diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 122697e8e3332..a3e74a7feacdc 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -146,11 +146,13 @@ export function restartContinuationRun( undefined, ); if (!run) return; + const promptSource = + run.status === "starting" + ? restartPromptSource(run, projection.runs, projection.providerTurns, projection.attempts) + : undefined; const preparedContinuation = run.status === "starting" && - (run.restartContinuationOfRunId !== undefined || - restartPromptSource(run, projection.runs, projection.providerTurns, projection.attempts) !== - undefined); + (run.restartContinuationOfRunId !== undefined || promptSource !== undefined); const pendingSteering = (run.restartPendingMessageIds?.length ?? 0) > 0; if ( run.status !== "running" && @@ -170,12 +172,24 @@ export function restartContinuationRun( providerThread.appThreadId !== projection.thread.id || providerThread.ownerNodeId !== null || providerThread.providerInstanceId !== run.providerInstanceId || + providerThread.status === "closed" || + providerThread.status === "archived" + ) + return; + // Admission can hold an accepted prompt before its first native session exists. + if ( + preparedContinuation && + promptSource !== undefined && + projection.attempts.some( + (attempt) => attempt.id === promptSource.activeAttemptId && attempt.reason === "initial", + ) + ) + return run; + if ( providerThread.nativeThreadRef?.nativeId == null || providerThread.nativeThreadRef.strength !== "strong" || providerThread.nativeThreadRef.driver !== providerThread.driver || - (liveTurnRequired && providerThread.status !== "active") || - providerThread.status === "closed" || - providerThread.status === "archived" + (liveTurnRequired && providerThread.status !== "active") ) return; const session = projection.providerSessions.find( From 23bb02acc4d10c4a6b696b137c5fb446d7c1188b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:51:54 +0000 Subject: [PATCH 115/149] fix(server): orphan cleanup never stops a provider install and survives an empty process record --- .../src/process/ProviderProcessLedger.ts | 49 +++++++++++++++++-- apps/server/src/process/agentScope.test.ts | 42 ++++++++++++++++ apps/server/src/process/agentScope.ts | 6 ++- 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/apps/server/src/process/ProviderProcessLedger.ts b/apps/server/src/process/ProviderProcessLedger.ts index 7344c13045865..dca592fe6a7f1 100644 --- a/apps/server/src/process/ProviderProcessLedger.ts +++ b/apps/server/src/process/ProviderProcessLedger.ts @@ -10,6 +10,8 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import { installLeasePath } from "../provider/ProviderMaintenanceHostLock.ts"; + const execFile = NodeUtil.promisify(NodeChildProcess.execFile); const Identity = Schema.Struct({ @@ -192,6 +194,8 @@ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { readonly stateDir: string; readonly ownerPid?: number; readonly controller?: ProcessController; + /** Host install lease directory. Its supervisor's process group is never stopped. */ + readonly installLeaseDirectory?: string; }) { const platform = yield* HostProcess.Platform; const environment = yield* HostProcess.Environment; @@ -247,6 +251,36 @@ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { if ((cause as NodeJS.ErrnoException).code !== "ESRCH") throw cause; } }; + // A server started from a provider or terminal can descend from it with its own + // detached install supervisor. Stopping that group would corrupt a shared CLI. + const withoutInstaller = (entry: Entry, table: ReadonlyArray): Entry => { + if (input.installLeaseDirectory === undefined) return entry; + let leader: number; + try { + leader = Number(NodeFS.readFileSync(installLeasePath(input.installLeaseDirectory), "utf8")); + } catch { + return entry; + } + if (!Number.isSafeInteger(leader) || leader <= 0) return entry; + // Windows has no process groups, so its installer is found through parents. + const installer = new Set([leader]); + let changed = true; + while (changed) { + changed = false; + for (const p of table) { + if (!installer.has(p.pid) && (p.pgid === leader || installer.has(p.ppid))) { + installer.add(p.pid); + changed = true; + } + } + } + return { + ...entry, + processes: entry.processes.filter( + (p) => !table.some((observed) => installer.has(observed.pid) && same(p, observed)), + ), + }; + }; const reap = (file: string, entry: Entry) => Effect.gen(function* () { let table = yield* Effect.promise(async () => controller.snapshot()); @@ -260,7 +294,7 @@ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { ) { throw new Error("Could not verify the recorded provider owner"); } - let next = discover(entry, table); + let next = withoutInstaller(discover(entry, table), table); if (platform !== "win32") { // Freeze to a fixed point. A child may fork while its parent is being stopped. const stopped = new Set(); @@ -274,7 +308,7 @@ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { stopped.add(`${p.pid}:${p.startTime}`); } table = yield* Effect.promise(async () => controller.snapshot()); - next = discover(next, table); + next = withoutInstaller(discover(next, table), table); if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); } } @@ -306,7 +340,16 @@ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { for (const name of NodeFS.readdirSync(directory)) { if (!/^[\da-f-]+\.json$/.test(name)) continue; const file = NodePath.join(directory, name); - const entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); + let entry: Entry; + try { + entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); + } catch (cause) { + // Records are not fsynced, so power loss can leave an empty one. It names + // no process to reap, and keeping it would fail every later startup. + yield* Effect.logWarning("Discarding an unreadable provider process record", { cause }); + NodeFS.rmSync(file, { force: true }); + continue; + } if (entry.hostname !== hostname || entry.stateDir !== input.stateDir) { NodeFS.rmSync(file, { force: true }); continue; diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index a1d7a15ca26ba..87dccd153335b 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -289,6 +289,48 @@ describe("provider process restart ownership", () => { }), ), ); + it.effect("never stops a nested server's install supervisor or its installer", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + // A server launched from the provider started an install that outlives it. + processes.set(70, { pid: 70, ppid: 20, pgid: 70, startTime: "supervisor", zombie: false }); + processes.set(71, { pid: 71, ppid: 70, pgid: 70, startTime: "installer", zombie: false }); + yield* record(directory, controller); + processes.delete(10); + processes.set(70, { ...processes.get(70)!, ppid: 1 }); + const leases = NodePath.join(directory, "leases"); + NodeFS.mkdirSync(leases); + NodeFS.writeFileSync(NodePath.join(leases, "install.json"), "70"); + yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + controller, + installLeaseDirectory: leases, + }).pipe(Effect.scoped); + expect(processes.has(20)).toBe(false); + expect(processes.has(70)).toBe(true); + expect(processes.has(71)).toBe(true); + expect(signals.some((s) => /^(70|71):/.test(s))).toBe(false); + }), + ), + ); + it.effect("starts after power loss leaves an empty process record", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + const records = NodePath.join(directory, "provider-processes"); + NodeFS.writeFileSync( + NodePath.join(records, "00000000-0000-0000-0000-000000000000-1.json"), + "", + ); + yield* restart(directory, controller); + expect(processes.has(20)).toBe(false); + expect(signals).toContain("20:SIGKILL"); + expect(NodeFS.readdirSync(records)).toEqual([]); + }), + ), + ); it.effect("rejects copied state without signalling the original environment", () => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index b306a12eb68c1..9c838441aa966 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -17,6 +17,7 @@ import * as Path from "effect/Path"; import * as ProcessRunner from "../processRunner.ts"; import * as ServerConfig from "../config.ts"; +import * as HostLock from "../provider/ProviderMaintenanceHostLock.ts"; import * as ProviderProcessLedger from "./ProviderProcessLedger.ts"; /** @@ -321,7 +322,10 @@ export const layer = Layer.effect( AgentScope, Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - const ledger = yield* ProviderProcessLedger.make({ stateDir: config.stateDir }); + const ledger = yield* ProviderProcessLedger.make({ + stateDir: config.stateDir, + installLeaseDirectory: HostLock.userDirectory(), + }); const scope = yield* make; return { ...scope, trackProcess: ledger.track }; }), From ccfa8cc4fee233e3762da3db06a36f45e98a1d62 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:51:55 +0000 Subject: [PATCH 116/149] docs: describe orphaned provider cleanup before continuation --- docs/internals/server-updates.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index c86901c6b753c..968cac9adf49d 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -70,6 +70,12 @@ Queued runs never started, so recovery holds them and continues the run they wai behind. Cancelled background work is reported on the next user turn; it does not wake a finished run. Undelivered steering can still qualify as unfinished input. +A crashed server can leave its providers and terminals running. The next server +stops the ones its [process ledger](../../apps/server/src/process/ProviderProcessLedger.ts) +recorded before any continuation can launch, so a continued run never races its +orphaned original. Each signal rechecks the process's creation identity. A detached +provider installer is never stopped, even when it descends from a recorded process. + Recovery retires effects tied to the lost process and records continuation intent in the durable outbox. That intent survives another restart before provider startup. Continuation effects wait for activation; a slow provider must not delay the server's From a57a275a95a4d9e2ca293b5843b12ab6f0ed0bab Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:04:00 +0000 Subject: [PATCH 117/149] fix(server): an undelivered prompt that is also pending steering continues once, not twice --- apps/server/src/orchestration-v2/RestartContinuation.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index a3e74a7feacdc..94456c2290456 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -359,9 +359,12 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta sourceRecords.messages.find((message) => message.id === id), ); if (pendingMessages.some((message) => message === undefined)) return; + // An undelivered initial prompt can also be pending steering; send it once. const promptMessages = [ ...(replacementMessage ? [replacementMessage] : []), - ...pendingMessages.filter((message) => message !== undefined), + ...pendingMessages + .filter((message) => message !== undefined) + .filter((message) => message.id !== replacementMessage?.id), ]; const prompt = promptMessages.length > 0 From 87199d08257aacb7d9c9677037ea3b1d46e12c43 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:42:13 +0000 Subject: [PATCH 118/149] fix(server): a provider install keeps admission until its last child exits, even when the server is removed or the disk is full --- .../src/provider/ProviderInstallSupervisor.ts | 29 ++++- .../provider/ProviderMaintenanceHostLock.ts | 19 ++- .../providerMaintenanceRunner.test.ts | 121 ++++++++++++++++++ 3 files changed, 159 insertions(+), 10 deletions(-) diff --git a/apps/server/src/provider/ProviderInstallSupervisor.ts b/apps/server/src/provider/ProviderInstallSupervisor.ts index 7d483678f6475..83ac1501251a0 100644 --- a/apps/server/src/provider/ProviderInstallSupervisor.ts +++ b/apps/server/src/provider/ProviderInstallSupervisor.ts @@ -264,6 +264,7 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< }; let heartbeat: NodeJS.Timeout | undefined; let ownsGate = false; + let installerFinished = false; const beat = (format: (line: string) => string | null) => { NodeFS.utimesSync(lease, new Date(), new Date()); NodeFS.utimesSync(NodePath.join(receipt.directory, "gate.lock"), new Date(), new Date()); @@ -278,6 +279,11 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< } }; try { + // Once readiness is durable the server or desktop can replace these files. + // Load every installer dependency before allowing that handoff. + const { runProviderMaintenanceCommandWithSpawner, toProgressLine } = + await import("./providerMaintenanceRunner.ts"); + const { ChildProcessSpawner } = await import("effect/process"); receipt = { ...receipt, pid: process.pid }; writeReceipt(file, receipt); const deadline = Date.now() + 30_000; @@ -291,12 +297,15 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< throw new Error("Provider install receipt replaced"); ownsGate = true; NodeFS.rmSync(`${file}.start`, { force: true }); - const { runProviderMaintenanceCommandWithSpawner, toProgressLine } = - await import("./providerMaintenanceRunner.ts"); - heartbeat = setInterval(() => beat(toProgressLine), 1_000); + heartbeat = setInterval(() => { + try { + beat(toProgressLine); + } catch { + // The live PID/process group still holds admission when disk writes fail. + } + }, 1_000); const result = await Effect.runPromise( Effect.gen(function* () { - const { ChildProcessSpawner } = yield* Effect.promise(() => import("effect/process")); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; return yield* runProviderMaintenanceCommandWithSpawner({ spawner, @@ -307,6 +316,9 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< }); }).pipe(Effect.provide(NodeServices.layer)), ); + while (HostLock.installChildrenAreAlive(process.pid)) + await new Promise((resolve) => setTimeout(resolve, 100)); + installerFinished = true; const stdout = output(".stdout"); const stderr = output(".stderr"); writeReceipt(file, { @@ -321,6 +333,11 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< }, }); } catch (error) { + if (ownsGate) { + while (HostLock.installChildrenAreAlive(process.pid)) + await new Promise((resolve) => setTimeout(resolve, 100)); + installerFinished = true; + } writeReceipt(file, { ...receipt, finishedAt: new Date().toISOString(), @@ -335,11 +352,11 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< }); } finally { clearInterval(heartbeat); - if (ownsGate) + if (ownsGate && installerFinished) NodeFS.rmSync(NodePath.join(receipt.directory, "gate.lock"), { recursive: true, force: true, }); - NodeFS.rmSync(lease, { force: true }); + if (!ownsGate || installerFinished) NodeFS.rmSync(lease, { force: true }); } } diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index 979b4a436eb47..aa835ec6df732 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -60,7 +60,11 @@ export const installIsAlive = (directory: string): boolean => { } if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid provider install lease"); if (isProcessAlive(pid)) return true; - // Snapshot only after an owner dies. A failed scan keeps the lease closed. + return installChildrenAreAlive(pid); +}; + +/** The installer can leave children writing after its root exits. Failed scans keep admission closed. */ +export const installChildrenAreAlive = (pid: number): boolean => { const windows = NodeOS.platform() === "win32"; const scan = windows ? NodeChildProcess.spawnSync( @@ -73,7 +77,7 @@ export const installIsAlive = (directory: string): boolean => { ], { encoding: "utf8", timeout: 10_000, windowsHide: true }, ) - : NodeChildProcess.spawnSync("ps", ["-ax", "-o", "pgid=", "-o", "stat="], { + : NodeChildProcess.spawnSync("ps", ["-ax", "-o", "pid=", "-o", "pgid=", "-o", "stat="], { encoding: "utf8", timeout: 10_000, }); @@ -92,6 +96,7 @@ export const installIsAlive = (directory: string): boolean => { if ( parent !== undefined && child !== undefined && + child !== scan.pid && descendants.has(parent) && !descendants.has(child) ) { @@ -103,8 +108,14 @@ export const installIsAlive = (directory: string): boolean => { if (descendants.size > 1) return true; } else if ( scan.stdout.split(/\r?\n/).some((line) => { - const [group, state] = line.trim().split(/\s+/); - return Number(group) === pid && state !== undefined && !state.startsWith("Z"); + const [child, group, state] = line.trim().split(/\s+/); + return ( + Number(child) !== pid && + Number(child) !== scan.pid && + Number(group) === pid && + state !== undefined && + !state.startsWith("Z") + ); }) ) return true; diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 7b4a8b4de7041..5d2fc4fcfebf2 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -2817,6 +2817,127 @@ describe("provider maintenance host leases", () => { 30_000, ); + it.live.each([ + "runtime-replaced", + "heartbeat-write-error", + "installer-killed-with-live-child", + ] as const)( + "finishes the installer after %s", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const runtime = NodePath.join(directory, "runtime"); + NodeFS.mkdirSync(NodePath.join(runtime, "provider"), { recursive: true }); + const supervisor = NodePath.join(runtime, "provider/ProviderInstallSupervisor.ts"); + NodeFS.copyFileSync( + NodePath.join(import.meta.dirname, "ProviderInstallSupervisor.ts"), + supervisor, + ); + for (const name of ["ProviderMaintenanceHostLock.ts", "providerMaintenanceRunner.ts"]) + NodeFS.symlinkSync( + NodePath.join(import.meta.dirname, name), + NodePath.join(runtime, "provider", name), + ); + NodeFS.symlinkSync( + NodePath.resolve(import.meta.dirname, "../config.ts"), + NodePath.join(runtime, "config.ts"), + ); + NodeFS.symlinkSync( + NodePath.resolve(import.meta.dirname, "../../node_modules"), + NodePath.join(runtime, "node_modules"), + ); + const file = NodePath.join(directory, "receipt.json"); + const installed = NodePath.join(directory, "installed"); + const dead = NodeChildProcess.spawn(process.execPath, ["-e", ""], { stdio: "ignore" }); + yield* Effect.promise( + () => new Promise((resolve) => dead.once("exit", () => resolve())), + ); + NodeFS.mkdirSync(NodePath.join(directory, "gate.lock")); + NodeFS.writeFileSync( + file, + JSON.stringify({ + id: "runtime-replaced", + provider: CODEX_DRIVER, + instanceId: CODEX_INSTANCE_ID, + instanceIds: [CODEX_INSTANCE_ID], + startedAt: "2026-04-10T00:00:00.000Z", + parentPid: process.pid, + directory, + command: process.execPath, + args: [ + "-e", + mode === "installer-killed-with-live-child" + ? `require("node:child_process").spawn(process.execPath, ["-e", ${JSON.stringify(`setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(installed)}, "installed"), 2500)`)}], {stdio: "ignore"}).on("spawn", () => process.kill(process.pid, "SIGKILL"))` + : `console.log("installing"); setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(installed)}, "installed"), ${mode === "heartbeat-write-error" ? 2500 : 0})`, + ], + }), + ); + for (const suffix of [".stdout", ".stderr"]) NodeFS.writeFileSync(file + suffix, ""); + const watcher = NodeFS.watch(directory, () => { + if (InstallSupervisor.readReceipt(file).pid === undefined) return; + watcher.close(); + if (mode === "runtime-replaced") + NodeFS.rmSync(NodePath.join(runtime, "provider/providerMaintenanceRunner.ts")); + NodeFS.writeFileSync(file + ".start", "runtime-replaced"); + }); + yield* Effect.addFinalizer(() => Effect.sync(() => watcher.close())); + const child = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import * as fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; + import { runInstallSubcommand } from ${JSON.stringify(supervisor)}; + ${ + mode === "heartbeat-write-error" + ? `const original = fs.default.utimesSync; let failed = false; + fs.default.utimesSync = (...args) => { if (!failed) { failed = true; throw Object.assign(new Error("full disk"), {code: "ENOSPC"}); } return original(...args); }; + syncBuiltinESMExports();` + : "" + } + await runInstallSubcommand("__provider-install", ${JSON.stringify([file, String(dead.pid)])});`, + ], + { stdio: "ignore", detached: true }, + ), + ), + (child) => + Effect.sync(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }), + ); + const code = yield* Effect.promise( + () => new Promise((resolve) => child.once("exit", resolve)), + ); + const finishedBeforeChildren = !NodeFS.existsSync(installed); + if (mode !== "runtime-replaced") + yield* Effect.callback((resume) => { + const complete = () => { + if (!NodeFS.existsSync(installed)) return; + watcher.close(); + resume(Effect.void); + }; + const watcher = NodeFS.watch(directory, complete); + complete(); + return Effect.sync(() => watcher.close()); + }); + assert.equal(code, 0); + if (mode === "installer-killed-with-live-child") { + assert.isFalse( + finishedBeforeChildren, + "lease released before installer child finished", + ); + assert.isFalse(HostLock.installIsAlive(directory)); + } else assert.equal(InstallSupervisor.readReceipt(file).result?.exitCode, 0); + assert.equal(NodeFS.readFileSync(installed, "utf8"), "installed"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + 30_000, + ); + it.live("reclaims a crashed server's turn and install leases", () => Effect.scoped( Effect.gen(function* () { From 0dbeb87d55000471d33fd9ba08811c29f944cc6f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:44:04 +0000 Subject: [PATCH 119/149] fix(server): the service launcher stops a crashed server's orphaned providers before starting another runtime --- apps/server/src/process/ownedProcesses.ts | 453 ++++++++++++++++++++++ apps/server/src/serviceLauncher.test.ts | 69 +++- apps/server/src/serviceLauncher.ts | 21 + 3 files changed, 541 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/process/ownedProcesses.ts diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts new file mode 100644 index 0000000000000..fc03f18e37b93 --- /dev/null +++ b/apps/server/src/process/ownedProcesses.ts @@ -0,0 +1,453 @@ +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off preferSchemaOverJson:off -- Standalone launcher shares this Node-only process ownership core. +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; + +const execFile = NodeUtil.promisify(NodeChildProcess.execFile); + +interface Identity { + readonly pid: number; + readonly startTime: string; +} +interface OwnedProcess extends Identity { + readonly pgid: number; +} +interface Entry { + readonly version: 1; + readonly instanceId: string; + readonly hostname: string; + readonly stateDir: string; + readonly owner: Identity; + readonly processes: ReadonlyArray; +} +interface ObservedProcess extends OwnedProcess { + readonly ppid: number; + readonly zombie: boolean; +} +export interface ProcessController { + readonly snapshot: () => ReadonlyArray | Promise>; + readonly identity?: (pid: number) => ObservedProcess | undefined; + readonly signal: (pid: number, signal: NodeJS.Signals) => void; + readonly exists: (pid: number) => boolean; +} +const same = (a: Identity, b: Identity) => a.pid === b.pid && a.startTime === b.startTime; +const object = (value: unknown): value is Record => + typeof value === "object" && value !== null; +const identity = (value: unknown): value is Identity => + object(value) && + Number.isSafeInteger(value.pid) && + Number(value.pid) > 0 && + typeof value.startTime === "string" && + value.startTime.length > 0; +const owned = (value: unknown): value is OwnedProcess => + identity(value) && "pgid" in value && Number.isSafeInteger(value.pgid); +function decodeEntry(text: string): Entry { + const value: unknown = JSON.parse(text); + if ( + !object(value) || + value.version !== 1 || + typeof value.instanceId !== "string" || + !value.instanceId || + typeof value.hostname !== "string" || + typeof value.stateDir !== "string" || + !identity(value.owner) || + !Array.isArray(value.processes) || + !value.processes.every(owned) + ) + throw new Error("Invalid provider process record"); + return value as unknown as Entry; +} +function decodeProcessTable(value: unknown): ReadonlyArray { + if ( + !Array.isArray(value) || + !value.every( + (p: unknown) => + owned(p) && + "ppid" in p && + Number.isSafeInteger(p.ppid) && + "zombie" in p && + typeof p.zombie === "boolean", + ) + ) + throw new Error("Invalid process table"); + return value as ReadonlyArray; +} + +/** Linux boot/start ticks, macOS ps and Windows CIM supply creation identities. */ +function controllerFor( + platform: NodeJS.Platform, + environment: NodeJS.ProcessEnv, +): ProcessController { + const boot = + platform === "linux" + ? NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() + : ""; + const parseLinux = (pid: number, stat: string): ObservedProcess => { + const fields = stat.slice(stat.lastIndexOf(") ") + 2).split(" "); + return { + pid, + ppid: Number(fields[1]), + pgid: Number(fields[2]), + startTime: `${boot}:${fields[19]}`, + zombie: fields[0] === "Z" || fields[0] === "X", + }; + }; + return { + ...(platform === "linux" + ? { + identity: (pid: number) => { + try { + return parseLinux(pid, NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8")); + } catch { + return undefined; + } + }, + } + : {}), + snapshot: async () => { + if (platform === "linux") { + const pids = (await NodeFSP.readdir("/proc")).filter((name) => /^[1-9][0-9]*$/.test(name)); + const table: Array = []; + // Bound concurrent reads without blocking the server on the whole process table. + for (let offset = 0; offset < pids.length; offset += 32) { + const rows = await Promise.all( + pids.slice(offset, offset + 32).map(async (name) => { + try { + return parseLinux( + Number(name), + await NodeFSP.readFile(`/proc/${name}/stat`, "utf8"), + ); + } catch { + return undefined; + } + }), + ); + for (const row of rows) if (row) table.push(row); + } + return table; + } + if (platform === "darwin") { + const { stdout: output } = await execFile( + "/bin/ps", + ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], + { + encoding: "utf8", + timeout: 5_000, + maxBuffer: 8 * 1024 * 1024, + env: { ...environment, LC_ALL: "C", TZ: "UTC" }, + }, + ); + return output.split("\n").flatMap((line) => { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); + return match + ? [ + { + pid: Number(match[1]), + ppid: Number(match[2]), + pgid: Number(match[3]), + zombie: match[4]!.startsWith("Z"), + startTime: match[5]!, + }, + ] + : []; + }); + } + if (platform === "win32") { + const { stdout: output } = await execFile( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference='Stop'; @(Get-CimInstance Win32_Process | Where-Object { $_.CreationDate -and $_.ProcessId -gt 0 } | ForEach-Object { @{pid=[int]$_.ProcessId; ppid=[int]$_.ParentProcessId; pgid=0; startTime=$_.CreationDate.ToUniversalTime().Ticks.ToString(); zombie=$false} }) | ConvertTo-Json -Compress", + ], + { + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + maxBuffer: 8 * 1024 * 1024, + env: environment, + }, + ); + return decodeProcessTable(JSON.parse(output)); + } + throw new Error(`Provider process ownership is unsupported on ${platform}`); + }, + signal: (pid, signal) => { + process.kill(pid, signal); + }, + exists: (pid) => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException).code !== "ESRCH"; + } + }, + }; +} + +function creationTime(identity: Identity): bigint | undefined { + const ticks = identity.startTime.split(":").at(-1)!; + if (/^\d+$/.test(ticks)) return BigInt(ticks); + const parsed = Date.parse(`${identity.startTime} UTC`); + return Number.isFinite(parsed) ? BigInt(parsed) : undefined; +} + +/** Admits descendants only through a live, matching identity, or its still-owned private process group. */ +function discover(entry: Entry, table: ReadonlyArray): Entry { + const found = new Map(entry.processes.map((p) => [`${p.pid}:${p.startTime}`, p])); + const live = table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); + const groups = new Set( + live + .filter( + (p) => + p.pgid > 0 && entry.processes.some((owned) => owned.pgid > 0 && owned.pgid === p.pgid), + ) + .map((p) => p.pgid), + ); + const parents = new Map(live.map((p) => [p.pid, p])); + let changed = true; + while (changed) { + changed = false; + for (const p of table) { + if (p.zombie || found.has(`${p.pid}:${p.startTime}`)) continue; + const parent = parents.get(p.ppid); + const parentBirth = parent ? creationTime(parent) : undefined; + const childBirth = creationTime(p); + const childOfParent = + parent !== undefined && + (parentBirth === undefined || childBirth === undefined || childBirth >= parentBirth); + if (!childOfParent && !groups.has(p.pgid)) continue; + found.set(`${p.pid}:${p.startTime}`, { + pid: p.pid, + startTime: p.startTime, + pgid: groups.has(p.pgid) ? p.pgid : 0, + }); + parents.set(p.pid, p); + changed = true; + } + } + return { ...entry, processes: [...found.values()] }; +} + +/** Node-only core shared with the launcher, including older-runtime fallback. */ +export async function make(input: { + readonly stateDir: string; + readonly ownerPid?: number; + readonly controller?: ProcessController; + readonly installLeaseDirectory?: string; + readonly platform: NodeJS.Platform; + readonly environment: NodeJS.ProcessEnv; + readonly hostname: string; +}) { + const { platform, environment, hostname } = input; + const controller = input.controller ?? controllerFor(platform, environment); + const directory = NodePath.join(input.stateDir, "provider-processes"); + NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const ownerPid = input.ownerPid ?? process.pid; + const owner = (await controller.snapshot()).find((p) => p.pid === ownerPid && !p.zombie); + if (!owner) throw new Error("Could not identify the provider process owner"); + const instanceId = NodeCrypto.randomUUID(); + const tracked = new Map(); + const write = (file: string, entry: Entry) => { + const temporary = `${file}.tmp`; + NodeFS.writeFileSync(temporary, JSON.stringify(entry), { mode: 0o600 }); + NodeFS.renameSync(temporary, file); + }; + const survivors = (entry: Entry, table: ReadonlyArray) => + table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); + const refresh = async () => { + if (tracked.size === 0) return; + const table = await controller.snapshot(); + for (const [file, entry] of tracked) { + const discovered = discover(entry, table); + const next = { + ...discovered, + processes: discovered.processes.filter( + (p) => + table.some((observed) => same(p, observed) && !observed.zombie) || + (!table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid)), + ), + }; + if (next.processes.length === 0) { + NodeFS.rmSync(file, { force: true }); + tracked.delete(file); + } else if (JSON.stringify(next.processes) !== JSON.stringify(entry.processes)) { + write(file, next); + tracked.set(file, next); + } + } + }; + const signal = async (p: Identity, signal: NodeJS.Signals) => { + // Refresh immediately before each signal, including escalation. Never signal by group id or taskkill /t. + const observed = controller.identity + ? controller.identity(p.pid) + : (await controller.snapshot()).find((candidate) => candidate.pid === p.pid); + if (!observed || observed.zombie || !same(p, observed)) return; + try { + controller.signal(p.pid, signal); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "ESRCH") throw cause; + } + }; + // A server started from a provider or terminal can descend from it with its own + // detached install supervisor. Stopping that group would corrupt a shared CLI. + const withoutInstaller = (entry: Entry, table: ReadonlyArray): Entry => { + if (input.installLeaseDirectory === undefined) return entry; + let leader: number; + try { + leader = Number( + NodeFS.readFileSync(NodePath.join(input.installLeaseDirectory, "install.json"), "utf8"), + ); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return entry; + throw cause; + } + if (!Number.isSafeInteger(leader) || leader <= 0) + throw new Error("Invalid provider install lease"); + // Windows has no process groups, so its installer is found through parents. + const installer = new Set([leader]); + let changed = true; + while (changed) { + changed = false; + for (const p of table) { + if (!installer.has(p.pid) && (p.pgid === leader || installer.has(p.ppid))) { + installer.add(p.pid); + changed = true; + } + } + } + return { + ...entry, + processes: entry.processes.filter( + (p) => !table.some((observed) => installer.has(observed.pid) && same(p, observed)), + ), + }; + }; + const reap = async (file: string, entry: Entry) => { + let table = await controller.snapshot(); + const liveOwner = table.find((p) => same(entry.owner, p)); + if (liveOwner && !liveOwner.zombie) return; + // A restricted/failed process query cannot establish that an owner died. + if ( + !liveOwner && + !table.some((p) => p.pid === entry.owner.pid) && + controller.exists(entry.owner.pid) + ) { + throw new Error("Could not verify the recorded provider owner"); + } + let next = withoutInstaller(discover(entry, table), table); + if (platform !== "win32") { + // Freeze to a fixed point. A child may fork while its parent is being stopped. + const stopped = new Set(); + for (let attempt = 0; attempt < 100; attempt++) { + const pending = survivors(next, table).filter( + (p) => !stopped.has(`${p.pid}:${p.startTime}`), + ); + if (pending.length === 0) break; + for (const p of pending) { + await signal(p, "SIGSTOP"); + stopped.add(`${p.pid}:${p.startTime}`); + } + table = await controller.snapshot(); + next = withoutInstaller(discover(next, table), table); + if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); + } + } + // Persist newly discovered children before signalling: recovery can itself be killed. + write(file, next); + for (const p of survivors(next, await controller.snapshot()).toReversed()) + await signal(p, "SIGKILL"); + for (let attempt = 0; attempt < 100; attempt++) { + table = await controller.snapshot(); + if ( + next.processes.some( + (p) => !table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid), + ) + ) { + throw new Error("Could not verify a recorded provider process"); + } + if (survivors(next, table).length === 0) { + NodeFS.rmSync(file, { force: true }); + return; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("Recorded provider processes did not stop"); + }; + // Await every reap before exposing the spawn hook, and therefore before restart recovery launches work. + for (const name of NodeFS.readdirSync(directory)) { + if (!/^[\da-f-]+\.json$/.test(name)) continue; + const file = NodePath.join(directory, name); + let entry: Entry; + try { + entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); + } catch { + // Records are not fsynced, so power loss can leave an empty one. It names + // no process to reap, and keeping it would fail every later startup. + NodeFS.rmSync(file, { force: true }); + continue; + } + if (entry.hostname !== hostname || entry.stateDir !== input.stateDir) { + NodeFS.rmSync(file, { force: true }); + continue; + } + await reap(file, entry); + } + let pendingSnapshot: Promise> | undefined; + const spawnSnapshot = () => { + pendingSnapshot ??= Promise.resolve() + .then(() => controller.snapshot()) + .finally(() => { + pendingSnapshot = undefined; + }); + return pendingSnapshot; + }; + const track = async (pid: number) => { + const direct = controller.identity?.(pid); + const table = direct && !direct.zombie ? [direct] : await spawnSnapshot(); + const root = + table.find((p) => p.pid === pid && !p.zombie) ?? + table.find((p) => p.pgid === pid && !p.zombie); + if (!root) return; + // Process groups shared with the server are never a source of ownership. + let entry = discover( + { + version: 1, + instanceId, + hostname, + stateDir: input.stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes: [ + { pid: root.pid, startTime: root.startTime, pgid: root.pgid === pid ? pid : 0 }, + ], + }, + table, + ); + const file = NodePath.join(directory, `${instanceId}-${pid}.json`); + try { + write(file, entry); + tracked.set(file, entry); + if (direct) { + // Persist the root before awaiting I/O, then capture its initial children. + entry = discover(entry, await spawnSnapshot()); + write(file, entry); + tracked.set(file, entry); + } + } catch (cause) { + // The first root write may fail before the initial descendant query. + try { + entry = discover(entry, await spawnSnapshot()); + } catch { + /* Still stop the identities already established. */ + } + for (const child of entry.processes.toReversed()) await signal(child, "SIGKILL"); + throw cause; + } + }; + return { track, refresh }; +} diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 5781fc0e3926a..eb9437957167d 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -210,6 +210,8 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { it.effect.each([ "unchanged", + "orphan", + "orphan-restart", "restart", "clock-jump", "before-prepared", @@ -217,12 +219,20 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { "wal-migration", "unreadable", "untracked-ddl", + "reverted-migration", + "same-count-schema", + "same-count-ledger", ] as const)( "supervises committed crash loops with schema %s without restoring accepted work", (schema) => Effect.gen(function* () { const compatible = - schema === "unchanged" || schema === "restart" || schema === "clock-jump"; + schema === "unchanged" || + schema === "restart" || + schema === "clock-jump" || + schema === "orphan" || + schema === "orphan-restart" || + schema === "reverted-migration"; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-crash-loop-" }); @@ -242,11 +252,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }; const migrate = "db.exec(\"INSERT INTO effect_sql_migrations VALUES (2, 'candidate'); CREATE TABLE candidate_data (id TEXT)\");"; + const orphans = schema === "orphan" || schema === "orphan-restart"; + const processes = path.join(root, "userdata", "provider-processes"); const source = ` const fs = require("node:fs"); const { DatabaseSync } = require("node:sqlite"); const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); fs.appendFileSync(${JSON.stringify(path.join(root, "starts"))}, context.childVersion + "\\n"); +${ + orphans + ? ` +if (context.childVersion === "1.1.0") { + const child = require("node:child_process").spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + child.unref(); + const boot = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + const identity = pid => ({ pid, startTime: boot + ":" + fs.readFileSync("/proc/" + pid + "/stat", "utf8").split(") ").at(-1).split(" ")[19] }); + fs.mkdirSync(${JSON.stringify(processes)}, { recursive: true }); + fs.writeFileSync(${JSON.stringify(processes)} + "/" + process.pid + ".json", JSON.stringify({ version: 1, instanceId: "candidate", hostname: require("node:os").hostname(), stateDir: ${JSON.stringify(path.dirname(processes))}, owner: identity(process.pid), processes: [{ ...identity(child.pid), pgid: child.pid }] })); + fs.appendFileSync(${JSON.stringify(path.join(root, "orphans"))}, child.pid + "\\n"); +} +` + : "" +} if (context.update.status === "rolled-back" || context.update.status === "failed") { const ready = ${JSON.stringify(path.join(root, "ready"))}; fs.writeFileSync(ready + ".staging", JSON.stringify(context)); @@ -263,6 +290,9 @@ if (context.update.status === "rolled-back" || context.update.status === "failed ${schema === "wal-migration" ? 'db.exec("PRAGMA journal_mode=WAL");' : ""} db.exec("INSERT INTO receipts VALUES ('accepted')"); ${schema === "after-commit" || schema === "wal-migration" ? migrate : schema === "unreadable" ? 'db.exec("DROP TABLE effect_sql_migrations");' : schema === "untracked-ddl" ? 'db.exec("CREATE TABLE untracked (id TEXT)");' : ""} + ${schema === "reverted-migration" ? "db.exec(\"INSERT INTO effect_sql_migrations VALUES (2, 'candidate'); CREATE TABLE candidate_data (id TEXT); DROP TABLE candidate_data; DELETE FROM effect_sql_migrations WHERE migration_id = 2\");" : ""} + ${schema === "same-count-schema" ? 'db.exec("ALTER TABLE receipts ADD COLUMN candidate TEXT");' : ""} + ${schema === "same-count-ledger" ? "db.exec(\"UPDATE effect_sql_migrations SET name = 'replacement' WHERE migration_id = 1\");" : ""} ${schema === "wal-migration" ? "" : "db.close();"} process.exit(42); }); @@ -285,6 +315,21 @@ if (context.update.status === "rolled-back" || context.update.status === "failed update: pending, }), ); + // Ensure a failing assertion also cleans up every child spawned by this case. + if (orphans) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + const file = path.join(root, "orphans"); + if (!NodeFS.existsSync(file)) return; + for (const pid of NodeFS.readFileSync(file, "utf8").trim().split("\n").map(Number)) { + try { + process.kill(pid, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + }), + ); const ready = Promise.withResolvers(); yield* Effect.acquireRelease( Effect.sync(() => @@ -301,7 +346,7 @@ if (context.update.status === "rolled-back" || context.update.status === "failed (clock) => Effect.sync(() => clock.mockRestore()), ); } - if (schema === "restart") { + if (schema === "restart" || schema === "orphan-restart") { const crashed = Promise.withResolvers(); yield* Effect.acquireRelease( Effect.sync(() => @@ -341,6 +386,26 @@ if (context.update.status === "rolled-back" || context.update.status === "failed () => Effect.promise(() => launcher.stop("SIGTERM")), ); yield* Effect.promise(() => ready.promise); + if (orphans) { + const pids = (yield* fs.readFileString(path.join(root, "orphans"))) + .trim() + .split("\n") + .map(Number); + assert.lengthOf(pids, 3); + for (const pid of pids) { + let stat: string | undefined; + try { + stat = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + assert.isTrue( + stat === undefined || stat.split(") ").at(-1)!.startsWith("Z "), + `orphan ${pid} survived previous-runtime fallback`, + ); + } + assert.deepEqual(NodeFS.readdirSync(processes), []); + } const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, compatible ? "1.0.0" : "1.1.0"); assert.equal(state.update?.status, compatible ? "rolled-back" : "failed"); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index f1ee8fd5068d9..0b85bd262c284 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -9,6 +9,7 @@ import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; +import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; import type { @@ -31,6 +32,8 @@ import { SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; +import * as OwnedProcesses from "./process/ownedProcesses.ts"; + const HANDOFF_DELAY_MS = 2_000; const PREPARED_TIMEOUT_MS = 120_000; const TERMINATE_GRACE_MS = 5_000; @@ -492,8 +495,25 @@ export class Launcher { } } + async #reapProviders(): Promise { + await OwnedProcesses.make({ + stateDir: NodePath.join(this.#baseDir, "userdata"), + // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. + platform: process.platform, + environment: process.env, + hostname: NodeOS.hostname(), + installLeaseDirectory: NodePath.join( + process.env.XDG_CACHE_HOME || NodePath.join(NodeOS.homedir(), ".cache"), + "t3", + "provider-maintenance", + ), + }); + } + async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise { if (this.#stopping) return; + // The previous runtime may predate reaping, including after a launcher restart. + await this.#reapProviders(); if (!(await runtimeExists(this.#baseDir, version))) { throw new Error(`Selected t3@${version} runtime is missing or incomplete.`); } @@ -772,6 +792,7 @@ export class Launcher { this.#child = null; await terminateChild(child.process); } + await this.#reapProviders(); await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); const { rollback: _rollback, ...state } = this.#state; From 0b6bfb8c89948f9f8923594d491a7b3f445e9db2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:44:09 +0000 Subject: [PATCH 120/149] fix(client-runtime): a failed or rolled-back background server update stays visible, and a manual retry error takes priority --- .../client-runtime/src/state/server.test.ts | 78 +++++++++++++++++++ packages/client-runtime/src/state/server.ts | 57 +++++++++++--- 2 files changed, 124 insertions(+), 11 deletions(-) diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index 72bb028ac9b67..f11ce13a884b3 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -122,6 +122,84 @@ it("treats an announced background restart as resuming and clears it on ready, i ).toBeUndefined(); }); +it.each(["rolled-back", "failed"] as const)( + "keeps a background %s outcome visible after reconnect until the next successful update", + (status) => { + const active = {} as RpcSession; + const stale = {} as RpcSession; + const outcome = { + id: "background", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status, + reason: "candidate-crash-loop", + }; + const state = { currentSession: active, welcomeSession: active, welcome: null }; + const ready = applyServerWelcomeEvent(state, active, { + type: "ready", + payload: { updateOutcome: outcome }, + }); + const version = status === "failed" ? "1.1.0" : "1.0.0"; + expect( + serverUpdateStateForServerVersion( + serverUpdateStateForAnnouncedRestart( + { status: "idle" }, + undefined, + ready.updateOutcome, + version, + ), + version, + ), + ).toEqual({ + status: "failed", + stage: "resuming", + fromVersion: version, + targetVersion: "1.1.0", + message: "candidate-crash-loop", + }); + expect( + serverUpdateStateForAnnouncedRestart({ status: "idle" }, undefined, outcome, version), + ).toBe(serverUpdateStateForAnnouncedRestart({ status: "idle" }, undefined, outcome, version)); + expect( + applyServerWelcomeEvent(state, stale, { + type: "ready", + payload: { updateOutcome: outcome }, + }), + ).toBe(state); + const running = { + status: "running", + stage: "installing", + fromVersion: version, + targetVersion: "1.2.0", + } as const; + expect(serverUpdateStateForAnnouncedRestart(running, undefined, outcome, version)).toBe( + running, + ); + const manualFailure = { + status: "failed", + stage: "installing", + fromVersion: version, + targetVersion: outcome.targetVersion, + message: "manual retry failed", + } as const; + expect(serverUpdateStateForAnnouncedRestart(manualFailure, undefined, outcome, version)).toBe( + manualFailure, + ); + const success = applyServerWelcomeEvent(ready, active, { + type: "ready", + payload: { updateOutcome: { ...outcome, status: "committed", targetVersion: "1.2.0" } }, + }); + expect( + serverUpdateStateForAnnouncedRestart( + { status: "idle" }, + undefined, + success.updateOutcome, + "1.2.0", + ), + ).toEqual({ status: "idle" }); + }, +); + describe("update restart reconnect nudges", () => { it.effect("retries a desktop commit that was lost before delivery", () => Effect.gen(function* () { diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 08f1e6f17f326..806a63dfc6bc0 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -7,6 +7,7 @@ import { type ServerLifecycleStreamUpdateRestartEvent, type ServerSelfUpdateProgressEvent, type ServerSelfUpdateResult, + type ServerSelfUpdateOutcome, WS_METHODS, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -309,18 +310,42 @@ export function serverUpdateStateForServerVersion( : IDLE_SERVER_UPDATE_STATE; } +const updateOutcomeStates = new WeakMap(); + export function serverUpdateStateForAnnouncedRestart( state: ServerUpdateState, restart: ServerLifecycleStreamUpdateRestartEvent["payload"] | undefined, + outcome?: ServerSelfUpdateOutcome, + serverVersion: string | null = null, ): ServerUpdateState { - return restart !== undefined && state.status !== "running" - ? { - status: "running", - stage: "resuming", - fromVersion: restart.fromVersion, - targetVersion: restart.targetVersion, - } - : state; + if (restart !== undefined && state.status !== "running") + return { + status: "running", + stage: "resuming", + fromVersion: restart.fromVersion, + targetVersion: restart.targetVersion, + }; + if ( + state.status === "running" || + outcome === undefined || + outcome.status === "committed" || + (serverVersion !== outcome.fromVersion && serverVersion !== outcome.targetVersion) || + state.status === "failed" + ) + return state; + const cached = updateOutcomeStates.get(outcome); + if (cached?.status === "failed" && cached.fromVersion === serverVersion) return cached; + const failure: ServerUpdateState = { + status: "failed", + stage: "resuming", + fromVersion: serverVersion, + targetVersion: outcome.targetVersion, + message: outcome.reason ?? `The t3@${outcome.targetVersion} update ${outcome.status}.`, + }; + // Existing notice dismissal tracks the state object, so config refreshes must + // retain it until a different launcher outcome arrives. + updateOutcomeStates.set(outcome, failure); + return failure; } export function serverUpdateFailureMessage(error: unknown): string { @@ -510,8 +535,12 @@ export function applyServerWelcomeEvent( restart: event.payload as ServerLifecycleStreamUpdateRestartEvent["payload"], }; } - if (event.type === "ready") - return current.restart === undefined ? current : { ...current, restart: undefined }; + if (event.type === "ready") { + const payload = event.payload as ServerLifecycleStreamReadyEvent["payload"]; + return current.restart === undefined && current.updateOutcome === payload.updateOutcome + ? current + : { ...current, restart: undefined, updateOutcome: payload.updateOutcome }; + } return event.type === "welcome" && current.currentSession === session ? { ...current, @@ -526,6 +555,7 @@ export interface EnvironmentServerWelcomeState { readonly welcomeSession: RpcSession | null; readonly welcome: ServerLifecycleWelcomePayload | null; readonly restart?: ServerLifecycleStreamUpdateRestartEvent["payload"] | undefined; + readonly updateOutcome?: ServerSelfUpdateOutcome | undefined; } export function resolveServerWelcomeState( @@ -724,7 +754,12 @@ export function createServerEnvironmentAtoms( const lifecycle = Option.getOrNull( AsyncResult.value(get(welcomeStateFamily(environmentId))), ); - return serverUpdateStateForAnnouncedRestart(manual, lifecycle?.restart); + return serverUpdateStateForAnnouncedRestart( + manual, + lifecycle?.restart, + lifecycle?.updateOutcome, + get(configValueAtom(environmentId))?.environment.serverVersion ?? null, + ); })(), get(configValueAtom(environmentId))?.environment.serverVersion ?? null, ), From 7854446669a410d7016fa8beb1d977d399d017b8 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:44:09 +0000 Subject: [PATCH 121/149] fix(server): a provider cleanup failure no longer stops the service launcher --- apps/server/src/serviceLauncher.test.ts | 44 +++++++++++++++++++++++++ apps/server/src/serviceLauncher.ts | 3 ++ 2 files changed, 47 insertions(+) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index eb9437957167d..a50207ed25f3d 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -208,6 +208,50 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect("starts the server when orphaned provider cleanup cannot run", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-reap-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const leases = path.join(root, "cache", "t3", "provider-maintenance"); + yield* fs.makeDirectory(leases, { recursive: true }); + yield* fs.writeFileString(path.join(leases, "install.json"), "corrupt"); + yield* Effect.acquireRelease( + Effect.sync(() => vi.stubEnv("XDG_CACHE_HOME", path.join(root, "cache"))), + () => Effect.sync(() => vi.unstubAllEnvs()), + ); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + `require("node:fs").writeFileSync(${JSON.stringify(path.join(root, "started"))}, "");\nsetInterval(() => {}, 1_000);\n`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const started = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + NodeFS.watch(root, (_event, name) => { + if (name === "started") started.resolve(); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), + ); + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const running = launcher.run(); + // A failed launcher would otherwise leave the child promise pending forever. + yield* Effect.promise(() => Promise.race([started.promise, running])); + assert.isTrue(yield* fs.exists(path.join(root, "started"))); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }), + ); + it.effect.each([ "unchanged", "orphan", diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 0b85bd262c284..e7f8fc7037027 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -507,6 +507,9 @@ export class Launcher { "t3", "provider-maintenance", ), + }).catch(() => { + // Never let cleanup stop the launcher. A newer child repeats it and a + // failing child stays in crash-loop recovery, which keeps the service up. }); } From 8507ed64497bc20e52f8d0d08034f91bbcd86cad Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:57:56 +0000 Subject: [PATCH 122/149] fix(server): pull request review guides wait for provider installs like other text generation --- .../src/textGeneration/TextGeneration.test.ts | 109 ++++++++++-------- .../src/textGeneration/TextGeneration.ts | 4 +- 2 files changed, 66 insertions(+), 47 deletions(-) diff --git a/apps/server/src/textGeneration/TextGeneration.test.ts b/apps/server/src/textGeneration/TextGeneration.test.ts index f61b471c3d341..9699cdc48a7a3 100644 --- a/apps/server/src/textGeneration/TextGeneration.test.ts +++ b/apps/server/src/textGeneration/TextGeneration.test.ts @@ -239,50 +239,69 @@ it.effect("holds the host lease until the text generation provider completes", ( }), ); -it.effect("holds shared admission through CLI-backed text generation", () => - Effect.gen(function* () { - const admission = yield* Maintenance.ProviderMaintenanceAdmission; - const instanceId = ProviderInstanceId.make("codex"); - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - const instance = makeStubInstance( - instanceId, - makeStubTextGeneration({ - generateBranchName: () => - Deferred.succeed(entered, undefined).pipe( - Effect.andThen(Deferred.await(release)), - Effect.as({ branch: "generated" }), - ), - }), - ); - const generation = yield* TextGeneration.make.pipe( - Effect.provideService( - ProviderInstanceRegistry.ProviderInstanceRegistry, - makeStubRegistry([instance]), - ), - Effect.provide( - Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ - resolveLink: () => undefined, +it.effect.each(["generateBranchName", "generateReviewGuide"] as const)( + "holds shared admission through CLI-backed %s", + (operation) => + Effect.gen(function* () { + const admission = yield* Maintenance.ProviderMaintenanceAdmission; + const instanceId = ProviderInstanceId.make("codex"); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const instance = makeStubInstance( + instanceId, + makeStubTextGeneration({ + generateBranchName: () => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as({ branch: "generated" }), + ), + generateReviewGuide: () => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as({ stops: [] }), + ), }), - ), - ); - const running = yield* generation - .generateBranchName({ - cwd: "/workspace", - message: "make a branch", - modelSelection: createModelSelection(instanceId, "gpt-5"), - }) - .pipe(Effect.forkScoped); - yield* Deferred.await(entered); - // Launches share admission, so a long generation never delays a turn start. - expect(yield* admission.withLaunchPermit(Effect.succeed("turn"))).toBe("turn"); - const install = yield* admission - .withPermit(Effect.succeed("installed")) - .pipe(Effect.forkScoped); - yield* Effect.yieldNow; - expect(install.pollUnsafe() === undefined).toBe(true); - yield* Deferred.succeed(release, undefined); - expect(yield* Fiber.join(running)).toEqual({ branch: "generated" }); - expect(yield* Fiber.join(install)).toBe("installed"); - }), + ); + const generation = yield* TextGeneration.make.pipe( + Effect.provideService( + ProviderInstanceRegistry.ProviderInstanceRegistry, + makeStubRegistry([instance]), + ), + Effect.provide( + Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ + resolveLink: () => undefined, + }), + ), + ); + const modelSelection = createModelSelection(instanceId, "gpt-5"); + const running = yield* ( + operation === "generateBranchName" + ? generation + .generateBranchName({ cwd: "/workspace", message: "make a branch", modelSelection }) + .pipe(Effect.as("generated")) + : generation + .generateReviewGuide({ + cwd: "/workspace", + number: 1, + title: "Guide", + description: "", + baseBranch: "main", + baseCommit: "base", + settledReview: "", + modelSelection, + }) + .pipe(Effect.as("generated")) + ).pipe(Effect.forkScoped); + yield* Deferred.await(entered); + // Launches share admission, so a long generation never delays a turn start. + expect(yield* admission.withLaunchPermit(Effect.succeed("turn"))).toBe("turn"); + const install = yield* admission + .withPermit(Effect.succeed("installed")) + .pipe(Effect.forkScoped); + yield* Effect.yieldNow; + expect(install.pollUnsafe() === undefined).toBe(true); + yield* Deferred.succeed(release, undefined); + expect(yield* Fiber.join(running)).toBe("generated"); + expect(yield* Fiber.join(install)).toBe("installed"); + }), ); diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index 2364d60aca0d9..59b847eaf9264 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -82,8 +82,8 @@ export const make = Effect.gen(function* () { Effect.flatMap((textGeneration) => launch(textGeneration.generateBranchName(input))), ), generateReviewGuide: (input) => - resolveInstance(registry, "generateReviewGuide", input.modelSelection.instanceId).pipe( - Effect.flatMap((textGeneration) => textGeneration.generateReviewGuide(input)), + instanceFor("generateReviewGuide", input.modelSelection.instanceId).pipe( + Effect.flatMap((textGeneration) => launch(textGeneration.generateReviewGuide(input))), ), generateThreadTitle: (input) => instanceFor("generateThreadTitle", input.modelSelection.instanceId).pipe( From 6452e91eee6ed5fc7193758d9e18286e4e08eac2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:59:04 +0000 Subject: [PATCH 123/149] docs: describe the launcher stopping orphaned providers before it starts a runtime --- docs/internals/server-updates.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 0d804acf9210d..2a948c72ca33b 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -33,7 +33,9 @@ is never restored: commit opens the gates, so a restore would erase accepted wor If the committed target exits three times within five minutes, the launcher returns to the previous runtime only when the migration ledger and schema still match the pre-trial database. Otherwise it keeps restarting the target with backoff and -records the update as failed. +records the update as failed. Before it starts any runtime, the launcher stops the +providers a crashed server recorded, because an older runtime cannot. A cleanup +failure never stops the launcher; a runtime with cleanup repeats it. ## Database rollback From ab709bbeece73dadf59d49319b38b7d70ba7aae8 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:00:47 +0000 Subject: [PATCH 124/149] fix(server): orphan cleanup samples processes without blocking the server and never stops another environment's server or launcher --- .../src/process/ProviderProcessLedger.ts | 391 +----------------- apps/server/src/process/agentScope.test.ts | 281 ++++++++++++- apps/server/src/process/ownedProcesses.ts | 147 +++++-- 3 files changed, 411 insertions(+), 408 deletions(-) diff --git a/apps/server/src/process/ProviderProcessLedger.ts b/apps/server/src/process/ProviderProcessLedger.ts index dca592fe6a7f1..c0d49ac49222c 100644 --- a/apps/server/src/process/ProviderProcessLedger.ts +++ b/apps/server/src/process/ProviderProcessLedger.ts @@ -1,395 +1,30 @@ -// @effect-diagnostics nodeBuiltinImport:off -- Process ownership survives server crashes. -import * as NodeChildProcess from "node:child_process"; -import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; -import * as NodeUtil from "node:util"; - -import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; -import * as Schema from "effect/Schema"; - -import { installLeasePath } from "../provider/ProviderMaintenanceHostLock.ts"; - -const execFile = NodeUtil.promisify(NodeChildProcess.execFile); - -const Identity = Schema.Struct({ - pid: Schema.Int.check(Schema.isGreaterThan(0)), - startTime: Schema.NonEmptyString, -}); -const OwnedProcess = Schema.Struct({ ...Identity.fields, pgid: Schema.Int }); -const Entry = Schema.Struct({ - version: Schema.Literal(1), - instanceId: Schema.NonEmptyString, - hostname: Schema.String, - stateDir: Schema.String, - owner: Identity, - processes: Schema.Array(OwnedProcess), -}); -type Identity = typeof Identity.Type; -type Entry = typeof Entry.Type; -interface ObservedProcess extends Identity { - readonly ppid: number; - readonly pgid: number; - readonly zombie: boolean; -} - -export interface ProcessController { - readonly snapshot: () => ReadonlyArray | Promise>; - readonly identity?: (pid: number) => ObservedProcess | undefined; - readonly signal: (pid: number, signal: NodeJS.Signals) => void; - readonly exists: (pid: number) => boolean; -} -const same = (a: Identity, b: Identity) => a.pid === b.pid && a.startTime === b.startTime; -const decodeEntry = Schema.decodeUnknownSync(Schema.fromJsonString(Entry)); -const decodeProcessTable = Schema.decodeUnknownSync( - Schema.Array( - Schema.Struct({ - ...Identity.fields, - ppid: Schema.Int, - pgid: Schema.Int, - zombie: Schema.Boolean, - }), - ), -); - -/** Uses the ACP Linux identities; macOS ps and Windows CIM supply the same creation identity. */ -function controllerFor( - platform: NodeJS.Platform, - environment: NodeJS.ProcessEnv, -): ProcessController { - const linux = - platform === "linux" ? AcpSessionRuntime.makePosixProcessTreeController(platform) : undefined; - // Start ticks repeat after reboot. Including boot id keeps power-loss records safe. - const boot = linux ? NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() : ""; - return { - ...(linux - ? { - identity: (pid: number) => { - const p = linux.identity(pid); - return p - ? { - pid, - ppid: p.ppid, - pgid: p.pgid, - startTime: `${boot}:${p.startTime}`, - zombie: AcpSessionRuntime.posixProcessIsZombie(p), - } - : undefined; - }, - } - : {}), - snapshot: async () => { - if (linux) - return linux.snapshot().map((p) => ({ - pid: p.pid, - ppid: p.ppid, - pgid: p.pgid, - startTime: `${boot}:${p.startTime}`, - zombie: AcpSessionRuntime.posixProcessIsZombie(p), - })); - if (platform === "darwin") { - const { stdout: output } = await execFile( - "/bin/ps", - ["-axo", "pid=,ppid=,pgid=,stat=,lstart="], - { - encoding: "utf8", - timeout: 5_000, - maxBuffer: 8 * 1024 * 1024, - env: { ...environment, LC_ALL: "C", TZ: "UTC" }, - }, - ); - return output.split("\n").flatMap((line) => { - const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line); - return match - ? [ - { - pid: Number(match[1]), - ppid: Number(match[2]), - pgid: Number(match[3]), - zombie: match[4]!.startsWith("Z"), - startTime: match[5]!, - }, - ] - : []; - }); - } - if (platform === "win32") { - const output = NodeChildProcess.execFileSync( - "powershell.exe", - [ - "-NoProfile", - "-NonInteractive", - "-Command", - "$ErrorActionPreference='Stop'; @(Get-CimInstance Win32_Process | Where-Object { $_.CreationDate -and $_.ProcessId -gt 0 } | ForEach-Object { @{pid=[int]$_.ProcessId; ppid=[int]$_.ParentProcessId; pgid=0; startTime=$_.CreationDate.ToUniversalTime().Ticks.ToString(); zombie=$false} }) | ConvertTo-Json -Compress", - ], - { encoding: "utf8", timeout: 10_000, windowsHide: true, maxBuffer: 8 * 1024 * 1024 }, - ); - return decodeProcessTable(JSON.parse(output)); - } - throw new Error(`Provider process ownership is unsupported on ${platform}`); - }, - signal: (pid, signal) => { - process.kill(pid, signal); - }, - exists: (pid) => { - try { - process.kill(pid, 0); - return true; - } catch (cause) { - return (cause as NodeJS.ErrnoException).code !== "ESRCH"; - } - }, - }; -} - -function creationTime(identity: Identity): bigint | undefined { - const ticks = identity.startTime.split(":").at(-1)!; - if (/^\d+$/.test(ticks)) return BigInt(ticks); - const parsed = Date.parse(`${identity.startTime} UTC`); - return Number.isFinite(parsed) ? BigInt(parsed) : undefined; -} - -/** Admits descendants only through a live, matching identity, or its still-owned private process group. */ -function discover(entry: Entry, table: ReadonlyArray): Entry { - const found = new Map(entry.processes.map((p) => [`${p.pid}:${p.startTime}`, p])); - const live = table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); - const groups = new Set( - live - .filter( - (p) => - p.pgid > 0 && entry.processes.some((owned) => owned.pgid > 0 && owned.pgid === p.pgid), - ) - .map((p) => p.pgid), - ); - const parents = new Map(live.map((p) => [p.pid, p])); - let changed = true; - while (changed) { - changed = false; - for (const p of table) { - if (p.zombie || found.has(`${p.pid}:${p.startTime}`)) continue; - const parent = parents.get(p.ppid); - const parentBirth = parent ? creationTime(parent) : undefined; - const childBirth = creationTime(p); - const childOfParent = - parent !== undefined && - (parentBirth === undefined || childBirth === undefined || childBirth >= parentBirth); - if (!childOfParent && !groups.has(p.pgid)) continue; - found.set(`${p.pid}:${p.startTime}`, { - pid: p.pid, - startTime: p.startTime, - pgid: groups.has(p.pgid) ? p.pgid : 0, - }); - parents.set(p.pid, p); - changed = true; - } - } - return { ...entry, processes: [...found.values()] }; -} +import * as OwnedProcesses from "./ownedProcesses.ts"; +export type { ProcessController } from "./ownedProcesses.ts"; /** One instance's local children only. External servers, launchers and install supervisors never register. */ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { readonly stateDir: string; readonly ownerPid?: number; - readonly controller?: ProcessController; - /** Host install lease directory. Its supervisor's process group is never stopped. */ + readonly controller?: OwnedProcesses.ProcessController; readonly installLeaseDirectory?: string; }) { const platform = yield* HostProcess.Platform; const environment = yield* HostProcess.Environment; - const controller = input.controller ?? controllerFor(platform, environment); - const directory = NodePath.join(input.stateDir, "provider-processes"); - NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); const hostname = yield* HostProcess.Hostname; - const ownerPid = input.ownerPid ?? process.pid; - const owner = (yield* Effect.promise(async () => controller.snapshot())).find( - (p) => p.pid === ownerPid && !p.zombie, + const ledger = yield* Effect.acquireRelease( + Effect.promise(() => + OwnedProcesses.make({ ...input, registerServer: true, platform, environment, hostname }), + ), + (ledger) => Effect.sync(ledger.release), ); - if (!owner) throw new Error("Could not identify the provider process owner"); - const instanceId = NodeCrypto.randomUUID(); - const tracked = new Map(); - const write = (file: string, entry: Entry) => { - const temporary = `${file}.tmp`; - NodeFS.writeFileSync(temporary, JSON.stringify(entry), { mode: 0o600 }); - NodeFS.renameSync(temporary, file); - }; - const survivors = (entry: Entry, table: ReadonlyArray) => - table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); - const refresh = async () => { - if (tracked.size === 0) return; - const table = await controller.snapshot(); - for (const [file, entry] of tracked) { - const discovered = discover(entry, table); - const next = { - ...discovered, - processes: discovered.processes.filter( - (p) => - table.some((observed) => same(p, observed) && !observed.zombie) || - (!table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid)), - ), - }; - if (next.processes.length === 0) { - NodeFS.rmSync(file, { force: true }); - tracked.delete(file); - } else if (JSON.stringify(next.processes) !== JSON.stringify(entry.processes)) { - write(file, next); - tracked.set(file, next); - } - } - }; - const signal = async (p: Identity, signal: NodeJS.Signals) => { - // Refresh immediately before each signal, including escalation. Never signal by group id or taskkill /t. - const observed = controller.identity - ? controller.identity(p.pid) - : (await controller.snapshot()).find((candidate) => candidate.pid === p.pid); - if (!observed || observed.zombie || !same(p, observed)) return; - try { - controller.signal(p.pid, signal); - } catch (cause) { - if ((cause as NodeJS.ErrnoException).code !== "ESRCH") throw cause; - } - }; - // A server started from a provider or terminal can descend from it with its own - // detached install supervisor. Stopping that group would corrupt a shared CLI. - const withoutInstaller = (entry: Entry, table: ReadonlyArray): Entry => { - if (input.installLeaseDirectory === undefined) return entry; - let leader: number; - try { - leader = Number(NodeFS.readFileSync(installLeasePath(input.installLeaseDirectory), "utf8")); - } catch { - return entry; - } - if (!Number.isSafeInteger(leader) || leader <= 0) return entry; - // Windows has no process groups, so its installer is found through parents. - const installer = new Set([leader]); - let changed = true; - while (changed) { - changed = false; - for (const p of table) { - if (!installer.has(p.pid) && (p.pgid === leader || installer.has(p.ppid))) { - installer.add(p.pid); - changed = true; - } - } - } - return { - ...entry, - processes: entry.processes.filter( - (p) => !table.some((observed) => installer.has(observed.pid) && same(p, observed)), - ), - }; - }; - const reap = (file: string, entry: Entry) => - Effect.gen(function* () { - let table = yield* Effect.promise(async () => controller.snapshot()); - const liveOwner = table.find((p) => same(entry.owner, p)); - if (liveOwner && !liveOwner.zombie) return; - // A restricted/failed process query cannot establish that an owner died. - if ( - !liveOwner && - !table.some((p) => p.pid === entry.owner.pid) && - controller.exists(entry.owner.pid) - ) { - throw new Error("Could not verify the recorded provider owner"); - } - let next = withoutInstaller(discover(entry, table), table); - if (platform !== "win32") { - // Freeze to a fixed point. A child may fork while its parent is being stopped. - const stopped = new Set(); - for (let attempt = 0; attempt < 100; attempt++) { - const pending = survivors(next, table).filter( - (p) => !stopped.has(`${p.pid}:${p.startTime}`), - ); - if (pending.length === 0) break; - for (const p of pending) { - yield* Effect.promise(() => signal(p, "SIGSTOP")); - stopped.add(`${p.pid}:${p.startTime}`); - } - table = yield* Effect.promise(async () => controller.snapshot()); - next = withoutInstaller(discover(next, table), table); - if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); - } - } - // Persist newly discovered children before signalling: recovery can itself be killed. - write(file, next); - for (const p of survivors( - next, - yield* Effect.promise(async () => controller.snapshot()), - ).toReversed()) - yield* Effect.promise(() => signal(p, "SIGKILL")); - for (let attempt = 0; attempt < 100; attempt++) { - table = yield* Effect.promise(async () => controller.snapshot()); - if ( - next.processes.some( - (p) => !table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid), - ) - ) { - throw new Error("Could not verify a recorded provider process"); - } - if (survivors(next, table).length === 0) { - NodeFS.rmSync(file, { force: true }); - return; - } - yield* Effect.sleep("20 millis"); - } - throw new Error("Recorded provider processes did not stop"); - }); - // Await every reap before exposing the spawn hook, and therefore before restart recovery launches work. - for (const name of NodeFS.readdirSync(directory)) { - if (!/^[\da-f-]+\.json$/.test(name)) continue; - const file = NodePath.join(directory, name); - let entry: Entry; - try { - entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); - } catch (cause) { - // Records are not fsynced, so power loss can leave an empty one. It names - // no process to reap, and keeping it would fail every later startup. - yield* Effect.logWarning("Discarding an unreadable provider process record", { cause }); - NodeFS.rmSync(file, { force: true }); - continue; - } - if (entry.hostname !== hostname || entry.stateDir !== input.stateDir) { - NodeFS.rmSync(file, { force: true }); - continue; - } - yield* reap(file, entry); - } - const track = (pid: number) => - Effect.promise(async () => { - const table = await controller.snapshot(); - const root = - table.find((p) => p.pid === pid && !p.zombie) ?? - table.find((p) => p.pgid === pid && !p.zombie); - if (!root) return; - // Process groups shared with the server are never a source of ownership. - const entry = discover( - { - version: 1, - instanceId, - hostname, - stateDir: input.stateDir, - owner: { pid: owner.pid, startTime: owner.startTime }, - processes: [ - { pid: root.pid, startTime: root.startTime, pgid: root.pgid === pid ? pid : 0 }, - ], - }, - table, - ); - const file = NodePath.join(directory, `${instanceId}-${pid}.json`); - try { - write(file, entry); - tracked.set(file, entry); - } catch (cause) { - for (const child of entry.processes.toReversed()) await signal(child, "SIGKILL"); - throw cause; - } - }); + // Registration records the PID immediately; one shared sampler discovers descendants. yield* Effect.forever( - Effect.sleep("1 second").pipe( + Effect.sleep("30 seconds").pipe( Effect.andThen( - Effect.tryPromise(refresh).pipe( + Effect.tryPromise(ledger.refresh).pipe( Effect.catch((cause) => Effect.logWarning("Could not record provider descendants", { cause }), ), @@ -398,6 +33,6 @@ export const make = Effect.fn("ProviderProcessLedger.make")(function* (input: { ), ).pipe(Effect.forkScoped); // Leave surviving entries for the next startup. A graceful provider stop makes refresh remove them. - yield* Effect.addFinalizer(() => Effect.tryPromise(refresh).pipe(Effect.ignore)); - return { track }; + yield* Effect.addFinalizer(() => Effect.tryPromise(ledger.refresh).pipe(Effect.ignore)); + return { track: (pid: number) => Effect.promise(() => ledger.track(pid)) }; }); diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 87dccd153335b..08c3a1b87b369 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -7,10 +7,15 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; +import { vi } from "vite-plus/test"; + import { ChildProcessSpawner } from "effect/process"; import * as ProcessRunner from "../processRunner.ts"; import * as ProviderProcessLedger from "./ProviderProcessLedger.ts"; +import { SERVICE_LAUNCHER_CONTEXT_ENV } from "../cloud/serviceProtocol.ts"; import { agentSliceMemoryLimits, classifyScope, make, type ScopeState } from "./agentScope.ts"; const GiB = 1024 ** 3; @@ -191,6 +196,109 @@ describe("provider process restart ownership", () => { Effect.scoped, ); + it.effect("samples all tracked trees once per 30 seconds, not once per second", () => + scenario(({ directory, controller }) => + Effect.gen(function* () { + let snapshots = 0; + const counted = { + ...controller, + snapshot: () => { + snapshots++; + return controller.snapshot(); + }, + }; + const ledger = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 10, + controller: counted, + }); + yield* ledger.track(20); + const initial = snapshots; + yield* Effect.yieldNow; + yield* TestClock.adjust("29 seconds"); + expect(snapshots).toBe(initial); + yield* TestClock.adjust("1 second"); + expect(snapshots).toBe(initial + 1); + }).pipe(Effect.scoped), + ), + ); + + it.effect("shares one process query between concurrent spawn registrations", () => + scenario(({ directory, controller }) => + Effect.gen(function* () { + let snapshots = 0; + let pending: Promise>> | undefined; + const counted = { + ...controller, + snapshot: () => { + snapshots++; + return pending ?? controller.snapshot(); + }, + }; + const ledger = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 10, + controller: counted, + }); + const query = Promise.withResolvers>>(); + pending = query.promise; + const registrations = yield* Effect.all([ledger.track(20), ledger.track(30)], { + concurrency: "unbounded", + }).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + query.resolve(yield* Effect.promise(async () => controller.snapshot())); + yield* Fiber.join(registrations); + expect(snapshots).toBe(2); // Owner query plus one shared spawn query. + pending = undefined; + }).pipe(Effect.scoped), + ), + ); + + it.effect("keeps the server responsive while a Windows process query is pending", () => + scenario(({ directory, processes }) => + Effect.gen(function* () { + const started = NodePath.join(directory, "query-started"); + const released = NodePath.join(directory, "query-release"); + const executable = NodePath.join(directory, "powershell.exe"); + // Stand in for CIM with a real child blocked on the server's acknowledgement. + NodeFS.writeFileSync( + executable, + `#!${process.execPath} +const fs = require("node:fs"); +const finish = () => { + if (!fs.existsSync(${JSON.stringify(released)})) return; + process.stdout.write(${JSON.stringify(JSON.stringify([...processes.values()]))}); + watcher.close(); +}; +const watcher = fs.watch(${JSON.stringify(directory)}, finish); +fs.writeFileSync(${JSON.stringify(started)}, "queried"); +finish(); +`, + { mode: 0o700 }, + ); + const queried = Promise.withResolvers(); + const watcher = NodeFS.watch(directory, () => { + if (NodeFS.existsSync(started)) queried.resolve(); + }); + vi.stubEnv("PATH", `${directory}${NodePath.delimiter}${process.env.PATH ?? ""}`); + try { + const fiber = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 10, + }).pipe(Effect.provideService(HostProcess.Platform, "win32"), Effect.forkScoped); + yield* Effect.promise(() => queried.promise); + // The server must be able to acknowledge the query before it returns. + NodeFS.writeFileSync(released, "responsive"); + yield* Fiber.join(fiber); + expect(NodeFS.readFileSync(released, "utf8")).toBe("responsive"); + } finally { + watcher.close(); + vi.unstubAllEnvs(); + } + }).pipe(Effect.scoped), + ), + ); + it.effect("reaps an orphan provider and its forked children before exposing the spawn hook", () => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { @@ -314,6 +422,121 @@ describe("provider process restart ownership", () => { }), ), ); + it.effect.each(["empty", "corrupt", "unreadable"] as const)( + "refuses orphan cleanup when the install lease is %s", + (mode) => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + processes.set(70, { + pid: 70, + ppid: 20, + pgid: 70, + startTime: "supervisor", + zombie: false, + }); + processes.set(71, { pid: 71, ppid: 70, pgid: 70, startTime: "installer", zombie: false }); + yield* record(directory, controller); + processes.delete(10); + const leases = NodePath.join(directory, "leases"); + NodeFS.mkdirSync(leases); + const lease = NodePath.join(leases, "install.json"); + if (mode === "unreadable") NodeFS.mkdirSync(lease); + else NodeFS.writeFileSync(lease, mode === "empty" ? "" : "70 partial"); + const result = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + controller, + installLeaseDirectory: leases, + }).pipe(Effect.scoped, Effect.exit); + expect(result._tag).toBe("Failure"); + expect(signals).toEqual([]); + expect(processes.has(71)).toBe(true); + }), + ), + ); + it.effect("never reaps the replacement server launched from an orphaned terminal", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + processes.set(90, { ...processes.get(90)!, ppid: 20, pgid: 20 }); + processes.set(91, { + pid: 91, + ppid: 90, + pgid: 20, + startTime: "new-provider", + zombie: false, + }); + yield* record(directory, controller); + processes.delete(10); + yield* restart(directory, controller); + expect(processes.has(20)).toBe(false); + expect(processes.has(90)).toBe(true); + expect(processes.has(91)).toBe(true); + expect(signals.some((s) => /^(90|91):/.test(s))).toBe(false); + }), + ), + ); + it.effect.each([ + { platform: "linux", handoff: false }, + { platform: "win32", handoff: false }, + { platform: "linux", handoff: true }, + { platform: "win32", handoff: true }, + ] as const)( + "preserves another environment's nested server and launcher on $platform, handoff=$handoff", + ({ platform, handoff }) => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + processes.set(60, { pid: 60, ppid: 20, pgid: 60, startTime: "launcher", zombie: false }); + processes.set(70, { + pid: 70, + ppid: 60, + pgid: 60, + startTime: "second-server", + zombie: false, + }); + processes.set(71, { + pid: 71, + ppid: 70, + pgid: 71, + startTime: "second-provider", + zombie: false, + }); + if (platform === "win32") + for (const [pid, p] of processes) processes.set(pid, { ...p, pgid: 0 }); + yield* record(directory, controller); + const leases = NodePath.join(directory, "leases"); + const second = ProviderProcessLedger.make({ + stateDir: NodePath.join(directory, "second"), + ownerPid: 70, + controller, + installLeaseDirectory: leases, + }).pipe( + Effect.tap(() => + Effect.sync(() => { + if (handoff) { + processes.delete(70); + processes.delete(71); + } + }), + ), + Effect.provideService(HostProcess.Environment, { + [SERVICE_LAUNCHER_CONTEXT_ENV]: "managed", + }), + ); + yield* handoff ? second.pipe(Effect.scoped) : second; + processes.delete(10); + yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + controller, + installLeaseDirectory: leases, + }); + expect(processes.has(20)).toBe(false); + for (const pid of handoff ? [60, 90] : [60, 70, 71, 90]) + expect(processes.has(pid)).toBe(true); + expect(signals.some((s) => /^(60|70|71|90):/.test(s))).toBe(false); + }).pipe(Effect.scoped, Effect.provideService(HostProcess.Platform, platform)), + ), + ); it.effect("starts after power loss leaves an empty process record", () => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { @@ -416,22 +639,68 @@ describe("provider process restart ownership", () => { }), ), ); - it.effect("stops the new tree if its durable ownership record cannot be written", () => - scenario(({ directory, processes, controller }) => + it.effect.each([false, true])( + "stops the new tree if its durable ownership record cannot be written (direct identity: %s)", + (direct) => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + const ledger = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 10, + controller: direct + ? { ...controller, identity: (pid) => processes.get(pid) } + : controller, + }).pipe(Effect.scoped); + const records = NodePath.join(directory, "provider-processes"); + NodeFS.rmSync(records, { recursive: true }); + NodeFS.writeFileSync(records, "unwritable records directory"); + const result = yield* ledger.track(20).pipe(Effect.exit); + expect(result._tag).toBe("Failure"); + expect(processes.has(20)).toBe(false); + expect(processes.has(30)).toBe(false); + }), + ), + ); + it.effect("a failed process record write leaves a nested live server running", () => + scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { + const leases = NodePath.join(directory, "leases"); const ledger = yield* ProviderProcessLedger.make({ stateDir: directory, ownerPid: 10, controller, - }).pipe(Effect.scoped); + installLeaseDirectory: leases, + }); + processes.set(70, { + pid: 70, + ppid: 20, + pgid: 70, + startTime: "second-server", + zombie: false, + }); + processes.set(71, { + pid: 71, + ppid: 70, + pgid: 70, + startTime: "second-provider", + zombie: false, + }); + yield* ProviderProcessLedger.make({ + stateDir: NodePath.join(directory, "second"), + ownerPid: 70, + controller, + installLeaseDirectory: leases, + }); const records = NodePath.join(directory, "provider-processes"); NodeFS.rmSync(records, { recursive: true }); - NodeFS.writeFileSync(records, "unwritable records directory"); + NodeFS.writeFileSync(records, "full disk"); const result = yield* ledger.track(20).pipe(Effect.exit); expect(result._tag).toBe("Failure"); expect(processes.has(20)).toBe(false); - expect(processes.has(30)).toBe(false); - }), + expect(processes.has(70)).toBe(true); + expect(processes.has(71)).toBe(true); + expect(signals.some((s) => /^(70|71):/.test(s))).toBe(false); + }).pipe(Effect.scoped), ), ); }); diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index fc03f18e37b93..9ab01f0a91939 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -6,6 +6,8 @@ import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; +import { SERVICE_LAUNCHER_CONTEXT_ENV } from "../cloud/serviceProtocol.ts"; + const execFile = NodeUtil.promisify(NodeChildProcess.execFile); interface Identity { @@ -240,6 +242,8 @@ export async function make(input: { readonly ownerPid?: number; readonly controller?: ProcessController; readonly installLeaseDirectory?: string; + /** Servers publish their identity so cleanup in another environment never stops them. */ + readonly registerServer?: boolean; readonly platform: NodeJS.Platform; readonly environment: NodeJS.ProcessEnv; readonly hostname: string; @@ -249,8 +253,13 @@ export async function make(input: { const directory = NodePath.join(input.stateDir, "provider-processes"); NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 }); const ownerPid = input.ownerPid ?? process.pid; - const owner = (await controller.snapshot()).find((p) => p.pid === ownerPid && !p.zombie); + const initial = await controller.snapshot(); + const owner = initial.find((p) => p.pid === ownerPid && !p.zombie); if (!owner) throw new Error("Could not identify the provider process owner"); + const launcher = + environment[SERVICE_LAUNCHER_CONTEXT_ENV] === undefined + ? undefined + : initial.find((p) => p.pid === owner.ppid && !p.zombie); const instanceId = NodeCrypto.randomUUID(); const tracked = new Map(); const write = (file: string, entry: Entry) => { @@ -258,6 +267,34 @@ export async function make(input: { NodeFS.writeFileSync(temporary, JSON.stringify(entry), { mode: 0o600 }); NodeFS.renameSync(temporary, file); }; + // Servers can be launched from a tracked terminal in another environment. + // Publish their identities before cleanup so neither server owns the other. + const servers = + input.installLeaseDirectory === undefined + ? undefined + : NodePath.join(input.installLeaseDirectory, "servers"); + const registration = + servers === undefined || !input.registerServer + ? undefined + : NodePath.join(servers, `${instanceId}.json`); + if (servers !== undefined) NodeFS.mkdirSync(servers, { recursive: true, mode: 0o700 }); + if (registration !== undefined) + write(registration, { + version: 1, + instanceId, + hostname, + stateDir: input.stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes: + launcher === undefined + ? [] + : [{ pid: launcher.pid, startTime: launcher.startTime, pgid: launcher.pgid }], + }); + const release = () => { + // The managed launcher stays alive between replacement children. + if (registration !== undefined && launcher === undefined) + NodeFS.rmSync(registration, { force: true }); + }; const survivors = (entry: Entry, table: ReadonlyArray) => table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); const refresh = async () => { @@ -328,6 +365,53 @@ export async function make(input: { ), }; }; + const withoutServers = (entry: Entry, table: ReadonlyArray): Entry => { + const owners: Array = same(entry.owner, owner) + ? [] + : launcher === undefined + ? [owner] + : [owner, launcher]; + if (servers !== undefined) { + for (const name of NodeFS.readdirSync(servers)) { + if (!name.endsWith(".json")) continue; + let server: Entry; + try { + server = decodeEntry(NodeFS.readFileSync(NodePath.join(servers, name), "utf8")); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") continue; + throw cause; + } + if (server.hostname !== hostname || same(server.owner, entry.owner)) continue; + const live = table.find((p) => same(p, server.owner) && !p.zombie); + const managed = table.filter( + (p) => !p.zombie && server.processes.some((owned) => same(p, owned)), + ); + if (live || managed.length > 0) { + if (live) owners.push(live); + owners.push(...managed); + } else if ( + !table.some((p) => p.pid === server.owner.pid) && + controller.exists(server.owner.pid) + ) + throw new Error("Could not verify a live server during provider cleanup"); + else NodeFS.rmSync(NodePath.join(servers, name), { force: true }); + } + } + const protectedPids = new Set(owners.map((p) => p.pid)); + let changed = true; + while (changed) { + changed = false; + for (const p of table) { + if (!protectedPids.has(p.pid) && protectedPids.has(p.ppid)) { + protectedPids.add(p.pid); + changed = true; + } + } + } + return { ...entry, processes: entry.processes.filter((p) => !protectedPids.has(p.pid)) }; + }; + const protectedEntry = (entry: Entry, table: ReadonlyArray) => + withoutServers(withoutInstaller(entry, table), table); const reap = async (file: string, entry: Entry) => { let table = await controller.snapshot(); const liveOwner = table.find((p) => same(entry.owner, p)); @@ -340,7 +424,7 @@ export async function make(input: { ) { throw new Error("Could not verify the recorded provider owner"); } - let next = withoutInstaller(discover(entry, table), table); + let next = protectedEntry(discover(entry, table), table); if (platform !== "win32") { // Freeze to a fixed point. A child may fork while its parent is being stopped. const stopped = new Set(); @@ -354,14 +438,15 @@ export async function make(input: { stopped.add(`${p.pid}:${p.startTime}`); } table = await controller.snapshot(); - next = withoutInstaller(discover(next, table), table); + next = protectedEntry(discover(next, table), table); if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); } } // Persist newly discovered children before signalling: recovery can itself be killed. write(file, next); - for (const p of survivors(next, await controller.snapshot()).toReversed()) - await signal(p, "SIGKILL"); + table = await controller.snapshot(); + next = protectedEntry(next, table); + for (const p of survivors(next, table).toReversed()) await signal(p, "SIGKILL"); for (let attempt = 0; attempt < 100; attempt++) { table = await controller.snapshot(); if ( @@ -380,23 +465,28 @@ export async function make(input: { throw new Error("Recorded provider processes did not stop"); }; // Await every reap before exposing the spawn hook, and therefore before restart recovery launches work. - for (const name of NodeFS.readdirSync(directory)) { - if (!/^[\da-f-]+\.json$/.test(name)) continue; - const file = NodePath.join(directory, name); - let entry: Entry; - try { - entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); - } catch { - // Records are not fsynced, so power loss can leave an empty one. It names - // no process to reap, and keeping it would fail every later startup. - NodeFS.rmSync(file, { force: true }); - continue; - } - if (entry.hostname !== hostname || entry.stateDir !== input.stateDir) { - NodeFS.rmSync(file, { force: true }); - continue; + try { + for (const name of NodeFS.readdirSync(directory)) { + if (!/^[\da-f-]+\.json$/.test(name)) continue; + const file = NodePath.join(directory, name); + let entry: Entry; + try { + entry = decodeEntry(NodeFS.readFileSync(file, "utf8")); + } catch { + // Records are not fsynced, so power loss can leave an empty one. It names + // no process to reap, and keeping it would fail every later startup. + NodeFS.rmSync(file, { force: true }); + continue; + } + if (entry.hostname !== hostname || entry.stateDir !== input.stateDir) { + NodeFS.rmSync(file, { force: true }); + continue; + } + await reap(file, entry); } - await reap(file, entry); + } catch (cause) { + release(); + throw cause; } let pendingSnapshot: Promise> | undefined; const spawnSnapshot = () => { @@ -440,14 +530,23 @@ export async function make(input: { } } catch (cause) { // The first root write may fail before the initial descendant query. + let latest = table; try { - entry = discover(entry, await spawnSnapshot()); + latest = await spawnSnapshot(); + entry = discover(entry, latest); } catch { /* Still stop the identities already established. */ } - for (const child of entry.processes.toReversed()) await signal(child, "SIGKILL"); + let safe: Entry; + try { + safe = protectedEntry(entry, latest); + } catch { + // Only the new root is certainly ours when other servers cannot be verified. + safe = { ...entry, processes: entry.processes.filter((p) => p.pid === root.pid) }; + } + for (const child of safe.processes.toReversed()) await signal(child, "SIGKILL"); throw cause; } }; - return { track, refresh }; + return { track, refresh, release }; } From 08d86d17581608e5a2246d1c35dc334eaae21d74 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:00:47 +0000 Subject: [PATCH 125/149] test(server): an initial prompt also saved as pending steering continues once --- .../orchestration-v2/RestartContinuation.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index b4102721d75cb..eb409c0dced5d 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -957,6 +957,19 @@ it.effect("delivers the original prompt when a restart cuts initial provider adm }), ); +it.effect("delivers an initial prompt also saved as pending steering exactly once", () => + Effect.gen(function* () { + const messageId = MessageId.make("message:user"); + const projection = { + ...cutMidTurn({ restartPendingMessageIds: [messageId] }), + attempts: [{ id: attemptId, runId, reason: "initial" }], + providerTurns: [], + messages: [{ id: messageId, text: "Do the accepted work.", attachments: [] }], + } as unknown as OrchestrationV2ThreadProjection; + assert.deepEqual(yield* continuationTexts(projection), ["Do the accepted work."]); + }), +); + it.effect( "delivers the replacement prompt when a steering restart never reached the provider", () => From 019b369eb4cf1f4c7901fd6ba337b351108399d5 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:00:53 +0000 Subject: [PATCH 126/149] docs: point orphan cleanup at the shared process core and its server protections --- docs/internals/server-updates.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 968cac9adf49d..9bdc7639b19c5 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -71,10 +71,11 @@ behind. Cancelled background work is reported on the next user turn; it does not wake a finished run. Undelivered steering can still qualify as unfinished input. A crashed server can leave its providers and terminals running. The next server -stops the ones its [process ledger](../../apps/server/src/process/ProviderProcessLedger.ts) +stops the ones its [process ledger](../../apps/server/src/process/ownedProcesses.ts) recorded before any continuation can launch, so a continued run never races its orphaned original. Each signal rechecks the process's creation identity. A detached -provider installer is never stopped, even when it descends from a recorded process. +provider installer, another environment's registered server and its launcher are +never stopped, even when they descend from a recorded process. Recovery retires effects tied to the lost process and records continuation intent in the durable outbox. That intent survives another restart before provider startup. From 5c8be8aeeb6dac8e91fdb6c1fb32382b28e7a0d2 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 15:16:24 +0000 Subject: [PATCH 127/149] test(server): the launcher cleanup failure case reaps a crashed server's record --- apps/server/src/serviceLauncher.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index a50207ed25f3d..a791596c5b8fb 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as NodeFSP from "node:fs/promises"; import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; vi.mock("node:fs/promises", { spy: true }); @@ -217,6 +218,20 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const leases = path.join(root, "cache", "t3", "provider-maintenance"); yield* fs.makeDirectory(leases, { recursive: true }); yield* fs.writeFileString(path.join(leases, "install.json"), "corrupt"); + // A crashed server's record makes cleanup read the corrupt lease. + const records = path.join(root, "userdata", "provider-processes"); + yield* fs.makeDirectory(records, { recursive: true }); + yield* fs.writeFileString( + path.join(records, "0.json"), + JSON.stringify({ + version: 1, + instanceId: "crashed", + hostname: NodeOS.hostname(), + stateDir: path.join(root, "userdata"), + owner: { pid: 2 ** 30, startTime: "exited" }, + processes: [], + }), + ); yield* Effect.acquireRelease( Effect.sync(() => vi.stubEnv("XDG_CACHE_HOME", path.join(root, "cache"))), () => Effect.sync(() => vi.unstubAllEnvs()), From bf396d8ff77afc16edc157b3792c026b41757e33 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:03:18 +0000 Subject: [PATCH 128/149] test(server): launcher tests never touch the real provider maintenance cache --- apps/server/src/serviceLauncher.test.ts | 30 ++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index a791596c5b8fb..24b4bb6602716 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,16 +1,18 @@ // @effect-diagnostics nodeBuiltinImport:off globalDateInEffect:off - Launcher recovery uses native filesystem failures and wall clocks. import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; -import { vi } from "vite-plus/test"; +import { afterEach, beforeEach, vi } from "vite-plus/test"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as NodeFSP from "node:fs/promises"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import * as NodeSqlite from "node:sqlite"; vi.mock("node:fs/promises", { spy: true }); +vi.mock("node:fs", { spy: true }); import { Launcher, @@ -27,6 +29,32 @@ import { SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; +let cache: string; +beforeEach(() => { + cache = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-launcher-cache-")); + vi.stubEnv("XDG_CACHE_HOME", cache); + vi.clearAllMocks(); +}); +afterEach(() => { + try { + const realCache = NodePath.join(NodeOS.homedir(), ".cache", "t3", "provider-maintenance"); + for (const operation of [ + NodeFS.readdirSync, + NodeFS.readFileSync, + NodeFS.mkdirSync, + NodeFS.rmSync, + ]) { + assert.isFalse( + vi.mocked(operation).mock.calls.some(([path]) => String(path).startsWith(realCache)), + "launcher accessed the real maintenance cache", + ); + } + } finally { + vi.unstubAllEnvs(); + NodeFS.rmSync(cache, { recursive: true, force: true }); + } +}); + it("accepts only exact semantic versions", () => { for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) { assert.isTrue(isExactServiceVersion(version), version); From 7ebddc7fddfd942bfb6fc4029907b437ec50d4eb Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:03:18 +0000 Subject: [PATCH 129/149] test(server): keep the host install gate test in a temporary cache --- .../provider/providerMaintenanceRunner.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 5d2fc4fcfebf2..b45a1ba89521b 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -2509,6 +2509,16 @@ describe("provider maintenance host leases", () => { it.effect("keeps anything but a server off the user-wide gate", () => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cache = yield* fs.makeTempDirectoryScoped(); + const previous = process.env.XDG_CACHE_HOME; + process.env.XDG_CACHE_HOME = cache; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (previous === undefined) delete process.env.XDG_CACHE_HOME; + else process.env.XDG_CACHE_HOME = previous; + }), + ); const fallback = yield* HostLock.ProviderMaintenanceHostLock; assert.isTrue(fallback.directory.startsWith(NodeOS.tmpdir())); assert.notStrictEqual(fallback.directory, HostLock.userDirectory()); @@ -2516,7 +2526,10 @@ describe("provider maintenance host leases", () => { Effect.provide(HostLock.layer), ); assert.strictEqual(served.directory, HostLock.userDirectory()); - }), + assert.strictEqual(served.directory, NodePath.join(cache, "t3", "provider-maintenance")); + yield* served.withRead(Effect.void); + assert.isTrue(NodeFS.existsSync(NodePath.join(served.directory, "readers"))); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); it.effect("shows a manual update waiting for turns and cancels it before installing", () => { From ac5fd46178c9bd2093ee214698fcd36ff38938c5 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:03:19 +0000 Subject: [PATCH 130/149] fix(server): stale server registrations are pruned without dropping a live one --- apps/server/src/process/agentScope.test.ts | 149 +++++++++++++++++++++ apps/server/src/process/ownedProcesses.ts | 38 +++++- 2 files changed, 186 insertions(+), 1 deletion(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 08c3a1b87b369..34e6210b51afa 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -15,6 +15,7 @@ import { ChildProcessSpawner } from "effect/process"; import * as ProcessRunner from "../processRunner.ts"; import * as ProviderProcessLedger from "./ProviderProcessLedger.ts"; +import * as OwnedProcesses from "./ownedProcesses.ts"; import { SERVICE_LAUNCHER_CONTEXT_ENV } from "../cloud/serviceProtocol.ts"; import { agentSliceMemoryLimits, classifyScope, make, type ScopeState } from "./agentScope.ts"; @@ -523,6 +524,20 @@ finish(); }), ); yield* handoff ? second.pipe(Effect.scoped) : second; + if (handoff) { + yield* Effect.promise(() => + OwnedProcesses.make({ + stateDir: NodePath.join(directory, "second"), + ownerPid: 60, + controller, + installLeaseDirectory: leases, + platform, + environment: {}, + hostname: NodeOS.hostname(), + }), + ); + expect(NodeFS.readdirSync(NodePath.join(leases, "servers"))).toHaveLength(1); + } processes.delete(10); yield* ProviderProcessLedger.make({ stateDir: directory, @@ -537,6 +552,140 @@ finish(); }).pipe(Effect.scoped, Effect.provideService(HostProcess.Platform, platform)), ), ); + it.effect.each(["gone", "reused", "zombie"] as const)( + "prunes managed server registrations whose owner is %s without dropping live servers", + (mode) => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + const leases = NodePath.join(directory, "leases"); + processes.set(60, { pid: 60, ppid: 20, pgid: 60, startTime: "launcher", zombie: false }); + const publish = (pid: number) => + Effect.promise(() => + OwnedProcesses.make({ + stateDir: NodePath.join(directory, String(pid)), + ownerPid: pid, + controller, + installLeaseDirectory: leases, + registerServer: true, + platform: "linux", + environment: { [SERVICE_LAUNCHER_CONTEXT_ENV]: "managed" }, + hostname: "test-host", + }), + ); + processes.set(70, { + pid: 70, + ppid: 60, + pgid: 60, + startTime: "live-server", + zombie: false, + }); + const live = yield* publish(70); + const liveRegistration = NodePath.join( + leases, + "servers", + NodeFS.readdirSync(NodePath.join(leases, "servers"))[0]!, + ); + for (let generation = 0; generation < 5; generation++) { + processes.set(80, { + pid: 80, + ppid: 60, + pgid: 60, + startTime: String(generation), + zombie: false, + }); + const next = yield* publish(80); + expect(NodeFS.readdirSync(NodePath.join(leases, "servers"))).toHaveLength(2); + expect(NodeFS.existsSync(liveRegistration)).toBe(true); + next.release(); + if (mode === "gone") processes.delete(80); + else + processes.set(80, { + ...processes.get(80)!, + startTime: "reused", + zombie: mode === "zombie", + }); + } + // A launcher handoff retains protection until the replacement publishes. + yield* publish(90); + expect(NodeFS.readdirSync(NodePath.join(leases, "servers"))).toHaveLength(2); + expect(signals).toEqual([]); + live.release(); + }), + ), + ); + + it.effect( + "retains a server registration when its live owner is missing from the process query", + () => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + const leases = NodePath.join(directory, "leases"); + const input = { + stateDir: directory, + ownerPid: 10, + controller, + installLeaseDirectory: leases, + }; + const server = yield* ProviderProcessLedger.make(input); + yield* ProviderProcessLedger.make({ + ...input, + stateDir: NodePath.join(directory, "second"), + ownerPid: 90, + controller: { + ...controller, + snapshot: () => [...processes.values()].filter((p) => p.pid !== 10), + }, + }); + expect(NodeFS.readdirSync(NodePath.join(leases, "servers"))).toHaveLength(2); + yield* server.track(20); + }).pipe(Effect.scoped), + ), + ); + + it.effect("does not delete a server registration published after the cleanup process query", () => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + const leases = NodePath.join(directory, "leases"); + const registration = NodePath.join(leases, "servers", "new-server.json"); + processes.set(70, { pid: 70, ppid: 20, pgid: 70, startTime: "old-pid", zombie: false }); + let publishing = 0; + const ledger = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 10, + installLeaseDirectory: leases, + controller: { + ...controller, + snapshot: () => { + const table = [...processes.values()]; + if (publishing > 0 && --publishing === 0) { + processes.set(70, { ...processes.get(70)!, startTime: "new-server" }); + NodeFS.writeFileSync( + registration, + JSON.stringify({ + version: 1, + instanceId: "new", + hostname: NodeOS.hostname(), + stateDir: NodePath.join(directory, "second"), + owner: { pid: 70, startTime: "new-server" }, + processes: [], + }), + ); + } + return table; + }, + }, + }); + const records = NodePath.join(directory, "provider-processes"); + NodeFS.rmSync(records, { recursive: true }); + NodeFS.writeFileSync(records, "unwritable"); + publishing = 2; + expect((yield* ledger.track(20).pipe(Effect.exit))._tag).toBe("Failure"); + expect(processes.has(70)).toBe(true); + expect(NodeFS.existsSync(registration)).toBe(true); + }).pipe(Effect.scoped), + ), + ); + it.effect("starts after power loss leaves an empty process record", () => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index 9ab01f0a91939..96cba76da11c7 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -295,6 +295,42 @@ export async function make(input: { if (registration !== undefined && launcher === undefined) NodeFS.rmSync(registration, { force: true }); }; + const pruneServers = async () => { + if (servers === undefined) return; + const records = NodeFS.readdirSync(servers).flatMap((name) => { + if (!name.endsWith(".json")) return []; + const file = NodePath.join(servers, name); + try { + return [{ file, entry: decodeEntry(NodeFS.readFileSync(file, "utf8")) }]; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return []; + throw cause; + } + }); + if (records.every(({ entry }) => entry.instanceId === instanceId)) return; + // Read records before sampling: a newer registration must not be judged + // against a process table taken before its server was born. + const table = await controller.snapshot(); + const live = (p: Identity) => table.some((observed) => same(p, observed) && !observed.zombie); + const unknown = (p: Identity) => + !table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid); + const protectedLaunchers = records.flatMap(({ entry }) => + entry.hostname === hostname && (live(entry.owner) || unknown(entry.owner)) + ? entry.processes.filter(live) + : [], + ); + for (const { file, entry } of records) { + if (entry.hostname !== hostname || live(entry.owner) || unknown(entry.owner)) continue; + if (entry.processes.some(unknown)) continue; + const managed = entry.processes.filter(live); + // Keep one handoff record until a replacement protects the live launcher. + if (managed.some((p) => !protectedLaunchers.some((other) => same(p, other)))) { + protectedLaunchers.push(...managed); + continue; + } + NodeFS.rmSync(file, { force: true }); + } + }; const survivors = (entry: Entry, table: ReadonlyArray) => table.filter((p) => !p.zombie && entry.processes.some((owned) => same(owned, p))); const refresh = async () => { @@ -394,7 +430,6 @@ export async function make(input: { controller.exists(server.owner.pid) ) throw new Error("Could not verify a live server during provider cleanup"); - else NodeFS.rmSync(NodePath.join(servers, name), { force: true }); } } const protectedPids = new Set(owners.map((p) => p.pid)); @@ -466,6 +501,7 @@ export async function make(input: { }; // Await every reap before exposing the spawn hook, and therefore before restart recovery launches work. try { + await pruneServers(); for (const name of NodeFS.readdirSync(directory)) { if (!/^[\da-f-]+\.json$/.test(name)) continue; const file = NodePath.join(directory, name); From 740a721216af256a740b3775f2ba2dccd2b1644a Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:56:33 +0000 Subject: [PATCH 131/149] fix(server): local file release mirrors still pass the download origin check --- packages/shared/src/cliRelease.test.ts | 1 + packages/shared/src/cliRelease.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/cliRelease.test.ts b/packages/shared/src/cliRelease.test.ts index b2ebc1607097b..1aa30be895257 100644 --- a/packages/shared/src/cliRelease.test.ts +++ b/packages/shared/src/cliRelease.test.ts @@ -110,6 +110,7 @@ describe("cliRelease", () => { "http://localhost:4321", "http://127.0.0.2:4321", "http://[::1]:4321", + "file:///tmp/t3-mirror", ]) { expect(cliReleaseDownloadBaseUrl("1.0.0", origin)).toBe(`${origin}/v1.0.0`); } diff --git a/packages/shared/src/cliRelease.ts b/packages/shared/src/cliRelease.ts index 4faf07127bbbd..520f45381cdb9 100644 --- a/packages/shared/src/cliRelease.ts +++ b/packages/shared/src/cliRelease.ts @@ -65,14 +65,21 @@ export function cliReleaseDownloadBaseUrl( return `${origin.href.replace(/\/+$/, "")}/v${version}`; } -/** Checks both configured mirrors and the final URL after HTTP redirects. */ +/** + * Checks both configured mirrors and the final URL after HTTP redirects. Local + * `file:` mirrors (SSH remote installs) never cross the network. + */ export function validateCliReleaseDownloadUrl(url: string): URL { const origin = new URL(url); const loopback = origin.hostname === "localhost" || origin.hostname === "[::1]" || /^127\.\d+\.\d+\.\d+$/.test(origin.hostname); - if (origin.protocol !== "https:" && !(origin.protocol === "http:" && loopback)) { + if ( + origin.protocol !== "https:" && + origin.protocol !== "file:" && + !(origin.protocol === "http:" && loopback) + ) { throw new Error("T3 release downloads require HTTPS (HTTP is allowed only on loopback)."); } return origin; From b367a977ae4704087c4e3eaffb83f98a7acd0b29 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:56:33 +0000 Subject: [PATCH 132/149] fix(server): provider cleanup resumes a tree it froze when it cannot finish --- apps/server/src/process/ownedProcesses.ts | 38 +++++++++++++---------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index fc03f18e37b93..03c5049092a6d 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -341,25 +341,31 @@ export async function make(input: { throw new Error("Could not verify the recorded provider owner"); } let next = withoutInstaller(discover(entry, table), table); - if (platform !== "win32") { - // Freeze to a fixed point. A child may fork while its parent is being stopped. - const stopped = new Set(); - for (let attempt = 0; attempt < 100; attempt++) { - const pending = survivors(next, table).filter( - (p) => !stopped.has(`${p.pid}:${p.startTime}`), - ); - if (pending.length === 0) break; - for (const p of pending) { - await signal(p, "SIGSTOP"); - stopped.add(`${p.pid}:${p.startTime}`); + const stopped = new Map(); + try { + if (platform !== "win32") { + // Freeze to a fixed point. A child may fork while its parent is being stopped. + for (let attempt = 0; attempt < 100; attempt++) { + const pending = survivors(next, table).filter( + (p) => !stopped.has(`${p.pid}:${p.startTime}`), + ); + if (pending.length === 0) break; + for (const p of pending) { + await signal(p, "SIGSTOP"); + stopped.set(`${p.pid}:${p.startTime}`, p); + } + table = await controller.snapshot(); + next = withoutInstaller(discover(next, table), table); + if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); } - table = await controller.snapshot(); - next = withoutInstaller(discover(next, table), table); - if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); } + // Persist newly discovered children before signalling: recovery can itself be killed. + write(file, next); + } catch (error) { + // Never leave a frozen tree behind: resume it so the next cleanup can retry. + for (const p of stopped.values()) await signal(p, "SIGCONT").catch(() => undefined); + throw error; } - // Persist newly discovered children before signalling: recovery can itself be killed. - write(file, next); for (const p of survivors(next, await controller.snapshot()).toReversed()) await signal(p, "SIGKILL"); for (let attempt = 0; attempt < 100; attempt++) { From 35b7f96ebdefd008c09277a721430c90649f0a7b Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:11:41 +0000 Subject: [PATCH 133/149] fix(server): installs drain readers, ignore foreign symlinks, and bound installer logs A waiting install now holds back new readers so it can drain, while helpers of an admitted turn still join it. bun and pnpm ownership follows the real path, so a symlink into another installation is not updated. The supervisor compacts both installer streams. The web update toast settles when a queued provider disappears mid-batch. --- .../ProviderSessionManager.ts | 18 ++- .../src/provider/ProviderInstallSupervisor.ts | 4 +- .../provider/ProviderMaintenanceHostLock.ts | 93 +++++++----- .../providerMaintenanceRunner.test.ts | 143 ++++++++++++++++-- .../ProviderUpdateEnvironmentRows.test.tsx | 45 ++++++ .../ProviderUpdatePrimaryNotification.tsx | 11 +- .../src/server/maintenanceResolver.test.ts | 27 ++++ .../src/server/maintenanceResolver.ts | 10 +- 8 files changed, 288 insertions(+), 63 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index db0c3f4ca334a..04e2519877590 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1771,8 +1771,11 @@ export const layerWithOptions = ( ), ), () => - (compact ? runtime.compactThread!(input) : runtime.startTurn(input)).pipe( - turnMetrics("send", input.modelSelection.model), + hostLock.withRead( + (compact ? runtime.compactThread!(input) : runtime.startTurn(input)).pipe( + turnMetrics("send", input.modelSelection.model), + ), + hostLeases.get(runtime), ), (_, exit) => Exit.isFailure(exit) @@ -1783,6 +1786,7 @@ export const layerWithOptions = ( : Effect.void, ), ), + hostLock.withRead, ); return { ...runtime, @@ -1908,20 +1912,24 @@ export const layerWithOptions = ( steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen( - hostLock.withRead(runtime.steerTurn(input)).pipe(turnMetrics("steer")), + hostLock + .withRead(runtime.steerTurn(input), hostLeases.get(runtime)) + .pipe(turnMetrics("steer")), ), ), interruptTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen( - hostLock.withRead(runtime.interruptTurn(input)).pipe(turnMetrics("interrupt")), + hostLock + .withRead(runtime.interruptTurn(input), hostLeases.get(runtime)) + .pipe(turnMetrics("interrupt")), ), ), respondToRuntimeRequest: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen( hostLock - .withRead(runtime.respondToRuntimeRequest(input)) + .withRead(runtime.respondToRuntimeRequest(input), hostLeases.get(runtime)) .pipe(turnMetrics("runtime-request-response")), ), ), diff --git a/apps/server/src/provider/ProviderInstallSupervisor.ts b/apps/server/src/provider/ProviderInstallSupervisor.ts index 83ac1501251a0..9cbab85efff5e 100644 --- a/apps/server/src/provider/ProviderInstallSupervisor.ts +++ b/apps/server/src/provider/ProviderInstallSupervisor.ts @@ -268,7 +268,9 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< const beat = (format: (line: string) => string | null) => { NodeFS.utimesSync(lease, new Date(), new Date()); NodeFS.utimesSync(NodePath.join(receipt.directory, "gate.lock"), new Date(), new Date()); - const text = output(".stderr").text || output(".stdout").text; + const stderr = output(".stderr"); + const stdout = output(".stdout"); + const text = stderr.text || stdout.text; const progress = text .split(/\r?\n|\r/) .map(format) diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index aa835ec6df732..75df067d23d26 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -19,6 +19,15 @@ const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } { defaultValue: () => null }, ); +interface ReadToken { + readonly lease: { readonly owner: object; readonly file: string; users: number }; + active: boolean; +} +const HeldRead = Context.Reference( + "t3/provider/ProviderMaintenanceHostLock/HeldRead", + { defaultValue: () => null }, +); + const HeldGate = Context.Reference<{ readonly handoff: Effect.Effect; readonly resume: Effect.Effect; @@ -234,8 +243,7 @@ export const make = (directory: string) => { ); const readPermit = Semaphore.makeUnsafe(1); - let readCount = 0; - let readFile: string | undefined; + const readTokens = new WeakMap, ReadToken>(); const registerRead = Effect.sync(() => { const file = NodePath.join(readers, `${process.pid}-${NodeCrypto.randomUUID()}`); NodeFS.writeFileSync(file, processIdentity(process.pid) ?? "", { flag: "wx" }); @@ -243,44 +251,51 @@ export const make = (directory: string) => { }); // A provider can launch helper CLIs while its turn holds a read lease. // Joining that lease must not wait behind an installer draining this turn. - const acquireRead = Effect.flatMap(HeldWrite, (held) => - held?.owner === owner && held.active - ? // A scoped verification can outlive the install. Its own reader remains - // visible to the next writer without admitting unrelated launches here. - registerRead.pipe( - Effect.map((file) => Effect.sync(() => NodeFS.rmSync(file, { force: true }))), - ) - : Effect.acquireUseRelease( - Effect.interruptible(readPermit.take(1)), - () => - Effect.uninterruptible( - Effect.gen(function* () { - if (readCount === 0) { - readFile = yield* withGate(registerRead); - } - readCount++; - let released = false; - return readPermit.withPermits(1)( - Effect.sync(() => { - if (released) return; - released = true; - if (--readCount === 0 && readFile !== undefined) { - NodeFS.rmSync(readFile, { force: true }); - readFile = undefined; - } - }), - ); - }), - ), - () => Effect.asVoid(readPermit.release(1)), - ), + const acquireRead = Effect.uninterruptible( + Effect.gen(function* () { + const inherited = yield* HeldRead; + const held = yield* HeldWrite; + const lease = + inherited?.lease.owner === owner && inherited.active + ? inherited.lease + : { + owner, + // An install's scoped verification keeps its own reader if it outlives the writer. + file: yield* held?.owner === owner && held.active + ? registerRead + : Effect.acquireUseRelease( + Effect.interruptible(readPermit.take(1)), + () => withGate(registerRead), + () => Effect.asVoid(readPermit.release(1)), + ), + users: 0, + }; + lease.users++; + const token = { lease, active: true }; + const release = Effect.sync(() => { + if (!token.active) return; + token.active = false; + if (--lease.users === 0) NodeFS.rmSync(lease.file, { force: true }); + }); + readTokens.set(release, token); + return release; + }), ); - const withRead = (effect: Effect.Effect) => - Effect.acquireUseRelease( + const withRead = ( + effect: Effect.Effect, + inheritedRelease?: Effect.Effect, + ) => { + const inherited = inheritedRelease === undefined ? undefined : readTokens.get(inheritedRelease); + const read = Effect.acquireUseRelease( acquireRead, - () => effect, + (release) => + effect.pipe( + Effect.provideService(HeldRead, inherited?.active ? inherited : readTokens.get(release)!), + ), (release) => release, ); + return inherited === undefined ? read : read.pipe(Effect.provideService(HeldRead, inherited)); + }; const drained = Effect.sync(() => { let busy = false; for (const file of NodeFS.readdirSync(readers)) { @@ -371,7 +386,11 @@ export const layer = Layer.effectContext( withScopedLaunchPermit: (effect) => Effect.acquireRelease(host.acquireRead, (release) => release, { interruptible: true, - }).pipe(Effect.andThen(admission.withScopedLaunchPermit(effect))), + }).pipe( + Effect.flatMap((release) => + host.withRead(admission.withScopedLaunchPermit(effect), release), + ), + ), }), ); }), diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index b45a1ba89521b..5c40c46aaf4bb 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -2343,7 +2343,7 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.forkScoped); yield* gate; assert.isFalse(yield* Deferred.isDone(entered)); - yield* local.withRead(Effect.void); + yield* local.withRead(Effect.void, release); yield* release; assert.isFalse(yield* Deferred.isDone(entered)); holder.child.stdin.end("release"); @@ -2354,6 +2354,85 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live("blocks unrelated readers behind a waiting writer while admitting turn helpers", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const host = HostLock.make(yield* fs.makeTempDirectoryScoped()); + const entered = yield* Deferred.make(); + const helper = yield* Deferred.make(); + const helperEntered = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const turn = yield* host + .withRead( + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(helper)), + Effect.andThen(host.withRead(Deferred.succeed(helperEntered, undefined))), + Effect.andThen(Deferred.await(finish)), + ), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const waiting = yield* Deferred.make(); + const installed = yield* Deferred.make(); + const releaseWriter = yield* Deferred.make(); + const writer = yield* host + .withWrite( + Deferred.succeed(installed, undefined).pipe( + Effect.andThen(Deferred.await(releaseWriter)), + ), + Deferred.succeed(waiting, undefined), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(waiting); + const unrelatedEntered = yield* Deferred.make(); + const unrelated = yield* Effect.forkChild( + host.withRead(Deferred.succeed(unrelatedEntered, undefined)), + { startImmediately: true }, + ); + yield* Deferred.succeed(helper, undefined); + yield* Deferred.await(helperEntered); + assert.isFalse(yield* Deferred.isDone(unrelatedEntered)); + yield* Deferred.succeed(finish, undefined); + yield* Fiber.join(turn); + yield* Deferred.await(installed); + assert.isFalse(yield* Deferred.isDone(unrelatedEntered)); + yield* Deferred.succeed(releaseWriter, undefined); + yield* Fiber.join(writer); + yield* Fiber.join(unrelated); + assert.isTrue(yield* Deferred.isDone(unrelatedEntered)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("does not let an expired turn lease bypass a later writer", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const host = HostLock.make(yield* fs.makeTempDirectoryScoped()); + const release = yield* host.acquireRead; + const entered = yield* Deferred.make(); + const lateHelper = host.withRead(Deferred.succeed(entered, undefined), release); + yield* release; + const writing = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const writer = yield* host + .withWrite( + Deferred.succeed(writing, undefined).pipe(Effect.andThen(Deferred.await(finish))), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(writing); + const helper = yield* Effect.forkChild(lateHelper, { startImmediately: true }); + yield* Effect.yieldNow; + assert.isFalse(yield* Deferred.isDone(entered)); + yield* Deferred.succeed(finish, undefined); + yield* Fiber.join(writer); + yield* Fiber.join(helper); + assert.isTrue(yield* Deferred.isDone(entered)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("serializes installs across independent server processes", () => Effect.scoped( Effect.gen(function* () { @@ -2834,6 +2913,7 @@ describe("provider maintenance host leases", () => { "runtime-replaced", "heartbeat-write-error", "installer-killed-with-live-child", + "verbose-output", ] as const)( "finishes the installer after %s", (mode) => @@ -2863,6 +2943,8 @@ describe("provider maintenance host leases", () => { ); const file = NodePath.join(directory, "receipt.json"); const installed = NodePath.join(directory, "installed"); + const compacted = NodePath.join(directory, "compacted"); + let heartbeatOutputSize: number | undefined; const dead = NodeChildProcess.spawn(process.execPath, ["-e", ""], { stdio: "ignore" }); yield* Effect.promise( () => new Promise((resolve) => dead.once("exit", () => resolve())), @@ -2883,27 +2965,50 @@ describe("provider maintenance host leases", () => { "-e", mode === "installer-killed-with-live-child" ? `require("node:child_process").spawn(process.execPath, ["-e", ${JSON.stringify(`setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(installed)}, "installed"), 2500)`)}], {stdio: "ignore"}).on("spawn", () => process.kill(process.pid, "SIGKILL"))` - : `console.log("installing"); setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(installed)}, "installed"), ${mode === "heartbeat-write-error" ? 2500 : 0})`, + : mode === "verbose-output" + ? `const fs = require("node:fs"); console.error("e".repeat(30000) + "\\ninstalling"); console.log("x".repeat(30000)); const watcher = fs.watch(${JSON.stringify(directory)}, () => { if (fs.existsSync(${JSON.stringify(compacted)})) { watcher.close(); fs.writeFileSync(${JSON.stringify(installed)}, "installed"); } });` + : `console.log("installing"); setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(installed)}, "installed"), ${mode === "heartbeat-write-error" ? 2500 : 0})`, ], }), ); for (const suffix of [".stdout", ".stderr"]) NodeFS.writeFileSync(file + suffix, ""); + let authorized = false; const watcher = NodeFS.watch(directory, () => { if (InstallSupervisor.readReceipt(file).pid === undefined) return; + if (mode === "verbose-output" && !InstallSupervisor.readReceipt(file).progress) { + if (!authorized) { + authorized = true; + NodeFS.writeFileSync(file + ".start", "runtime-replaced"); + } + return; + } watcher.close(); + if (mode === "verbose-output") { + heartbeatOutputSize = Math.max( + NodeFS.statSync(file + ".stdout").size, + NodeFS.statSync(file + ".stderr").size, + ); + NodeFS.writeFileSync(compacted, "continue"); + return; + } if (mode === "runtime-replaced") NodeFS.rmSync(NodePath.join(runtime, "provider/providerMaintenanceRunner.ts")); NodeFS.writeFileSync(file + ".start", "runtime-replaced"); }); yield* Effect.addFinalizer(() => Effect.sync(() => watcher.close())); const child = yield* Effect.acquireRelease( - Effect.sync(() => - NodeChildProcess.spawn( - process.execPath, - [ - "--input-type=module", - "-e", - `import * as fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; + Effect.sync(() => { + const outputs = + mode === "verbose-output" + ? [NodeFS.openSync(file + ".stdout", "a"), NodeFS.openSync(file + ".stderr", "a")] + : []; + try { + return NodeChildProcess.spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import * as fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import { runInstallSubcommand } from ${JSON.stringify(supervisor)}; ${ mode === "heartbeat-write-error" @@ -2913,10 +3018,17 @@ describe("provider maintenance host leases", () => { : "" } await runInstallSubcommand("__provider-install", ${JSON.stringify([file, String(dead.pid)])});`, - ], - { stdio: "ignore", detached: true }, - ), - ), + ], + { + stdio: + mode === "verbose-output" ? ["ignore", outputs[0]!, outputs[1]!] : "ignore", + detached: true, + }, + ); + } finally { + for (const fd of outputs) NodeFS.closeSync(fd); + } + }), (child) => Effect.sync(() => { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); @@ -2938,6 +3050,11 @@ describe("provider maintenance host leases", () => { return Effect.sync(() => watcher.close()); }); assert.equal(code, 0); + if (mode === "verbose-output") { + assert.isAtMost(heartbeatOutputSize!, 20_000); + assert.isTrue(InstallSupervisor.readReceipt(file).result?.stdoutTruncated); + assert.equal(InstallSupervisor.readReceipt(file).progress, "installing"); + } if (mode === "installer-killed-with-live-child") { assert.isFalse( finishedBeforeChildren, diff --git a/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx b/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx index 10748a5553700..2b28a1da3496a 100644 --- a/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx +++ b/apps/web/src/components/ProviderUpdateEnvironmentRows.test.tsx @@ -224,6 +224,51 @@ async function flushPromises(): Promise { } describe("ProviderUpdatePrimaryNotification", () => { + it("settles a batch when a queued provider disappears during an earlier update", async () => { + hooks.reset(); + testState.permittedEnvironmentIds.add(environmentId); + testState.addToast.mockReset().mockReturnValue("removed-provider-toast"); + testState.updateToast.mockReset(); + testState.updateProvider.mockReset(); + const first = { ...provider(), instanceId: ProviderInstanceId.make("removal-first") }; + const second = { + ...provider(), + instanceId: ProviderInstanceId.make("removal-second"), + driver: ProviderDriverKind.make("claude"), + }; + const pending = + deferred>>(); + testState.updateProvider.mockReturnValueOnce(pending.promise); + testState.primaryProviders = [first, second]; + const render = () => { + hooks.beginRender(); + ProviderUpdatePrimaryNotification(); + hooks.flushEffects(); + }; + render(); + testState.addToast.mock.calls[0]![0].actionProps.onClick(); + expect(testState.updateProvider).toHaveBeenCalledTimes(1); + testState.primaryProviders = [first]; + render(); + pending.resolve( + AsyncResult.success({ + providers: [{ ...provider("succeeded"), instanceId: first.instanceId }], + }), + ); + await flushPromises(); + expect(testState.updateProvider).toHaveBeenCalledTimes(1); + expect(testState.addToast.mock.calls.at(-1)?.[0].title).toBe("Provider updated"); + testState.primaryProviders = [ + { + ...first, + instanceId: ProviderInstanceId.make("removal-next"), + versionAdvisory: { ...first.versionAdvisory!, latestVersion: "1.2.0" }, + }, + ]; + render(); + expect(testState.addToast.mock.calls.at(-1)?.[0].actionProps.children).toBe("Update"); + }); + it("removes the stale launch Update action when an unsolicited update starts", () => { hooks.reset(); testState.permittedEnvironmentIds.add(environmentId); diff --git a/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx b/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx index 6cc541de78609..b533ddd05b796 100644 --- a/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx +++ b/apps/web/src/components/ProviderUpdatePrimaryNotification.tsx @@ -45,7 +45,7 @@ type ActiveProviderUpdateToast = readonly kind: "update"; readonly key: string; readonly providerInstanceIds: ReadonlySet; - readonly providerCount: number; + providerCount: number; }; function ProviderUpdateToastIcon({ provider }: { provider: ServerProvider }) { @@ -237,7 +237,7 @@ export function ProviderUpdatePrimaryNotification() { } updateStarted = true; - const providerCount = oneClickProviders.length; + let providerCount = oneClickProviders.length; const providerInstanceIds = new Set(oneClickProviders.map((provider) => provider.instanceId)); const activeUpdate: ActiveProviderUpdateToast = { kind: "update", @@ -260,6 +260,8 @@ export function ProviderUpdatePrimaryNotification() { !isProviderUpdateCandidate(current) || !canOneClickUpdateProviderCandidate(current, latestProvidersRef.current) ) { + providerInstanceIds.delete(provider.instanceId); + activeUpdate.providerCount = --providerCount; continue; } if (!readEnvironmentScope(primaryEnvironment.environmentId, AuthProvidersManageScope)) { @@ -291,6 +293,11 @@ export function ProviderUpdatePrimaryNotification() { return; } + if (providerCount === 0) { + activeToastRef.current = null; + return; + } + const failedMessage = firstFailedProviderUpdateMessage(results); if (failedMessage) { addProviderUpdateToast({ diff --git a/packages/provider-core/src/server/maintenanceResolver.test.ts b/packages/provider-core/src/server/maintenanceResolver.test.ts index 3c63c8e4abef7..35b7fcb2a346b 100644 --- a/packages/provider-core/src/server/maintenanceResolver.test.ts +++ b/packages/provider-core/src/server/maintenanceResolver.test.ts @@ -1014,6 +1014,33 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { { skip: windowsHost }, ); + it.effect.each(["bun", "pnpm"] as const)( + "does not grant %s ownership to a symlink targeting another installation", + (manager) => + Effect.gen(function* () { + const root = yield* makeTempDir("r5-02-manager-link"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const home = NodePath.join(root, `custom-${manager}`); + const visible = NodePath.join(home, "bin", "package-tool"); + const real = NodePath.join(root, "foreign", "package-tool"); + writeExecutable(real); + NodeFS.mkdirSync(NodePath.dirname(visible), { recursive: true }); + NodeFS.symlinkSync(real, visible); + const result = yield* resolveProviderMaintenanceCapabilitiesEffect(packageToolUpdate, { + binaryPath: visible, + env: { + PATH: "", + BUN_INSTALL: manager === "bun" ? home : undefined, + PNPM_HOME: manager === "pnpm" ? home : undefined, + }, + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)); + expect(result.update).toBeNull(); + }).pipe(Effect.scoped), + { skip: windowsHost || !symlinksSupported }, + ); + it.effect.each(["bun", "pnpm", "yarn", "vp"] as const)( "does not treat a foreign SDK binary under %s as an owned provider install", (manager) => diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index e014020d287a3..f5b685d99235f 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -497,10 +497,10 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( } const bunBin = context.env.BUN_INSTALL?.trim().replace(/[\\/]+$/, ""); if ( - commandPaths.some(isBunGlobalCommandPath) || + isBunGlobalCommandPath(context.realCommandPath) || (bunBin && - commandPaths.some((value) => - normalizeCommandPath(value).startsWith(`${normalizeCommandPath(bunBin)}/bin/`), + normalizeCommandPath(context.realCommandPath).startsWith( + `${normalizeCommandPath(bunBin)}/bin/`, )) ) { return makeProviderMaintenanceCapabilities({ @@ -516,8 +516,8 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( } const pnpmHome = context.env.PNPM_HOME?.trim().replace(/[\\/]+$/, ""); if ( - commandPaths.some(isPnpmGlobalCommandPath) || - (pnpmHome && commandPaths.some((value) => identity(value).startsWith(`${identity(pnpmHome)}/`))) + isPnpmGlobalCommandPath(context.realCommandPath) || + (pnpmHome && identity(context.realCommandPath).startsWith(`${identity(pnpmHome)}/`)) ) { return makeProviderMaintenanceCapabilities({ provider: definition.provider, From 651b3a24eb0409b24666f88e1265bd7b56759734 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:37:52 +0000 Subject: [PATCH 134/149] fix(server): invalid server registrations no longer block startup, and OpenCode keeps a joined child turn's history An empty or corrupt server registration is discarded before process recovery instead of failing startup. A subagent call that joins an already running OpenCode 2 child turn keeps that turn's history boundary, so replays keep its earlier text and tools. The Pi steering replay fixture records the persisted steering exchange, and a case covers cleanup resuming a tree it froze. --- .../Adapters/OpenCode2AdapterV2.test.ts | 52 +++++++++++- .../fixtures/message_steering/pi_output.ts | 7 ++ .../message_steering/pi_transcript.ndjson | 2 + apps/server/src/process/agentScope.test.ts | 81 +++++++++++++++++++ apps/server/src/process/ownedProcesses.ts | 8 +- .../src/server/v2/adapter.ts | 18 +++-- 6 files changed, 153 insertions(+), 15 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index 34c39205d5b53..b5e812060559d 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -1020,6 +1020,34 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { { id: "msg_child_prompt", type: "user", time: { created: 1 }, text: "Choose a color" }, ], }; + // A second call joins the same execution after another user message + // landed. Its history still includes work before that message. + const joinedHistory = { + ...childHistory, + data: [ + { id: "msg_child_steer", type: "user", time: { created: 3 }, text: "Keep going" }, + { + ...childHistory.data[0], + content: [ + { type: "text", text: "Pre gap" }, + { + type: "tool", + id: "call_child_gap", + name: "shell", + executed: true, + state: { + status: "completed", + input: { command: "echo CHILD_GAP" }, + content: [{ type: "text", text: "CHILD_GAP" }], + metadata: {}, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + }, + ...childHistory.data.slice(1), + ], + }; const { runtime, thread } = yield* resumed( [ out("session.active"), @@ -1044,9 +1072,9 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ...(execution === "joined" ? [ out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), - reply("message.list", childHistory), + reply("message.list", joinedHistory), out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), - reply("message.list", childHistory), + reply("message.list", joinedHistory), out("permission.list", { sessionID: CHILD }), replyData("permission.list", []), out("session.form.list", { sessionID: CHILD }), @@ -1069,7 +1097,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { sessionID: CHILD, assistantMessageID: "msg_child", ordinal: 0, - text: "Prefix", + text: execution === "joined" ? "Pre gapfix" : "Prefix", }), event("session.execution.succeeded", { sessionID: CHILD }), ...(execution === "joined" @@ -1145,7 +1173,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { (event) => event.type === "message.updated" && event.message.threadId !== threadId && - event.message.text === "Prefix", + event.message.text === (execution === "joined" ? "Pre gapfix" : "Prefix"), ), ); assert.isFalse( @@ -1154,6 +1182,22 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ), ); if (execution === "joined") { + assert.isTrue( + collected.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "completed" && + event.turnItem.output === "CHILD_GAP", + ), + "joining a running child must backfill tools before its latest user message", + ); + assert.isTrue( + collected.some( + (event) => event.type === "message.updated" && event.message.text === "Pre gap", + ), + "joining a running child must backfill text before its latest user message", + ); const childTurns = collected.flatMap((event) => event.type === "provider_turn.updated" && event.providerTurn.nativeTurnRef?.nativeId?.startsWith(CHILD) diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_output.ts index cb48f13643843..b250dbf3785c1 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_output.ts @@ -55,6 +55,13 @@ export function assertPiMessageSteeringOutput( MESSAGE_STEERING_INITIAL_PROMPT, MESSAGE_STEERING_STEER_PROMPT, ]); + assert.lengthOf( + projection.turnItems.filter( + (item) => item.type === "user_message" && item.text === MESSAGE_STEERING_STEER_PROMPT, + ), + 1, + "steering must appear once in the saved conversation", + ); assertUserMessageInputIntents(projection, ["turn_start", "steer"]); assertAssistantTextIncludes(projection, "steering fixture observed"); assert.lengthOf(projection.providerTurns, 1, "active steering must not start a provider turn"); diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_transcript.ndjson index 32ffdd3987c4b..c931887fc6079 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering/pi_transcript.ndjson @@ -40,6 +40,8 @@ {"type":"emit_inbound","label":"queue_update","frame":{"type":"queue_update","steering":[],"followUp":[]}} {"type":"emit_inbound","label":"message_start","frame":{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"Actually, respond with exactly: steering fixture observed"}],"timestamp":1767225600462}}} {"type":"emit_inbound","label":"message_end","frame":{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"Actually, respond with exactly: steering fixture observed"}],"timestamp":1767225600462}}} +{"type":"expect_outbound","label":"get_state","frame":{"type":"get_state","id":"t3-9"}} +{"type":"emit_inbound","label":"response:get_state","frame":{"id":"t3-9","type":"response","command":"get_state","success":true,"data":{"model":{"type":"chat","id":"deepseek/deepseek-v4-flash","name":"DeepSeek: DeepSeek V4 Flash 0423","api":"openai-completions","baseUrl":"https://openrouter.ai/api/v1","provider":"openrouter","reasoning":true,"thinkingLevelMap":{"off":"none","minimal":null,"low":null,"medium":null,"high":"high","xhigh":"xhigh","max":null},"input":["text"],"cost":{"input":0.04186,"output":0.08372,"cacheRead":0.008372,"cacheWrite":0},"contextWindow":1048576,"maxTokens":131072,"compat":{"supportsDeveloperRole":false,"thinkingFormat":"openrouter","supportsStrictMode":true,"sendSessionAffinityHeaders":true,"requiresReasoningContentOnAssistantMessages":true}},"thinkingLevel":"high","isStreaming":true,"isCompacting":false,"steeringMode":"one-at-a-time","followUpMode":"one-at-a-time","sessionFile":"/pi-sessions/session-1.jsonl","sessionId":"00000000-0000-4000-8000-000000000002","sessionName":"Replay fixture: message_steering","autoCompactionEnabled":true,"messageCount":4,"pendingMessageCount":0}}} {"type":"emit_inbound","label":"message_start","frame":{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"openrouter","model":"deepseek/deepseek-v4-flash","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1767225601757}}} {"type":"emit_inbound","label":"message_update","frame":{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_start","contentIndex":0}}} {"type":"emit_inbound","label":"message_update","frame":{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_delta","contentIndex":0,"delta":"The"}}} diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 34e6210b51afa..9d6295447d009 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -313,6 +313,37 @@ finish(); }), ), ); + it.effect("resumes a provider tree it froze when cleanup cannot finish", () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + processes.delete(10); + // Each stopped process forks again, so the tree never reaches a fixed point. + let nextPid = 1000; + const forking: ProviderProcessLedger.ProcessController = { + ...controller, + signal: (pid, signal) => { + controller.signal(pid, signal); + if (signal !== "SIGSTOP") return; + const child = nextPid++; + processes.set(child, { + pid: child, + ppid: pid, + pgid: 20, + startTime: `fork-${child}`, + zombie: false, + }); + }, + }; + yield* Effect.exit(restart(directory, forking)); + const stopped = signals.filter((s) => s.endsWith(":SIGSTOP")).map((s) => s.split(":")[0]); + const resumed = signals.filter((s) => s.endsWith(":SIGCONT")).map((s) => s.split(":")[0]); + expect(stopped.length).toBeGreaterThan(0); + expect(resumed.toSorted()).toEqual(stopped.toSorted()); + expect(signals.some((s) => s.endsWith(":SIGKILL"))).toBe(false); + }), + ), + ); it.effect("leaves every child of a live owning server alone", () => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { @@ -614,6 +645,56 @@ finish(); ), ); + it.effect.each(["", "{", "{}"])( + "discards an invalid server registration %j before recovering orphaned providers", + (contents) => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + const leases = NodePath.join(directory, "leases"); + processes.set(70, { + pid: 70, + ppid: 20, + pgid: 70, + startTime: "live-server", + zombie: false, + }); + processes.set(71, { + pid: 71, + ppid: 70, + pgid: 71, + startTime: "live-provider", + zombie: false, + }); + yield* ProviderProcessLedger.make({ + stateDir: NodePath.join(directory, "second"), + ownerPid: 70, + controller, + installLeaseDirectory: leases, + }); + const servers = NodePath.join(leases, "servers"); + const liveRegistration = NodePath.join(servers, NodeFS.readdirSync(servers)[0]!); + const invalid = NodePath.join(servers, "invalid.json"); + NodeFS.writeFileSync(invalid, contents); + processes.delete(10); + yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + controller, + installLeaseDirectory: leases, + }); + expect(NodeFS.existsSync(invalid)).toBe(false); + expect(NodeFS.existsSync(liveRegistration)).toBe(true); + expect(processes.has(20)).toBe(false); + expect(processes.has(30)).toBe(false); + expect(signals).toContain("20:SIGKILL"); + expect(processes.has(70)).toBe(true); + expect(processes.has(71)).toBe(true); + expect(signals.some((s) => /^(70|71|90):/.test(s))).toBe(false); + }).pipe(Effect.scoped), + ), + ); + it.effect( "retains a server registration when its live owner is missing from the process query", () => diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index 79e80f044b71b..2e0bb0945e04a 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -302,9 +302,11 @@ export async function make(input: { const file = NodePath.join(servers, name); try { return [{ file, entry: decodeEntry(NodeFS.readFileSync(file, "utf8")) }]; - } catch (cause) { - if ((cause as NodeJS.ErrnoException).code === "ENOENT") return []; - throw cause; + } catch { + // A partial registration names no server to protect and must not + // block recovery in every environment sharing the install directory. + NodeFS.rmSync(file, { force: true }); + return []; } }); if (records.every(({ entry }) => entry.instanceId === instanceId)) return; diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 40c2579ed50db..2b718980c4644 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -2852,14 +2852,16 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr child.subagent!.turns, history.filter((entry) => entry.type === "idle").length, ); - yield* startChildTurn(child); - child.active!.before = - history.find( - (entry) => - entry.type === "user" || - entry.type === "synthetic" || - entry.type === "idle", - )?.id ?? null; + if (child.active === undefined) { + yield* startChildTurn(child); + child.active!.before = + history.find( + (entry) => + entry.type === "user" || + entry.type === "synthetic" || + entry.type === "idle", + )?.id ?? null; + } yield* backfill(childId!, child, activeSessions); } const [permissions, forms] = yield* Effect.all([ From 2934544e9f71cfa18be32aff69e7d499719a1b17 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:05:22 +0000 Subject: [PATCH 135/149] fix(server): turns already holding the install lock never wait behind a waiting install Start, resume, ensure, and snapshot calls for a session that holds a host lease join that lease, and helpers inherit their own active token, so a waiting install cannot deadlock against them while unrelated new turns still queue behind it. An acquired gate is released when the follow-up liveness check or lease removal fails, and a pinned npm install never gets an installation key. --- .../ProviderSessionManager.test.ts | 196 ++++++++++++++++++ .../ProviderSessionManager.ts | 37 ++-- .../provider/ProviderMaintenanceHostLock.ts | 16 +- .../providerMaintenanceRunner.test.ts | 70 ++++++- .../src/server/maintenanceResolver.test.ts | 46 +++- .../src/server/maintenanceResolver.ts | 2 +- 6 files changed, 335 insertions(+), 32 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 8e6a88624f480..ffdc16fa47da1 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -4837,3 +4837,199 @@ it.effect("ProviderSessionManagerV2 holds the host lease through terminal backgr ); }), ); + +it.live.each(["start", "resume", "ensure", "snapshot"] as const)( + "a background session %s joins its lease ahead of a waiting maintenance writer", + (operation) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const pending = yield* Ref.make(true); + const fs = yield* FileSystem.FileSystem; + const hostLock = HostLock.make(yield* fs.makeTempDirectoryScoped()); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-host-maintenance-background"); + const providerSessionId = idAllocator.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: "native-host-maintenance", + }); + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + const turnInput = { + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + } satisfies ProviderAdapter.ProviderAdapterV2TurnInput; + const nextRunId = idAllocator.derive.run({ threadId, ordinal: 2 }); + const continuation = runtime.startTurn({ + ...turnInput, + runId: nextRunId, + runOrdinal: 2, + providerTurnOrdinal: 2, + attemptId: idAllocator.derive.runAttempt({ runId: nextRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: nextRunId }), + }); + yield* runtime.startTurn(turnInput); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId))!; + const terminal = { + type: "turn.terminal" as const, + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-host-turn", + }), + runOrdinal: 1, + status: "completed" as const, + failure: null, + threadDisposition: "reusable" as const, + }; + const subscription = yield* runtime.subscribeEvents!; + const observed = yield* subscription.events.pipe( + Stream.take(1), + Stream.runDrain, + Effect.forkChild, + ); + yield* Queue.offer(queue, terminal); + yield* Fiber.join(observed); + const unrelatedThreadId = ThreadId.make("thread-unrelated-maintenance"); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId: unrelatedThreadId, now }), + ], + }); + const unrelatedSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: unrelatedThreadId, + }); + const unrelatedRuntime = yield* manager.open({ + threadId: unrelatedThreadId, + providerSessionId: unrelatedSessionId, + modelSelection, + runtimePolicy, + }); + const unrelatedRunId = idAllocator.derive.run({ threadId: unrelatedThreadId, ordinal: 1 }); + const waiting = yield* Deferred.make(); + const installed = yield* Deferred.make(); + const releaseWriter = yield* Deferred.make(); + const writer = yield* hostLock + .withWrite( + Deferred.succeed(installed, undefined).pipe( + Effect.andThen(Deferred.await(releaseWriter)), + ), + Deferred.succeed(waiting, undefined), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(waiting); + const unrelatedEntered = yield* Deferred.make(); + const unrelated = yield* Effect.forkChild( + unrelatedRuntime + .startTurn({ + ...turnInput, + appThread: (yield* projectionStore.getThreadProjection(unrelatedThreadId)).thread, + threadId: unrelatedThreadId, + runId: unrelatedRunId, + attemptId: idAllocator.derive.runAttempt({ + runId: unrelatedRunId, + attemptOrdinal: 1, + }), + rootNodeId: idAllocator.derive.rootNode({ runId: unrelatedRunId }), + providerThread: makeProviderThread({ + idAllocator, + threadId: unrelatedThreadId, + providerSessionId: unrelatedSessionId, + now, + nativeThreadId: "native-unrelated-maintenance", + }), + }) + .pipe(Effect.andThen(Deferred.succeed(unrelatedEntered, undefined))), + { startImmediately: true }, + ); + if (operation === "resume") { + yield* runtime.resumeThread({ + threadId, + providerThread, + modelSelection: { ...modelSelection, model: "changed-model" }, + runtimePolicy, + }); + assert.equal((yield* Ref.get(state)).resumeCount, 1); + } + if (operation === "ensure") { + const error = yield* runtime + .ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: providerThread, + }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.equal( + error.message, + `${CODEX_DRIVER} provider protocol error: ensureThread unused in test.`, + ); + } + if (operation === "snapshot") { + const error = yield* runtime.readThreadSnapshot({ providerThread }).pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.equal( + error.message, + `${CODEX_DRIVER} provider protocol error: readThreadSnapshot unused in test.`, + ); + } + // A continuation constructed before the lease existed must use the current lease. + yield* continuation; + assert.isFalse(yield* Deferred.isDone(installed)); + assert.isFalse(yield* Deferred.isDone(unrelatedEntered)); + yield* manager.close(providerSessionId); + yield* Deferred.await(installed); + assert.isFalse(yield* Deferred.isDone(unrelatedEntered)); + yield* Deferred.succeed(releaseWriter, undefined); + yield* Fiber.join(writer); + yield* Fiber.join(unrelated); + assert.isTrue(yield* Deferred.isDone(unrelatedEntered)); + yield* manager.close(unrelatedSessionId); + }).pipe( + Effect.provide( + layerTest({ + state, + hostLock, + hasPendingBackgroundWork: Ref.get(pending), + idleTimeoutMs: 60_000, + }), + ), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 04e2519877590..d71cf45c85c54 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1732,6 +1732,8 @@ export const layerWithOptions = ( ): ProviderAdapter.ProviderAdapterV2SessionRuntime => { const providerSessionId = runtime.providerSessionId; const subscribeEvents = makeEventSubscription(eventSubscribers); + const withSessionRead = (effect: Effect.Effect) => + Effect.suspend(() => hostLock.withRead(effect, hostLeases.get(runtime))); // Every provider's turn operations pass through here, so this is where they are // counted. Only turn starts are timed: until the provider accepts the turn. const turnMetrics = (operation: string, model?: string) => @@ -1771,11 +1773,10 @@ export const layerWithOptions = ( ), ), () => - hostLock.withRead( + withSessionRead( (compact ? runtime.compactThread!(input) : runtime.startTurn(input)).pipe( turnMetrics("send", input.modelSelection.model), ), - hostLeases.get(runtime), ), (_, exit) => Exit.isFailure(exit) @@ -1786,7 +1787,7 @@ export const layerWithOptions = ( : Effect.void, ), ), - hostLock.withRead, + withSessionRead, ); return { ...runtime, @@ -1803,7 +1804,7 @@ export const layerWithOptions = ( providerInstanceId: runtime.instanceId, }), ).pipe( - Effect.andThen(hostLock.withRead(runtime.ensureThread(input))), + Effect.andThen(withSessionRead(runtime.ensureThread(input))), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, @@ -1819,7 +1820,7 @@ export const layerWithOptions = ( resumeThread: (input) => { const threadId = input.threadId ?? input.providerThread.appThreadId; if (threadId === null || threadId === undefined) { - return hostLock.withRead(runtime.resumeThread(input)); + return withSessionRead(runtime.resumeThread(input)); } const providerThreadKey = providerThreadLoadKey({ providerThread: input.providerThread, @@ -1849,7 +1850,7 @@ export const layerWithOptions = ( Effect.flatMap((loaded) => loaded ? Effect.succeed(input.providerThread) - : hostLock.withRead(runtime.resumeThread(input)), + : withSessionRead(runtime.resumeThread(input)), ), Effect.tap((providerThread) => markProviderThreadLoaded({ @@ -1877,7 +1878,7 @@ export const layerWithOptions = ( providerInstanceId: runtime.instanceId, }), ).pipe( - Effect.andThen(hostLock.withRead(runtime.forkThread(input))), + Effect.andThen(withSessionRead(runtime.forkThread(input))), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, @@ -1894,13 +1895,13 @@ export const layerWithOptions = ( }), ), ), - readThreadSnapshot: (input) => hostLock.withRead(runtime.readThreadSnapshot(input)), - rollbackThread: (input) => hostLock.withRead(runtime.rollbackThread(input)), + readThreadSnapshot: (input) => withSessionRead(runtime.readThreadSnapshot(input)), + rollbackThread: (input) => withSessionRead(runtime.rollbackThread(input)), ...(runtime.injectHistory === undefined ? {} : { injectHistory: (input: Parameters>[0]) => - hostLock.withRead(runtime.injectHistory!(input)), + withSessionRead(runtime.injectHistory!(input)), }), startTurn: (input) => startTurn(input), ...(runtime.compactThread === undefined @@ -1911,26 +1912,20 @@ export const layerWithOptions = ( }), steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( - Effect.andThen( - hostLock - .withRead(runtime.steerTurn(input), hostLeases.get(runtime)) - .pipe(turnMetrics("steer")), - ), + Effect.andThen(withSessionRead(runtime.steerTurn(input)).pipe(turnMetrics("steer"))), ), interruptTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen( - hostLock - .withRead(runtime.interruptTurn(input), hostLeases.get(runtime)) - .pipe(turnMetrics("interrupt")), + withSessionRead(runtime.interruptTurn(input)).pipe(turnMetrics("interrupt")), ), ), respondToRuntimeRequest: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen( - hostLock - .withRead(runtime.respondToRuntimeRequest(input), hostLeases.get(runtime)) - .pipe(turnMetrics("runtime-request-response")), + withSessionRead(runtime.respondToRuntimeRequest(input)).pipe( + turnMetrics("runtime-request-response"), + ), ), ), }; diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index 75df067d23d26..07ab341055f6b 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -172,13 +172,16 @@ export const make = (directory: string) => { if (result._tag === "Success") { // An installer may have registered after our first check, before its // server died and this gate's heartbeat expired. - if (yield* Effect.sync(() => installIsAlive(directory))) { + const alive = yield* Effect.sync(() => { + const alive = installIsAlive(directory); + // Reclaim under the gate so no scan can erase a newer install lease. + if (!alive) NodeFS.rmSync(installLeasePath(directory), { force: true }); + return alive; + }).pipe(Effect.onError(() => Effect.promise(() => result.success()))); + if (alive) { yield* Effect.promise(() => result.success()); continue; } - // Reclaim only under the gate, so a concurrent stale-owner scan - // cannot erase a newer supervisor's lease. - yield* Effect.sync(() => NodeFS.rmSync(installLeasePath(directory), { force: true })); return { release: result.success, handoff: async () => { @@ -288,10 +291,7 @@ export const make = (directory: string) => { const inherited = inheritedRelease === undefined ? undefined : readTokens.get(inheritedRelease); const read = Effect.acquireUseRelease( acquireRead, - (release) => - effect.pipe( - Effect.provideService(HeldRead, inherited?.active ? inherited : readTokens.get(release)!), - ), + (release) => effect.pipe(Effect.provideService(HeldRead, readTokens.get(release)!)), (release) => release, ); return inherited === undefined ? read : read.pipe(Effect.provideService(HeldRead, inherited)); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 5c40c46aaf4bb..ecbdb14430b2a 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -6,9 +6,10 @@ import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionMana import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as Option from "effect/Option"; import * as ThreadContracts from "@t3tools/contracts"; -import { describe, it, assert } from "@effect/vitest"; +import { describe, it, assert, vi } from "@effect/vitest"; import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeModule from "node:module"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -2289,6 +2290,73 @@ describe("provider maintenance host leases", () => { return Effect.sync(() => watcher.close()); }); + it.live.each(["second check", "lease removal"] as const)( + "releases the acquired gate when %s fails", + (failure) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const host = HostLock.make(directory); + const leaseFile = HostLock.installLeasePath(directory); + const injected = new Error("injected lease failure"); + let checks = 0; + const mutableFS = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const readFile = NodeFS.readFileSync; + const remove = NodeFS.rmSync; + const spy = + failure === "second check" + ? vi + .spyOn(mutableFS, "readFileSync") + .mockImplementation((...args: Parameters) => { + if (args[0] === leaseFile && ++checks === 2) throw injected; + return readFile(...args); + }) + : vi + .spyOn(mutableFS, "rmSync") + .mockImplementation((...args: Parameters) => { + if (args[0] === leaseFile) throw injected; + return remove(...args); + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + spy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + const { unlock } = await import("proper-lockfile"); + await unlock(NodePath.join(directory, "gate"), { realpath: false }).catch(() => {}); + }), + ); + const exit = yield* Effect.exit(host.withRead(Effect.void)); + spy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) assert.strictEqual(Cause.squash(exit.cause), injected); + assert.isFalse(NodeFS.existsSync(NodePath.join(directory, "gate.lock"))); + yield* HostLock.make(directory).withWrite(Effect.void); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("a joined helper keeps reentrancy after the original session lease releases", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const host = HostLock.make(yield* fs.makeTempDirectoryScoped()); + const release = yield* host.acquireRead; + const waiting = yield* Deferred.make(); + const installed = yield* Deferred.make(); + const writer = yield* host + .withWrite(Deferred.succeed(installed, undefined), Deferred.succeed(waiting, undefined)) + .pipe(Effect.forkScoped); + yield* Deferred.await(waiting); + yield* host.withRead(release.pipe(Effect.andThen(host.withRead(Effect.void))), release); + yield* Fiber.join(writer); + assert.isTrue(yield* Deferred.isDone(installed)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("server admission keeps probes behind another server's install", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/provider-core/src/server/maintenanceResolver.test.ts b/packages/provider-core/src/server/maintenanceResolver.test.ts index 35b7fcb2a346b..c851804a6125d 100644 --- a/packages/provider-core/src/server/maintenanceResolver.test.ts +++ b/packages/provider-core/src/server/maintenanceResolver.test.ts @@ -262,7 +262,11 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { }, ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)); - expect(capabilities.update?.installationKey).toBeDefined(); + if (prefix.startsWith("Cellar/")) { + expect(capabilities.update?.installationKey).toBeUndefined(); + } else { + expect(capabilities.update?.installationKey).toBeDefined(); + } expect(capabilities).toMatchObject({ provider: driver("packageTool"), packageName: "@example/package-tool", @@ -284,6 +288,46 @@ it.layer(NodeServices.layer)("providerMaintenance", (it) => { { skip: windowsHost || !symlinksSupported }, ); + it.effect( + "excludes explicitly pinned npm keg binaries from automatic updates", + () => + Effect.gen(function* () { + const root = yield* makeTempDir("t3-pinned-npm-capabilities"); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true })), + ); + const prefix = NodePath.join(root, "Cellar/node/24"); + const alias = linkIntoPackage(prefix, "package-tool", [ + "lib", + "node_modules", + "@example", + "package-tool", + ]); + const real = NodeFS.realpathSync(alias); + for (const binaryPath of ["package-tool", real]) { + const capabilities = yield* resolvePackageManagedProviderMaintenance( + { + provider: driver("packageTool"), + npmPackageName: "@example/package-tool", + nativeUpdate: null, + }, + { + binaryPath, + resolvedCommandPath: alias, + realCommandPath: real, + env: { PATH: "" }, + platform: "linux", + }, + ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)); + expect(capabilities.update?.executable).toBe("npm"); + expect(capabilities.update?.installationKey).toBe( + binaryPath === real ? undefined : `@example/package-tool:${prefix}`, + ); + } + }).pipe(Effect.scoped), + { skip: windowsHost || !symlinksSupported }, + ); + it("derives the npm prefix only from the global lib/node_modules layout", () => { expect( npmGlobalPrefixFromCommandPath( diff --git a/packages/provider-core/src/server/maintenanceResolver.ts b/packages/provider-core/src/server/maintenanceResolver.ts index f5b685d99235f..94683db26de55 100644 --- a/packages/provider-core/src/server/maintenanceResolver.ts +++ b/packages/provider-core/src/server/maintenanceResolver.ts @@ -581,7 +581,7 @@ export const resolvePackageManagedProviderMaintenance = Effect.fn( `${packageName}@latest`, ], updateLockKey: `npm-global:${normalizeCommandPath(npmPrefix)}`, - installationKey: `${packageName}:${identity(npmPrefix)}`, + installationKey: keyFor(npmPrefix), }); } From 9039982899d4d20141de4997b9f9f5a21f031372 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:08:52 +0000 Subject: [PATCH 136/149] fix(server): restarts keep parent requests, frozen trees, and finished child transcripts Following a surviving OpenCode 2 execution restores and surfaces the parent's pending permission and form requests. A failure in the final protection check after freezing resumes the frozen tree. A foreground subagent that finished while the stream was disconnected gets its transcript restored before it settles. --- .../Adapters/OpenCode2AdapterV2.test.ts | 214 ++++++++++++++++++ apps/server/src/process/agentScope.test.ts | 43 ++++ apps/server/src/process/ownedProcesses.ts | 4 +- .../src/server/v2/adapter.ts | 66 ++++-- 4 files changed, 304 insertions(+), 23 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index b5e812060559d..6c7e3e3631d2b 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -424,6 +424,66 @@ const history = { }; it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { + it.effect.each(["permission", "form"] as const)( + "restores a parent %s arriving between resume and restart attachment", + (kind) => + Effect.gen(function* () { + const duringRestart = { data: history.data.slice(0, 2).toReversed(), cursor: {} }; + const { runtime, thread } = yield* resumed( + [ + kind === "permission" ? shellAskEvent : event("form.created", { form: colorForm }), + out("session.active"), + replyData("session.active", { [SESSION]: { type: "running" } }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", duringRestart), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", duringRestart), + out("permission.list", { sessionID: SESSION }), + replyData("permission.list", kind === "permission" ? [shellAsk.data] : []), + out("session.form.list", { sessionID: SESSION }), + replyData("session.form.list", kind === "form" ? [colorForm] : []), + ...(kind === "permission" + ? [ + out("permission.reply", { + sessionID: SESSION, + requestID: shellAsk.data.id, + decision: "once", + }), + reply("permission.reply", null), + ] + : [ + out("session.form.reply", { + sessionID: SESSION, + formID: colorForm.id, + answer: { q0: "Red" }, + }), + reply("session.form.reply", null), + ]), + event("session.execution.succeeded", { sessionID: SESSION }), + ], + { external: true }, + ); + const requested = yield* requestOf(runtime).pipe(Effect.forkScoped); + const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); + yield* runtime.startTurn({ + ...turnInput(thread), + restartContinuationOfRunId: RunId.make("cut-run"), + }); + const request = yield* Fiber.join(requested); + assert.equal(request?.status, "pending"); + assert.equal( + request?.nativeRequestRef?.nativeId, + kind === "permission" ? shellAsk.data.id : colorForm.id, + ); + yield* runtime.respondToRuntimeRequest({ + requestId: request!.id, + decision: "accept", + ...(kind === "form" ? { answers: { q0: "Red" } } : {}), + }); + assert.equal((yield* Fiber.join(terminal))?.status, "completed"); + }).pipe(Effect.scoped), + ); + it.effect.each([false, true, "note"])( "follows a surviving execution after a T3 restart (buffered=%s) without another prompt", (buffered) => @@ -438,6 +498,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { reply("message.list", duringRestart), out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), reply("message.list", duringRestart), + ...noOpenRequests, event("session.execution.succeeded", { sessionID: SESSION }), ], { external: true }, @@ -1081,6 +1142,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { replyData("session.form.list", [{ ...colorForm, sessionID: CHILD }]), ] : []), + ...noOpenRequests, out("session.form.reply", { sessionID: CHILD, formID: colorForm.id, @@ -3609,6 +3671,158 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { event.type === "turn_item.updated" ? [`${event.turnItem.type}:${event.turnItem.status}`] : [], ); + it.effect.each(["first", "reused", "attached"] as const)( + "replays a foreground child's %s turn when its entire execution was missed", + (execution) => + Effect.gen(function* () { + const ordinal = execution === "reused" ? 2 : 1; + const childHistory = { + data: [ + { id: "msg_child_idle", type: "idle", time: { created: 4 }, outcome: "succeeded" }, + { + ...history.data[1], + id: "msg_child_answer", + content: [{ type: "text", text: "Recovered child answer" }], + }, + { id: "msg_child_prompt", type: "user", time: { created: 2 }, text: "Find the answer" }, + ...(ordinal === 2 + ? [ + { + id: "msg_previous_idle", + type: "idle", + time: { created: 1 }, + outcome: "succeeded", + }, + { + ...history.data[1], + id: "msg_previous_answer", + content: [{ type: "text", text: "Earlier child answer" }], + }, + { + id: "msg_previous_prompt", + type: "user", + time: { created: 0 }, + text: "Earlier question", + }, + ] + : []), + ], + cursor: {}, + }; + const parentHistory = { + data: [ + { + ...history.data[1], + id: "msg_parent_answer", + content: [ + { + type: "tool", + id: "call_missed_child", + name: "subagent", + executed: true, + state: { + status: "completed", + input: { prompt: "Find the answer", description: "Answer" }, + content: [{ type: "text", text: "Recovered child answer" }], + metadata: execution === "attached" ? {} : { sessionID: CHILD }, + }, + time: { created: 2, ran: 2, completed: 4 }, + }, + ], + }, + { id: PROMPT_ID, type: "user", time: { created: 1 }, text: "hi" }, + ], + cursor: {}, + }; + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, id: PROMPT_ID, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + ...(execution === "attached" + ? [ + event("session.tool.input.started", { + sessionID: SESSION, + assistantMessageID: "msg_parent_answer", + id: "call_missed_child", + name: "subagent", + }), + event("session.tool.called", { + sessionID: SESSION, + assistantMessageID: "msg_parent_answer", + id: "call_missed_child", + name: "subagent", + input: { prompt: "Find the answer", description: "Answer" }, + executed: false, + }), + event("session.created", childCreated(CHILD)), + event("session.tool.progress", { + sessionID: SESSION, + assistantMessageID: "msg_parent_answer", + id: "call_missed_child", + metadata: { sessionID: CHILD }, + }), + ] + : []), + { type: "runtime_exit", status: "success" }, + out("event.subscribe"), + out("session.active"), + replyData("session.active", { [SESSION]: { type: "running" } }), + out("message.list", { sessionID: SESSION, order: "desc", limit: "50" }), + reply("message.list", parentHistory), + ...(execution === "attached" + ? [] + : [ + out("session.update", { sessionID: CHILD, permissions: t3Rules }), + reply("session.update", null), + ]), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", childHistory), + out("message.list", { sessionID: CHILD, order: "desc", limit: "50" }), + reply("message.list", childHistory), + ...openRequests([]), + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn(withLineage(thread)); + const collected = yield* Fiber.join(events); + const child = collected.findLast((event) => event.type === "subagent.updated"); + assert.equal( + child?.type === "subagent.updated" ? child.subagent.status : undefined, + "completed", + ); + const childThreadId = + child?.type === "subagent.updated" ? child.subagent.childThreadId : undefined; + assert.isNotNull(childThreadId); + assert.isTrue( + collected.some( + (event) => + event.type === "message.updated" && + event.message.threadId === childThreadId && + event.message.text === "Recovered child answer", + ), + ); + assert.isTrue( + collected.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === `${CHILD}:turn:${ordinal}` && + event.providerTurn.status === "completed", + ), + ); + assert.isFalse( + collected.some( + (event) => + event.type === "message.updated" && event.message.text === "Earlier child answer", + ), + ); + assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" }); + }).pipe(Effect.scoped), + ); + it.effect("does not end a turn from an earlier turn's history when its prompt is not there", () => Effect.gen(function* () { // The stream drops before the turn's prompt reached the history: what the diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 9d6295447d009..576a92afeb3cd 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -344,6 +344,49 @@ finish(); }), ), ); + it.effect( + "resumes frozen providers when the final server protection query cannot verify a live pid", + () => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + yield* record(directory, controller); + const leases = NodePath.join(directory, "leases"); + processes.set(70, { + pid: 70, + ppid: 1, + pgid: 70, + startTime: "live-server", + zombie: false, + }); + yield* ProviderProcessLedger.make({ + stateDir: NodePath.join(directory, "second"), + ownerPid: 70, + controller, + installLeaseDirectory: leases, + }); + processes.delete(10); + let frozenQueries = 0; + const result = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + installLeaseDirectory: leases, + controller: { + ...controller, + identity: (pid) => processes.get(pid), + snapshot: () => { + if (signals.filter((signal) => signal.endsWith(":SIGSTOP")).length === 2) + frozenQueries++; + return [...processes.values()].filter((p) => frozenQueries < 2 || p.pid !== 70); + }, + }, + }).pipe(Effect.exit); + expect(result._tag).toBe("Failure"); + expect(signals).toEqual(["20:SIGSTOP", "30:SIGSTOP", "20:SIGCONT", "30:SIGCONT"]); + expect(processes.has(20)).toBe(true); + expect(processes.has(30)).toBe(true); + }).pipe(Effect.scoped), + ), + ); it.effect("leaves every child of a live owning server alone", () => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index 2e0bb0945e04a..9ecdee30c6320 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -482,13 +482,13 @@ export async function make(input: { } // Persist newly discovered children before signalling: recovery can itself be killed. write(file, next); + table = await controller.snapshot(); + next = protectedEntry(next, table); } catch (error) { // Never leave a frozen tree behind: resume it so the next cleanup can retry. for (const p of stopped.values()) await signal(p, "SIGCONT").catch(() => undefined); throw error; } - table = await controller.snapshot(); - next = protectedEntry(next, table); // A process found to be protected after it was frozen is left running, not stopped. for (const p of stopped.values()) { if (!next.processes.some((owned) => same(owned, p))) { diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 2b718980c4644..2aa51361efecb 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -2823,16 +2823,15 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr call.background || stringField(metadata, "status") === "running" || input?.["background"] === true; - const childId = stringField(metadata, "sessionID"); + const childId = stringField(metadata, "sessionID") ?? call.child?.sessionId; const restoringChild = childId !== undefined && call.child === undefined; if (childId !== undefined) yield* attachChild(call, childId); if (part.state.status === "error") { yield* settleCall(call, "failed", part.state.error.message); - } else if (part.state.status === "completed" && !call.background) { - yield* settleCall(call, "completed", subagentOutput(textOf(part.state.content))); } else { + const completed = part.state.status === "completed" && !call.background; yield* emitSubagent(call); - if (restoringChild) { + if (restoringChild || (completed && call.child?.subagent?.turns === 0)) { if ( call.background && activeSessions !== undefined && @@ -2842,20 +2841,24 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr continue; } const sessionID = Session.ID.make(childId!); - if (activeSessions?.has(childId!) && call.child !== undefined) { + if ((activeSessions?.has(childId!) || completed) && call.child !== undefined) { const history = yield* OpenCode2Client.paginate( { sessionID, order: "desc" as const, limit: 50 }, client.message.list, ).pipe(Stream.runCollect); + // A completed execution's final idle ends this turn, not the + // earlier turn its history must begin after. + const recent = + completed && history[0]?.type === "idle" ? history.slice(1) : history; const child = call.child; child.subagent!.turns = Math.max( child.subagent!.turns, - history.filter((entry) => entry.type === "idle").length, + recent.filter((entry) => entry.type === "idle").length, ); if (child.active === undefined) { yield* startChildTurn(child); child.active!.before = - history.find( + recent.find( (entry) => entry.type === "user" || entry.type === "synthetic" || @@ -2863,13 +2866,20 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr )?.id ?? null; } yield* backfill(childId!, child, activeSessions); + if (completed) yield* finishTurn(child, { status: "completed" }); } - const [permissions, forms] = yield* Effect.all([ - client.permission.list({ sessionID }), - client.session.form.list({ sessionID }), - ]); - for (const request of permissions) yield* onAsked({ type: "permission", request }); - for (const form of forms) yield* onAsked({ type: "form", form }); + if (!completed) { + const [permissions, forms] = yield* Effect.all([ + client.permission.list({ sessionID }), + client.session.form.list({ sessionID }), + ]); + for (const request of permissions) + yield* onAsked({ type: "permission", request }); + for (const form of forms) yield* onAsked({ type: "form", form }); + } + } + if (part.state.status === "completed" && completed) { + yield* settleCall(call, "completed", subagentOutput(textOf(part.state.content))); } } continue; @@ -4120,15 +4130,29 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr nativeTurnRef: ref(prompt.id, "weak"), }; yield* emitProviderTurn(state, turn, turn.providerTurn); - yield* backfill(sessionId, state, new Set(Object.keys(active))).pipe( - Effect.tapError((cause) => - finishTurn(state, { - status: "failed", - failure: makeProviderFailure({ cause, class: "provider_error" }), - }), - ), - ); } + yield* Effect.gen(function* () { + if (prompt?.type === "user") { + yield* backfill(sessionId, state, new Set(Object.keys(active))); + } + if (sessionId in active) { + const sessionID = Session.ID.make(sessionId); + const [permissions, forms] = yield* Effect.all([ + client.permission.list({ sessionID }), + client.session.form.list({ sessionID }), + ]); + for (const request of permissions) + yield* onAsked({ type: "permission", request }); + for (const form of forms) yield* onAsked({ type: "form", form }); + } + }).pipe( + Effect.tapError((cause) => + finishTurn(state, { + status: "failed", + failure: makeProviderFailure({ cause, class: "provider_error" }), + }), + ), + ); if (finishedWhileDown && !(sessionId in active)) { yield* finishTurn(state, { status: "completed" }); } From b574fd0b3a2358cbab0987f6bc84583b62d96b0f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:28:43 +0000 Subject: [PATCH 137/149] fix(server): a reused process id never keeps a stale install lease alive Install leases record the supervisor's process start identity (with the boot identity on Linux), so a lease whose pid now belongs to an unrelated process is reclaimed instead of blocking every provider launch. Leases written without an identity, or whose identity cannot be read, stay conservative. --- .../src/provider/ProviderInstallSupervisor.ts | 25 ++- .../provider/ProviderMaintenanceHostLock.ts | 64 ++++++- .../providerMaintenanceRunner.test.ts | 164 +++++++++++++++++- 3 files changed, 241 insertions(+), 12 deletions(-) diff --git a/apps/server/src/provider/ProviderInstallSupervisor.ts b/apps/server/src/provider/ProviderInstallSupervisor.ts index 9cbab85efff5e..78bc655d10ae2 100644 --- a/apps/server/src/provider/ProviderInstallSupervisor.ts +++ b/apps/server/src/provider/ProviderInstallSupervisor.ts @@ -38,6 +38,7 @@ const Receipt = Schema.Struct({ args: Schema.Array(Schema.String), env: Schema.optional(Schema.Record(Schema.String, Schema.UndefinedOr(Schema.String))), parentPid: Schema.Number, + parentIdentity: Schema.optional(Schema.NullOr(Schema.String)), directory: Schema.String, pid: Schema.optional(Schema.Number), progress: Schema.optional(Schema.String), @@ -103,6 +104,7 @@ export const layer = Layer.effect( ...input, id: NodeCrypto.randomUUID(), parentPid: process.pid, + parentIdentity: HostLock.readProcessIdentity(process.pid), directory: host.directory, }; yield* Effect.try(() => { @@ -211,7 +213,14 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< const stderr = NodeFS.openSync(`${file}.stderr`, "a", 0o600); const child = NodeChildProcess.spawn( invocation.command, - [...selfInvocationArgs(invocation, ["__provider-install", file, String(process.pid)])], + [ + ...selfInvocationArgs(invocation, [ + "__provider-install", + file, + String(process.pid), + HostLock.readProcessIdentity(process.pid) ?? "", + ]), + ], { detached: true, stdio: ["ignore", stdout, stderr], @@ -231,10 +240,15 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< let receipt = readReceipt(file); const relayPid = Number(args[1]); // The live relay is the only link into the desktop/server's Windows tree. - while (HostLock.isProcessAlive(relayPid)) await new Promise((resolve) => setTimeout(resolve, 25)); + while (HostLock.isProcessAlive(relayPid, args[2] || undefined)) + await new Promise((resolve) => setTimeout(resolve, 25)); const lease = HostLock.installLeasePath(receipt.directory); const candidate = `${lease}.${process.pid}`; - NodeFS.writeFileSync(candidate, String(process.pid), { flag: "wx", mode: 0o600 }); + NodeFS.writeFileSync( + candidate, + JSON.stringify({ pid: process.pid, identity: HostLock.readProcessIdentity(process.pid) }), + { flag: "wx", mode: 0o600 }, + ); try { // A delayed relay must never overwrite an already running install's owner. NodeFS.linkSync(candidate, lease); @@ -290,7 +304,10 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< writeReceipt(file, receipt); const deadline = Date.now() + 30_000; while (!NodeFS.existsSync(`${file}.start`)) { - if (!HostLock.isProcessAlive(receipt.parentPid) || Date.now() >= deadline) { + if ( + !HostLock.isProcessAlive(receipt.parentPid, receipt.parentIdentity) || + Date.now() >= deadline + ) { throw new Error("Server exited before authorizing provider installation."); } await new Promise((resolve) => setTimeout(resolve, 50)); diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index 07ab341055f6b..f93464906801e 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -12,6 +12,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Deferred from "effect/Deferred"; import * as Semaphore from "effect/Semaphore"; +import * as Schema from "effect/Schema"; import * as Maintenance from "@t3tools/provider-core/server/maintenance"; const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } | null>( @@ -42,10 +43,45 @@ const processIdentity = (pid: number) => { export const installLeasePath = (directory: string) => NodePath.join(directory, "install.json"); -export const isProcessAlive = (pid: number) => { +/** Start identity includes the Linux boot so leases cannot survive a reboot's PID reuse. */ +export const readProcessIdentity = (pid: number): string | null => { + if (!Number.isSafeInteger(pid) || pid <= 0) return null; + try { + const platform = NodeOS.platform(); + if (platform === "linux") { + const start = processIdentity(pid); + return start + ? `${NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` + : null; + } + if (platform !== "darwin" && platform !== "win32") return null; + const scan = + platform === "win32" + ? NodeChildProcess.spawnSync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + `(Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}' -ErrorAction Stop).CreationDate.ToUniversalTime().ToString('o')`, + ], + { encoding: "utf8", timeout: 10_000, windowsHide: true }, + ) + : NodeChildProcess.spawnSync("ps", ["-p", String(pid), "-o", "lstart="], { + encoding: "utf8", + timeout: 10_000, + env: { ...process.env, LC_ALL: "C", TZ: "UTC" }, + }); + return scan.error || scan.status !== 0 ? null : scan.stdout.trim() || null; + } catch { + return null; + } +}; + +export const isProcessAlive = (pid: number, identity?: string | null) => { try { process.kill(pid, 0); - return processIdentity(pid) !== ""; + if (processIdentity(pid) === "") return false; } catch (error) { if ( error instanceof Error && @@ -53,22 +89,40 @@ export const isProcessAlive = (pid: number) => { (error.code === "ESRCH" || error.code === "ENOENT") ) return false; - throw error; + return true; } + if (identity == null) return true; + const current = readProcessIdentity(pid); + return current === null || current === identity; }; +const decodeInstallLease = Schema.decodeSync( + Schema.fromJsonString( + Schema.Union([ + Schema.Number, + Schema.Struct({ + pid: Schema.Number, + identity: Schema.optional(Schema.NullOr(Schema.String)), + }), + ]), + ), +); + /** A dead supervisor's installer still owns admission until its entire process group exits. */ export const installIsAlive = (directory: string): boolean => { const file = installLeasePath(directory); let pid: number; + let identity: string | null | undefined; try { - pid = Number(NodeFS.readFileSync(file, "utf8")); + const lease = decodeInstallLease(NodeFS.readFileSync(file, "utf8")); + pid = typeof lease === "number" ? lease : lease.pid; + identity = typeof lease === "number" ? undefined : lease.identity; } catch (error) { if (error instanceof Error && "code" in error && error.code === "ENOENT") return false; throw error; } if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid provider install lease"); - if (isProcessAlive(pid)) return true; + if (isProcessAlive(pid, identity)) return true; return installChildrenAreAlive(pid); }; diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index ecbdb14430b2a..b01efc077b601 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -66,6 +66,11 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; const isServerProviderUpdateError = Schema.is(ServerProviderUpdateError); +const decodeInstallLease = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ pid: Schema.Number, identity: Schema.NullOr(Schema.String) }), + ), +); it.effect.each([ "claude", @@ -2237,6 +2242,148 @@ it.effect.each(["interaction", "opt-out"] as const)( ); describe("provider maintenance host leases", () => { + it.live("reclaims an install lease whose live PID belongs to another process", () => + Effect.scoped( + Effect.gen(function* () { + assert.isFalse(HostLock.isProcessAlive(process.pid, "previous-process-start")); + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + NodeFS.writeFileSync( + HostLock.installLeasePath(directory), + JSON.stringify({ pid: process.pid, identity: "previous-process-start" }), + ); + assert.isFalse(HostLock.installIsAlive(directory)); + const host = HostLock.make(directory); + assert.equal(yield* host.withRead(Effect.succeed("launched")), "launched"); + assert.equal(yield* host.withWrite(Effect.succeed("installed")), "installed"); + assert.isFalse(NodeFS.existsSync(HostLock.installLeasePath(directory))); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live.each(["legacy", "matching", "missing", "unknown"] as const)( + "keeps a live install exclusive with %s identity", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const identity = HostLock.readProcessIdentity(process.pid); + assert.isNotNull(identity); + NodeFS.writeFileSync( + HostLock.installLeasePath(directory), + mode === "legacy" + ? String(process.pid) + : JSON.stringify({ + pid: process.pid, + ...(mode === "missing" ? {} : { identity: mode === "unknown" ? null : identity }), + }), + ); + assert.isTrue(HostLock.installIsAlive(directory)); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.each(["darwin", "win32"] as const)( + "checks %s process creation identity conservatively", + (platform) => { + const require = NodeModule.createRequire(import.meta.url); + const mutableOS = require("node:os") as typeof NodeOS; + const mutableChildProcess = require("node:child_process") as typeof NodeChildProcess; + const osSpy = vi.spyOn(mutableOS, "platform").mockReturnValue(platform); + const scan = vi.spyOn(mutableChildProcess, "spawnSync").mockReturnValue({ + pid: 1, + output: [null, "current-start\n", ""], + stdout: "current-start\n", + stderr: "", + status: 0, + signal: null, + }); + NodeModule.syncBuiltinESMExports(); + try { + assert.equal(HostLock.readProcessIdentity(process.pid), "current-start"); + assert.isTrue(HostLock.isProcessAlive(process.pid, "current-start")); + assert.isFalse(HostLock.isProcessAlive(process.pid, "previous-start")); + scan.mockReturnValue({ + pid: 1, + output: [null, "", ""], + stdout: "", + stderr: "", + status: 1, + signal: null, + }); + assert.isNull(HostLock.readProcessIdentity(process.pid)); + assert.isTrue(HostLock.isProcessAlive(process.pid, "previous-start")); + } finally { + scan.mockRestore(); + osSpy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + } + }, + ); + + it.effect("keeps an install alive when its Linux identity cannot be read", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) !== "linux") return; + const mutableFS = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const readFile = NodeFS.readFileSync; + const spy = vi + .spyOn(mutableFS, "readFileSync") + .mockImplementation((...args: Parameters) => { + if ( + args[0] === "/proc/sys/kernel/random/boot_id" || + args[0] === `/proc/${process.pid}/stat` + ) + throw Object.assign(new Error("denied"), { code: "EACCES" }); + return readFile(...args); + }); + NodeModule.syncBuiltinESMExports(); + try { + assert.isNull(HostLock.readProcessIdentity(process.pid)); + assert.isTrue(HostLock.isProcessAlive(process.pid, "previous-start")); + } finally { + spy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + } + }), + ); + + it.live.each(["live children", "failed scan"] as const)( + "a reused supervisor PID keeps admission closed for %s", + (mode) => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) !== "linux") return; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + NodeFS.writeFileSync( + HostLock.installLeasePath(directory), + JSON.stringify({ pid: process.pid, identity: "previous-start" }), + ); + const mutableChildProcess = NodeModule.createRequire(import.meta.url)( + "node:child_process", + ) as typeof NodeChildProcess; + const scan = vi.spyOn(mutableChildProcess, "spawnSync").mockReturnValue({ + pid: 1, + output: [null, "", ""], + stdout: `2 ${process.pid} S\n`, + stderr: "", + status: mode === "failed scan" ? 1 : 0, + signal: null, + }); + NodeModule.syncBuiltinESMExports(); + try { + if (mode === "failed scan") + assert.throws(() => HostLock.installIsAlive(directory), /Could not inspect/); + else assert.isTrue(HostLock.installIsAlive(directory)); + } finally { + scan.mockRestore(); + NodeModule.syncBuiltinESMExports(); + } + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + const coordinatedWrite = (directory: string, run: Effect.Effect) => Effect.gen(function* () { const coordinator = yield* makeProviderMaintenanceCommandCoordinator({ @@ -2751,7 +2898,7 @@ describe("provider maintenance host leases", () => { ).pipe(Effect.provide(NodeServices.layer)), ); - it.live.each(["unacknowledged", "existing owner"] as const)( + it.live.each(["unacknowledged", "existing owner", "reused parent", "reused relay"] as const)( "a late supervisor preserves the current host gate (%s)", (mode) => Effect.scoped( @@ -2773,7 +2920,8 @@ describe("provider maintenance host leases", () => { instanceId: CODEX_INSTANCE_ID, instanceIds: [CODEX_INSTANCE_ID], startedAt: "2026-04-10T00:00:00.000Z", - parentPid: dead.pid, + parentPid: mode === "reused parent" ? process.pid : dead.pid, + ...(mode === "reused parent" ? { parentIdentity: "previous-start" } : {}), directory, command: process.execPath, args: ["-e", ""], @@ -2789,7 +2937,8 @@ describe("provider maintenance host leases", () => { NodePath.resolve(import.meta.dirname, "../bin.ts"), "__provider-install", file, - String(dead.pid), + String(mode === "reused relay" ? process.pid : dead.pid), + "previous-start", ], { stdio: "ignore" }, ); @@ -2940,6 +3089,15 @@ describe("provider maintenance host leases", () => { return Effect.sync(() => watcher.close()); }); const receipt = InstallSupervisor.readReceipt(receiptFile); + assert.ok(receipt.pid); + assert.equal(receipt.parentIdentity, HostLock.readProcessIdentity(receipt.parentPid)); + const lease = yield* decodeInstallLease( + NodeFS.readFileSync(HostLock.installLeasePath(directory), "utf8"), + ); + assert.deepEqual(lease, { + pid: receipt.pid, + identity: HostLock.readProcessIdentity(receipt.pid), + }); assert.isTrue(HostLock.installIsAlive(directory)); if (mode === "supervisor") { assert.ok(receipt.pid); From 29b0303b997cfa505087ecd39a1d252a8c698510 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:52:57 +0000 Subject: [PATCH 138/149] fix(server): background server restarts wait for review guides, clones, and git refreshes Every pull request review guide generation is tracked through fetch, checkout, generation, storage and cleanup, including superseded generations and their finalizers, and holds back a server restart until it ends. Project clones, workspace preparation and background git refreshes do the same. The counts show in the update window's blocker log. --- .../ThreadLaunchService.test.ts | 36 +++ .../orchestration-v2/ThreadLaunchService.ts | 20 +- .../src/project/ProjectCloneTracker.test.ts | 13 +- .../server/src/project/ProjectCloneTracker.ts | 12 +- .../providerMaintenanceRunner.test.ts | 46 ++++ .../PullRequestReviewGuideService.test.ts | 256 +++++++++++++++++- .../PullRequestReviewGuideService.ts | 22 +- apps/server/src/server.ts | 4 +- apps/server/src/updates/UpdateWindow.ts | 45 +++ .../src/vcs/VcsStatusBroadcaster.test.ts | 43 +++ apps/server/src/vcs/VcsStatusBroadcaster.ts | 14 +- 11 files changed, 499 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 3012e09411ac1..be93f5ab78581 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -32,6 +32,7 @@ import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; @@ -401,6 +402,40 @@ it.effect("retains automation and sender attribution while a message waits in th }).pipe(Effect.provide(harness.layer)); }); +it.effect("tracks workspace preparation for an empty thread with no active run", () => + Effect.gen(function* () { + const entered = yield* Deferred.make>(); + const release = yield* Deferred.make(); + const harness = makeHarness({ + createWorktree: (input) => + Effect.gen(function* () { + yield* Deferred.succeed(entered, yield* Effect.fiber); + yield* Deferred.await(release); + return { + worktree: { path: "/repo-worktrees/empty", refName: input.newRefName, headSha: "abc" }, + } as never; + }), + }); + yield* Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const result = yield* launches.launch( + launchInput({ + command: "command:launch:empty-preparation", + thread: "thread:launch:empty-preparation", + workspace: { type: "worktree", baseRef: "main", branch: "empty" }, + }), + ); + const fiber = yield* Deferred.await(entered); + assert.equal((yield* threads.getThreadProjection(result.threadId)).runs.length, 0); + assert.equal(yield* launches.activePreparationCount, 1); + yield* Deferred.succeed(release, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.await(fiber))); + assert.equal(yield* launches.activePreparationCount, 0); + }).pipe(Effect.provide(harness.layer)); + }), +); + it.effect("returns a visible preparing message while provisioning is still blocked", () => Effect.gen(function* () { const worktreeEntered = yield* Deferred.make(); @@ -2263,6 +2298,7 @@ it.effect("shared intake preserves durable attachment bytes after a lost launch ), checkWorktreeBase: launches.checkWorktreeBase, retryPreparation: launches.retryPreparation, + activePreparationCount: launches.activePreparationCount, }), Effect.flip, ); diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 42b1d9cef1e42..f49e629a61690 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -23,6 +23,7 @@ import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as FiberSet from "effect/FiberSet"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; @@ -47,6 +48,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import type * as Orchestrator from "./Orchestrator.ts"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid"; +import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; import * as ThreadManagement from "./ThreadManagementService.ts"; export type ThreadLaunchWorkspaceStrategy = @@ -187,6 +189,8 @@ export class ThreadLaunchService extends Context.Service< | Orchestrator.OrchestratorV2Error | ThreadManagement.ThreadManagementProviderInstanceDisabledError >; + /** Workspace preparation and branch naming can outlive a thread's run. */ + readonly activePreparationCount: Effect.Effect; } >()("t3/orchestration-v2/ThreadLaunchService") {} @@ -215,9 +219,13 @@ const make = Effect.gen(function* () { const ids = yield* IdAllocator.IdAllocatorV2; const threads = yield* ThreadManagement.ThreadManagementService; const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const preparationScope = yield* Scope.make("sequential"); const scheduledLaunches = yield* Ref.make>(new Set()); yield* Effect.addFinalizer(() => Scope.close(preparationScope, Exit.void)); + const preparations = yield* FiberSet.make().pipe( + Effect.provideService(Scope.Scope, preparationScope), + ); const mapError = (input: PreparationInput, operation: ThreadLaunchError["operation"], threadId?: ThreadId) => @@ -531,7 +539,7 @@ const make = Effect.gen(function* () { cause, }), ), - Effect.forkIn(preparationScope), + FiberSet.run(preparations, { startImmediately: false }), ); } @@ -745,7 +753,8 @@ const make = Effect.gen(function* () { ), Effect.ignoreCause, Effect.ensuring(releasePreparation(input.commandId)), - Effect.forkIn(preparationScope), + FiberSet.run(preparations, { startImmediately: false }), + admission.withLaunchPermit, ); }); @@ -1050,7 +1059,12 @@ const make = Effect.gen(function* () { ); }; - return ThreadLaunchService.of({ launch, checkWorktreeBase, retryPreparation }); + return ThreadLaunchService.of({ + launch, + checkWorktreeBase, + retryPreparation, + activePreparationCount: FiberSet.size(preparations), + }); }); export const layer = Layer.effect(ThreadLaunchService, make); diff --git a/apps/server/src/project/ProjectCloneTracker.test.ts b/apps/server/src/project/ProjectCloneTracker.test.ts index d327ca4436f08..df14a343a8f9b 100644 --- a/apps/server/src/project/ProjectCloneTracker.test.ts +++ b/apps/server/src/project/ProjectCloneTracker.test.ts @@ -8,6 +8,7 @@ import { import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; @@ -162,20 +163,28 @@ describe("ProjectCloneTracker", () => { it.effect("a cancel that lands after git finished keeps the checkout", () => { const gate = Deferred.makeUnsafe(); + const entered = Deferred.makeUnsafe>(); const harness = makeHarness(); // The clone itself completes instantly; the post-clone hook is what hangs. const hooks: ProjectCloneTracker.ProjectCloneHooks = { ...harness.hooks, - onCloned: () => Deferred.await(gate), + onCloned: () => + Effect.fiber.pipe( + Effect.flatMap((fiber) => Deferred.succeed(entered, fiber)), + Effect.andThen(Deferred.await(gate)), + ), }; return Effect.gen(function* () { const tracker = yield* ProjectCloneTracker.ProjectCloneTracker; yield* tracker.start(startInput, hooks); - yield* Effect.yieldNow; + const fiber = yield* Deferred.await(entered); expect((yield* tracker.get(projectId))?.phase).toBe("done"); + expect(yield* tracker.activeCloneCount).toBe(1); expect(yield* tracker.cancel(projectId)).toBe(false); expect(harness.discarded).toEqual([]); yield* Deferred.succeed(gate, undefined); + expect(Exit.isSuccess(yield* Fiber.await(fiber))).toBe(true); + expect(yield* tracker.activeCloneCount).toBe(0); }).pipe(Effect.provide(harness.layer)); }); diff --git a/apps/server/src/project/ProjectCloneTracker.ts b/apps/server/src/project/ProjectCloneTracker.ts index 092a0376942be..6b729655aa92d 100644 --- a/apps/server/src/project/ProjectCloneTracker.ts +++ b/apps/server/src/project/ProjectCloneTracker.ts @@ -19,6 +19,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as FiberSet from "effect/FiberSet"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; @@ -29,6 +30,7 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts"; +import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; /** * Runs repository clones that back newly added projects and tracks their @@ -69,6 +71,8 @@ export class ProjectCloneTracker extends Context.Service< readonly get: (projectId: ProjectId) => Effect.Effect; /** Emits every tracked clone first, then the full list after each change. */ readonly stream: Stream.Stream>; + /** Includes post-clone hooks after the progress snapshot has settled. */ + readonly activeCloneCount: Effect.Effect; } >()("t3/project/ProjectCloneTracker") {} @@ -115,6 +119,7 @@ interface TrackedClone { /** @public Service construction is part of the canonical Effect module API. */ export const make = Effect.gen(function* () { const repositories = yield* SourceControlRepositoryService.SourceControlRepositoryService; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const clones = yield* Ref.make(new Map()); const changes = yield* PubSub.unbounded>(); const retentionFibers = new Map>(); @@ -122,6 +127,7 @@ export const make = Effect.gen(function* () { // Clone fibers outlive the RPC that started them but not the server. const cloneScope = yield* Scope.make("parallel"); yield* Effect.addFinalizer(() => Scope.close(cloneScope, Exit.void)); + const cloneFibers = yield* FiberSet.make().pipe(Effect.provideService(Scope.Scope, cloneScope)); // start/cancel/retry/discard mutate the same entry and the same directory; // one at a time keeps a double-clicked Retry from racing two clones into it. const actionLock = yield* Semaphore.make(1); @@ -249,7 +255,10 @@ export const make = Effect.gen(function* () { const current = yield* Ref.get(clones); const tracked = current.get(projectId); if (!tracked) return; - const fiber = yield* runClone(projectId, tracked).pipe(Effect.forkIn(cloneScope)); + const fiber = yield* runClone(projectId, tracked).pipe( + FiberSet.run(cloneFibers, { startImmediately: false }), + admission.withLaunchPermit, + ); yield* Ref.update(clones, (map) => { const existing = map.get(projectId); if (!existing) return map; @@ -430,6 +439,7 @@ export const make = Effect.gen(function* () { discard: (projectId) => locked(discard(projectId)), get, stream, + activeCloneCount: FiberSet.size(cloneFibers), }); }); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 40b21a360a5a3..68fcf6cdc7308 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -3,6 +3,10 @@ import * as Maintenance from "@t3tools/provider-core/server/maintenance"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; +import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; +import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts"; +import * as PullRequestReviewGuideService from "../pullRequest/PullRequestReviewGuideService.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as Option from "effect/Option"; import * as ThreadContracts from "@t3tools/contracts"; @@ -3511,6 +3515,26 @@ it.effect.each([ expected: false, updatedInstance: ProviderInstanceId.make("other"), }, + ...( + [ + "review-guides", + "project-clones", + "workspace-preparation", + "git-refreshes", + "missing-guide-tracker", + ] as const + ).flatMap((work) => + [true, false].map((restart) => ({ + name: `${work} during ${restart ? "restart" : "provider install"}`, + restart, + busy: false, + installing: false, + interaction: false, + expected: !restart, + updatedInstance: null, + work, + })), + ), ])("rechecks the update window for $name", (scenario) => Effect.gen(function* () { let quietReads = 0; @@ -3519,6 +3543,28 @@ it.effect.each([ UpdateWindow.layer.pipe( Layer.provide( Layer.mergeAll( + "work" in scenario && scenario.work === "missing-guide-tracker" + ? Layer.empty + : Layer.mock(PullRequestReviewGuideService.PullRequestReviewGuideService)({ + activeGenerationCount: Effect.succeed( + "work" in scenario && scenario.work === "review-guides" ? 2 : 0, + ), + }), + Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({ + activeCloneCount: Effect.succeed( + "work" in scenario && scenario.work === "project-clones" ? 1 : 0, + ), + }), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + activePreparationCount: Effect.succeed( + "work" in scenario && scenario.work === "workspace-preparation" ? 1 : 0, + ), + }), + Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ + activeRefreshCount: Effect.succeed( + "work" in scenario && scenario.work === "git-refreshes" ? 1 : 0, + ), + }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ getRecoveryThreadIds: () => Effect.succeed([]), getLimitRecoveryCandidates: () => Effect.succeed([]), diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts index 51591f031f1c4..a4c17139b4956 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts @@ -3,12 +3,19 @@ import { assert, it } from "@effect/vitest"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Duration from "effect/Duration"; +import * as Fiber from "effect/Fiber"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; +import * as Scope from "effect/Scope"; import { ProjectId, + DEFAULT_SERVER_SETTINGS, + ProviderDriverKind, type PullRequestActivity, type PullRequestDetail, type PullRequestStack, @@ -17,6 +24,16 @@ import { import type { ReviewGuideGenerationInput } from "@t3tools/provider-core/server/textGeneration"; import * as ServerConfig from "../config.ts"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; +import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; +import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts"; +import * as CodexInstallation from "../provider/CodexInstallation.ts"; +import * as AntigravityInstallation from "../provider/AntigravityInstallation.ts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import * as UpdateWindow from "../updates/UpdateWindow.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as PullRequestReviewGuides from "../persistence/PullRequestReviewGuides.ts"; import * as ProjectService from "../project/ProjectService.ts"; @@ -50,6 +67,18 @@ const reference: PullRequestRef = { number: 7, }; +const idleInstallationState = (driver: string) => ({ + driver: ProviderDriverKind.make(driver), + phase: "idle" as const, + operationId: null, + downloadedBytes: 0, + totalBytes: null, + version: null, + installedVersion: null, + canRemove: false, + message: null, +}); + const git = (cwd: string, args: ReadonlyArray) => Effect.gen(function* () { const driver = yield* GitVcsDriver.GitVcsDriver; @@ -167,6 +196,7 @@ const makeLayer = (input: { readonly currentHead?: Ref.Ref; /** The project's own repository; defaults to the one the test reference names. */ readonly identity?: typeof ACME_APP; + readonly number?: number; }) => PullRequestReviewGuideService.layer.pipe( Layer.provide( @@ -190,7 +220,9 @@ const makeLayer = (input: { (input.currentHead === undefined ? Effect.succeed(input.headSha) : Ref.get(input.currentHead) - ).pipe(Effect.map((head) => detail(input.workspace, head))), + ).pipe( + Effect.map((head) => ({ ...detail(input.workspace, head), number: input.number ?? 7 })), + ), activity: () => Effect.succeed(activity), invalidate: () => Effect.void, stack: () => Effect.succeed(stack(input)), @@ -230,6 +262,228 @@ const layerBase = Layer.mergeAll( // Runs drive real git, which the test clock does not advance, so waits use real time. it.layer(layerBase, { excludeTestServices: true })("PullRequestReviewGuideService", (it) => { + it.effect.each([ + "pre-model fetch", + "post-model cleanup", + "cleanup during scope shutdown", + ] as const)( + "blocks restart admission during a superseded generation's %s after the newer guide settles", + (stage) => + Effect.scoped( + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const reviewReference = { + ...reference, + number: stage === "pre-model fetch" ? 70 : stage === "post-model cleanup" ? 71 : 72, + }; + const stored = yield* Deferred.make(); + const held = yield* Deferred.make(); + const release = yield* Deferred.make(); + const oldFiber = yield* Deferred.make>(); + const newFiber = yield* Deferred.make>(); + const currentHead = yield* Ref.make(repositories.headSha); + const driver = yield* GitVcsDriver.GitVcsDriver; + const serviceScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(serviceScope, Exit.void)); + let oldCheckout: string | undefined; + const gate = Deferred.succeed(held, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + number: reviewReference.number, + stored, + currentHead, + generate: () => Effect.succeed({ stops: [] }), + }).pipe( + Layer.provide( + Layer.succeed(GitVcsDriver.GitVcsDriver, { + ...driver, + execute: (input) => + Effect.gen(function* () { + if (input.operation === "PullRequestReviewGuideService.fetch") { + if (input.args.some((arg) => arg.endsWith("/0/base"))) { + yield* Deferred.succeed(oldFiber, yield* Effect.fiber); + if (stage === "pre-model fetch") yield* gate; + } + if (input.args.some((arg) => arg.endsWith("/1/base"))) { + yield* Deferred.succeed(newFiber, yield* Effect.fiber); + } + } + if ( + input.operation === "PullRequestReviewGuideService.addWorktree" && + oldCheckout === undefined && + stage !== "pre-model fetch" + ) { + oldCheckout = input.args.at(-2); + } + return yield* driver.execute(input); + }), + removeWorktree: (input) => + (stage !== "pre-model fetch" && input.path === oldCheckout + ? gate + : Effect.void + ).pipe(Effect.andThen(driver.removeWorktree(input))), + }), + ), + ), + ).pipe(Effect.provideService(Scope.Scope, serviceScope)); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const memoMap = yield* Layer.makeMemoMap; + const scope = yield* Effect.scope; + // Production constructs the provider window before routes provide guides. + yield* Layer.buildWithMemoMap( + Layer.fresh(UpdateWindow.layer).pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({}), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.succeed(Duration.minutes(14)), + }), + Layer.mock(ServerSettings.ServerSettingsService)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + ), + ), + ), + memoMap, + scope, + ); + const windowContext = yield* Layer.buildWithMemoMap( + UpdateWindow.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed( + PullRequestReviewGuideService.PullRequestReviewGuideService, + service, + ), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([]), + getLimitRecoveryCandidates: () => Effect.succeed([]), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + hasPendingBackgroundWork: Effect.succeed(false), + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + sinceLastClientInteraction: Effect.succeed(Duration.minutes(15)), + }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed([]), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS), + }), + Layer.mock(CodexInstallation.CodexInstallation)({ + managedDirectory: "/managed/codex", + state: Effect.succeed(idleInstallationState("codex")), + }), + Layer.mock(AntigravityInstallation.AntigravityInstallation)({ + managedDirectory: "/managed/antigravity", + state: Effect.succeed(idleInstallationState("antigravity")), + }), + Layer.mock(TerminalManager.TerminalManager)({ + withIdleTerminals: (effect) => Effect.asSome(effect), + }), + Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({ + activeCloneCount: Effect.succeed(0), + }), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + activePreparationCount: Effect.succeed(0), + }), + Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ + activeRefreshCount: Effect.succeed(0), + }), + ), + ), + ), + memoMap, + scope, + ); + const window = Context.get(windowContext, UpdateWindow.UpdateWindow); + yield* Effect.gen(function* () { + // Once a restart owns admission, even a not-yet-started guide waits. + const handoffEntered = yield* Deferred.make(); + const handoffRelease = yield* Deferred.make(); + const handoff = yield* window + .runIfOpen( + Deferred.succeed(handoffEntered, undefined).pipe( + Effect.andThen(Deferred.await(handoffRelease)), + ), + { restartsServer: true }, + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(handoffEntered); + const waitingGuide = yield* service + .generate(reviewReference) + .pipe(Effect.forkScoped({ startImmediately: true })); + assert.isUndefined(waitingGuide.pollUnsafe()); + assert.strictEqual(yield* service.activeGenerationCount, 0); + yield* Deferred.succeed(handoffRelease, undefined); + yield* Fiber.join(handoff); + yield* Fiber.join(waitingGuide); + yield* Deferred.await(held); + assert.strictEqual(yield* service.activeGenerationCount, 1); + // Same-head presses join the run rather than inflating the count. + assert.isFalse((yield* service.generate(reviewReference)).started); + assert.strictEqual(yield* service.activeGenerationCount, 1); + yield* Ref.set(currentHead, repositories.upperSha); + yield* service.generate(reviewReference); + yield* Deferred.await(stored); + assert.isTrue(Exit.isSuccess(yield* Fiber.await(yield* Deferred.await(newFiber)))); + const latest = yield* service.get(reviewReference); + assert.isNull(latest.run); + assert.strictEqual(latest.guide?.headSha, repositories.upperSha); + const closing = + stage === "cleanup during scope shutdown" + ? yield* Scope.close(serviceScope, Exit.void).pipe( + Effect.forkScoped({ startImmediately: true }), + ) + : null; + if (closing !== null) assert.isUndefined(closing.pollUnsafe()); + assert.strictEqual(yield* service.activeGenerationCount, 1); + assert.isTrue( + Option.isNone( + yield* window.runIfOpen(Effect.die("must not restart"), { restartsServer: true }), + ), + ); + // Fetch/setup and cleanup do not launch a CLI outside TextGeneration's permit. + assert.isTrue(Option.isSome(yield* window.runIfOpen(Effect.void))); + yield* Deferred.succeed(release, undefined); + const old = yield* Deferred.await(oldFiber); + const oldExit = yield* Fiber.await(old); + if (closing === null) assert.isTrue(Exit.isSuccess(oldExit)); + if (closing !== null) yield* Fiber.join(closing); + assert.strictEqual(yield* service.activeGenerationCount, 0); + assert.isTrue( + Option.isSome(yield* window.runIfOpen(Effect.void, { restartsServer: true })), + ); + assert.strictEqual( + (yield* service.get(reviewReference)).guide?.headSha, + repositories.upperSha, + ); + assert.strictEqual( + yield* git(repositories.workspace, [ + "for-each-ref", + "--format=%(refname)", + "refs/t3/review-guide", + ]), + "", + ); + assert.deepEqual( + (yield* git(repositories.workspace, ["worktree", "list", "--porcelain"])).match( + /^worktree .+$/gmu, + ), + [`worktree ${repositories.workspace}`], + ); + }).pipe(Effect.ensuring(Deferred.succeed(release, undefined))); + }), + ), + ); + it.effect("briefs the change against its own base in a checkout it then removes", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.ts index f074054c3a99a..f6cbed7845e37 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.ts @@ -36,6 +36,7 @@ import { type ThreadPullRequestKey, } from "@t3tools/contracts"; import type { ReviewGuideStackLayer } from "@t3tools/provider-core/server/textGeneration"; +import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { sourceControlRepositorySelector } from "@t3tools/shared/sourceControl"; @@ -71,6 +72,8 @@ export class PullRequestReviewGuideService extends Context.Service< PullRequestReviewGuideResult & { readonly started: boolean }, PullRequestService.PullRequestError >; + /** All generations, including superseded runs and checkout cleanup. */ + readonly activeGenerationCount: Effect.Effect; } >()("t3/pullRequest/PullRequestReviewGuideService") {} @@ -90,8 +93,10 @@ const make = Effect.gen(function* () { const git = yield* GitVcsDriver.GitVcsDriver; const fileSystem = yield* FileSystem.FileSystem; const config = yield* ServerConfig.ServerConfig; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; // Runs outlive the request that started them and end with the server. const serviceScope = yield* Effect.scope; + const activeGenerations = yield* Ref.make(0); // In memory on purpose: a restart drops a run in progress, and a reader presses Generate again. const runs = yield* Ref.make>(new Map()); // Names each run's private git refs; unique for the server's lifetime, like the runs map. @@ -380,6 +385,7 @@ const make = Effect.gen(function* () { return "The review guide could not be written."; }; + // Register under admission so a restart cannot read zero then race a new guide. const generate: PullRequestReviewGuideService["Service"]["generate"] = Effect.fn( "PullRequestReviewGuideService.generate", )(function* (input) { @@ -478,12 +484,22 @@ const make = Effect.gen(function* () { ), ), ), - Effect.forkIn(serviceScope), + (work) => + Effect.acquireUseRelease( + Ref.update(activeGenerations, (count) => count + 1), + () => work, + () => Ref.update(activeGenerations, (count) => count - 1), + ), + Effect.forkIn(serviceScope, { startImmediately: true }), ); return { guide: stored, run: running, currentHeadSha, started: true }; - }); + }, admission.withLaunchPermit); - return PullRequestReviewGuideService.of({ get, generate }); + return PullRequestReviewGuideService.of({ + get, + generate, + activeGenerationCount: Ref.get(activeGenerations), + }); }); export const layer = Layer.effect(PullRequestReviewGuideService, make); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 27b82054dc376..279455c9c8c35 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -556,7 +556,9 @@ const layerProviderInstallationRefresh = Layer.effectDiscard( // One maintenance runner for RPCs and background updates, so both queue // behind the same per-installer lock. const layerProviderUpdates = ProviderAutoUpdater.layer.pipe( - Layer.provide(UpdateWindow.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), + // The route window also sees guide work. Do not memoize this earlier, + // provider-only window as the server restart window. + Layer.provide(Layer.fresh(UpdateWindow.layer).pipe(Layer.provide(ProjectionStoreV2.layer))), Layer.provideMerge(ProviderMaintenanceRunner.layer.pipe(Layer.provide(InstallSupervisor.layer))), ); diff --git a/apps/server/src/updates/UpdateWindow.ts b/apps/server/src/updates/UpdateWindow.ts index 1bce23dbf3ff9..73f08f08d7fed 100644 --- a/apps/server/src/updates/UpdateWindow.ts +++ b/apps/server/src/updates/UpdateWindow.ts @@ -10,13 +10,17 @@ import type { ProviderInstanceId } from "@t3tools/contracts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "../orchestration-v2/ProviderSessionManager.ts"; +import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; +import * as ProjectCloneTracker from "../project/ProjectCloneTracker.ts"; import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as PullRequestReviewGuideService from "../pullRequest/PullRequestReviewGuideService.ts"; import * as AntigravityInstallation from "../provider/AntigravityInstallation.ts"; import * as CodexInstallation from "../provider/CodexInstallation.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TerminalManager from "../terminal/Manager.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; /** Work due this soon keeps the window closed, so it does not start late. */ const UPDATE_LOOKAHEAD = Duration.minutes(5); @@ -64,6 +68,38 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { const providers = yield* ProviderRegistry.ProviderRegistry; const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const terminals = yield* Effect.serviceOption(TerminalManager.TerminalManager); + // The server updater sees the route and runtime services; provider updates + // need only CLI admission, so those layers need not supply these services. + const restartWork = [ + [ + "review-guides", + Option.map( + yield* Effect.serviceOption(PullRequestReviewGuideService.PullRequestReviewGuideService), + (service) => service.activeGenerationCount, + ), + ], + [ + "project-clones", + Option.map( + yield* Effect.serviceOption(ProjectCloneTracker.ProjectCloneTracker), + (service) => service.activeCloneCount, + ), + ], + [ + "workspace-preparation", + Option.map( + yield* Effect.serviceOption(ThreadLaunchService.ThreadLaunchService), + (service) => service.activePreparationCount, + ), + ], + [ + "git-refreshes", + Option.map( + yield* Effect.serviceOption(VcsStatusBroadcaster.VcsStatusBroadcaster), + (service) => service.activeRefreshCount, + ), + ], + ] as const; const managedInstalls = [ Option.map( yield* Effect.serviceOption(CodexInstallation.CodexInstallation), @@ -108,6 +144,15 @@ const make = Effect.fn("updates.UpdateWindow.make")(function* () { if (providerUpdating) return ["provider-update"]; // A first install downloads inside this server, so a restart would discard it. if (restartsServer) { + const activeWork: string[] = []; + for (const [name, work] of restartWork) { + if (Option.isNone(work)) activeWork.push(`${name}:unknown`); + else { + const count = yield* work.value; + if (count > 0) activeWork.push(`${name}:${count}`); + } + } + if (activeWork.length > 0) return activeWork; for (const installation of managedInstalls) { if (Option.isNone(installation)) return ["provider-install"]; const { phase } = yield* installation.value; diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index 9e9f2e4d2cca0..10f58265e0919 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -152,6 +152,49 @@ function layerBackgroundPolicy(shouldRunScopeWork: (scope: BackgroundScope) => b } describe("VcsStatusBroadcaster", () => { + it.effect( + "tracks background fetches through interrupted cleanup without counting idle pollers", + () => { + const entered = Deferred.makeUnsafe(); + const cleanup = Deferred.makeUnsafe(); + const releaseCleanup = Deferred.makeUnsafe(); + const state = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + beforeRemoteStatus: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Effect.never), + Effect.ensuring( + Deferred.succeed(cleanup, undefined).pipe( + Effect.andThen(Deferred.await(releaseCleanup)), + ), + ), + ), + }; + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + assert.equal(yield* broadcaster.activeRefreshCount, 0); + const reader = yield* broadcaster + .streamStatus({ cwd: "/repo" }) + .pipe(Stream.runDrain, Effect.forkScoped); + yield* Deferred.await(entered); + assert.equal(yield* broadcaster.activeRefreshCount, 1); + const stopping = yield* Fiber.interrupt(reader).pipe(Effect.forkScoped); + yield* Deferred.await(cleanup); + assert.equal(yield* broadcaster.activeRefreshCount, 1); + yield* Deferred.succeed(releaseCleanup, undefined); + yield* Fiber.join(stopping); + assert.equal(yield* broadcaster.activeRefreshCount, 0); + }).pipe( + Effect.ensuring(Deferred.succeed(releaseCleanup, undefined)), + Effect.provide(layerTestFor(state)), + ); + }, + ); + it.effect.skipIf(!symlinksSupported)( "automatically pulls an enabled clean default branch when status detects it is behind", () => { diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.ts index 468bc3301bfcb..c054fdf37b874 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts @@ -27,6 +27,7 @@ import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import * as ProviderMaintenanceCoordinator from "@t3tools/provider-core/server/maintenance"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -205,6 +206,8 @@ export class VcsStatusBroadcaster extends Context.Service< input: VcsStatusSubscriptionInput, options?: StreamStatusOptions, ) => Stream.Stream; + /** Remote refreshes include background fetches and automatic pulls. */ + readonly activeRefreshCount: Effect.Effect; } >()("t3/vcs/VcsStatusBroadcaster") {} @@ -222,6 +225,7 @@ const normalizeCwd = (cwd: string) => export const make = Effect.gen(function* () { const autoPullPolicy = yield* VcsAutoPullPolicy; const workflow = yield* GitWorkflowService.GitWorkflowService; + const admission = yield* ProviderMaintenanceCoordinator.ProviderMaintenanceAdmission; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const fs = yield* FileSystem.FileSystem; const changesPubSub = yield* Effect.acquireRelease( @@ -236,8 +240,15 @@ export const make = Effect.gen(function* () { // periodic poll that started before `gh pr create` can finish after the // turn-end refresh and overwrite the fresh PR with its stale `pr: null`. const remoteWriteLocks = yield* KeyedLock.make(); + const activeRefreshes = yield* Ref.make(0); const withRemoteWriteLock = (cwd: string, effect: Effect.Effect) => - remoteWriteLocks.withLock(cwd, effect); + Effect.uninterruptibleMask((restore) => + Effect.acquireUseRelease( + restore(admission.withLaunchPermit(Ref.update(activeRefreshes, (count) => count + 1))), + () => restore(remoteWriteLocks.withLock(cwd, effect)), + () => Ref.update(activeRefreshes, (count) => count - 1), + ), + ); const pollersRef = yield* SynchronizedRef.make(new Map()); const getCachedStatus = Effect.fn("VcsStatusBroadcaster.getCachedStatus")(function* ( @@ -743,6 +754,7 @@ export const make = Effect.gen(function* () { refreshStatus, refreshPullRequestStatus, streamStatus, + activeRefreshCount: Ref.get(activeRefreshes), }); }); From 84c937c74ca5b727b2ff61d7ec0b2922a94510d6 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:40:43 +0000 Subject: [PATCH 139/149] test(server): drop a duplicate Fiber import left by the merge --- apps/server/src/orchestration-v2/ThreadLaunchService.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index a0ba8535f258c..0b8416f89a968 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -27,7 +27,6 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; -import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; From 92972640b36286a7e1ea47853e007bf86694bd8e Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:08:23 +0000 Subject: [PATCH 140/149] fix(provider): retain install ownership through process exit and cleanup --- .../src/provider/ProviderInstallSupervisor.ts | 232 +++--- .../provider/ProviderMaintenanceHostLock.ts | 446 +++++++--- .../providerMaintenanceRunner.test.ts | 761 +++++++++++++++++- 3 files changed, 1218 insertions(+), 221 deletions(-) diff --git a/apps/server/src/provider/ProviderInstallSupervisor.ts b/apps/server/src/provider/ProviderInstallSupervisor.ts index 78bc655d10ae2..3103094da8673 100644 --- a/apps/server/src/provider/ProviderInstallSupervisor.ts +++ b/apps/server/src/provider/ProviderInstallSupervisor.ts @@ -49,8 +49,12 @@ const decodeReceipt = Schema.decodeSync(Schema.fromJsonString(Receipt)); export const readReceipt = (file: string) => decodeReceipt(NodeFS.readFileSync(file, "utf8")); const writeReceipt = (file: string, receipt: InstallReceipt) => { const temp = `${file}.${process.pid}.tmp`; - NodeFS.writeFileSync(temp, JSON.stringify(receipt), { mode: 0o600 }); - NodeFS.renameSync(temp, file); + try { + NodeFS.writeFileSync(temp, JSON.stringify(receipt), { mode: 0o600 }); + NodeFS.renameSync(temp, file); + } finally { + NodeFS.rmSync(temp, { force: true }); + } }; type InstallInput = Pick< @@ -79,6 +83,15 @@ const installError = (cause: unknown) => cause, }); +const failedResult = (error: unknown) => ({ + stdout: "", + stderr: error instanceof Error ? error.message : String(error), + exitCode: null, + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, +}); + export class ProviderInstallSupervisor extends Context.Service< ProviderInstallSupervisor, { @@ -153,28 +166,17 @@ export const layer = Layer.effect( } return yield* host.handoff( Effect.gen(function* () { - let started = false; let progress: string | undefined; while (true) { const current = yield* readCurrent; if (current.result) return current.result; - if (current.pid !== undefined && !started) { - // Only acknowledge after the durable lease is visible. A server dying - // before this write cannot authorize a late installer to start. - yield* Effect.try(() => - NodeFS.writeFileSync(`${file}.start`, receipt.id, { mode: 0o600 }), - ); - started = true; - } if (current.progress && current.progress !== progress) { progress = current.progress; if (input.onProgress) yield* input.onProgress(progress); } - if (!started && (yield* Clock.currentTimeMillis) >= deadline) - return yield* Effect.fail( - new ProviderInstallError({ message: "Provider install supervisor did not start." }), - ); - if (started && !(yield* Effect.sync(() => HostLock.installIsAlive(host.directory)))) { + if (!(yield* Effect.sync(() => HostLock.installIsAlive(host.directory)))) { + const finished = yield* readCurrent; + if (finished.result) return finished.result; return yield* Effect.fail( new ProviderInstallError({ message: "Provider install supervisor exited without a result.", @@ -184,6 +186,8 @@ export const layer = Layer.effect( yield* Effect.sleep("1 second"); } }), + // Authorize immediately after stopping the server heartbeat, before cancellation can interrupt polling. + Effect.try(() => NodeFS.writeFileSync(`${file}.start`, receipt.id, { mode: 0o600 })), ); }, Effect.mapError(installError)); return ProviderInstallSupervisor.of({ @@ -209,32 +213,39 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< ); NodeFS.writeFileSync(`${file}.stdout`, "", { mode: 0o600 }); NodeFS.writeFileSync(`${file}.stderr`, "", { mode: 0o600 }); - const stdout = NodeFS.openSync(`${file}.stdout`, "a", 0o600); - const stderr = NodeFS.openSync(`${file}.stderr`, "a", 0o600); - const child = NodeChildProcess.spawn( - invocation.command, - [ - ...selfInvocationArgs(invocation, [ - "__provider-install", - file, - String(process.pid), - HostLock.readProcessIdentity(process.pid) ?? "", - ]), - ], - { - detached: true, - stdio: ["ignore", stdout, stderr], - windowsHide: true, - env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, - }, - ); - NodeFS.closeSync(stdout); - NodeFS.closeSync(stderr); - await new Promise((resolve, reject) => { - child.once("spawn", resolve); - child.once("error", reject); - }); - child.unref(); + const outputs: number[] = []; + try { + outputs.push(NodeFS.openSync(`${file}.stdout`, "a", 0o600)); + outputs.push(NodeFS.openSync(`${file}.stderr`, "a", 0o600)); + const child = NodeChildProcess.spawn( + invocation.command, + [ + ...selfInvocationArgs(invocation, [ + "__provider-install", + file, + String(process.pid), + HostLock.readProcessIdentity(process.pid) ?? "", + ]), + ], + { + detached: true, + stdio: ["ignore", outputs[0]!, outputs[1]!], + windowsHide: true, + env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, + }, + ); + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + child.unref(); + } finally { + try { + if (outputs[1] !== undefined) NodeFS.closeSync(outputs[1]); + } finally { + if (outputs[0] !== undefined) NodeFS.closeSync(outputs[0]); + } + } return; } let receipt = readReceipt(file); @@ -244,16 +255,26 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< await new Promise((resolve) => setTimeout(resolve, 25)); const lease = HostLock.installLeasePath(receipt.directory); const candidate = `${lease}.${process.pid}`; - NodeFS.writeFileSync( - candidate, - JSON.stringify({ pid: process.pid, identity: HostLock.readProcessIdentity(process.pid) }), - { flag: "wx", mode: 0o600 }, - ); + const identity = HostLock.readProcessIdentity(process.pid); + const descendants: HostLock.InstallDescendants = new Map(); + const job = `t3-provider-install-${NodeCrypto.randomUUID()}`; + let registered = false; try { + NodeFS.writeFileSync(candidate, JSON.stringify({ pid: process.pid, identity }), { + flag: "wx", + mode: 0o600, + }); // A delayed relay must never overwrite an already running install's owner. NodeFS.linkSync(candidate, lease); - } finally { + registered = true; NodeFS.rmSync(candidate, { force: true }); + } catch (error) { + try { + if (registered) NodeFS.rmSync(lease, { force: true }); + } finally { + NodeFS.rmSync(candidate, { force: true }); + } + throw error; } // Quit signals must never interrupt an install. SIGKILL still leaves the // process group in the lease, which other servers inspect before reclaiming. @@ -278,7 +299,26 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< }; let heartbeat: NodeJS.Timeout | undefined; let ownsGate = false; - let installerFinished = false; + const childrenAreAlive = () => + HostLock.installChildrenAreAlive(process.pid, identity, descendants, job); + const persistChildren = () => { + const temp = `${candidate}.tmp`; + try { + NodeFS.writeFileSync( + temp, + JSON.stringify({ + pid: process.pid, + identity, + job, + descendants: Array.from(descendants, ([pid, identity]) => ({ pid, identity })), + }), + { mode: 0o600 }, + ); + NodeFS.renameSync(temp, lease); + } finally { + NodeFS.rmSync(temp, { force: true }); + } + }; const beat = (format: (line: string) => string | null) => { NodeFS.utimesSync(lease, new Date(), new Date()); NodeFS.utimesSync(NodePath.join(receipt.directory, "gate.lock"), new Date(), new Date()); @@ -294,7 +334,10 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< writeReceipt(file, receipt); } }; + let result: ProviderMaintenanceCommandResult | undefined; try { + HostLock.createInstallJob(process.pid, job); + persistChildren(); // Once readiness is durable the server or desktop can replace these files. // Load every installer dependency before allowing that handoff. const { runProviderMaintenanceCommandWithSpawner, toProgressLine } = @@ -317,13 +360,19 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< ownsGate = true; NodeFS.rmSync(`${file}.start`, { force: true }); heartbeat = setInterval(() => { + try { + childrenAreAlive(); + persistChildren(); + } catch { + // Retry failed observation/persistence while the supervisor keeps admission closed. + } try { beat(toProgressLine); } catch { // The live PID/process group still holds admission when disk writes fail. } }, 1_000); - const result = await Effect.runPromise( + result = await Effect.runPromise( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; return yield* runProviderMaintenanceCommandWithSpawner({ @@ -335,47 +384,50 @@ export async function runInstallSubcommand(command: string, args: ReadonlyArray< }); }).pipe(Effect.provide(NodeServices.layer)), ); - while (HostLock.installChildrenAreAlive(process.pid)) - await new Promise((resolve) => setTimeout(resolve, 100)); - installerFinished = true; - const stdout = output(".stdout"); - const stderr = output(".stderr"); - writeReceipt(file, { - ...receipt, - finishedAt: new Date().toISOString(), - result: { - ...result, - stdout: stdout.text, - stderr: stderr.text, - stdoutTruncated: stdout.truncated, - stderrTruncated: stderr.truncated, - }, - }); } catch (error) { - if (ownsGate) { - while (HostLock.installChildrenAreAlive(process.pid)) - await new Promise((resolve) => setTimeout(resolve, 100)); - installerFinished = true; - } - writeReceipt(file, { - ...receipt, - finishedAt: new Date().toISOString(), - result: { - stdout: "", - stderr: error instanceof Error ? error.message : String(error), - exitCode: null, - timedOut: false, - stdoutTruncated: false, - stderrTruncated: false, - }, - }); + result = failedResult(error); } finally { - clearInterval(heartbeat); - if (ownsGate && installerFinished) - NodeFS.rmSync(NodePath.join(receipt.directory, "gate.lock"), { - recursive: true, - force: true, - }); - if (!ownsGate || installerFinished) NodeFS.rmSync(lease, { force: true }); + try { + if (ownsGate) { + while (true) { + try { + if (!childrenAreAlive()) break; + } catch { + // An unavailable process table cannot authorize admission. + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + } + clearInterval(heartbeat); + if (result) { + if (result.exitCode !== null) { + try { + const stdout = output(".stdout"); + const stderr = output(".stderr"); + result = { + ...result, + stdout: stdout.text, + stderr: stderr.text, + stdoutTruncated: stdout.truncated, + stderrTruncated: stderr.truncated, + }; + } catch (error) { + result = failedResult(error); + } + } + writeReceipt(file, { ...receipt, finishedAt: new Date().toISOString(), result }); + } + } finally { + clearInterval(heartbeat); + try { + if (ownsGate) + NodeFS.rmSync(NodePath.join(receipt.directory, "gate.lock"), { + recursive: true, + force: true, + }); + } finally { + NodeFS.rmSync(lease, { force: true }); + } + } } } diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index f93464906801e..c903505dd0a7b 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -13,6 +13,7 @@ import * as Layer from "effect/Layer"; import * as Deferred from "effect/Deferred"; import * as Semaphore from "effect/Semaphore"; import * as Schema from "effect/Schema"; +import * as Schedule from "effect/Schedule"; import * as Maintenance from "@t3tools/provider-core/server/maintenance"; const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } | null>( @@ -34,13 +35,6 @@ const HeldGate = Context.Reference<{ readonly resume: Effect.Effect; } | null>("t3/provider/ProviderMaintenanceHostLock/HeldGate", { defaultValue: () => null }); -// Linux exposes a process start identity, so recycled PIDs cannot pin a dead reader. -const processIdentity = (pid: number) => { - if (NodeOS.platform() !== "linux") return null; - const fields = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").at(-1)!.split(" "); - return fields[0] === "Z" ? "" : fields[19]!; -}; - export const installLeasePath = (directory: string) => NodePath.join(directory, "install.json"); /** Start identity includes the Linux boot so leases cannot survive a reboot's PID reuse. */ @@ -49,7 +43,12 @@ export const readProcessIdentity = (pid: number): string | null => { try { const platform = NodeOS.platform(); if (platform === "linux") { - const start = processIdentity(pid); + const fields = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8") + .split(") ") + .at(-1)! + .split(" "); + if (fields[0] === "Z" || fields[0] === "X") return ""; + const start = fields[19]; return start ? `${NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` : null; @@ -67,68 +66,164 @@ export const readProcessIdentity = (pid: number): string | null => { ], { encoding: "utf8", timeout: 10_000, windowsHide: true }, ) - : NodeChildProcess.spawnSync("ps", ["-p", String(pid), "-o", "lstart="], { + : NodeChildProcess.spawnSync("ps", ["-p", String(pid), "-o", "stat=", "-o", "lstart="], { encoding: "utf8", timeout: 10_000, env: { ...process.env, LC_ALL: "C", TZ: "UTC" }, }); - return scan.error || scan.status !== 0 ? null : scan.stdout.trim() || null; + if (scan.error || scan.status !== 0) return null; + const value = scan.stdout.trim(); + if (platform === "darwin") { + const state = /^([A-Z][A-Z+ { - try { - process.kill(pid, 0); - if (processIdentity(pid) === "") return false; - } catch (error) { - if ( - error instanceof Error && - "code" in error && - (error.code === "ESRCH" || error.code === "ENOENT") - ) - return false; - return true; +/** Unknown identities keep admission closed; only death or a different start proves release safe. */ +export const isProcessAlive = (pid: number, identity?: string | null, observed?: string | null) => { + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + // A live table row is already a liveness observation. Rechecking death here + // could lose a helper spawned between the scan and its parent's exit. + if (observed === undefined) { + try { + process.kill(pid, 0); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ESRCH") return false; + // Permission to signal and permission to read creation identity can differ. + } } - if (identity == null) return true; - const current = readProcessIdentity(pid); - return current === null || current === identity; + const current = observed === undefined ? readProcessIdentity(pid) : observed; + if (current === "") return false; + if (!identity || current === null) return true; + // Readers written by older Linux servers contain only the start tick. + return ( + current === identity || + (NodeOS.platform() === "linux" && /^\d+$/.test(identity) && current.endsWith(`:${identity}`)) + ); }; +// A named Windows job retains every descendant in the kernel, even when a +// short-lived parent exits between observations or the supervisor is killed. +const windowsJobApi = ` +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +public static class T3InstallJob { + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr CreateJobObject(IntPtr attributes, string name); + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr OpenJobObject(uint access, bool inherit, string name); + [DllImport("kernel32.dll", SetLastError=true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job, int kind, IntPtr info, uint size, IntPtr returned); + [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr handle); + public static int[] Processes(string name, int pid, bool create) { + IntPtr job = create ? CreateJobObject(IntPtr.Zero, name) : OpenJobObject(4, false, name); + if (job == IntPtr.Zero) { + int error = Marshal.GetLastWin32Error(); + if (!create && error == 2) return new int[0]; + throw new Win32Exception(error); + } + try { + if (create) { + IntPtr process = OpenProcess(0x101, false, pid); + if (process == IntPtr.Zero) throw new Win32Exception(Marshal.GetLastWin32Error()); + try { + if (!AssignProcessToJobObject(job, process)) throw new Win32Exception(Marshal.GetLastWin32Error()); + } finally { CloseHandle(process); } + } + IntPtr info = Marshal.AllocHGlobal(65536); + try { + if (!QueryInformationJobObject(job, 3, info, 65536, IntPtr.Zero)) throw new Win32Exception(Marshal.GetLastWin32Error()); + int count = Marshal.ReadInt32(info, 4); + int[] pids = new int[count]; + for (int i = 0; i < count; i++) pids[i] = Marshal.ReadIntPtr(info, 8 + i * IntPtr.Size).ToInt32(); + return pids; + } finally { Marshal.FreeHGlobal(info); } + } finally { CloseHandle(job); } + } +}`; +const ownedJobs = new Set(); +const windowsJobProcesses = ( + pid: number, + job: string, + create: boolean, + includeRoot = false, + identity?: string | null, +) => { + if (!/^t3-provider-install-[a-f0-9-]+$/.test(job) || !Number.isSafeInteger(pid) || pid <= 0) + throw new Error("Invalid provider install job"); + const scan = NodeChildProcess.spawnSync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + `$ErrorActionPreference = 'Stop'; Add-Type -TypeDefinition @'\n${windowsJobApi}\n'@; [T3InstallJob]::Processes('Global\\${job}', ${pid}, $${create})`, + ], + { encoding: "utf8", timeout: 10_000, windowsHide: true }, + ); + if (scan.error || scan.status !== 0) + throw new Error("Could not inspect provider installer processes"); + const children = scan.stdout + .split(/\r?\n/) + .filter((line) => line.trim()) + .map(Number); + if (children.some((child) => !Number.isSafeInteger(child) || child <= 0)) + throw new Error("Could not inspect provider installer processes"); + const local = pid === process.pid && ownedJobs.has(job); + const current = includeRoot || local || !identity ? null : readProcessIdentity(pid); + const supervisor = + !includeRoot && + (local || (identity != null && current !== null && isProcessAlive(pid, identity, current))); + return children.some( + (child) => child !== scan.pid && (includeRoot || child !== pid || !supervisor), + ); +}; +export const createInstallJob = (pid: number, job: string) => { + if (NodeOS.platform() !== "win32") return; + windowsJobProcesses(pid, job, true); + if (pid === process.pid) ownedJobs.add(job); +}; + +const ProcessIdentity = Schema.Struct({ + pid: Schema.Number, + identity: Schema.optional(Schema.NullOr(Schema.String)), +}); const decodeInstallLease = Schema.decodeSync( Schema.fromJsonString( Schema.Union([ Schema.Number, Schema.Struct({ - pid: Schema.Number, - identity: Schema.optional(Schema.NullOr(Schema.String)), + ...ProcessIdentity.fields, + descendants: Schema.optional(Schema.Array(ProcessIdentity)), + job: Schema.optional(Schema.String), }), ]), ), ); - -/** A dead supervisor's installer still owns admission until its entire process group exits. */ -export const installIsAlive = (directory: string): boolean => { - const file = installLeasePath(directory); - let pid: number; - let identity: string | null | undefined; - try { - const lease = decodeInstallLease(NodeFS.readFileSync(file, "utf8")); - pid = typeof lease === "number" ? lease : lease.pid; - identity = typeof lease === "number" ? undefined : lease.identity; - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") return false; - throw error; +export type InstallDescendants = Map; +const installs = new Map< + string, + { + pid: number; + identity: string | null | undefined; + job: string | undefined; + descendants: InstallDescendants; } - if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid provider install lease"); - if (isProcessAlive(pid, identity)) return true; - return installChildrenAreAlive(pid); -}; +>(); -/** The installer can leave children writing after its root exits. Failed scans keep admission closed. */ -export const installChildrenAreAlive = (pid: number): boolean => { +/** Retain ancestors for the lease's whole lifetime, including after they exit or are reparented. */ +export const installChildrenAreAlive = ( + pid: number, + identity?: string | null, + descendants: InstallDescendants = new Map(), + job?: string, +): boolean => { const windows = NodeOS.platform() === "win32"; + if (windows && job) return windowsJobProcesses(pid, job, false, false, identity); const scan = windows ? NodeChildProcess.spawnSync( "powershell.exe", @@ -136,7 +231,7 @@ export const installChildrenAreAlive = (pid: number): boolean => { "-NoProfile", "-NonInteractive", "-Command", - 'Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output "$($_.ProcessId)|$($_.ParentProcessId)" }', + "Get-CimInstance Win32_Process -ErrorAction Stop | ForEach-Object { Write-Output \"$($_.ProcessId)|$($_.ParentProcessId)|$($_.CreationDate.ToUniversalTime().ToString('o'))\" }", ], { encoding: "utf8", timeout: 10_000, windowsHide: true }, ) @@ -146,43 +241,118 @@ export const installChildrenAreAlive = (pid: number): boolean => { }); if (scan.error || scan.status !== 0) throw new Error("Could not inspect provider installer processes"); - if (windows) { - const descendants = new Set([pid]); - const rows = scan.stdout - .trim() - .split(/\r?\n/) - .map((line) => line.split("|").map(Number)); - let changed = true; - while (changed) { - changed = false; - for (const [child, parent] of rows) { - if ( - parent !== undefined && - child !== undefined && - child !== scan.pid && - descendants.has(parent) && - !descendants.has(child) - ) { - descendants.add(child); - changed = true; - } + const rows = scan.stdout + .split(/\r?\n/) + .filter((line) => line.trim()) + .map((line) => { + const [child, parent, state] = windows ? line.split("|") : line.trim().split(/\s+/); + if ( + !Number.isSafeInteger(Number(child)) || + Number(child) <= 0 || + !Number.isSafeInteger(Number(parent)) || + !state + ) + throw new Error("Could not inspect provider installer processes"); + return { + pid: Number(child), + parent: Number(parent), + identity: windows ? state : undefined, + zombie: !windows && state.startsWith("Z"), + }; + }); + const groupAlive = + !windows && + rows.some( + (row) => row.pid !== pid && row.pid !== scan.pid && row.parent === pid && !row.zombie, + ); + const roots = new Map(descendants); + roots.set(pid, identity ?? null); + let changed = true; + while (changed) { + changed = false; + for (const row of rows) { + const retained = descendants.has(row.pid); + if ( + row.pid === pid || + row.pid === scan.pid || + row.zombie || + (retained && (windows ? descendants.get(row.pid) === row.identity : row.parent !== pid)) + ) + continue; + if (!windows) { + if (row.parent !== pid) continue; + } else { + if (!roots.has(row.parent)) continue; + const parent = rows.find((candidate) => candidate.pid === row.parent); + const expected = roots.get(row.parent); + // Replacing a retained PID needs live ancestry, not an orphan's stale parent PID. + if (retained && !parent) continue; + // A recycled ancestor cannot introduce descendants belonging to its new owner. + if (parent && expected && parent.identity !== expected) continue; + const parentStart = expected ? Date.parse(expected) : NaN; + if (Number.isFinite(parentStart) && Date.parse(row.identity!) < parentStart) continue; } + const start = row.identity ?? readProcessIdentity(row.pid); + if (start === "" || (retained && descendants.get(row.pid) === start)) continue; + descendants.set(row.pid, start); + roots.set(row.pid, start); + changed = true; } - if (descendants.size > 1) return true; - } else if ( - scan.stdout.split(/\r?\n/).some((line) => { - const [child, group, state] = line.trim().split(/\s+/); + } + return ( + groupAlive || + Array.from(descendants).some(([child, start]) => { + const row = rows.find((candidate) => candidate.pid === child); return ( - Number(child) !== pid && - Number(child) !== scan.pid && - Number(group) === pid && - state !== undefined && - !state.startsWith("Z") + !row?.zombie && + isProcessAlive(child, start, row ? (row.identity ?? readProcessIdentity(child)) : undefined) ); }) - ) - return true; - return false; + ); +}; + +/** Observe even live supervisors so a later scan can retain orphaned Windows helpers. */ +export const installIsAlive = (directory: string): boolean => { + let lease; + try { + lease = decodeInstallLease(NodeFS.readFileSync(installLeasePath(directory), "utf8")); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + installs.delete(directory); + return false; + } + throw error; + } + const pid = typeof lease === "number" ? lease : lease.pid; + const identity = typeof lease === "number" ? undefined : lease.identity; + if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Invalid provider install lease"); + const job = typeof lease === "number" ? undefined : lease.job; + if (NodeOS.platform() === "win32" && job) return windowsJobProcesses(pid, job, false, true); + let tracked = installs.get(directory); + if (!tracked || tracked.pid !== pid || tracked.identity !== identity || tracked.job !== job) { + tracked = { pid, identity, job, descendants: new Map() }; + installs.set(directory, tracked); + } + const uncertain: number[] = []; + if (typeof lease !== "number") { + for (const descendant of lease.descendants ?? []) { + if (!Number.isSafeInteger(descendant.pid) || descendant.pid <= 0) + throw new Error("Invalid provider install descendant"); + const durable = descendant.identity ?? null; + // A stale receipt cannot replace a newer observation of the same helper PID. + if (!tracked.descendants.has(descendant.pid)) { + tracked.descendants.set(descendant.pid, durable); + } else if (durable !== null && tracked.descendants.get(descendant.pid) !== durable) { + const current = readProcessIdentity(descendant.pid); + if (current === durable) tracked.descendants.set(descendant.pid, durable); + // Retry conflicting generations next time without erasing either ownership observation. + else if (current === null) uncertain.push(descendant.pid); + } + } + } + const supervisor = isProcessAlive(pid, identity); + const children = installChildrenAreAlive(pid, identity, tracked.descendants, job); + return supervisor || children || uncertain.some((child) => isProcessAlive(child)); }; /** @@ -283,12 +453,16 @@ export const make = (directory: string) => { ); // Stop this process's heartbeat and exit hook without opening the gate. The // acknowledged supervisor maintains the same gate, then releases it on exit. - const handoff = (effect: Effect.Effect) => + const handoff = ( + effect: Effect.Effect, + acknowledge: Effect.Effect = Effect.void, + ) => Effect.flatMap(HeldGate, (gate) => gate === null ? Effect.die(new Error("Provider install requires the host write lease")) : Effect.uninterruptibleMask((restore) => gate.handoff.pipe( + Effect.andThen(acknowledge), Effect.andThen(restore(effect)), Effect.onExit((exit) => exit._tag === "Failure" && Cause.hasInterrupts(exit.cause) @@ -301,41 +475,70 @@ export const make = (directory: string) => { const readPermit = Semaphore.makeUnsafe(1); const readTokens = new WeakMap, ReadToken>(); - const registerRead = Effect.sync(() => { - const file = NodePath.join(readers, `${process.pid}-${NodeCrypto.randomUUID()}`); - NodeFS.writeFileSync(file, processIdentity(process.pid) ?? "", { flag: "wx" }); + // A stale reader file can pin admission for this process's entire lifetime. + const retryReadCleanup = (cleanup: () => A) => + Effect.try(cleanup).pipe(Effect.retry(Schedule.spaced("1 second")), Effect.orDie); + const registerRead = Effect.gen(function* () { + const file = yield* Effect.sync(() => + NodePath.join(readers, `${process.pid}-${NodeCrypto.randomUUID()}`), + ); + yield* Effect.sync(() => + NodeFS.writeFileSync(file, readProcessIdentity(process.pid) ?? "", { flag: "wx" }), + ).pipe( + Effect.onError((cause) => { + const error = Cause.squash(cause); + return error instanceof Error && "code" in error && error.code === "EEXIST" + ? Effect.void + : retryReadCleanup(() => NodeFS.rmSync(file, { force: true })); + }), + ); return file; }); // A provider can launch helper CLIs while its turn holds a read lease. // Joining that lease must not wait behind an installer draining this turn. const acquireRead = Effect.uninterruptible( Effect.gen(function* () { - const inherited = yield* HeldRead; - const held = yield* HeldWrite; - const lease = - inherited?.lease.owner === owner && inherited.active - ? inherited.lease - : { - owner, - // An install's scoped verification keeps its own reader if it outlives the writer. - file: yield* held?.owner === owner && held.active - ? registerRead - : Effect.acquireUseRelease( - Effect.interruptible(readPermit.take(1)), - () => withGate(registerRead), - () => Effect.asVoid(readPermit.release(1)), - ), - users: 0, - }; - lease.users++; - const token = { lease, active: true }; - const release = Effect.sync(() => { - if (!token.active) return; - token.active = false; - if (--lease.users === 0) NodeFS.rmSync(lease.file, { force: true }); - }); - readTokens.set(release, token); - return release; + let created: string | undefined; + const register = Effect.tap(registerRead, (file) => + Effect.sync(() => { + created = file; + }), + ); + return yield* Effect.gen(function* () { + const inherited = yield* HeldRead; + const held = yield* HeldWrite; + const lease = + inherited?.lease.owner === owner && inherited.active + ? inherited.lease + : { + owner, + // An install's scoped verification keeps its own reader if it outlives the writer. + file: yield* held?.owner === owner && held.active + ? register + : Effect.acquireUseRelease( + Effect.interruptible(readPermit.take(1)), + () => withGate(register), + () => Effect.asVoid(readPermit.release(1)), + ), + users: 0, + }; + lease.users++; + const token = { lease, active: true }; + const release = retryReadCleanup(() => { + if (!token.active) return; + if (lease.users === 1) NodeFS.rmSync(lease.file, { force: true }); + lease.users--; + token.active = false; + }); + readTokens.set(release, token); + return release; + }).pipe( + Effect.onError(() => + retryReadCleanup(() => { + if (created) NodeFS.rmSync(created, { force: true }); + }), + ), + ); }), ); const withRead = ( @@ -353,28 +556,17 @@ export const make = (directory: string) => { const drained = Effect.sync(() => { let busy = false; for (const file of NodeFS.readdirSync(readers)) { + const path = NodePath.join(readers, file); const pid = Number(file.split("-")[0]); + let identity; try { - process.kill(pid, 0); - const identity = processIdentity(pid); - if ( - identity !== null && - identity !== NodeFS.readFileSync(NodePath.join(readers, file), "utf8") - ) { - NodeFS.rmSync(NodePath.join(readers, file), { force: true }); - } else { - busy = true; - } + identity = NodeFS.readFileSync(path, "utf8"); } catch (error) { - if ( - !(error instanceof Error) || - !("code" in error) || - (error.code !== "ESRCH" && error.code !== "ENOENT") - ) { - throw error; - } - NodeFS.rmSync(NodePath.join(readers, file), { force: true }); + if (error instanceof Error && "code" in error && error.code === "ENOENT") continue; + throw error; } + if (isProcessAlive(pid, identity)) busy = true; + else NodeFS.rmSync(path, { force: true }); } return !busy; }); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index b01efc077b601..47b6b6f6a166c 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -2242,6 +2242,706 @@ it.effect.each(["interaction", "opt-out"] as const)( ); describe("provider maintenance host leases", () => { + it("compares creation identity when the PID probe is denied", () => { + const identity = HostLock.readProcessIdentity(process.pid); + assert.isNotNull(identity); + const kill = vi.spyOn(process, "kill").mockImplementation(() => { + throw Object.assign(new Error("permission denied"), { code: "EPERM" }); + }); + try { + assert.isFalse(HostLock.isProcessAlive(process.pid, "previous-process-start")); + assert.isTrue(HostLock.isProcessAlive(process.pid, identity)); + } finally { + kill.mockRestore(); + } + }); + + it.live("retains Unix group observations across child exit and PID reuse", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) !== "linux") return; + const require = NodeModule.createRequire(import.meta.url); + const cp = require("node:child_process") as typeof NodeChildProcess; + const fs = require("node:fs") as typeof NodeFS; + const readFile = NodeFS.readFileSync; + const fields = readFile(`/proc/${process.pid}/stat`, "utf8").split(") ").at(-1)!.split(" "); + let liveRow = true; + let childExited = true; + let group = process.pid; + const read = vi + .spyOn(fs, "readFileSync") + .mockImplementation((...args: Parameters) => { + if (args[0] === "/proc/201/stat") + return childExited ? "201 (exited child) Z" : `201 (helper) ${fields.join(" ")}`; + return readFile(...args); + }); + const scan = vi.spyOn(cp, "spawnSync").mockImplementation(() => { + const stdout = liveRow ? `201 ${group} S\n` : ""; + return { pid: 1, output: [null, stdout, ""], stdout, stderr: "", status: 0, signal: null }; + }); + NodeModule.syncBuiltinESMExports(); + try { + const descendants: HostLock.InstallDescendants = new Map(); + assert.isTrue( + HostLock.installChildrenAreAlive(process.pid, undefined, descendants), + "a child may have spawned a helper after its table row was read", + ); + liveRow = false; + assert.isFalse(HostLock.installChildrenAreAlive(process.pid, undefined, descendants)); + liveRow = true; + childExited = false; + fields[19] = "10"; + assert.isTrue(HostLock.installChildrenAreAlive(process.pid, undefined, descendants)); + fields[19] = "11"; + assert.isTrue(HostLock.installChildrenAreAlive(process.pid, undefined, descendants)); + group = process.pid + 1; + assert.isTrue( + HostLock.installChildrenAreAlive(process.pid, undefined, descendants), + "an observed replacement helper stays owned after leaving the group", + ); + fields[19] = "12"; + assert.isFalse( + HostLock.installChildrenAreAlive(process.pid, undefined, descendants), + "an unrelated reused PID outside the group cannot pin admission", + ); + } finally { + scan.mockRestore(); + read.mockRestore(); + NodeModule.syncBuiltinESMExports(); + } + }), + ); + + it.live("retries cleanup of a partially registered reader without masking its write error", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const readers = NodePath.join(directory, "readers"); + const mutableFS = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const writeFile = NodeFS.writeFileSync; + const remove = NodeFS.rmSync; + const writeError = new Error("partial reader write"); + let failed = false; + const write = vi + .spyOn(mutableFS, "writeFileSync") + .mockImplementation((...args: Parameters) => { + writeFile(...args); + if (String(args[0]).startsWith(readers + NodePath.sep)) throw writeError; + }); + const unlink = vi + .spyOn(mutableFS, "rmSync") + .mockImplementation((...args: Parameters) => { + if (!failed && String(args[0]).startsWith(readers + NodePath.sep)) { + failed = true; + throw new Error("temporary unlink failure"); + } + return remove(...args); + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + write.mockRestore(); + unlink.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + const host = HostLock.make(directory); + const exit = yield* Effect.exit(host.acquireRead); + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) assert.strictEqual(Cause.squash(exit.cause), writeError); + assert.deepEqual(NodeFS.readdirSync(readers), []); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("retries a failed scoped reader unlink before returning", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const readers = NodePath.join(directory, "readers"); + const mutableFS = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const remove = NodeFS.rmSync; + let failed = false; + const spy = vi + .spyOn(mutableFS, "rmSync") + .mockImplementation((...args: Parameters) => { + if (!failed && String(args[0]).startsWith(readers + NodePath.sep)) { + failed = true; + throw new Error("transient unlink failure"); + } + return remove(...args); + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + spy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + const host = HostLock.make(directory); + assert.equal(yield* host.withRead(Effect.succeed("completed")), "completed"); + assert.isTrue(failed); + assert.deepEqual(NodeFS.readdirSync(readers), []); + yield* host.withWrite( + Effect.void, + Effect.die(new Error("finished reader pinned admission")), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("acknowledges a transferred gate before an interrupted wait", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const host = HostLock.make(directory); + const acknowledged = NodePath.join(directory, "acknowledged"); + const exit = yield* Effect.exit( + host.withWrite( + host.handoff( + Effect.interrupt, + Effect.sync(() => NodeFS.writeFileSync(acknowledged, "authorized")), + ), + ), + ); + assert.isTrue(Exit.isFailure(exit)); + assert.equal(NodeFS.readFileSync(acknowledged, "utf8"), "authorized"); + NodeFS.utimesSync(NodePath.join(directory, "gate.lock"), 0, 0); + yield* host.withWrite(Effect.void); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.each(["linux", "darwin"] as const)("does not let a %s zombie retain admission", (platform) => { + const require = NodeModule.createRequire(import.meta.url); + const os = require("node:os") as typeof NodeOS; + const fs = require("node:fs") as typeof NodeFS; + const cp = require("node:child_process") as typeof NodeChildProcess; + const readFile = NodeFS.readFileSync; + const osSpy = vi.spyOn(os, "platform").mockReturnValue(platform); + const read = vi + .spyOn(fs, "readFileSync") + .mockImplementation((...args: Parameters) => { + if (args[0] === `/proc/${process.pid}/stat`) return `${process.pid} (zombie) Z`; + return readFile(...args); + }); + const scan = vi.spyOn(cp, "spawnSync").mockReturnValue({ + pid: 1, + output: [null, "Z Fri Oct 11 00:00:00 2026", ""], + stdout: "Z Fri Oct 11 00:00:00 2026", + stderr: "", + status: 0, + signal: null, + }); + NodeModule.syncBuiltinESMExports(); + try { + assert.isFalse(HostLock.isProcessAlive(process.pid)); + } finally { + scan.mockRestore(); + read.mockRestore(); + osSpy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + } + }); + + it.live.each(["second output", "spawn", "output close"] as const)( + "closes relay output descriptors when %s fails", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const require = NodeModule.createRequire(import.meta.url); + const mutableFS = require("node:fs") as typeof NodeFS; + const childProcess = require("node:child_process") as typeof NodeChildProcess; + const open = NodeFS.openSync; + const close = NodeFS.closeSync; + const descriptors = new Set(); + let spawned = false; + let closeFailed = false; + const file = NodePath.join(directory, "receipt.json"); + const openSpy = vi + .spyOn(mutableFS, "openSync") + .mockImplementation((...args: Parameters) => { + if (mode === "second output" && args[0] === file + ".stderr" && args[1] === "a") + throw new Error("cannot open stderr"); + const fd = open(...args); + descriptors.add(fd); + return fd; + }); + const closeSpy = vi.spyOn(mutableFS, "closeSync").mockImplementation((fd) => { + descriptors.delete(fd); + close(fd); + if (mode === "output close" && spawned && !closeFailed) { + closeFailed = true; + throw new Error("output close failed"); + } + }); + const spawn = vi.spyOn(childProcess, "spawn").mockImplementation(() => { + if (mode !== "output close") throw new Error("cannot spawn relay child"); + spawned = true; + const child = new childProcess.ChildProcess(); + queueMicrotask(() => child.emit("spawn")); + return child; + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + openSpy.mockRestore(); + closeSpy.mockRestore(); + spawn.mockRestore(); + NodeModule.syncBuiltinESMExports(); + for (const fd of descriptors) close(fd); + }), + ); + const exit = yield* Effect.exit( + Effect.tryPromise(() => + InstallSupervisor.runInstallSubcommand("__provider-install-relay", [file]), + ), + ); + assert.isTrue(Exit.isFailure(exit)); + assert.equal(descriptors.size, 0, "relay leaked an output descriptor"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live.each(["partial write", "gate release", "lease release"] as const)( + "cleans up or retries reader release after %s fails", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const readers = NodePath.join(directory, "readers"); + const mutableFS = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const writeFile = NodeFS.writeFileSync; + const rmdir = NodeFS.rmdir; + const remove = NodeFS.rmSync; + const injected = new Error("injected reader failure"); + const host = HostLock.make(directory); + const release = mode === "lease release" ? yield* host.acquireRead : undefined; + let failed = false; + const spy = + mode === "partial write" + ? vi + .spyOn(mutableFS, "writeFileSync") + .mockImplementation((...args: Parameters) => { + writeFile(...args); + if (String(args[0]).startsWith(readers + NodePath.sep)) throw injected; + }) + : mode === "gate release" + ? vi + .spyOn(mutableFS, "rmdir") + .mockImplementation((...args: Parameters) => { + if (!failed && args[0] === NodePath.join(directory, "gate.lock")) { + failed = true; + const callback = args.at(-1); + if (typeof callback === "function") callback(injected); + return; + } + return rmdir(...args); + }) + : vi + .spyOn(mutableFS, "rmSync") + .mockImplementation((...args: Parameters) => { + if (!failed && String(args[0]).startsWith(readers + NodePath.sep)) { + failed = true; + throw injected; + } + return remove(...args); + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + spy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + const exit = yield* Effect.exit(release ?? host.acquireRead); + assert.equal(Exit.isFailure(exit), mode !== "lease release"); + spy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + if (release) yield* release; + assert.deepEqual(NodeFS.readdirSync(readers), []); + if (mode === "gate release") + NodeFS.utimesSync(NodePath.join(directory, "gate.lock"), 0, 0); + yield* host.withWrite(Effect.void, Effect.die(new Error("reader leaked"))); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live.each([ + "live", + "empty", + "failed", + "malformed", + "reused root", + "reused foreign root", + ] as const)("uses Windows job lifetime membership for %s descendants", (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const job = "t3-provider-install-00000000-0000-0000-0000-000000000001"; + NodeFS.writeFileSync( + HostLock.installLeasePath(directory), + JSON.stringify({ pid: 101, identity: "root-start", job }), + ); + const require = NodeModule.createRequire(import.meta.url); + const os = require("node:os") as typeof NodeOS; + const childProcess = require("node:child_process") as typeof NodeChildProcess; + const platform = vi.spyOn(os, "platform").mockReturnValue("win32"); + const kill = vi.spyOn(process, "kill").mockImplementation(() => { + if (mode === "reused foreign root") return true; + throw Object.assign(new Error("root exited"), { code: "ESRCH" }); + }); + const stdout = + mode === "live" + ? "103\n" + : mode === "reused root" + ? "101\n" + : mode === "malformed" + ? "broken" + : ""; + const scan = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + pid: 1, + output: [null, stdout, ""], + stdout, + stderr: "", + status: mode === "failed" ? 1 : 0, + signal: null, + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + scan.mockRestore(); + kill.mockRestore(); + platform.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + if (mode === "failed" || mode === "malformed") + assert.throws(() => HostLock.installIsAlive(directory), /Could not inspect/); + else + assert.equal( + HostLock.installIsAlive(directory), + mode === "live" || mode === "reused root", + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.each([NaN, 0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + "rejects invalid process id %s instead of waiting forever", + (pid) => { + assert.isFalse(HostLock.isProcessAlive(pid)); + }, + ); + + it.live("keeps a legacy Linux reader active until it releases", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) !== "linux") return; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const readers = NodePath.join(directory, "readers"); + NodeFS.mkdirSync(readers); + NodeFS.writeFileSync( + NodePath.join(readers, `${process.pid}-legacy`), + HostLock.readProcessIdentity(process.pid)!.split(":").at(-1)!, + ); + const exit = yield* Effect.exit( + HostLock.make(directory).withWrite( + Effect.void, + Effect.die(new Error("legacy reader is active")), + ), + ); + assert.isTrue(Exit.isFailure(exit)); + assert.equal(NodeFS.readdirSync(readers).length, 1); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live.each(["darwin", "win32"] as const)( + "stores and reclaims cross-platform reader identities on %s", + (platform) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const require = NodeModule.createRequire(import.meta.url); + const os = require("node:os") as typeof NodeOS; + const childProcess = require("node:child_process") as typeof NodeChildProcess; + const platformSpy = vi.spyOn(os, "platform").mockReturnValue(platform); + const scan = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + pid: 1, + output: [null, "current-start", ""], + stdout: "current-start", + stderr: "", + status: 0, + signal: null, + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + scan.mockRestore(); + platformSpy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + const host = HostLock.make(directory); + const release = yield* host.acquireRead; + const readers = NodePath.join(directory, "readers"); + const [reader] = NodeFS.readdirSync(readers); + assert.equal( + NodeFS.readFileSync(NodePath.join(readers, reader!), "utf8"), + "current-start", + ); + yield* release; + NodeFS.writeFileSync(NodePath.join(readers, `${process.pid}-stale`), "previous-start"); + assert.equal( + yield* host.withWrite( + Effect.succeed("installed"), + Effect.die(new Error("reused reader blocked installation")), + ), + "installed", + ); + assert.deepEqual(NodeFS.readdirSync(readers), []); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live.each(["legacy", "durable", "durable unknown"] as const)( + "retains Windows descendants after their parent and supervisor exit (%s)", + (mode) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + NodeFS.writeFileSync( + HostLock.installLeasePath(directory), + JSON.stringify({ + pid: 101, + identity: "root-start", + ...(mode !== "legacy" + ? { descendants: [{ pid: 103, identity: "helper-start" }] } + : {}), + }), + ); + const require = NodeModule.createRequire(import.meta.url); + const os = require("node:os") as typeof NodeOS; + const childProcess = require("node:child_process") as typeof NodeChildProcess; + const platformSpy = vi.spyOn(os, "platform").mockReturnValue("win32"); + let rootAlive = true; + let helperAlive = true; + let helperStart = "helper-start"; + let unknownIdentity = false; + const kill = vi.spyOn(process, "kill").mockImplementation((pid) => { + if ( + (pid === 101 && rootAlive) || + (pid === 103 && helperAlive) || + (pid === 102 && rootAlive) + ) + return true; + throw Object.assign(new Error("gone"), { code: "ESRCH" }); + }); + const scan = vi.spyOn(childProcess, "spawnSync").mockImplementation((_command, args) => { + const command = args?.join(" ") ?? ""; + const stdout = command.includes("ProcessId =") + ? command.includes("101") + ? "root-start" + : command.includes("102") + ? "parent-start" + : unknownIdentity + ? "" + : helperStart + : rootAlive + ? `101|1|root-start\n102|101|parent-start\n103|102|${helperStart}\n` + : helperAlive + ? `103|102|${helperStart}\n` + : ""; + return { + pid: 1, + output: [null, stdout, ""], + stdout, + stderr: "", + status: 0, + signal: null, + }; + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + scan.mockRestore(); + kill.mockRestore(); + platformSpy.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + assert.isTrue(HostLock.installIsAlive(directory)); + rootAlive = false; + assert.isTrue(HostLock.installIsAlive(directory), "orphaned grandchild still writes"); + const recovered = yield* fs.makeTempDirectoryScoped(); + NodeFS.writeFileSync( + HostLock.installLeasePath(recovered), + JSON.stringify({ + pid: 101, + identity: "root-start", + descendants: [ + { pid: 102, identity: "parent-start" }, + { pid: 103, identity: "helper-start" }, + ], + }), + ); + assert.isTrue( + HostLock.installIsAlive(recovered), + "durable identities survive a new observer", + ); + helperStart = "reused-helper-start"; + assert.isFalse( + HostLock.installIsAlive(directory), + "reused grandchild cannot pin the old lease", + ); + const replacing = yield* fs.makeTempDirectoryScoped(); + NodeFS.copyFileSync( + HostLock.installLeasePath(directory), + HostLock.installLeasePath(replacing), + ); + rootAlive = true; + helperStart = "helper-start"; + assert.isTrue(HostLock.installIsAlive(replacing)); + helperStart = "replacement-helper-start"; + assert.isTrue(HostLock.installIsAlive(replacing)); + rootAlive = false; + unknownIdentity = mode === "durable unknown"; + assert.isTrue( + HostLock.installIsAlive(replacing), + "a replacement helper observed under the live installer remains owned", + ); + helperAlive = false; + assert.isFalse(HostLock.installIsAlive(replacing)); + assert.isFalse(HostLock.installIsAlive(directory)); + if (mode === "durable unknown") { + helperAlive = true; + rootAlive = true; + unknownIdentity = false; + helperStart = "helper-start"; + const fresh = yield* fs.makeTempDirectoryScoped(); + NodeFS.copyFileSync( + HostLock.installLeasePath(directory), + HostLock.installLeasePath(fresh), + ); + assert.isTrue(HostLock.installIsAlive(fresh)); + rootAlive = false; + helperStart = "replacement-helper-start"; + unknownIdentity = true; + NodeFS.writeFileSync( + HostLock.installLeasePath(fresh), + JSON.stringify({ + pid: 101, + identity: "root-start", + descendants: [{ pid: 103, identity: helperStart }], + }), + ); + assert.isTrue( + HostLock.installIsAlive(fresh), + "a new durable generation retains admission when its identity query fails", + ); + helperAlive = false; + assert.isFalse(HostLock.installIsAlive(fresh)); + helperAlive = true; + unknownIdentity = false; + helperStart = "unrelated-helper-start"; + assert.isFalse( + HostLock.installIsAlive(fresh), + "transient identity uncertainty cannot pin a later unrelated PID", + ); + } + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("returns the durable result when the supervisor exits during the liveness check", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const file = NodePath.join(directory, "provider-updates", "codex.json"); + const result = { + stdout: "installed", + stderr: "", + exitCode: 0, + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; + const require = NodeModule.createRequire(import.meta.url); + const childProcess = require("node:child_process") as typeof NodeChildProcess; + const mutableFS = require("node:fs") as typeof NodeFS; + const readFile = NodeFS.readFileSync; + const spawn = vi.spyOn(childProcess, "spawn").mockImplementation(() => { + NodeFS.writeFileSync( + file, + JSON.stringify({ ...InstallSupervisor.readReceipt(file), pid: 101 }), + ); + const child = new childProcess.ChildProcess(); + queueMicrotask(() => child.emit("spawn")); + return child; + }); + const read = vi + .spyOn(mutableFS, "readFileSync") + .mockImplementation((...args: Parameters) => { + if (args[0] === HostLock.installLeasePath(directory)) { + NodeFS.writeFileSync( + file, + JSON.stringify({ ...InstallSupervisor.readReceipt(file), result }), + ); + throw Object.assign(new Error("supervisor finished"), { code: "ENOENT" }); + } + return readFile(...args); + }); + NodeModule.syncBuiltinESMExports(); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + spawn.mockRestore(); + read.mockRestore(); + NodeModule.syncBuiltinESMExports(); + }), + ); + const supervisor = yield* InstallSupervisor.ProviderInstallSupervisor.pipe( + Effect.provide( + InstallSupervisor.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ServerConfig.ServerConfig, { + stateDir: directory, + } as ServerConfig.ServerConfig["Service"]), + Layer.succeed(HostLock.ProviderMaintenanceHostLock, { + ...HostLock.make(directory), + handoff: (effect) => effect, + }), + ), + ), + ), + ), + ); + assert.deepEqual( + yield* supervisor.run({ + provider: CODEX_DRIVER, + instanceId: CODEX_INSTANCE_ID, + instanceIds: [CODEX_INSTANCE_ID], + startedAt: "2026-04-10T00:00:00.000Z", + command: "npm", + args: [], + }), + result, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("reclaims an install lease whose live PID belongs to another process", () => Effect.scoped( Effect.gen(function* () { @@ -2327,6 +3027,9 @@ describe("provider maintenance host leases", () => { if ((yield* HostProcess.Platform) !== "linux") return; const mutableFS = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; const readFile = NodeFS.readFileSync; + const kill = vi.spyOn(process, "kill").mockImplementation(() => { + throw Object.assign(new Error("denied"), { code: "EPERM" }); + }); const spy = vi .spyOn(mutableFS, "readFileSync") .mockImplementation((...args: Parameters) => { @@ -2343,6 +3046,7 @@ describe("provider maintenance host leases", () => { assert.isTrue(HostLock.isProcessAlive(process.pid, "previous-start")); } finally { spy.mockRestore(); + kill.mockRestore(); NodeModule.syncBuiltinESMExports(); } }), @@ -3140,6 +3844,11 @@ describe("provider maintenance host leases", () => { "heartbeat-write-error", "installer-killed-with-live-child", "verbose-output", + "receipt-write-error", + "gate-release-error", + "descendant-scan-error", + "job-registration-error", + "output-read-error", ] as const)( "finishes the installer after %s", (mode) => @@ -3241,7 +3950,17 @@ describe("provider maintenance host leases", () => { ? `const original = fs.default.utimesSync; let failed = false; fs.default.utimesSync = (...args) => { if (!failed) { failed = true; throw Object.assign(new Error("full disk"), {code: "ENOSPC"}); } return original(...args); }; syncBuiltinESMExports();` - : "" + : mode === "receipt-write-error" + ? `const original = fs.default.renameSync; fs.default.renameSync = (...args) => { if (fs.existsSync(${JSON.stringify(installed)}) && args[1] === ${JSON.stringify(file)}) throw new Error("receipt write failed"); return original(...args); }; syncBuiltinESMExports();` + : mode === "gate-release-error" + ? `const original = fs.default.rmSync; fs.default.rmSync = (...args) => { if (args[0] === ${JSON.stringify(NodePath.join(directory, "gate.lock"))}) throw new Error("gate removal failed"); return original(...args); }; syncBuiltinESMExports();` + : mode === "descendant-scan-error" + ? `const cp = await import("node:child_process"); const original = cp.default.spawnSync; let failed = false; cp.default.spawnSync = (...args) => { if (!failed && fs.existsSync(${JSON.stringify(installed)})) { failed = true; throw new Error("process table unavailable"); } return original(...args); }; syncBuiltinESMExports();` + : mode === "job-registration-error" + ? `const os = await import("node:os"); os.default.platform = () => "win32"; syncBuiltinESMExports();` + : mode === "output-read-error" + ? `const original = fs.default.readSync; fs.default.readSync = (...args) => { if (fs.existsSync(${JSON.stringify(installed)})) throw new Error("output read failed"); return original(...args); }; syncBuiltinESMExports();` + : "" } await runInstallSubcommand("__provider-install", ${JSON.stringify([file, String(dead.pid)])});`, ], @@ -3264,7 +3983,7 @@ describe("provider maintenance host leases", () => { () => new Promise((resolve) => child.once("exit", resolve)), ); const finishedBeforeChildren = !NodeFS.existsSync(installed); - if (mode !== "runtime-replaced") + if (mode !== "runtime-replaced" && mode !== "job-registration-error") yield* Effect.callback((resume) => { const complete = () => { if (!NodeFS.existsSync(installed)) return; @@ -3275,7 +3994,40 @@ describe("provider maintenance host leases", () => { complete(); return Effect.sync(() => watcher.close()); }); - assert.equal(code, 0); + assert.equal( + code, + mode === "receipt-write-error" || mode === "gate-release-error" ? 1 : 0, + ); + if (mode === "job-registration-error") { + assert.isFalse( + NodeFS.existsSync(installed), + "installer started without lifetime tracking", + ); + assert.isFalse(NodeFS.existsSync(HostLock.installLeasePath(directory))); + assert.isTrue( + NodeFS.existsSync(NodePath.join(directory, "gate.lock")), + "unacknowledged supervisor opened the gate", + ); + assert.equal(InstallSupervisor.readReceipt(file).result?.exitCode, null); + return; + } + if (mode === "output-read-error") { + assert.equal(InstallSupervisor.readReceipt(file).result?.exitCode, null); + assert.equal(InstallSupervisor.readReceipt(file).result?.stderr, "output read failed"); + assert.isFalse(NodeFS.existsSync(HostLock.installLeasePath(directory))); + assert.isFalse(NodeFS.existsSync(NodePath.join(directory, "gate.lock"))); + assert.equal(NodeFS.readFileSync(installed, "utf8"), "installed"); + return; + } + if (mode === "receipt-write-error" || mode === "gate-release-error") { + assert.isFalse( + NodeFS.existsSync(HostLock.installLeasePath(directory)), + "install lease leaked after cleanup error", + ); + if (mode === "gate-release-error") + NodeFS.utimesSync(NodePath.join(directory, "gate.lock"), 0, 0); + yield* HostLock.make(directory).withWrite(Effect.void); + } if (mode === "verbose-output") { assert.isAtMost(heartbeatOutputSize!, 20_000); assert.isTrue(InstallSupervisor.readReceipt(file).result?.stdoutTruncated); @@ -3287,7 +4039,8 @@ describe("provider maintenance host leases", () => { "lease released before installer child finished", ); assert.isFalse(HostLock.installIsAlive(directory)); - } else assert.equal(InstallSupervisor.readReceipt(file).result?.exitCode, 0); + } else if (mode !== "receipt-write-error") + assert.equal(InstallSupervisor.readReceipt(file).result?.exitCode, 0); assert.equal(NodeFS.readFileSync(installed, "utf8"), "installed"); }), ).pipe(Effect.provide(NodeServices.layer)), From 61622194aba6c80c91c5cf514058d3f9291d42ae Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:37:11 +0000 Subject: [PATCH 141/149] fix(server): preserve active installer leases during startup cleanup --- apps/server/src/process/agentScope.test.ts | 222 +++++++++++++----- apps/server/src/process/ownedProcesses.ts | 96 +++++++- apps/server/src/process/windowsInstallJob.ts | 90 +++++++ .../provider/ProviderMaintenanceHostLock.ts | 63 +---- 4 files changed, 340 insertions(+), 131 deletions(-) create mode 100644 apps/server/src/process/windowsInstallJob.ts diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 576a92afeb3cd..7908ab37cd6a7 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -255,49 +255,74 @@ describe("provider process restart ownership", () => { ), ); - it.effect("keeps the server responsive while a Windows process query is pending", () => - scenario(({ directory, processes }) => - Effect.gen(function* () { - const started = NodePath.join(directory, "query-started"); - const released = NodePath.join(directory, "query-release"); - const executable = NodePath.join(directory, "powershell.exe"); - // Stand in for CIM with a real child blocked on the server's acknowledgement. - NodeFS.writeFileSync( - executable, - `#!${process.execPath} + it.effect.each(["table", "job"] as const)( + "keeps the server responsive while a Windows %s query is pending", + (query) => + scenario(({ directory, processes, controller }) => + Effect.gen(function* () { + const started = NodePath.join(directory, "query-started"); + const released = NodePath.join(directory, "query-release"); + const executable = NodePath.join(directory, "powershell.exe"); + const leases = NodePath.join(directory, "leases"); + if (query === "job") { + processes.set(71, { + pid: 71, + ppid: 20, + pgid: 0, + startTime: "installer", + zombie: false, + }); + yield* record(directory, controller); + processes.delete(10); + processes.set(71, { ...processes.get(71)!, ppid: 1 }); + NodeFS.mkdirSync(leases); + NodeFS.writeFileSync( + NodePath.join(leases, "install.json"), + JSON.stringify({ pid: 70, job: "t3-provider-install-abcd", descendants: [] }), + ); + } + // Stand in for CIM with a real child blocked on the server's acknowledgement. + NodeFS.writeFileSync( + executable, + `#!${process.execPath} const fs = require("node:fs"); +if (${JSON.stringify(query)} === "job" && !process.argv.some(arg => arg.includes("T3InstallJob"))) { + process.stdout.write(${JSON.stringify(JSON.stringify([...processes.values()]))}); + process.exit(0); +} const finish = () => { if (!fs.existsSync(${JSON.stringify(released)})) return; - process.stdout.write(${JSON.stringify(JSON.stringify([...processes.values()]))}); + process.stdout.write(${JSON.stringify(query === "job" ? "71\n" : JSON.stringify([...processes.values()]))}); watcher.close(); }; const watcher = fs.watch(${JSON.stringify(directory)}, finish); fs.writeFileSync(${JSON.stringify(started)}, "queried"); finish(); `, - { mode: 0o700 }, - ); - const queried = Promise.withResolvers(); - const watcher = NodeFS.watch(directory, () => { - if (NodeFS.existsSync(started)) queried.resolve(); - }); - vi.stubEnv("PATH", `${directory}${NodePath.delimiter}${process.env.PATH ?? ""}`); - try { - const fiber = yield* ProviderProcessLedger.make({ - stateDir: directory, - ownerPid: 10, - }).pipe(Effect.provideService(HostProcess.Platform, "win32"), Effect.forkScoped); - yield* Effect.promise(() => queried.promise); - // The server must be able to acknowledge the query before it returns. - NodeFS.writeFileSync(released, "responsive"); - yield* Fiber.join(fiber); - expect(NodeFS.readFileSync(released, "utf8")).toBe("responsive"); - } finally { - watcher.close(); - vi.unstubAllEnvs(); - } - }).pipe(Effect.scoped), - ), + { mode: 0o700 }, + ); + const queried = Promise.withResolvers(); + const watcher = NodeFS.watch(directory, () => { + if (NodeFS.existsSync(started)) queried.resolve(); + }); + vi.stubEnv("PATH", `${directory}${NodePath.delimiter}${process.env.PATH ?? ""}`); + try { + const fiber = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: query === "job" ? 90 : 10, + ...(query === "job" ? { controller, installLeaseDirectory: leases } : {}), + }).pipe(Effect.provideService(HostProcess.Platform, "win32"), Effect.forkScoped); + yield* Effect.promise(() => queried.promise); + // The server must be able to acknowledge the query before it returns. + NodeFS.writeFileSync(released, "responsive"); + yield* Fiber.join(fiber); + expect(NodeFS.readFileSync(released, "utf8")).toBe("responsive"); + } finally { + watcher.close(); + vi.unstubAllEnvs(); + } + }).pipe(Effect.scoped), + ), ); it.effect("reaps an orphan provider and its forked children before exposing the spawn hook", () => @@ -472,30 +497,111 @@ finish(); }), ), ); - it.effect("never stops a nested server's install supervisor or its installer", () => - scenario(({ directory, processes, signals, controller }) => - Effect.gen(function* () { - // A server launched from the provider started an install that outlives it. - processes.set(70, { pid: 70, ppid: 20, pgid: 70, startTime: "supervisor", zombie: false }); - processes.set(71, { pid: 71, ppid: 70, pgid: 70, startTime: "installer", zombie: false }); - yield* record(directory, controller); - processes.delete(10); - processes.set(70, { ...processes.get(70)!, ppid: 1 }); - const leases = NodePath.join(directory, "leases"); - NodeFS.mkdirSync(leases); - NodeFS.writeFileSync(NodePath.join(leases, "install.json"), "70"); - yield* ProviderProcessLedger.make({ - stateDir: directory, - ownerPid: 90, - controller, - installLeaseDirectory: leases, - }).pipe(Effect.scoped); - expect(processes.has(20)).toBe(false); - expect(processes.has(70)).toBe(true); - expect(processes.has(71)).toBe(true); - expect(signals.some((s) => /^(70|71):/.test(s))).toBe(false); - }), - ), + it.effect.each([ + { platform: "linux", lease: "legacy" }, + { platform: "win32", lease: "legacy" }, + { platform: "linux", lease: "structured" }, + { platform: "win32", lease: "structured" }, + { platform: "linux", lease: "orphan" }, + { platform: "win32", lease: "orphan" }, + { platform: "linux", lease: "reused" }, + { platform: "win32", lease: "reused" }, + { platform: "linux", lease: "root-reused" }, + { platform: "win32", lease: "root-reused" }, + { platform: "linux", lease: "group" }, + { platform: "win32", lease: "job" }, + { platform: "win32", lease: "job-failed" }, + ] as const)( + "protects a nested installer on $platform with a $lease lease", + ({ platform, lease }) => + scenario(({ directory, processes, signals, controller }) => + Effect.gen(function* () { + // A server launched from the provider started an install that outlives it. + const childIdentity = platform === "win32" ? "2026-10-11T20:00:00.1234567Z" : "boot:123"; + const childStart = platform === "win32" ? "639273456001234567" : childIdentity; + const rootReused = lease === "root-reused"; + processes.set(70, { + pid: 70, + ppid: 20, + pgid: 70, + startTime: rootReused ? "replacement" : "supervisor", + zombie: false, + }); + const reused = lease === "reused" || rootReused; + processes.set(71, { + pid: 71, + ppid: 70, + pgid: 70, + startTime: reused + ? platform === "win32" + ? "639273456001234568" + : "replacement" + : childStart, + zombie: false, + }); + if (platform === "win32") + for (const [pid, p] of processes) processes.set(pid, { ...p, pgid: 0 }); + yield* record(directory, controller); + processes.delete(10); + processes.set(70, { ...processes.get(70)!, ppid: 1 }); + const job = lease === "job" || lease === "job-failed"; + const orphan = lease === "orphan" || lease === "reused" || lease === "group" || job; + if (orphan) { + processes.delete(70); + processes.set(71, { + ...processes.get(71)!, + ppid: 1, + pgid: platform === "win32" ? 0 : lease === "group" ? 70 : 71, + }); + } + const leases = NodePath.join(directory, "leases"); + NodeFS.mkdirSync(leases); + NodeFS.writeFileSync( + NodePath.join(leases, "install.json"), + lease === "legacy" + ? "70" + : JSON.stringify({ + pid: 70, + identity: "supervisor", + ...(job ? { job: "t3-provider-install-abcd" } : {}), + descendants: + job || rootReused || lease === "group" + ? [] + : [{ pid: 71, identity: childIdentity }], + }), + ); + if (job) + NodeFS.writeFileSync( + NodePath.join(directory, "powershell.exe"), + `#!${process.execPath}\nprocess.stdout.write("71\\n"); process.exit(${lease === "job-failed" ? 1 : 0});\n`, + { mode: 0o700 }, + ); + const result = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + controller, + installLeaseDirectory: leases, + }).pipe( + Effect.provideService(HostProcess.Platform, platform), + Effect.provideService(HostProcess.Environment, { + ...process.env, + PATH: `${directory}${NodePath.delimiter}${process.env.PATH ?? ""}`, + }), + Effect.scoped, + Effect.exit, + ); + expect(result._tag).toBe(lease === "job-failed" ? "Failure" : "Success"); + if (lease === "job-failed") { + expect(signals).toEqual([]); + return; + } + expect(processes.has(20)).toBe(false); + expect(processes.has(70)).toBe(!orphan && !rootReused); + expect(processes.has(71)).toBe(!reused); + if (reused) expect(signals).toContain("71:SIGKILL"); + else expect(signals.some((s) => /^(70|71):/.test(s))).toBe(false); + }), + ), ); it.effect.each(["empty", "corrupt", "unreadable"] as const)( "refuses orphan cleanup when the install lease is %s", diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index 9ecdee30c6320..c1719272ee75d 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -7,6 +7,7 @@ import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; import { SERVICE_LAUNCHER_CONTEXT_ENV } from "../cloud/serviceProtocol.ts"; +import * as WindowsInstallJob from "./windowsInstallJob.ts"; const execFile = NodeUtil.promisify(NodeChildProcess.execFile); @@ -46,6 +47,32 @@ const identity = (value: unknown): value is Identity => value.startTime.length > 0; const owned = (value: unknown): value is OwnedProcess => identity(value) && "pgid" in value && Number.isSafeInteger(value.pgid); +interface InstallProcess { + readonly pid: number; + readonly identity?: string | null; +} +const installProcess = (value: unknown): value is InstallProcess => + object(value) && + Number.isSafeInteger(value.pid) && + Number(value.pid) > 0 && + (value.identity === undefined || value.identity === null || typeof value.identity === "string"); +function decodeInstallLease(text: string) { + const value: unknown = JSON.parse(text); + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) + return { pid: value, descendants: [] as ReadonlyArray, job: undefined }; + if ( + !installProcess(value) || + ("job" in value && typeof value.job !== "string") || + ("descendants" in value && + (!Array.isArray(value.descendants) || !value.descendants.every(installProcess))) + ) + throw new Error("Invalid provider install lease"); + return { + ...value, + job: "job" in value ? (value.job as string) : undefined, + descendants: "descendants" in value ? (value.descendants as ReadonlyArray) : [], + }; +} function decodeEntry(text: string): Entry { const value: unknown = JSON.parse(text); if ( @@ -371,26 +398,69 @@ export async function make(input: { }; // A server started from a provider or terminal can descend from it with its own // detached install supervisor. Stopping that group would corrupt a shared CLI. - const withoutInstaller = (entry: Entry, table: ReadonlyArray): Entry => { + const withoutInstaller = async ( + entry: Entry, + table: ReadonlyArray, + ): Promise => { if (input.installLeaseDirectory === undefined) return entry; - let leader: number; + let lease: ReturnType; try { - leader = Number( + lease = decodeInstallLease( NodeFS.readFileSync(NodePath.join(input.installLeaseDirectory, "install.json"), "utf8"), ); } catch (cause) { if ((cause as NodeJS.ErrnoException).code === "ENOENT") return entry; throw cause; } - if (!Number.isSafeInteger(leader) || leader <= 0) - throw new Error("Invalid provider install lease"); + const leader = lease.pid; + const matches = (record: InstallProcess, p: ObservedProcess) => { + if (record.pid !== p.pid || p.zombie) return false; + const start = record.identity; + if (start === undefined || start === null) return true; + if (start === p.startTime) return true; + // CIM leases use ISO dates; the ownership ledger stores .NET ticks. + if (platform !== "win32" || !/^\d+$/.test(p.startTime)) return false; + const milliseconds = Date.parse(start); + if (!Number.isFinite(milliseconds)) return false; + const fraction = /\.(\d{1,7})(?:Z|[+-]\d{2}:\d{2})$/.exec(start)?.[1] ?? ""; + const ticks = + BigInt(milliseconds) * 10_000n + + 621355968000000000n + + BigInt(fraction.padEnd(7, "0").slice(3)); + return ticks.toString() === p.startTime; + }; + const root = table.find((p) => p.pid === leader); + const records: ReadonlyArray = [lease, ...lease.descendants]; + if ( + records.some( + (record) => !table.some((p) => p.pid === record.pid) && controller.exists(record.pid), + ) + ) + throw new Error("Could not verify a provider installer during cleanup"); + const installer = new Set( + table.filter((p) => records.some((record) => matches(record, p))).map((p) => p.pid), + ); + if (platform === "win32" && lease.job) + for (const pid of await WindowsInstallJob.processesAsync(leader, lease.job, environment)) + installer.add(pid); + // A surviving private POSIX group reserves its leader's PID after exit. + if (platform !== "win32" && (!root || matches(lease, { ...root, zombie: false }))) + for (const p of table) if (p.pgid === leader && !p.zombie) installer.add(p.pid); // Windows has no process groups, so its installer is found through parents. - const installer = new Set([leader]); let changed = true; + const parents = new Map(table.map((p) => [p.pid, p])); while (changed) { changed = false; for (const p of table) { - if (!installer.has(p.pid) && (p.pgid === leader || installer.has(p.ppid))) { + const parent = installer.has(p.ppid) ? parents.get(p.ppid) : undefined; + const parentBirth = parent ? creationTime(parent) : undefined; + const childBirth = creationTime(p); + if ( + !p.zombie && + !installer.has(p.pid) && + parent && + (parentBirth === undefined || childBirth === undefined || childBirth >= parentBirth) + ) { installer.add(p.pid); changed = true; } @@ -447,8 +517,8 @@ export async function make(input: { } return { ...entry, processes: entry.processes.filter((p) => !protectedPids.has(p.pid)) }; }; - const protectedEntry = (entry: Entry, table: ReadonlyArray) => - withoutServers(withoutInstaller(entry, table), table); + const protectedEntry = async (entry: Entry, table: ReadonlyArray) => + withoutServers(await withoutInstaller(entry, table), table); const reap = async (file: string, entry: Entry) => { let table = await controller.snapshot(); const liveOwner = table.find((p) => same(entry.owner, p)); @@ -461,7 +531,7 @@ export async function make(input: { ) { throw new Error("Could not verify the recorded provider owner"); } - let next = protectedEntry(discover(entry, table), table); + let next = await protectedEntry(discover(entry, table), table); const stopped = new Map(); try { if (platform !== "win32") { @@ -476,14 +546,14 @@ export async function make(input: { stopped.set(`${p.pid}:${p.startTime}`, p); } table = await controller.snapshot(); - next = protectedEntry(discover(next, table), table); + next = await protectedEntry(discover(next, table), table); if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); } } // Persist newly discovered children before signalling: recovery can itself be killed. write(file, next); table = await controller.snapshot(); - next = protectedEntry(next, table); + next = await protectedEntry(next, table); } catch (error) { // Never leave a frozen tree behind: resume it so the next cleanup can retry. for (const p of stopped.values()) await signal(p, "SIGCONT").catch(() => undefined); @@ -589,7 +659,7 @@ export async function make(input: { } let safe: Entry; try { - safe = protectedEntry(entry, latest); + safe = await protectedEntry(entry, latest); } catch { // Only the new root is certainly ours when other servers cannot be verified. safe = { ...entry, processes: entry.processes.filter((p) => p.pid === root.pid) }; diff --git a/apps/server/src/process/windowsInstallJob.ts b/apps/server/src/process/windowsInstallJob.ts new file mode 100644 index 0000000000000..838eedea98679 --- /dev/null +++ b/apps/server/src/process/windowsInstallJob.ts @@ -0,0 +1,90 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Shared with the standalone Node launcher. +import * as NodeChildProcess from "node:child_process"; +import * as NodeUtil from "node:util"; + +const execFile = NodeUtil.promisify(NodeChildProcess.execFile); + +const windowsJobApi = ` +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +public static class T3InstallJob { + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr CreateJobObject(IntPtr attributes, string name); + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr OpenJobObject(uint access, bool inherit, string name); + [DllImport("kernel32.dll", SetLastError=true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job, int kind, IntPtr info, uint size, IntPtr returned); + [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr handle); + public static int[] Processes(string name, int pid, bool create) { + IntPtr job = create ? CreateJobObject(IntPtr.Zero, name) : OpenJobObject(4, false, name); + if (job == IntPtr.Zero) { + int error = Marshal.GetLastWin32Error(); + if (!create && error == 2) return new int[0]; + throw new Win32Exception(error); + } + try { + if (create) { + IntPtr process = OpenProcess(0x101, false, pid); + if (process == IntPtr.Zero) throw new Win32Exception(Marshal.GetLastWin32Error()); + try { + if (!AssignProcessToJobObject(job, process)) throw new Win32Exception(Marshal.GetLastWin32Error()); + } finally { CloseHandle(process); } + } + IntPtr info = Marshal.AllocHGlobal(65536); + try { + if (!QueryInformationJobObject(job, 3, info, 65536, IntPtr.Zero)) throw new Win32Exception(Marshal.GetLastWin32Error()); + int count = Marshal.ReadInt32(info, 4); + int[] pids = new int[count]; + for (int i = 0; i < count; i++) pids[i] = Marshal.ReadIntPtr(info, 8 + i * IntPtr.Size).ToInt32(); + return pids; + } finally { Marshal.FreeHGlobal(info); } + } finally { CloseHandle(job); } + } +}`; + +function argsFor(pid: number, job: string, create: boolean) { + if (!/^t3-provider-install-[a-f0-9-]+$/.test(job) || !Number.isSafeInteger(pid) || pid <= 0) + throw new Error("Invalid provider install job"); + return [ + "-NoProfile", + "-NonInteractive", + "-Command", + `$ErrorActionPreference = 'Stop'; Add-Type -TypeDefinition @'\n${windowsJobApi}\n'@; [T3InstallJob]::Processes('Global\\${job}', ${pid}, $${create})`, + ]; +} +function decode(stdout: string, queryPid?: number): ReadonlyArray { + const children = stdout + .split(/\r?\n/) + .filter((line) => line.trim()) + .map(Number); + if (children.some((child) => !Number.isSafeInteger(child) || child <= 0)) + throw new Error("Could not inspect provider installer processes"); + return children.filter((child) => child !== queryPid); +} +/** The kernel retains install helpers even after their parents exit. */ +export function processes(pid: number, job: string, create = false) { + const scan = NodeChildProcess.spawnSync("powershell.exe", argsFor(pid, job, create), { + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + }); + if (scan.error || scan.status !== 0) + throw new Error("Could not inspect provider installer processes"); + return decode(scan.stdout, scan.pid); +} +/** Startup cleanup must keep the server responsive while PowerShell inspects the job. */ +export async function processesAsync(pid: number, job: string, environment: NodeJS.ProcessEnv) { + const args = argsFor(pid, job, false); + try { + const query = execFile("powershell.exe", args, { + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + env: environment, + }); + const { stdout } = await query; + return decode(stdout, query.child.pid); + } catch { + throw new Error("Could not inspect provider installer processes"); + } +} diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index c903505dd0a7b..6424bb042bd48 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -15,6 +15,7 @@ import * as Semaphore from "effect/Semaphore"; import * as Schema from "effect/Schema"; import * as Schedule from "effect/Schedule"; import * as Maintenance from "@t3tools/provider-core/server/maintenance"; +import * as WindowsInstallJob from "../process/windowsInstallJob.ts"; const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } | null>( "t3/provider/ProviderMaintenanceHostLock/HeldWrite", @@ -108,43 +109,6 @@ export const isProcessAlive = (pid: number, identity?: string | null, observed?: // A named Windows job retains every descendant in the kernel, even when a // short-lived parent exits between observations or the supervisor is killed. -const windowsJobApi = ` -using System; -using System.ComponentModel; -using System.Runtime.InteropServices; -public static class T3InstallJob { - [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr CreateJobObject(IntPtr attributes, string name); - [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr OpenJobObject(uint access, bool inherit, string name); - [DllImport("kernel32.dll", SetLastError=true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid); - [DllImport("kernel32.dll", SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); - [DllImport("kernel32.dll", SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job, int kind, IntPtr info, uint size, IntPtr returned); - [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr handle); - public static int[] Processes(string name, int pid, bool create) { - IntPtr job = create ? CreateJobObject(IntPtr.Zero, name) : OpenJobObject(4, false, name); - if (job == IntPtr.Zero) { - int error = Marshal.GetLastWin32Error(); - if (!create && error == 2) return new int[0]; - throw new Win32Exception(error); - } - try { - if (create) { - IntPtr process = OpenProcess(0x101, false, pid); - if (process == IntPtr.Zero) throw new Win32Exception(Marshal.GetLastWin32Error()); - try { - if (!AssignProcessToJobObject(job, process)) throw new Win32Exception(Marshal.GetLastWin32Error()); - } finally { CloseHandle(process); } - } - IntPtr info = Marshal.AllocHGlobal(65536); - try { - if (!QueryInformationJobObject(job, 3, info, 65536, IntPtr.Zero)) throw new Win32Exception(Marshal.GetLastWin32Error()); - int count = Marshal.ReadInt32(info, 4); - int[] pids = new int[count]; - for (int i = 0; i < count; i++) pids[i] = Marshal.ReadIntPtr(info, 8 + i * IntPtr.Size).ToInt32(); - return pids; - } finally { Marshal.FreeHGlobal(info); } - } finally { CloseHandle(job); } - } -}`; const ownedJobs = new Set(); const windowsJobProcesses = ( pid: number, @@ -153,34 +117,13 @@ const windowsJobProcesses = ( includeRoot = false, identity?: string | null, ) => { - if (!/^t3-provider-install-[a-f0-9-]+$/.test(job) || !Number.isSafeInteger(pid) || pid <= 0) - throw new Error("Invalid provider install job"); - const scan = NodeChildProcess.spawnSync( - "powershell.exe", - [ - "-NoProfile", - "-NonInteractive", - "-Command", - `$ErrorActionPreference = 'Stop'; Add-Type -TypeDefinition @'\n${windowsJobApi}\n'@; [T3InstallJob]::Processes('Global\\${job}', ${pid}, $${create})`, - ], - { encoding: "utf8", timeout: 10_000, windowsHide: true }, - ); - if (scan.error || scan.status !== 0) - throw new Error("Could not inspect provider installer processes"); - const children = scan.stdout - .split(/\r?\n/) - .filter((line) => line.trim()) - .map(Number); - if (children.some((child) => !Number.isSafeInteger(child) || child <= 0)) - throw new Error("Could not inspect provider installer processes"); + const children = WindowsInstallJob.processes(pid, job, create); const local = pid === process.pid && ownedJobs.has(job); const current = includeRoot || local || !identity ? null : readProcessIdentity(pid); const supervisor = !includeRoot && (local || (identity != null && current !== null && isProcessAlive(pid, identity, current))); - return children.some( - (child) => child !== scan.pid && (includeRoot || child !== pid || !supervisor), - ); + return children.some((child) => includeRoot || child !== pid || !supervisor); }; export const createInstallJob = (pid: number, job: string) => { if (NodeOS.platform() !== "win32") return; From 3844019d0dfda0d996c76523b39068572e77631e Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:53:05 +0000 Subject: [PATCH 142/149] fix(server): preserve installer leases during launcher cleanup --- apps/server/src/process/ownedProcesses.ts | 100 ++++++++++++-- apps/server/src/process/windowsInstallJob.ts | 90 +++++++++++++ .../provider/ProviderMaintenanceHostLock.ts | 63 +-------- apps/server/src/serviceLauncher.test.ts | 126 ++++++++++-------- 4 files changed, 253 insertions(+), 126 deletions(-) create mode 100644 apps/server/src/process/windowsInstallJob.ts diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index 03c5049092a6d..87685b9593dc3 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -6,6 +6,8 @@ import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; +import * as WindowsInstallJob from "./windowsInstallJob.ts"; + const execFile = NodeUtil.promisify(NodeChildProcess.execFile); interface Identity { @@ -44,6 +46,32 @@ const identity = (value: unknown): value is Identity => value.startTime.length > 0; const owned = (value: unknown): value is OwnedProcess => identity(value) && "pgid" in value && Number.isSafeInteger(value.pgid); +interface InstallProcess { + readonly pid: number; + readonly identity?: string | null; +} +const installProcess = (value: unknown): value is InstallProcess => + object(value) && + Number.isSafeInteger(value.pid) && + Number(value.pid) > 0 && + (value.identity === undefined || value.identity === null || typeof value.identity === "string"); +function decodeInstallLease(text: string) { + const value: unknown = JSON.parse(text); + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) + return { pid: value, descendants: [] as ReadonlyArray, job: undefined }; + if ( + !installProcess(value) || + ("job" in value && typeof value.job !== "string") || + ("descendants" in value && + (!Array.isArray(value.descendants) || !value.descendants.every(installProcess))) + ) + throw new Error("Invalid provider install lease"); + return { + ...value, + job: "job" in value ? (value.job as string) : undefined, + descendants: "descendants" in value ? (value.descendants as ReadonlyArray) : [], + }; +} function decodeEntry(text: string): Entry { const value: unknown = JSON.parse(text); if ( @@ -296,26 +324,69 @@ export async function make(input: { }; // A server started from a provider or terminal can descend from it with its own // detached install supervisor. Stopping that group would corrupt a shared CLI. - const withoutInstaller = (entry: Entry, table: ReadonlyArray): Entry => { + const withoutInstaller = async ( + entry: Entry, + table: ReadonlyArray, + ): Promise => { if (input.installLeaseDirectory === undefined) return entry; - let leader: number; + let lease: ReturnType; try { - leader = Number( + lease = decodeInstallLease( NodeFS.readFileSync(NodePath.join(input.installLeaseDirectory, "install.json"), "utf8"), ); } catch (cause) { if ((cause as NodeJS.ErrnoException).code === "ENOENT") return entry; throw cause; } - if (!Number.isSafeInteger(leader) || leader <= 0) - throw new Error("Invalid provider install lease"); + const leader = lease.pid; + const matches = (record: InstallProcess, p: ObservedProcess) => { + if (record.pid !== p.pid || p.zombie) return false; + const start = record.identity; + if (start === undefined || start === null) return true; + if (start === p.startTime) return true; + // CIM leases use ISO dates; the ownership ledger stores .NET ticks. + if (platform !== "win32" || !/^\d+$/.test(p.startTime)) return false; + const milliseconds = Date.parse(start); + if (!Number.isFinite(milliseconds)) return false; + const fraction = /\.(\d{1,7})(?:Z|[+-]\d{2}:\d{2})$/.exec(start)?.[1] ?? ""; + const ticks = + BigInt(milliseconds) * 10_000n + + 621355968000000000n + + BigInt(fraction.padEnd(7, "0").slice(3)); + return ticks.toString() === p.startTime; + }; + const root = table.find((p) => p.pid === leader); + const records: ReadonlyArray = [lease, ...lease.descendants]; + if ( + records.some( + (record) => !table.some((p) => p.pid === record.pid) && controller.exists(record.pid), + ) + ) + throw new Error("Could not verify a provider installer during cleanup"); + const installer = new Set( + table.filter((p) => records.some((record) => matches(record, p))).map((p) => p.pid), + ); + if (platform === "win32" && lease.job) + for (const pid of await WindowsInstallJob.processesAsync(leader, lease.job, environment)) + installer.add(pid); + // A surviving private POSIX group reserves its leader's PID after exit. + if (platform !== "win32" && (!root || matches(lease, { ...root, zombie: false }))) + for (const p of table) if (p.pgid === leader && !p.zombie) installer.add(p.pid); // Windows has no process groups, so its installer is found through parents. - const installer = new Set([leader]); let changed = true; + const parents = new Map(table.map((p) => [p.pid, p])); while (changed) { changed = false; for (const p of table) { - if (!installer.has(p.pid) && (p.pgid === leader || installer.has(p.ppid))) { + const parent = installer.has(p.ppid) ? parents.get(p.ppid) : undefined; + const parentBirth = parent ? creationTime(parent) : undefined; + const childBirth = creationTime(p); + if ( + !p.zombie && + !installer.has(p.pid) && + parent && + (parentBirth === undefined || childBirth === undefined || childBirth >= parentBirth) + ) { installer.add(p.pid); changed = true; } @@ -340,7 +411,7 @@ export async function make(input: { ) { throw new Error("Could not verify the recorded provider owner"); } - let next = withoutInstaller(discover(entry, table), table); + let next = await withoutInstaller(discover(entry, table), table); const stopped = new Map(); try { if (platform !== "win32") { @@ -355,19 +426,26 @@ export async function make(input: { stopped.set(`${p.pid}:${p.startTime}`, p); } table = await controller.snapshot(); - next = withoutInstaller(discover(next, table), table); + next = await withoutInstaller(discover(next, table), table); if (attempt === 99) throw new Error("Provider process tree did not stop for cleanup"); } } // Persist newly discovered children before signalling: recovery can itself be killed. write(file, next); + table = await controller.snapshot(); + next = await withoutInstaller(next, table); } catch (error) { // Never leave a frozen tree behind: resume it so the next cleanup can retry. for (const p of stopped.values()) await signal(p, "SIGCONT").catch(() => undefined); throw error; } - for (const p of survivors(next, await controller.snapshot()).toReversed()) - await signal(p, "SIGKILL"); + // A process found to be protected after it was frozen is left running, not stopped. + for (const p of stopped.values()) { + if (!next.processes.some((owned) => same(owned, p))) { + await signal(p, "SIGCONT").catch(() => undefined); + } + } + for (const p of survivors(next, table).toReversed()) await signal(p, "SIGKILL"); for (let attempt = 0; attempt < 100; attempt++) { table = await controller.snapshot(); if ( diff --git a/apps/server/src/process/windowsInstallJob.ts b/apps/server/src/process/windowsInstallJob.ts new file mode 100644 index 0000000000000..838eedea98679 --- /dev/null +++ b/apps/server/src/process/windowsInstallJob.ts @@ -0,0 +1,90 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Shared with the standalone Node launcher. +import * as NodeChildProcess from "node:child_process"; +import * as NodeUtil from "node:util"; + +const execFile = NodeUtil.promisify(NodeChildProcess.execFile); + +const windowsJobApi = ` +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +public static class T3InstallJob { + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr CreateJobObject(IntPtr attributes, string name); + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr OpenJobObject(uint access, bool inherit, string name); + [DllImport("kernel32.dll", SetLastError=true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job, int kind, IntPtr info, uint size, IntPtr returned); + [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr handle); + public static int[] Processes(string name, int pid, bool create) { + IntPtr job = create ? CreateJobObject(IntPtr.Zero, name) : OpenJobObject(4, false, name); + if (job == IntPtr.Zero) { + int error = Marshal.GetLastWin32Error(); + if (!create && error == 2) return new int[0]; + throw new Win32Exception(error); + } + try { + if (create) { + IntPtr process = OpenProcess(0x101, false, pid); + if (process == IntPtr.Zero) throw new Win32Exception(Marshal.GetLastWin32Error()); + try { + if (!AssignProcessToJobObject(job, process)) throw new Win32Exception(Marshal.GetLastWin32Error()); + } finally { CloseHandle(process); } + } + IntPtr info = Marshal.AllocHGlobal(65536); + try { + if (!QueryInformationJobObject(job, 3, info, 65536, IntPtr.Zero)) throw new Win32Exception(Marshal.GetLastWin32Error()); + int count = Marshal.ReadInt32(info, 4); + int[] pids = new int[count]; + for (int i = 0; i < count; i++) pids[i] = Marshal.ReadIntPtr(info, 8 + i * IntPtr.Size).ToInt32(); + return pids; + } finally { Marshal.FreeHGlobal(info); } + } finally { CloseHandle(job); } + } +}`; + +function argsFor(pid: number, job: string, create: boolean) { + if (!/^t3-provider-install-[a-f0-9-]+$/.test(job) || !Number.isSafeInteger(pid) || pid <= 0) + throw new Error("Invalid provider install job"); + return [ + "-NoProfile", + "-NonInteractive", + "-Command", + `$ErrorActionPreference = 'Stop'; Add-Type -TypeDefinition @'\n${windowsJobApi}\n'@; [T3InstallJob]::Processes('Global\\${job}', ${pid}, $${create})`, + ]; +} +function decode(stdout: string, queryPid?: number): ReadonlyArray { + const children = stdout + .split(/\r?\n/) + .filter((line) => line.trim()) + .map(Number); + if (children.some((child) => !Number.isSafeInteger(child) || child <= 0)) + throw new Error("Could not inspect provider installer processes"); + return children.filter((child) => child !== queryPid); +} +/** The kernel retains install helpers even after their parents exit. */ +export function processes(pid: number, job: string, create = false) { + const scan = NodeChildProcess.spawnSync("powershell.exe", argsFor(pid, job, create), { + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + }); + if (scan.error || scan.status !== 0) + throw new Error("Could not inspect provider installer processes"); + return decode(scan.stdout, scan.pid); +} +/** Startup cleanup must keep the server responsive while PowerShell inspects the job. */ +export async function processesAsync(pid: number, job: string, environment: NodeJS.ProcessEnv) { + const args = argsFor(pid, job, false); + try { + const query = execFile("powershell.exe", args, { + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + env: environment, + }); + const { stdout } = await query; + return decode(stdout, query.child.pid); + } catch { + throw new Error("Could not inspect provider installer processes"); + } +} diff --git a/apps/server/src/provider/ProviderMaintenanceHostLock.ts b/apps/server/src/provider/ProviderMaintenanceHostLock.ts index c903505dd0a7b..6424bb042bd48 100644 --- a/apps/server/src/provider/ProviderMaintenanceHostLock.ts +++ b/apps/server/src/provider/ProviderMaintenanceHostLock.ts @@ -15,6 +15,7 @@ import * as Semaphore from "effect/Semaphore"; import * as Schema from "effect/Schema"; import * as Schedule from "effect/Schedule"; import * as Maintenance from "@t3tools/provider-core/server/maintenance"; +import * as WindowsInstallJob from "../process/windowsInstallJob.ts"; const HeldWrite = Context.Reference<{ readonly owner: object; active: boolean } | null>( "t3/provider/ProviderMaintenanceHostLock/HeldWrite", @@ -108,43 +109,6 @@ export const isProcessAlive = (pid: number, identity?: string | null, observed?: // A named Windows job retains every descendant in the kernel, even when a // short-lived parent exits between observations or the supervisor is killed. -const windowsJobApi = ` -using System; -using System.ComponentModel; -using System.Runtime.InteropServices; -public static class T3InstallJob { - [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr CreateJobObject(IntPtr attributes, string name); - [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr OpenJobObject(uint access, bool inherit, string name); - [DllImport("kernel32.dll", SetLastError=true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid); - [DllImport("kernel32.dll", SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); - [DllImport("kernel32.dll", SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job, int kind, IntPtr info, uint size, IntPtr returned); - [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr handle); - public static int[] Processes(string name, int pid, bool create) { - IntPtr job = create ? CreateJobObject(IntPtr.Zero, name) : OpenJobObject(4, false, name); - if (job == IntPtr.Zero) { - int error = Marshal.GetLastWin32Error(); - if (!create && error == 2) return new int[0]; - throw new Win32Exception(error); - } - try { - if (create) { - IntPtr process = OpenProcess(0x101, false, pid); - if (process == IntPtr.Zero) throw new Win32Exception(Marshal.GetLastWin32Error()); - try { - if (!AssignProcessToJobObject(job, process)) throw new Win32Exception(Marshal.GetLastWin32Error()); - } finally { CloseHandle(process); } - } - IntPtr info = Marshal.AllocHGlobal(65536); - try { - if (!QueryInformationJobObject(job, 3, info, 65536, IntPtr.Zero)) throw new Win32Exception(Marshal.GetLastWin32Error()); - int count = Marshal.ReadInt32(info, 4); - int[] pids = new int[count]; - for (int i = 0; i < count; i++) pids[i] = Marshal.ReadIntPtr(info, 8 + i * IntPtr.Size).ToInt32(); - return pids; - } finally { Marshal.FreeHGlobal(info); } - } finally { CloseHandle(job); } - } -}`; const ownedJobs = new Set(); const windowsJobProcesses = ( pid: number, @@ -153,34 +117,13 @@ const windowsJobProcesses = ( includeRoot = false, identity?: string | null, ) => { - if (!/^t3-provider-install-[a-f0-9-]+$/.test(job) || !Number.isSafeInteger(pid) || pid <= 0) - throw new Error("Invalid provider install job"); - const scan = NodeChildProcess.spawnSync( - "powershell.exe", - [ - "-NoProfile", - "-NonInteractive", - "-Command", - `$ErrorActionPreference = 'Stop'; Add-Type -TypeDefinition @'\n${windowsJobApi}\n'@; [T3InstallJob]::Processes('Global\\${job}', ${pid}, $${create})`, - ], - { encoding: "utf8", timeout: 10_000, windowsHide: true }, - ); - if (scan.error || scan.status !== 0) - throw new Error("Could not inspect provider installer processes"); - const children = scan.stdout - .split(/\r?\n/) - .filter((line) => line.trim()) - .map(Number); - if (children.some((child) => !Number.isSafeInteger(child) || child <= 0)) - throw new Error("Could not inspect provider installer processes"); + const children = WindowsInstallJob.processes(pid, job, create); const local = pid === process.pid && ownedJobs.has(job); const current = includeRoot || local || !identity ? null : readProcessIdentity(pid); const supervisor = !includeRoot && (local || (identity != null && current !== null && isProcessAlive(pid, identity, current))); - return children.some( - (child) => child !== scan.pid && (includeRoot || child !== pid || !supervisor), - ); + return children.some((child) => includeRoot || child !== pid || !supervisor); }; export const createInstallJob = (pid: number, job: string) => { if (NodeOS.platform() !== "win32") return; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 24b4bb6602716..e6d1983acf5ed 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -237,62 +237,78 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); - it.effect("starts the server when orphaned provider cleanup cannot run", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-reap-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const leases = path.join(root, "cache", "t3", "provider-maintenance"); - yield* fs.makeDirectory(leases, { recursive: true }); - yield* fs.writeFileString(path.join(leases, "install.json"), "corrupt"); - // A crashed server's record makes cleanup read the corrupt lease. - const records = path.join(root, "userdata", "provider-processes"); - yield* fs.makeDirectory(records, { recursive: true }); - yield* fs.writeFileString( - path.join(records, "0.json"), - JSON.stringify({ - version: 1, - instanceId: "crashed", - hostname: NodeOS.hostname(), - stateDir: path.join(root, "userdata"), - owner: { pid: 2 ** 30, startTime: "exited" }, - processes: [], - }), - ); - yield* Effect.acquireRelease( - Effect.sync(() => vi.stubEnv("XDG_CACHE_HOME", path.join(root, "cache"))), - () => Effect.sync(() => vi.unstubAllEnvs()), - ); - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", "1.0.0"), - `require("node:fs").writeFileSync(${JSON.stringify(path.join(root, "started"))}, "");\nsetInterval(() => {}, 1_000);\n`, - ); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const started = Promise.withResolvers(); - yield* Effect.acquireRelease( - Effect.sync(() => - NodeFS.watch(root, (_event, name) => { - if (name === "started") started.resolve(); + it.effect.each(["corrupt", "legacy", "structured"] as const)( + "starts the server with a %s installer lease", + (lease) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-reap-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const leases = path.join(root, "cache", "t3", "provider-maintenance"); + yield* fs.makeDirectory(leases, { recursive: true }); + yield* fs.writeFileString( + path.join(leases, "install.json"), + lease === "corrupt" + ? "corrupt" + : lease === "legacy" + ? String(2 ** 30) + : JSON.stringify({ pid: 2 ** 30, identity: "exited", descendants: [] }), + ); + // A crashed server's record makes cleanup inspect the install lease. + const records = path.join(root, "userdata", "provider-processes"); + yield* fs.makeDirectory(records, { recursive: true }); + yield* fs.writeFileString( + path.join(records, "0.json"), + JSON.stringify({ + version: 1, + instanceId: "crashed", + hostname: NodeOS.hostname(), + stateDir: path.join(root, "userdata"), + owner: { pid: 2 ** 30, startTime: "exited" }, + processes: [], }), - ), - (watcher) => Effect.sync(() => watcher.close()), - ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); - const running = launcher.run(); - // A failed launcher would otherwise leave the child promise pending forever. - yield* Effect.promise(() => Promise.race([started.promise, running])); - assert.isTrue(yield* fs.exists(path.join(root, "started"))); - yield* Effect.promise(() => launcher.stop("SIGTERM")); - yield* Effect.promise(() => running); - }), + ); + yield* Effect.acquireRelease( + Effect.sync(() => vi.stubEnv("XDG_CACHE_HOME", path.join(root, "cache"))), + () => Effect.sync(() => vi.unstubAllEnvs()), + ); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + `require("node:fs").writeFileSync(${JSON.stringify(path.join(root, "started"))}, "");\nsetInterval(() => {}, 1_000);\n`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const started = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + NodeFS.watch(root, (_event, name) => { + if (name === "started") started.resolve(); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + const running = yield* Effect.acquireRelease( + Effect.sync(() => launcher.run()), + () => Effect.promise(() => launcher.stop("SIGTERM")), + ); + // A failed launcher would otherwise leave the child promise pending forever. + yield* Effect.promise(() => Promise.race([started.promise, running])); + assert.isTrue(yield* fs.exists(path.join(root, "started"))); + assert.equal(yield* fs.exists(path.join(records, "0.json")), lease === "corrupt"); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }), ); it.effect.each([ From b4d5600fca3c160b8f8302ab058a2a8163960699 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:53:12 +0000 Subject: [PATCH 143/149] test(server): cover installer leases created during cleanup --- apps/server/src/process/agentScope.test.ts | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 7908ab37cd6a7..ca4e37281b7bf 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -509,6 +509,7 @@ finish(); { platform: "linux", lease: "root-reused" }, { platform: "win32", lease: "root-reused" }, { platform: "linux", lease: "group" }, + { platform: "linux", lease: "late" }, { platform: "win32", lease: "job" }, { platform: "win32", lease: "job-failed" }, ] as const)( @@ -556,8 +557,7 @@ finish(); } const leases = NodePath.join(directory, "leases"); NodeFS.mkdirSync(leases); - NodeFS.writeFileSync( - NodePath.join(leases, "install.json"), + const contents = lease === "legacy" ? "70" : JSON.stringify({ @@ -568,8 +568,9 @@ finish(); job || rootReused || lease === "group" ? [] : [{ pid: 71, identity: childIdentity }], - }), - ); + }); + const installLease = NodePath.join(leases, "install.json"); + if (lease !== "late") NodeFS.writeFileSync(installLease, contents); if (job) NodeFS.writeFileSync( NodePath.join(directory, "powershell.exe"), @@ -579,7 +580,17 @@ finish(); const result = yield* ProviderProcessLedger.make({ stateDir: directory, ownerPid: 90, - controller, + controller: + lease === "late" + ? { + ...controller, + snapshot: () => { + if (signals.includes("70:SIGSTOP")) + NodeFS.writeFileSync(installLease, contents); + return controller.snapshot(); + }, + } + : controller, installLeaseDirectory: leases, }).pipe( Effect.provideService(HostProcess.Platform, platform), @@ -599,7 +610,13 @@ finish(); expect(processes.has(70)).toBe(!orphan && !rootReused); expect(processes.has(71)).toBe(!reused); if (reused) expect(signals).toContain("71:SIGKILL"); - else expect(signals.some((s) => /^(70|71):/.test(s))).toBe(false); + else if (lease === "late") { + for (const pid of [70, 71]) { + expect(signals).toContain(`${pid}:SIGSTOP`); + expect(signals).toContain(`${pid}:SIGCONT`); + expect(signals).not.toContain(`${pid}:SIGKILL`); + } + } else expect(signals.some((s) => /^(70|71):/.test(s))).toBe(false); }), ), ); From 70dcc18059a72921486266eeedc910ba5b5b1673 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:19:56 +0000 Subject: [PATCH 144/149] test(server): await launcher startup without nesting promises --- apps/server/src/serviceLauncher.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index e6d1983acf5ed..96fc1432f7da9 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -298,16 +298,16 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { root, yield* Effect.promise(() => readServiceState(statePath)), ); - const running = yield* Effect.acquireRelease( - Effect.sync(() => launcher.run()), + yield* Effect.acquireRelease( + Effect.sync(() => { + void launcher.run().catch(started.reject); + }), () => Effect.promise(() => launcher.stop("SIGTERM")), ); - // A failed launcher would otherwise leave the child promise pending forever. - yield* Effect.promise(() => Promise.race([started.promise, running])); + yield* Effect.promise(() => started.promise); assert.isTrue(yield* fs.exists(path.join(root, "started"))); assert.equal(yield* fs.exists(path.join(records, "0.json")), lease === "corrupt"); yield* Effect.promise(() => launcher.stop("SIGTERM")); - yield* Effect.promise(() => running); }), ); From 5fbd0b373df6133616398902c136312ac2340f60 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:43:31 +0000 Subject: [PATCH 145/149] fix(server): reject stale server ancestry during crash cleanup --- apps/server/src/process/agentScope.test.ts | 24 +++++++++++++++++----- apps/server/src/process/ownedProcesses.ts | 10 ++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index ca4e37281b7bf..1b6c159de9ff5 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -683,19 +683,28 @@ finish(); ({ platform, handoff }) => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { - processes.set(60, { pid: 60, ppid: 20, pgid: 60, startTime: "launcher", zombie: false }); + for (const [pid, p] of processes) + processes.set(pid, { ...p, startTime: String(pid * 10) }); + processes.set(60, { pid: 60, ppid: 20, pgid: 60, startTime: "600", zombie: false }); processes.set(70, { pid: 70, ppid: 60, pgid: 60, - startTime: "second-server", + startTime: "700", zombie: false, }); processes.set(71, { pid: 71, ppid: 70, pgid: 71, - startTime: "second-provider", + startTime: "800", + zombie: false, + }); + processes.set(72, { + pid: 72, + ppid: 60, + pgid: 60, + startTime: "800", zombie: false, }); if (platform === "win32") @@ -735,6 +744,8 @@ finish(); ); expect(NodeFS.readdirSync(NodePath.join(leases, "servers"))).toHaveLength(1); } + // A stale parent id must not make an older orphan belong to a reused server or launcher. + processes.set(30, { ...processes.get(30)!, ppid: handoff ? 60 : 70 }); processes.delete(10); yield* ProviderProcessLedger.make({ stateDir: directory, @@ -743,9 +754,12 @@ finish(); installLeaseDirectory: leases, }); expect(processes.has(20)).toBe(false); - for (const pid of handoff ? [60, 90] : [60, 70, 71, 90]) + expect(processes.has(30)).toBe(false); + expect(signals).toContain("30:SIGKILL"); + expect(signals).not.toContain("30:SIGCONT"); + for (const pid of handoff ? [60, 72, 90] : [60, 70, 71, 72, 90]) expect(processes.has(pid)).toBe(true); - expect(signals.some((s) => /^(60|70|71|90):/.test(s))).toBe(false); + expect(signals.some((s) => /^(60|70|71|72|90):/.test(s))).toBe(false); }).pipe(Effect.scoped, Effect.provideService(HostProcess.Platform, platform)), ), ); diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index c1719272ee75d..fd83da32eeea9 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -505,11 +505,19 @@ export async function make(input: { } } const protectedPids = new Set(owners.map((p) => p.pid)); + const parents = new Map(table.map((p) => [p.pid, p])); let changed = true; while (changed) { changed = false; for (const p of table) { - if (!protectedPids.has(p.pid) && protectedPids.has(p.ppid)) { + const parent = protectedPids.has(p.ppid) ? parents.get(p.ppid) : undefined; + const parentBirth = parent ? creationTime(parent) : undefined; + const childBirth = creationTime(p); + if ( + !protectedPids.has(p.pid) && + parent && + (parentBirth === undefined || childBirth === undefined || childBirth >= parentBirth) + ) { protectedPids.add(p.pid); changed = true; } From ed59b2af1a24c1c140bb22955518c7d50bbab243 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:52:20 +0000 Subject: [PATCH 146/149] fix(server): defer cleanup when a live launcher is unobservable --- apps/server/src/process/agentScope.test.ts | 44 +++++++++++++++++++--- apps/server/src/process/ownedProcesses.ts | 12 +++--- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 1b6c159de9ff5..b2f20dce6de33 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -369,26 +369,43 @@ finish(); }), ), ); - it.effect( - "resumes frozen providers when the final server protection query cannot verify a live pid", - () => + it.effect.each(["owner", "managed-owner", "launcher", "handoff"] as const)( + "resumes frozen providers when the final server protection query cannot verify a live %s", + (missing) => scenario(({ directory, processes, signals, controller }) => Effect.gen(function* () { yield* record(directory, controller); const leases = NodePath.join(directory, "leases"); + const managed = missing !== "owner"; + if (managed) + processes.set(60, { + pid: 60, + ppid: 1, + pgid: 60, + startTime: "launcher", + zombie: false, + }); processes.set(70, { pid: 70, - ppid: 1, + ppid: managed ? 60 : 1, pgid: 70, startTime: "live-server", zombie: false, }); - yield* ProviderProcessLedger.make({ + const second = ProviderProcessLedger.make({ stateDir: NodePath.join(directory, "second"), ownerPid: 70, controller, installLeaseDirectory: leases, }); + yield* managed + ? second.pipe( + Effect.provideService(HostProcess.Environment, { + [SERVICE_LAUNCHER_CONTEXT_ENV]: "managed", + }), + ) + : second; + if (missing === "handoff") processes.delete(70); processes.delete(10); let frozenQueries = 0; const result = yield* ProviderProcessLedger.make({ @@ -401,7 +418,8 @@ finish(); snapshot: () => { if (signals.filter((signal) => signal.endsWith(":SIGSTOP")).length === 2) frozenQueries++; - return [...processes.values()].filter((p) => frozenQueries < 2 || p.pid !== 70); + const hidden = missing === "owner" || missing === "managed-owner" ? 70 : 60; + return [...processes.values()].filter((p) => frozenQueries < 2 || p.pid !== hidden); }, }, }).pipe(Effect.exit); @@ -747,6 +765,20 @@ finish(); // A stale parent id must not make an older orphan belong to a reused server or launcher. processes.set(30, { ...processes.get(30)!, ppid: handoff ? 60 : 70 }); processes.delete(10); + if (handoff) { + const result = yield* ProviderProcessLedger.make({ + stateDir: directory, + ownerPid: 90, + installLeaseDirectory: leases, + controller: { + ...controller, + snapshot: () => [...processes.values()].filter((p) => p.pid !== 60), + }, + }).pipe(Effect.exit); + expect(result._tag).toBe("Failure"); + expect(signals).toEqual([]); + expect(processes.has(72)).toBe(true); + } yield* ProviderProcessLedger.make({ stateDir: directory, ownerPid: 90, diff --git a/apps/server/src/process/ownedProcesses.ts b/apps/server/src/process/ownedProcesses.ts index fd83da32eeea9..67fba33508caf 100644 --- a/apps/server/src/process/ownedProcesses.ts +++ b/apps/server/src/process/ownedProcesses.ts @@ -490,6 +490,12 @@ export async function make(input: { throw cause; } if (server.hostname !== hostname || same(server.owner, entry.owner)) continue; + if ( + [server.owner, ...server.processes].some( + (p) => !table.some((observed) => observed.pid === p.pid) && controller.exists(p.pid), + ) + ) + throw new Error("Could not verify a live server during provider cleanup"); const live = table.find((p) => same(p, server.owner) && !p.zombie); const managed = table.filter( (p) => !p.zombie && server.processes.some((owned) => same(p, owned)), @@ -497,11 +503,7 @@ export async function make(input: { if (live || managed.length > 0) { if (live) owners.push(live); owners.push(...managed); - } else if ( - !table.some((p) => p.pid === server.owner.pid) && - controller.exists(server.owner.pid) - ) - throw new Error("Could not verify a live server during provider cleanup"); + } } } const protectedPids = new Set(owners.map((p) => p.pid)); From 1429f1b5abfd7438369f9f7470d6b94eabf3b56f Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:06:09 +0000 Subject: [PATCH 147/149] fix(server): restarts keep steering messages and stop duplicate launches --- .../server/src/mcp/McpSessionRegistry.test.ts | 20 +++ apps/server/src/mcp/McpSessionRegistry.ts | 3 +- .../src/mcp/toolkits/project/handlers.test.ts | 91 +++++++++++ .../src/mcp/toolkits/project/handlers.ts | 15 +- apps/server/src/mcp/toolkits/project/tools.ts | 3 +- .../Adapters/CodexAdapterV2.test.ts | 148 ++++++++++++++++++ .../Adapters/CodexAdapterV2.ts | 27 +++- .../ProjectionRecovery.test.ts | 80 ++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 37 ++++- .../ProviderRuntimeRecoveryService.test.ts | 73 +++++++++ .../ProviderRuntimeRecoveryService.ts | 33 +++- .../RestartContinuation.test.ts | 23 +++ .../orchestration-v2/RestartContinuation.ts | 38 +++-- docs/internals/server-updates.md | 8 + packages/contracts/src/orchestrationV2.ts | 2 + 15 files changed, 577 insertions(+), 24 deletions(-) diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 6b6207dc26601..2e8e1c40ea318 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -59,6 +59,26 @@ it.effect("stores only a token hash, resolves the bearer token, and revokes by t }), ); +it.effect("keeps the request namespace across credential reissue and splits it per thread", () => + Effect.gen(function* () { + const registry = yield* makeRegistry(() => 1_000); + const request = { + threadId: ThreadId.make("thread-restarted"), + providerInstanceId: ProviderInstanceId.make("codex"), + }; + const scopeOf = (issued: { readonly config: { readonly authorizationHeader: string } }) => + registry.resolve(issued.config.authorizationHeader.replace(/^Bearer\s+/, "")); + const first = yield* scopeOf(yield* registry.issue(request)); + const second = yield* scopeOf(yield* registry.issue(request)); + const other = yield* scopeOf( + yield* registry.issue({ ...request, threadId: ThreadId.make("thread-other") }), + ); + expect(second?.thread.providerSessionId).not.toBe(first?.thread.providerSessionId); + expect(second?.requestNamespace).toBe(first?.requestNamespace); + expect(other?.requestNamespace).not.toBe(first?.requestNamespace); + }), +); + it.effect("always grants pull-requests and gates browser and device access independently", () => Effect.gen(function* () { const registry = yield* makeRegistry(() => 1_000); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index f1e027f9eb85f..80e36dbcfcc0a 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -130,7 +130,8 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( const browserToolsAvailable = request.browserToolsAvailable ?? true; const scope: McpInvocationContext.McpThreadInvocationScope = { environmentId, - requestNamespace: providerSessionId, + // Retrying an accepted write after a restart must reuse its receipt. + requestNamespace: `thread:${encodeURIComponent(request.threadId)}:${encodeURIComponent(request.providerInstanceId)}`, thread: { threadId: ThreadId.make(request.threadId), providerSessionId, diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index 7b045ae6b1893..ebb6c681603c6 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -101,6 +101,97 @@ it.effect("attributes a launched thread's first message to the calling thread", }), ); +it.effect( + "a launch retried with the same clientRequestId keeps its command after session rotation", + () => + Effect.gen(function* () { + const sourceThreadId = ThreadId.make("source-thread"); + const projectId = ProjectId.make("project"); + const providerInstanceId = ProviderInstanceId.make("codex"); + const modelSelection = { instanceId: providerInstanceId, model: "gpt-5" }; + const caller = { + id: sourceThreadId, + projectId, + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + activeRunId: "active-run", + archivedAt: null, + deletedAt: null, + } as OrchestrationV2ThreadShell; + const launches: Array = []; + const launch = (clientRequestId: string | undefined, requestNamespace: string) => + Effect.gen(function* () { + const dependencies = Layer.mergeAll( + ServerSettings.layerTest(), + NodeCrypto.layer, + Layer.succeed(McpInvocationContext.McpInvocationContext, { + environmentId: EnvironmentId.make("environment"), + requestNamespace, + thread: { + threadId: sourceThreadId, + providerSessionId: "session", + providerInstanceId, + }, + client: undefined, + issuedAt: 0, + capabilities: new Set(["orchestration" as const]), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(caller), + }), + Layer.mock(ThreadLaunch.ThreadLaunchService)({ + launch: (input) => { + launches.push(input); + return Effect.succeed({ + threadId: input.threadId, + projection: { + thread: { id: input.threadId, projectId, modelSelection }, + runs: [], + }, + resumed: false, + } as unknown as ThreadLaunch.ThreadLaunchResult); + }, + }), + Layer.mock(Project.ProjectService)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + }), + Layer.mock(GitVcsDriver.GitVcsDriver)({}), + NodeServices.layer, + ServerConfig.layerTest(process.cwd(), { prefix: "t3-launch-retry-" }).pipe( + Layer.provide(NodeServices.layer), + ), + ); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe( + Layer.provide(dependencies), + ), + ), + ); + yield* toolkit + .handle("t3_thread_launch", { + title: "Audit", + message: "Review the change", + ...(clientRequestId === undefined ? {} : { clientRequestId }), + }) + .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies)); + }); + // The namespace is stable per thread, so a rotated session still matches. + yield* launch("retry-launch", "thread:source-thread:codex"); + yield* launch("retry-launch", "thread:source-thread:codex"); + expect(launches[1]?.commandId).toBe(launches[0]?.commandId); + expect(launches[1]?.threadId).toBe(launches[0]?.threadId); + expect(launches[1]?.initialMessage?.messageId).toBe(launches[0]?.initialMessage?.messageId); + yield* launch("retry-launch", "thread:other-thread:codex"); + yield* launch(undefined, "thread:source-thread:codex"); + yield* launch(undefined, "thread:source-thread:codex"); + expect(new Set(launches.map((input) => input.threadId)).size).toBe(4); + }), +); + it.effect("launches a scratch thread into the Scratch project", () => Effect.gen(function* () { const sourceThreadId = ThreadId.make("source-thread"); diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index c346ad62001f3..3a522c179b585 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -1,4 +1,10 @@ -import { MessageId, ThreadId, OrchestratorMcpFailure, ProjectId } from "@t3tools/contracts"; +import { + CommandId, + MessageId, + ThreadId, + OrchestratorMcpFailure, + ProjectId, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; @@ -59,7 +65,12 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { Effect.gen(function* () { const context = yield* readCaller(); const { caller } = context; - const commandId = yield* newCommandId(); + const commandId = + input.clientRequestId === undefined + ? yield* newCommandId() + : CommandId.make( + `command:mcp:${encodeURIComponent(context.scope.requestNamespace)}:thread-launch:${encodeURIComponent(input.clientRequestId)}`, + ); const threadId = ThreadId.make(commandId); const messageId = MessageId.make(commandId); const attachments = input.attachments ?? []; diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index e455c041c4c31..85f04c04d79d4 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -104,8 +104,9 @@ const ProjectCloneTool = Tool.make("t3_project_clone", { const ThreadLaunchTool = Tool.make("t3_thread_launch", { ...shared, description: - 'Create an ordinary TOP-LEVEL thread with an explicit workspace binding before its agent starts. Use this when the user requests independent work, a new thread, or a PR stack in its own worktree; use delegate_task for child subagents. Set workspaceStrategy to {type:"worktree",baseRef:"parent-branch",branch:"new-branch",startFromOrigin:false} for a new worktree based on local commits, or {type:"existing_worktree",worktreePath:"/absolute/path",branch:"existing-branch"} to use an existing checkout. For upstream commits, set startFromOrigin:true. Omitted workspaceStrategy means the project root, NOT the caller\'s worktree. Omit projectId/modelSelection/modes to inherit those settings from the calling thread; a caller outside a T3 thread must pass projectId and gets the project\'s default model. Set scratch:true instead of projectId for a thread without a project: it runs in a fresh folder of its own, outside any repository. Put the task in message. Do not ask the agent to create its own worktree via shell: that does not update the thread binding. Each call creates a new launch with no retry key; retain threadId and use t3_thread_read/t3_thread_wait to follow preparation. To link a thread for the user, write `[title](t3-thread://v1/)` with the threadId exactly as returned, not URL-encoded; T3 Code shows the thread\'s current title. After errors or lost responses, inspect t3_thread_list before retrying. Attachments must be pending uploads. The new thread may not run with broader runtime or interaction modes than the caller: the calling T3 thread\'s own modes, or the permission mode an outside agent was approved with.', + 'Create an ordinary TOP-LEVEL thread with an explicit workspace binding before its agent starts. Use this when the user requests independent work, a new thread, or a PR stack in its own worktree; use delegate_task for child subagents. Set workspaceStrategy to {type:"worktree",baseRef:"parent-branch",branch:"new-branch",startFromOrigin:false} for a new worktree based on local commits, or {type:"existing_worktree",worktreePath:"/absolute/path",branch:"existing-branch"} to use an existing checkout. For upstream commits, set startFromOrigin:true. Omitted workspaceStrategy means the project root, NOT the caller\'s worktree. Omit projectId/modelSelection/modes to inherit those settings from the calling thread; a caller outside a T3 thread must pass projectId and gets the project\'s default model. Set scratch:true instead of projectId for a thread without a project: it runs in a fresh folder of its own, outside any repository. Put the task in message. Do not ask the agent to create its own worktree via shell: that does not update the thread binding. Pass a stable clientRequestId to retry the same launch after errors or restarts; retain threadId and use t3_thread_read/t3_thread_wait to follow preparation. To link a thread for the user, write `[title](t3-thread://v1/)` with the threadId exactly as returned, not URL-encoded; T3 Code shows the thread\'s current title. After errors or lost responses, inspect t3_thread_list before retrying. Attachments must be pending uploads. The new thread may not run with broader runtime or interaction modes than the caller: the calling T3 thread\'s own modes, or the permission mode an outside agent was approved with.', parameters: Schema.Struct({ + clientRequestId: Schema.optional(TrimmedNonEmptyString.check(Schema.isMaxLength(256))), projectId: Schema.optional(ProjectId), scratch: Schema.optional( Schema.Boolean.annotate({ diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 14bb6a03d8111..de3b00771a001 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -3256,6 +3256,154 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each(["active-goal", "no-goal"] as const)( + "a continuation Codex rejects as EmptyInput retries only on an active goal (%s)", + (state) => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-restart-empty-input"; + const nativeThreadId = "native-restart-empty-input"; + const nativeTurnId = "turn-restart-empty-input"; + const prompt = "Continue where you left off."; + const preamble = codexReplayPreamble({ + nativeThreadId, + nativeTurnId, + prompt: "unused", + }).slice(0, 5); + const turnParams = { + threadId: nativeThreadId, + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + }; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...preamble, + { + type: "expect_outbound", + label: "resume", + frame: { + id: 3, + method: "thread/resume", + params: { + threadId: nativeThreadId, + excludeTurns: true, + config: CodexAdapterV2.CODEX_THREAD_CONFIG, + }, + }, + }, + { + type: "emit_inbound", + label: "resume", + frame: { id: 3, result: { thread: { id: nativeThreadId, updatedAt: 1782622450 } } }, + }, + { + type: "expect_outbound", + label: "empty continue", + frame: { id: 4, method: "turn/start", params: { ...turnParams, input: [] } }, + }, + { + type: "emit_inbound", + label: "empty continue", + frame: { + id: 4, + error: { code: -32603, message: "failed to submit turn input: EmptyInput" }, + }, + }, + { + type: "expect_outbound", + label: "read goal", + frame: { id: 5, method: "thread/goal/get", params: { threadId: nativeThreadId } }, + }, + { + type: "emit_inbound", + label: "read goal", + frame: { + id: 5, + result: { + goal: + state === "no-goal" + ? null + : { + threadId: nativeThreadId, + objective: "Finish the work", + status: "active", + tokenBudget: null, + tokensUsed: 10, + timeUsedSeconds: 3, + createdAt: 1782622440, + updatedAt: 1782622450, + }, + }, + }, + }, + ...(state === "active-goal" + ? [ + { + type: "expect_outbound" as const, + label: "prompted continue", + frame: { + id: 6, + method: "turn/start", + params: { ...turnParams, input: [{ type: "text", text: prompt }] }, + }, + }, + { + type: "emit_inbound" as const, + label: "prompted continue", + frame: { + id: 6, + result: { + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "inProgress" }), + }, + }, + }, + ] + : []), + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + const resumed = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }); + const started = yield* harness.runtime + .startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: resumed, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-restart-empty-input"), + text: prompt, + }), + restartContinuationOfRunId: RunId.make("run-before-restart"), + }) + .pipe(Effect.result); + if (state === "active-goal") { + assert.equal(started._tag, "Success"); + return; + } + assert.equal(started._tag, "Failure"); + if (started._tag === "Failure") { + assert.nestedPropertyVal( + started.failure, + "cause.errorMessage", + "failed to submit turn input: EmptyInput", + ); + } + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("resolves retryable app-server errors on resumed provider activity", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 5a51136396ad8..6afbecf4b1737 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -6638,11 +6638,32 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( : null; if (goalCommand !== null) return runGoalCommand(turnInput, goalCommand); return Effect.gen(function* () { - yield* startNativeTurn( - turnInput, + const input = turnInput.restartContinuationOfRunId === undefined ? yield* toCodexInput(turnInput) - : [], + : []; + yield* startNativeTurn(turnInput, input).pipe( + Effect.catchTags({ + CodexAppServerRequestError: (cause) => { + // Codex rejects an empty continuation while a goal turn is + // still active. Send the continue prompt as real input. + if ( + input.length !== 0 || + cause.code !== -32603 || + cause.errorMessage !== "failed to submit turn input: EmptyInput" + ) + return Effect.fail(cause); + return Effect.gen(function* () { + const current = yield* client + .request("thread/goal/get", { + threadId: yield* getNativeThreadId(turnInput.providerThread), + }) + .pipe(Effect.orElseSucceed(() => null)); + if (current?.goal?.status !== "active") return yield* Effect.fail(cause); + yield* startNativeTurn(turnInput, yield* toCodexInput(turnInput)); + }); + }, + }), ); }).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index 95efab067f807..680f9fb946b43 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -12,6 +12,7 @@ import { ProviderInstanceId, ProviderSessionId, ProviderThreadId, + ProviderTurnId, RunId, RuntimeRequestId, ThreadId, @@ -611,3 +612,82 @@ it.effect("marks fork descendants unreadable when their source is missing or cor ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("surfaces steering still queued in the outbox and keeps its ids across run upserts", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const now = yield* DateTime.now; + const threadId = yield* createThread("steering"); + const runId = yield* createRun(threadId, "running"); + const steer = (name: string) => { + const messageId = MessageId.make(`message:steering:${name}`); + return { + messageId, + effect: { + id: `effect:steering:${name}`, + commandId: CommandId.make(`command:steering:${name}`), + threadId, + request: { + type: "provider-turn.steer" as const, + providerSessionId: ProviderSessionId.make("session:steering"), + providerThreadId: ProviderThreadId.make("provider-thread:steering"), + providerTurnId: ProviderTurnId.make("provider-turn:steering"), + messageId, + }, + }, + }; + }; + const delivered = steer("delivered"); + const queued = steer("queued"); + for (const { messageId } of [delivered, queued]) { + yield* projections.apply({ + id: EventId.make(`event:${messageId}`), + type: "message.updated", + threadId, + runId, + driver, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: messageId, + threadId, + runId, + nodeId: null, + role: "user", + text: "steer", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }, + }); + } + yield* outbox.enqueue([delivered.effect]); + const claimed = yield* outbox.claimNext({ workerId: "worker:steering", leaseDurationMs: 1000 }); + assert.isTrue( + yield* outbox.succeed({ effectId: delivered.effect.id, workerId: "worker:steering" }), + ); + assert.isTrue(claimed._tag === "Some"); + yield* outbox.enqueue([queued.effect]); + + const before = yield* projections.getRuntimeRecoveryProjection(threadId); + assert.deepEqual(before.pendingSteering, [{ runId, messageId: queued.messageId }]); + + const run = before.runs[0]!; + const apply = (payload: OrchestrationV2Run, name: string) => + projections.apply({ + id: EventId.make(`event:steering:${name}`), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload, + }); + yield* apply({ ...run, restartPendingMessageIds: [queued.messageId] }, "record"); + yield* apply({ ...run, status: "cancelled", completedAt: now }, "later-upsert"); + const after = yield* projections.getRuntimeRecoveryProjection(threadId); + assert.deepEqual(after.runs[0]?.restartPendingMessageIds, [queued.messageId]); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 4f10a0512fd53..256b632488990 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -275,7 +275,13 @@ export type ProjectionRuntimeRecoveryState = Pick< | "runtimeRequests" | "messages" | "turnItems" ->; +> & { + /** Steering whose delivery was still queued in the outbox, per run. */ + readonly pendingSteering?: ReadonlyArray<{ + readonly runId: RunId; + readonly messageId: MessageId; + }>; +}; export type ProjectionPendingUserInputs = Pick< OrchestrationV2ThreadProjection, @@ -652,6 +658,10 @@ function preserveRunRecordedFields( ...(next.delegatedCompletion === undefined && current.delegatedCompletion !== undefined ? { delegatedCompletion: current.delegatedCompletion } : {}), + ...(next.restartPendingMessageIds === undefined && + current.restartPendingMessageIds !== undefined + ? { restartPendingMessageIds: current.restartPendingMessageIds } + : {}), ...(next.restartCancelledBackgroundWork === undefined && current.restartCancelledBackgroundWork !== undefined ? { restartCancelledBackgroundWork: current.restartCancelledBackgroundWork } @@ -1934,7 +1944,10 @@ export const layer: Layer.Layer = requested_at = excluded.requested_at, completed_at = excluded.completed_at, payload_json = ${keepRecordedRunField( - keepRecordedRunField(sql`excluded.payload_json`, "$.delegatedCompletion"), + keepRecordedRunField( + keepRecordedRunField(sql`excluded.payload_json`, "$.delegatedCompletion"), + "$.restartPendingMessageIds", + ), "$.restartCancelledBackgroundWork", )} `; @@ -4247,7 +4260,27 @@ export const layer: Layer.Layer = decodeRows(decodeMessagePayload, threadId)(messageRows), decodeRows(decodeTurnItemPayload, threadId)(turnItemRows), ]); + // Delegated results are redelivered by their own cohort, not by the prompt. + const pendingSteering = yield* sql<{ + readonly run_id: string; + readonly message_id: string; + }>` + SELECT message.run_id, message.message_id + FROM orchestration_v2_effect_outbox AS effect + JOIN orchestration_v2_projection_messages AS message + ON message.thread_id = effect.thread_id + AND message.message_id = json_extract(effect.payload_json, '$.messageId') + WHERE effect.thread_id = ${threadId} AND effect.effect_type = 'provider-turn.steer' + AND effect.status IN ('pending', 'running') + AND message.run_id IS NOT NULL + AND json_extract(message.payload_json, '$.delegatedCompletion') IS NULL + ORDER BY effect.rowid ASC + `; return { + pendingSteering: pendingSteering.map((row) => ({ + runId: RunId.make(row.run_id), + messageId: MessageId.make(row.message_id), + })), thread, runs, attempts, diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 05c2b41c01164..00704b9a6fdc8 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -643,6 +643,79 @@ it.effect("closes a secret request card's form when its run is recovered", () => }).pipe(Effect.provide(layer)); }); +it.effect("records undelivered steering on the cancelled run and ends cut streams", () => { + const threadId = ThreadId.make("thread_steer_recovery"); + const runId = RunId.make("run_steer_recovery"); + const steerId = MessageId.make("message_steer_recovery"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + pendingSteering: [{ runId, messageId: steerId }], + runtimeRequests: [], + providerSessions: [], + providerThreads: [], + providerTurns: [], + runs: [{ id: runId, status: "running", providerInstanceId: ProviderInstanceId.make("codex") }], + attempts: [], + nodes: [], + subagents: [], + messages: [], + turnItems: ["assistant_message", "reasoning", "proposed_plan"].map((type) => ({ + id: `turn-item:${type}`, + threadId, + runId, + nodeId: null, + type, + status: "running", + streaming: true, + })), + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ + runRecoveryOnce: Effect.succeed(false), + }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: () => Effect.succeed([]), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + return Effect.gen(function* () { + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + const events = committedInput?.events ?? []; + const run = events.find((event) => event.type === "run.updated"); + assert.equal(run?.type === "run.updated" ? run.payload.status : null, "cancelled"); + assert.deepEqual(run?.type === "run.updated" ? run.payload.restartPendingMessageIds : null, [ + steerId, + ]); + const items = events.flatMap((event) => + event.type === "turn-item.updated" ? [event.payload] : [], + ); + assert.lengthOf(items, 3); + for (const item of items) { + assert.equal(item.status, "cancelled"); + assert.isFalse("streaming" in item ? item.streaming : true); + } + }).pipe(Effect.provide(layer)); +}); + it.effect("holds accepted queued work without cancelling its execution state after restart", () => { const threadId = ThreadId.make("thread_queued_restart"); const runId = RunId.make("run_queued_restart"); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index ad57247e03847..75208fb3a7914 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -156,7 +156,18 @@ const cancelledItem = ( ): OrchestrationV2ThreadProjection["turnItems"][number] => item.type === "secret_request" && item.secretStatus === "pending" ? { ...item, status: "cancelled", secretStatus: "cancelled", completedAt: now, updatedAt: now } - : { ...item, status: "cancelled", completedAt: now, updatedAt: now }; + : { + ...item, + status: "cancelled", + completedAt: now, + updatedAt: now, + // A cut stream never gets its closing event. + ...(item.type === "reasoning" || + item.type === "assistant_message" || + item.type === "proposed_plan" + ? { streaming: false } + : {}), + }; /** * A provider thread's latest started run: the last turn that provider saw. @@ -317,6 +328,16 @@ export const make = Effect.gen(function* () { }); } for (const run of runs) { + // Steering still queued in the outbox died with the process; the + // continuation prompt carries it. + const undeliveredSteering = [ + ...new Set([ + ...(run.restartPendingMessageIds ?? []), + ...(projection.pendingSteering ?? []) + .filter((steer) => steer.runId === run.id) + .map((steer) => steer.messageId), + ]), + ]; events.push({ id: yield* allocateEventId(), type: "run.updated", @@ -324,7 +345,15 @@ export const make = Effect.gen(function* () { runId: run.id, providerInstanceId: run.providerInstanceId, occurredAt: now, - payload: { ...run, status: "cancelled", queuePosition: null, completedAt: now }, + payload: { + ...run, + status: "cancelled", + queuePosition: null, + completedAt: now, + ...(undeliveredSteering.length === 0 + ? {} + : { restartPendingMessageIds: undeliveredSteering }), + }, }); for (const attempt of projection.attempts.filter( (candidate) => diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index eb03389adc626..5898d0c512341 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -771,6 +771,29 @@ it.effect("shows a continuation as a T3 Code notice whose detail is the prompt", }), ); +it.effect("carries steering the provider never received into the continuation", () => + Effect.gen(function* () { + const steerId = MessageId.make("message:steer"); + const attachment = { + type: "file" as const, + id: "attachment:steer", + name: "notes.txt", + mimeType: "text/plain", + sizeBytes: 12, + }; + const [resumed] = yield* continuationDispatches({ + ...cutMidTurn({ restartPendingMessageIds: [steerId] }), + messages: [ + { id: steerId, text: "Also update the docs.", attachments: [attachment] }, + { id: MessageId.make("message:unrelated"), text: "Not carried.", attachments: [] }, + ], + } as unknown as OrchestrationV2ThreadProjection); + assert.equal(resumed?.text, "Continue where you left off.\n\nAlso update the docs."); + assert.deepEqual(resumed?.attachments, [attachment]); + assert.isUndefined(resumed?.notification); + }), +); + it.effect( "carries the note forward when its continuation was cut before reaching the provider", () => diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index cb435d1043508..13295e3d00e19 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -211,7 +211,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta input.threadId, ["messages", "turnItems"], { - messageIds: [source.userMessageId], + messageIds: [source.userMessageId, ...(source.restartPendingMessageIds ?? [])], turnItemRunIds: [source.id], turnItemTypes: ["run_interrupt_request"], }, @@ -227,6 +227,13 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta (message) => message.id === source.userMessageId, ); if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return; + // Steering the provider never received before the process ended. + const pendingMessages = (source.restartPendingMessageIds ?? []).flatMap((id) => + sourceRecords.messages.filter((message) => message.id === id), + ); + const prompt = [CONTINUE_PROMPT, ...pendingMessages.map((message) => message.text)].join( + "\n\n", + ); const note = restartContinuationNote( source, projection.runs, @@ -237,10 +244,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work); const text = noteText === undefined - ? CONTINUE_PROMPT - : note.settled + ? prompt + : note.settled && pendingMessages.length === 0 ? noteText - : `${noteText}\n\n${CONTINUE_PROMPT}`; + : `${noteText}\n\n${prompt}`; yield* threads .dispatch({ type: "message.dispatch", @@ -249,15 +256,20 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta messageId, text, // Shown as a work log row; the prompt the agent got is its detail. - notification: { - source: { kind: "system" }, - outcome: "updated", - summary: note.settled - ? "T3 Code restarted and stopped background work" - : "T3 Code restarted and resumed this turn", - detail: text, - }, - attachments: [], + // Carried steering stays a visible message so its attachments show. + ...(pendingMessages.length === 0 + ? { + notification: { + source: { kind: "system" as const }, + outcome: "updated" as const, + summary: note.settled + ? "T3 Code restarted and stopped background work" + : "T3 Code restarted and resumed this turn", + detail: text, + }, + } + : {}), + attachments: pendingMessages.flatMap((message) => message.attachments), modelSelection: source.modelSelection, dispatchMode: { type: "start_immediately" }, createdBy: "agent", diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 51bc16775d08c..44e5ccbe6802e 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -86,6 +86,14 @@ command and message IDs prevent duplicate submissions after an outbox retry. Cod resumes without adding provider prompt text unless the turn lost background work; other adapters receive the continuation message through their normal turn path. +Steering is process-bound, so a steer still queued when the process ended never reached +the provider. Recovery records its message ids on the cancelled run +(`restartPendingMessageIds`) and the continuation appends their text and attachments to +its prompt. This carries them one hop: a second restart before delivery loses them, and a +steer delivered just before the crash may be repeated once. When Codex rejects the empty +continuation input while a goal turn is still active, the adapter reads the goal and, if +it is active, retries with the continue prompt as real input. + Delegated tasks (`delegate_task` child threads) are reconciled as their own threads, never as the parent's background work. The orchestrator settles child results and completion deliveries in a startup pass after reconciliation, because the terminal-run diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index b4db1038f4ebf..53f8e3c2c7835 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -561,6 +561,8 @@ export const OrchestrationV2Run = Schema.Struct({ contextHandoffId: Schema.NullOr(ContextHandoffId), /** Links server-generated restart continuations to the interrupted run. */ restartContinuationOfRunId: Schema.optional(RunId), + /** Steering input lost with a provider process, carried into the restart prompt. */ + restartPendingMessageIds: Schema.optional(Schema.Array(MessageId)), /** The delegated task this run answers, including a follow-up in an existing child thread. */ delegatedTaskId: Schema.optional(NodeId), /** From fdcd3cce2765d9e89acd84cc6fd52ac2adeaea8d Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:11:45 +0000 Subject: [PATCH 148/149] fix(server): restart recovery continues every thread on a shared session --- .../ProviderRuntimeRecoveryService.ts | 25 +++++- .../RestartContinuation.test.ts | 80 +++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 75208fb3a7914..1a9c4a71fca98 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -201,6 +201,10 @@ export const make = Effect.gen(function* () { projection: ProjectionStore.ProjectionRuntimeRecoveryState, trigger: "startup" | "shutdown", continueAfterRestart: boolean, + initialSessions: ReadonlyMap< + string, + OrchestrationV2ThreadProjection["providerSessions"][number] + >, ) { const now = yield* DateTime.now; const runs = [] as Array; @@ -590,7 +594,15 @@ export const make = Effect.gen(function* () { } const continuationRun = continueAfterRestart && trigger === "startup" - ? restartContinuationRun(projection, now) + ? restartContinuationRun( + { + ...projection, + providerSessions: projection.providerSessions.map( + (session) => initialSessions.get(session.id) ?? session, + ), + }, + now, + ) : undefined; const effects: Array = continuationRun ? [ @@ -676,6 +688,12 @@ export const make = Effect.gen(function* () { let stoppedSessions = 0; let closedRequests = 0; let retiredEffects = 0; + // Recovering one thread stops its shared session. Later threads must + // judge continuation against that session's state before recovery. + const initialSessions = new Map< + string, + OrchestrationV2ThreadProjection["providerSessions"][number] + >(); for (const threadId of threadIds) { const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe( Effect.mapError( @@ -687,11 +705,14 @@ export const make = Effect.gen(function* () { }), ), ); + for (const session of projection.providerSessions) { + if (!initialSessions.has(session.id)) initialSessions.set(session.id, session); + } const enabled = continueAfterRestart !== null && resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings .continueThreadsAfterServerUpdate; - const result = yield* reconcileProjection(projection, trigger, enabled); + const result = yield* reconcileProjection(projection, trigger, enabled, initialSessions); terminalizedRuns += result.terminalizedRuns; stoppedSessions += result.stoppedSessions; closedRequests += result.closedRequests; diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 5898d0c512341..dd6bb28921454 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -833,6 +833,86 @@ it.effect( }), ); +it.effect("continues every thread that shares one provider session", () => + Effect.gen(function* () { + const secondThreadId = ThreadId.make("thread:restart-second"); + const secondRunId = RunId.make("run:restart-second"); + const secondProviderThreadId = ProviderThreadId.make("provider-thread:restart-second"); + const secondAttemptId = RunAttemptId.make("attempt:restart-second"); + const commits: Array[0]> = []; + // Recovering the first thread stops the session they share. + const read = (id: ThreadId) => { + const base = makeProjection(); + const session = { + ...base.providerSessions[0]!, + status: commits.length === 0 ? ("ready" as const) : ("stopped" as const), + }; + return id === threadId + ? { ...base, providerSessions: [session] } + : { + ...base, + thread: { ...base.thread, id: secondThreadId }, + runs: [ + { + ...base.runs[0]!, + id: secondRunId, + providerThreadId: secondProviderThreadId, + activeAttemptId: secondAttemptId, + }, + ], + providerThreads: [ + { + ...base.providerThreads[0]!, + id: secondProviderThreadId, + appThreadId: secondThreadId, + }, + ], + providerSessions: [session], + providerTurns: [ + { + ...base.providerTurns[0]!, + id: ProviderTurnId.make("turn:restart-second"), + providerThreadId: secondProviderThreadId, + runAttemptId: secondAttemptId, + }, + ], + }; + }; + const recovery = yield* ProviderRuntimeRecovery.make.pipe( + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId, secondThreadId]), + getRuntimeRecoveryProjection: (id) => Effect.succeed(read(id)), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + commits.push(input); + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ + runRecoveryOnce: Effect.succeed(false), + }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + yield* recovery.reconcile("startup"); + assert.deepEqual( + commits.flatMap((commit) => commit.effects.map((effect) => effect.request)), + [ + { type: "provider-runtime.continue", sourceRunId: runId }, + { type: "provider-runtime.continue", sourceRunId: secondRunId }, + ], + ); + }), +); + it.effect("prepares later threads' continuations when one thread fails", () => Effect.gen(function* () { const brokenThreadId = ThreadId.make("thread:broken"); From 8fbd54821c79f9db2267941fbb70c14e57904136 Mon Sep 17 00:00:00 2001 From: maria-rcks <254055478+maria-rcks@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:17:44 +0000 Subject: [PATCH 149/149] fix(server): retain context in restart steering --- .../RestartContinuation.test.ts | 36 +++++++++++++++++-- .../orchestration-v2/RestartContinuation.ts | 13 +++++-- .../src/server/orchestrationInstructions.ts | 2 +- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 9573706cd0ab6..3f6c8208d3481 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -1069,6 +1069,7 @@ it.effect("shows a continuation as a T3 Code notice whose detail is the prompt", it.effect("carries steering the provider never received into the continuation", () => Effect.gen(function* () { const steerId = MessageId.make("message:steer"); + const contextSteerId = MessageId.make("message:context-steer"); const attachment = { type: "file" as const, id: "attachment:steer", @@ -1077,13 +1078,44 @@ it.effect("carries steering the provider never received into the continuation", sizeBytes: 12, }; const [resumed] = yield* continuationDispatches({ - ...cutMidTurn({ restartPendingMessageIds: [steerId] }), + ...cutMidTurn({ restartPendingMessageIds: [steerId, contextSteerId] }), messages: [ { id: steerId, text: "Also update the docs.", attachments: [attachment] }, + { + id: contextSteerId, + text: "Check [log](t3-context://v1/terminal/ctx_terminal).", + attachments: [], + context: { + version: 1, + records: [ + { + version: 1, + contextId: "ctx_terminal", + kind: "terminal", + label: "Terminal 1 lines 3-4", + terminalId: "term-1", + terminalLabel: "Terminal 1", + lineStart: 3, + lineEnd: 4, + text: "STATUS=queued\nMODE=restart", + }, + ], + }, + }, { id: MessageId.make("message:unrelated"), text: "Not carried.", attachments: [] }, ], } as unknown as OrchestrationV2ThreadProjection); - assert.equal(resumed?.text, "Continue where you left off.\n\nAlso update the docs."); + assert.include( + resumed?.text ?? "", + "Continue where you left off.\n\nAlso update the docs.\n\nCheck [Terminal: log; ref=ctx_terminal].", + ); + assert.include(resumed?.text ?? "", ''); + assert.include( + resumed?.text ?? "", + "terminal: Terminal 1\n3 | STATUS=queued\n4 | MODE=restart", + ); + assert.notInclude(resumed?.text ?? "", 'unavailable="true"'); + assert.notInclude(resumed?.text ?? "", "Not carried."); assert.deepEqual(resumed?.attachments, [attachment]); assert.isUndefined(resumed?.notification); }), diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 80cc058c9ef53..aec28b0197c29 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -1,5 +1,6 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; +import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { CommandId, latestProviderTurnForAttempt, @@ -265,9 +266,15 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta const pendingMessages = (source.restartPendingMessageIds ?? []).flatMap((id) => sourceRecords.messages.filter((message) => message.id === id), ); - const prompt = [CONTINUE_PROMPT, ...pendingMessages.map((message) => message.text)].join( - "\n\n", - ); + const prompt = [ + CONTINUE_PROMPT, + ...pendingMessages.map((message) => + projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }), + ), + ].join("\n\n"); const note = restartContinuationNote( source, projection.runs, diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 49d6d5440bdc0..67a55b771d587 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -28,7 +28,7 @@ For stacked work, set \`baseRef\` to the intended parent branch and \`startFromO \`t3_thread_launch\` is the single-thread launch tool. Use \`create_threads\` only for a batch of threads intentionally sharing the caller's checkout: it always inherits the caller's project, branch, and worktree and has no workspace override. Asking an agent to run \`git worktree add\` or \`cd\` in its prompt does not update T3's thread binding. Select the workspace in the launch call instead. \`t3_worktree_handoff\` moves the calling thread, not another thread, and cannot move a thread already attached to a worktree. -\`t3_thread_launch\` has no idempotency key. Retain its returned threadId and inspect it with \`t3_thread_read\` / \`t3_thread_wait\`; preparation can still be running after acceptance. If a launch fails or its response is lost, inspect \`t3_thread_list\` before retrying, since a thread may already exist.`, +Reuse a stable \`clientRequestId\` when retrying \`t3_thread_launch\`. Retain its returned threadId and inspect it with \`t3_thread_read\` / \`t3_thread_wait\`; preparation can still be running after acceptance. If a launch fails or its response is lost, inspect \`t3_thread_list\` before retrying, since a thread may already exist.`, `Tool names may include a harness-normalized MCP prefix, such as \`mcp__t3_code__delegate_task\`; the semantics are the same. Some harnesses attach optional MCP servers lazily: if an initial tool-catalog scan does not show T3 tools, do not conclude that cross-provider delegation is unavailable. Make one bounded direct attempt using the known T3 tool name on the next tool step. In Codex code mode, for example, call \`tools.mcp__t3_code__orchestrator_capabilities({})\` before reporting that the capability is absent. Keep polling/wait loops bounded, do not duplicate active work, and use stable \`clientRequestId\` values when retrying tools that accept them. ACP fallback: some ACP agents accept the injected MCP server but fail to expose its tools. When the T3 tools are absent and \`T3_ACP_MCP_NODE\` is present, call the same tools through the terminal: \`ELECTRON_RUN_AS_NODE=1 "$T3_ACP_MCP_NODE" \${T3_ACP_MCP_ENTRYPOINT:+"$T3_ACP_MCP_ENTRYPOINT"} acp-mcp-call orchestrator_capabilities '{}'\` (\`T3_ACP_MCP_ENTRYPOINT\` is unset when T3 runs as a standalone executable). Delegate with \`acp-mcp-call delegate_task '{"task":"...","target":{"providerInstanceId":"...","model":"..."},"mode":"async","clientRequestId":"..."}'\`. This is the supported T3 transport fallback, not an ordinary shell-based substitute for delegation.