From 000fe0357dd2dbc5badbb46a9101fa8f620a592a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 29 Oct 2026 23:20:32 +0000 Subject: [PATCH 1/7] test(server): reproduce #27230 slow Codex app-server status timeout A Codex app-server that answers the status probe correctly after ~25 s should report ready. Today checkCodexProviderStatus wraps the whole probe in a 20 s timeout and reports "Timed out while checking Codex app-server provider status." instead. This test reproduces #27230 and fails on main. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/ProviderRegistry.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 14b8dfb2ca150..fe999b972783c 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -422,6 +422,19 @@ it.layer( assert.strictEqual(status.auth.status, "unknown"); }), ); + + it.effect("returns ready when app-server answers slowly but correctly", () => + Effect.gen(function* () { + const fiber = yield* checkCodexProviderStatus(defaultCodexSettings, () => + Effect.sleep("25 seconds").pipe(Effect.as(makeCodexProbeSnapshot())), + ).pipe(Effect.forkChild); + yield* TestClock.adjust("25 seconds"); + const status = yield* Fiber.join(fiber); + + assert.strictEqual(status.message, undefined); + assert.strictEqual(status.status, "ready"); + }), + ); }); describe("ProviderRegistry.layer", () => { From 0d7d64737b0295389c21fa53ab1a424987d1d669 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 29 Oct 2026 23:25:05 +0000 Subject: [PATCH 2/7] fix(server): slow-starting Codex app-server reports ready instead of timing out On old machines `codex app-server` can take ~25 s to answer initialize, so the 20 s status-check budget always reported "Timed out while checking Codex app-server provider status." even though sessions worked. The status check now allows 60 s overall. Optional discovery (models, skills, config, usage) keeps a 20 s budget measured from initialize and capped at the overall limit, so fast machines still settle within ~20 s when discovery stalls, and slow starts still get time to list models. Fixes pingdotgg/t3code#27230 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/CodexProvider.ts | 13 ++++++++++--- .../server/src/provider/ProviderRegistry.test.ts | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 2b2d5501881b4..63f6cc1b66fdf 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -72,8 +72,10 @@ const CODEX_CATALOG_REFRESH_TIMEOUT_MS = 3_000; const CODEX_CONFIG_PROBE_TIMEOUT_MS = 3_000; const CODEX_CATALOG_REFRESH_POLL = "100 millis" as const; const CODEX_CLIENT_NAME = "T3 Code"; -// Handshake plus account/models/skills requests can exceed the shared 10s probe budget. -const CODEX_AUTH_PROBE_TIMEOUT_MS = 20_000; +// Old machines can take ~25s to start app-server and answer initialize. +const CODEX_AUTH_PROBE_TIMEOUT_MS = 60_000; +// Account/models/skills requests after initialize can exceed the shared 10s probe budget. +const CODEX_DISCOVERY_BUDGET_MS = 20_000; const CODEX_DISCOVERY_MARGIN_MS = 250; const CODEX_PRESENTATION = { @@ -518,6 +520,7 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun }) { const spawnedAtMs = DateTime.toEpochMillis(yield* DateTime.now); const { client, initialize } = yield* withCodexAppServerClient(input); + const initializedAtMs = DateTime.toEpochMillis(yield* DateTime.now); const versionMatch = initialize.userAgent.match(/\/([^\s]+)/); const version = versionMatch ? versionMatch[1] : undefined; @@ -533,7 +536,11 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun } // Optional discovery must finish before the outer timeout can discard the account result. - const deadlineMs = spawnedAtMs + CODEX_AUTH_PROBE_TIMEOUT_MS - CODEX_DISCOVERY_MARGIN_MS; + const deadlineMs = + Math.min( + spawnedAtMs + CODEX_AUTH_PROBE_TIMEOUT_MS, + initializedAtMs + CODEX_DISCOVERY_BUDGET_MS, + ) - CODEX_DISCOVERY_MARGIN_MS; const [skillsResponse, models, serviceTier, rateLimits] = yield* Effect.all( [ withCodexDiscoveryDeadline(client.request("skills/list", { cwds: [input.cwd] }), deadlineMs), diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index fe999b972783c..851a261ab068a 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -435,6 +435,22 @@ it.layer( assert.strictEqual(status.status, "ready"); }), ); + + it.effect("times out when app-server never answers", () => + Effect.gen(function* () { + const fiber = yield* checkCodexProviderStatus(defaultCodexSettings, () => + Effect.never.pipe(Effect.as(makeCodexProbeSnapshot())), + ).pipe(Effect.forkChild); + yield* TestClock.adjust("60 seconds"); + const status = yield* Fiber.join(fiber); + + assert.strictEqual(status.status, "error"); + assert.strictEqual( + status.message, + "Timed out while checking Codex app-server provider status.", + ); + }), + ); }); describe("ProviderRegistry.layer", () => { From fc6af35e384bb22991540d61ed7cc7c5549432f0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 29 Oct 2026 23:35:24 +0000 Subject: [PATCH 3/7] fix(server): give the Codex workspace skills scan the same startup allowance The per-project skills scan bounds spawn, initialize and skills/list with a fixed 20s, so on machines where codex app-server needs ~25s to answer initialize project skills silently never appear. Share the status check's 60s app-server allowance. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/CodexProvider.ts | 6 +++--- apps/server/src/provider/Drivers/CodexDriver.ts | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 63f6cc1b66fdf..b901cbd32fa2d 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -73,7 +73,7 @@ const CODEX_CONFIG_PROBE_TIMEOUT_MS = 3_000; const CODEX_CATALOG_REFRESH_POLL = "100 millis" as const; const CODEX_CLIENT_NAME = "T3 Code"; // Old machines can take ~25s to start app-server and answer initialize. -const CODEX_AUTH_PROBE_TIMEOUT_MS = 60_000; +export const CODEX_APP_SERVER_PROBE_TIMEOUT_MS = 60_000; // Account/models/skills requests after initialize can exceed the shared 10s probe budget. const CODEX_DISCOVERY_BUDGET_MS = 20_000; const CODEX_DISCOVERY_MARGIN_MS = 250; @@ -538,7 +538,7 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun // Optional discovery must finish before the outer timeout can discard the account result. const deadlineMs = Math.min( - spawnedAtMs + CODEX_AUTH_PROBE_TIMEOUT_MS, + spawnedAtMs + CODEX_APP_SERVER_PROBE_TIMEOUT_MS, initializedAtMs + CODEX_DISCOVERY_BUDGET_MS, ) - CODEX_DISCOVERY_MARGIN_MS; const [skillsResponse, models, serviceTier, rateLimits] = yield* Effect.all( @@ -738,7 +738,7 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu ...(managedAuth ? { skipNativeUsage: true } : {}), }).pipe( // Process teardown can outlast the work budget without discarding a completed snapshot. - Effect.timeoutOption(Duration.millis(CODEX_AUTH_PROBE_TIMEOUT_MS)), + Effect.timeoutOption(Duration.millis(CODEX_APP_SERVER_PROBE_TIMEOUT_MS)), Effect.scoped, Effect.result, ); diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index bbd910cf5d4a9..07c905e98b65f 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -22,6 +22,7 @@ import { ProviderDriverError } from "../Errors.ts"; import * as ResetCreditCoordinator from "../resetCreditCoordinator.ts"; import { checkCodexProviderStatus, + CODEX_APP_SERVER_PROBE_TIMEOUT_MS, makePendingCodexProvider, probeCodexSkillsForCwd, withCodexAppServerClient, @@ -280,7 +281,7 @@ export const CodexDriver: ProviderDriver< environment: processEnv, }).pipe( Effect.scoped, - Effect.timeout("20 seconds"), + Effect.timeout(CODEX_APP_SERVER_PROBE_TIMEOUT_MS), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ), ]).pipe( From 004f6a7c48225b54fadae4428f9e3c049351806b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 03:04:41 +0000 Subject: [PATCH 4/7] test(server): cover the Codex discovery deadline counted from initialize AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/CodexProvider.test.ts | 18 +++++++++++++++++- apps/server/src/provider/CodexProvider.ts | 15 +++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index 30f63b21f318e..014eb4d7417cf 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -1,6 +1,22 @@ import { assert, it } from "@effect/vitest"; -import { applyPreferredCodexDefaultModel, mapCodexModelCapabilities } from "./CodexProvider.ts"; +import { + applyPreferredCodexDefaultModel, + codexDiscoveryDeadlineMs, + mapCodexModelCapabilities, +} from "./CodexProvider.ts"; + +it("counts the Codex discovery budget from initialize, capped by the probe timeout (#27230)", () => { + // [initialized after spawn, deadline after spawn] + const cases = [ + [500, 20_250], + [25_000, 44_750], + [50_000, 59_750], + ] as const; + for (const [initializedAtMs, expected] of cases) { + assert.strictEqual(codexDiscoveryDeadlineMs(0, initializedAtMs), expected); + } +}); it("maps current Codex model capability fields", () => { const capabilities = mapCodexModelCapabilities({ diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 722232e7c380b..c5ba6bb4158fb 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -399,6 +399,14 @@ const readCodexModelsCacheFetchedAt = Effect.fn("readCodexModelsCacheFetchedAt") ); }); +// Discovery gets its own budget from `initialize`, so a slow start still lists models, and must +// finish before the outer timeout can discard the account result. +export const codexDiscoveryDeadlineMs = (spawnedAtMs: number, initializedAtMs: number) => + Math.min( + spawnedAtMs + CODEX_APP_SERVER_PROBE_TIMEOUT_MS, + initializedAtMs + CODEX_DISCOVERY_BUDGET_MS, + ) - CODEX_DISCOVERY_MARGIN_MS; + const withCodexDiscoveryDeadline = Effect.fnUntraced(function* ( effect: Effect.Effect, deadlineMs: number, @@ -553,12 +561,7 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun } satisfies CodexAppServerProviderSnapshot; } - // Optional discovery must finish before the outer timeout can discard the account result. - const deadlineMs = - Math.min( - spawnedAtMs + CODEX_APP_SERVER_PROBE_TIMEOUT_MS, - initializedAtMs + CODEX_DISCOVERY_BUDGET_MS, - ) - CODEX_DISCOVERY_MARGIN_MS; + const deadlineMs = codexDiscoveryDeadlineMs(spawnedAtMs, initializedAtMs); const [skillsResponse, models, serviceTier, rateLimits] = yield* Effect.all( [ withCodexDiscoveryDeadline(client.request("skills/list", { cwds: [input.cwd] }), deadlineMs), From 9b55e5c3d919e109dfe2c06feec3448c1184139a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 16:27:22 +0000 Subject: [PATCH 5/7] docs: Codex probe timeout example extends the new 60s default Co-Authored-By: Claude Opus 5.5 --- docs/user/install.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user/install.md b/docs/user/install.md index b594894efe377..71ffdae2edf0f 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -251,16 +251,16 @@ For provider-specific setup and accounts, see [Codex](./providers-codex.md), If a provider works in its CLI but T3 Code reports a timeout while checking it, allow more time in the T3 server's environment. For example, give Codex and -Claude readiness checks 60 seconds: +Claude readiness checks 120 seconds: ```bash -T3CODE_CODEX_PROBE_TIMEOUT_MS=60000 T3CODE_CLAUDE_PROBE_TIMEOUT_MS=60000 t3 serve +T3CODE_CODEX_PROBE_TIMEOUT_MS=120000 T3CODE_CLAUDE_PROBE_TIMEOUT_MS=120000 t3 serve ``` For a remote environment, set these on the machine running the T3 server. Keep them in your launcher or service configuration to retain them across updates, then restart the server. For systemd, add -`Environment=T3CODE_CODEX_PROBE_TIMEOUT_MS=60000` under `[Service]` in a service +`Environment=T3CODE_CODEX_PROBE_TIMEOUT_MS=120000` under `[Service]` in a service override, reload systemd, and restart the service. Provider instances can also set their own overrides in Settings → Providers. From 7139fa1f50c0ecc17ecae205e8374bb205bc9e0f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 16:35:25 +0000 Subject: [PATCH 6/7] fix(server): a raised Codex probe timeout extends discovery again T3CODE_CODEX_PROBE_TIMEOUT_MS above the 60s default now widens the post-initialize discovery budget by the same amount, instead of discovery stopping 20s after initialize whatever the override. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/CodexProvider.test.ts | 14 ++++++++------ apps/server/src/provider/CodexProvider.ts | 10 +++++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index d4a51dd82c8ae..ea11420b62868 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -8,14 +8,16 @@ import { } from "./CodexProvider.ts"; it("counts the Codex discovery budget from initialize, capped by the probe timeout (#27230)", () => { - // [initialized after spawn, deadline after spawn] + // [initialized after spawn, probe timeout, deadline after spawn] const cases = [ - [500, 20_250], - [25_000, 44_750], - [50_000, 59_750], + [500, 60_000, 20_250], + [25_000, 60_000, 44_750], + [50_000, 60_000, 59_750], + // A raised T3CODE_CODEX_PROBE_TIMEOUT_MS also extends discovery. + [2_000, 90_000, 51_750], ] as const; - for (const [initializedAtMs, expected] of cases) { - assert.strictEqual(codexDiscoveryDeadlineMs(0, initializedAtMs, 60_000), expected); + for (const [initializedAtMs, probeTimeoutMs, expected] of cases) { + assert.strictEqual(codexDiscoveryDeadlineMs(0, initializedAtMs, probeTimeoutMs), expected); } }); diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 569cb72daab5c..ff9d971e8d90c 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -537,14 +537,18 @@ const readCodexModelsCacheFetchedAt = Effect.fn("readCodexModelsCacheFetchedAt") }); // Discovery gets its own budget from `initialize`, so a slow start still lists models, and must -// finish before the outer timeout can discard the account result. +// finish before the outer timeout can discard the account result. A raised probe timeout extends it. export const codexDiscoveryDeadlineMs = ( spawnedAtMs: number, initializedAtMs: number, probeTimeoutMs: number, ) => - Math.min(spawnedAtMs + probeTimeoutMs, initializedAtMs + CODEX_DISCOVERY_BUDGET_MS) - - CODEX_DISCOVERY_MARGIN_MS; + Math.min( + spawnedAtMs + probeTimeoutMs, + initializedAtMs + + CODEX_DISCOVERY_BUDGET_MS + + Math.max(0, probeTimeoutMs - CODEX_APP_SERVER_PROBE_TIMEOUT_MS), + ) - CODEX_DISCOVERY_MARGIN_MS; const withCodexDiscoveryDeadline = Effect.fnUntraced(function* ( effect: Effect.Effect, From 0909016220ff3caa0f359102e498eca94509ee18 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 16:44:42 +0000 Subject: [PATCH 7/7] fix(server): Codex discovery keeps main's spawn-based deadline Drop the separate post-initialize discovery budget. It shortened discovery for anyone who had raised T3CODE_CODEX_PROBE_TIMEOUT_MS or a per-instance override; the 60s default alone covers a slow app-server start. Co-Authored-By: Claude Opus 5.5 --- .../server/src/provider/CodexProvider.test.ts | 15 ------------ apps/server/src/provider/CodexProvider.ts | 23 ++++--------------- 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index ea11420b62868..7eadf550eab28 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -2,25 +2,10 @@ import { assert, it } from "@effect/vitest"; import { applyPreferredCodexDefaultModel, - codexDiscoveryDeadlineMs, codexRoutesSpendNativeQuota, mapCodexModelCapabilities, } from "./CodexProvider.ts"; -it("counts the Codex discovery budget from initialize, capped by the probe timeout (#27230)", () => { - // [initialized after spawn, probe timeout, deadline after spawn] - const cases = [ - [500, 60_000, 20_250], - [25_000, 60_000, 44_750], - [50_000, 60_000, 59_750], - // A raised T3CODE_CODEX_PROBE_TIMEOUT_MS also extends discovery. - [2_000, 90_000, 51_750], - ] as const; - for (const [initializedAtMs, probeTimeoutMs, expected] of cases) { - assert.strictEqual(codexDiscoveryDeadlineMs(0, initializedAtMs, probeTimeoutMs), expected); - } -}); - it("maps current Codex model capability fields", () => { const capabilities = mapCodexModelCapabilities({ additionalSpeedTiers: [], diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index ff9d971e8d90c..e76e079ff4fbb 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -125,10 +125,9 @@ const CODEX_CATALOG_REFRESH_TIMEOUT_MS = 3_000; const CODEX_CONFIG_PROBE_TIMEOUT_MS = 3_000; const CODEX_CATALOG_REFRESH_POLL = "100 millis" as const; const CODEX_CLIENT_NAME = "T3 Code"; -// Old machines can take ~25s to start app-server and answer initialize. +// Old machines can take ~25s to start app-server and answer initialize, before +// account/models/skills discovery runs. export const CODEX_APP_SERVER_PROBE_TIMEOUT_MS = 60_000; -// Account/models/skills requests after initialize can exceed the shared 10s probe budget. -const CODEX_DISCOVERY_BUDGET_MS = 20_000; const CODEX_DISCOVERY_MARGIN_MS = 250; // `codex-cli 0.159.0`, `codex-cli 0.160.0-alpha.1` or the npm CLI's bare `0.1.4161006`. // Keep a prerelease suffix so compatibility stays unknown for it, as it does for `initialize`. @@ -536,20 +535,6 @@ const readCodexModelsCacheFetchedAt = Effect.fn("readCodexModelsCacheFetchedAt") ); }); -// Discovery gets its own budget from `initialize`, so a slow start still lists models, and must -// finish before the outer timeout can discard the account result. A raised probe timeout extends it. -export const codexDiscoveryDeadlineMs = ( - spawnedAtMs: number, - initializedAtMs: number, - probeTimeoutMs: number, -) => - Math.min( - spawnedAtMs + probeTimeoutMs, - initializedAtMs + - CODEX_DISCOVERY_BUDGET_MS + - Math.max(0, probeTimeoutMs - CODEX_APP_SERVER_PROBE_TIMEOUT_MS), - ) - CODEX_DISCOVERY_MARGIN_MS; - const withCodexDiscoveryDeadline = Effect.fnUntraced(function* ( effect: Effect.Effect, deadlineMs: number, @@ -714,7 +699,6 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun }) { const spawnedAtMs = DateTime.toEpochMillis(yield* DateTime.now); const { client, initialize } = yield* withCodexAppServerClient(input); - const initializedAtMs = DateTime.toEpochMillis(yield* DateTime.now); const versionMatch = initialize.userAgent.match(/\/([^\s]+)/); const version = versionMatch ? versionMatch[1] : undefined; @@ -729,7 +713,8 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun } satisfies CodexAppServerProviderSnapshot; } - const deadlineMs = codexDiscoveryDeadlineMs(spawnedAtMs, initializedAtMs, input.probeTimeoutMs); + // Optional discovery must finish before the outer timeout can discard the account result. + const deadlineMs = spawnedAtMs + input.probeTimeoutMs - CODEX_DISCOVERY_MARGIN_MS; const [skillsResponse, models, { config, nativeUsage, rateLimits }] = yield* Effect.all( [ withCodexDiscoveryDeadline(client.request("skills/list", { cwds: [input.cwd] }), deadlineMs),