From a13a5b8782d75c7d96b8abf1df76ac1d6c4169b7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 9 Nov 2026 08:55:01 +0000 Subject: [PATCH 1/7] test(provider-pi): reproduce #33247 Pi usage limits classified as provider errors These tests reproduce #33247: a Pi assistant message_end carrying a Codex usage-limit error fails the turn as provider_error with no resetAt, so usage-limit recovery never arms. Co-Authored-By: Claude Opus 5.5 --- .../provider-pi/src/server/adapter.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index 06ad17f0568bb..e2c917795a9d6 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -32,6 +32,7 @@ import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -4638,6 +4639,51 @@ describe("PiAdapterV2", () => { }).pipe(Effect.scoped, Effect.provide(layerTest)), ); + it.effect.each([ + { + name: "a streamed usage limit", + errorMessage: "Codex error: The usage limit has been reached", + resetsInMinutes: null, + }, + { + name: "a ChatGPT plan limit with a reset time", + errorMessage: "You have hit your ChatGPT usage limit (plus plan). Try again in ~137 min.", + resetsInMinutes: 137, + }, + ])("fails the turn as a usage limit on $name", ({ errorMessage, resetsInMinutes }) => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-11-09T12:00:00.000Z")); + const fake = yield* makeFakePi; + const { runtime, takeEvent } = yield* openRuntime(fake); + const providerThread = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + yield* startTurn(runtime, providerThread); + yield* fake.takeRequest("prompt"); + const receivedAt = yield* DateTime.now; + yield* fake.emit({ + type: "message_end", + message: { role: "assistant", stopReason: "error", errorMessage }, + }); + yield* fake.emit({ type: "agent_settled" }); + + const terminal = yield* takeEvent((event) => event.type === "turn.terminal"); + assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "failed"); + if (terminal.type !== "turn.terminal" || terminal.status !== "failed") return; + assert.strictEqual(terminal.failure.class, "usage_limit"); + if (resetsInMinutes === null) { + assert.isUndefined(terminal.failure.resetAt); + return; + } + const earliest = DateTime.toEpochMillis(receivedAt) + resetsInMinutes * 60_000; + const resetAt = Date.parse(terminal.failure.resetAt ?? ""); + assert.isAtLeast(resetAt, earliest); + assert.isAtMost(resetAt, earliest + 60_000); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + it.effect.each([ { pi: "clears its native queue", From 68759f5b986260d00a8f6725652741b11854e5c0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 9 Nov 2026 08:57:44 +0000 Subject: [PATCH 2/7] fix(provider-pi): Pi usage-limit failures are classified as usage limits Pi reports a failed model call as an assistant message_end with only an errorMessage, and the adapter always failed the turn as provider_error, so usage-limit recovery never armed. Messages with usage-limit wording now fail as usage_limit, with resetAt taken from Pi's "Try again in ~N min." when present. Transient throttles such as "Rate limited" stay provider_error. Fixes pingdotgg/t3code#33247 Co-Authored-By: Claude Opus 5.5 --- .../provider-pi/src/server/adapter.test.ts | 7 +++++ packages/provider-pi/src/server/adapter.ts | 31 ++++++++++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index e2c917795a9d6..d68d89ced880d 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -846,6 +846,13 @@ describe("PiAdapterV2", () => { } const terminal = yield* takeEvent((event) => event.type === "turn.terminal"); assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "failed"); + if ( + failure === "model error" && + terminal.type === "turn.terminal" && + terminal.status === "failed" + ) { + assert.strictEqual(terminal.failure.class, "provider_error"); + } assert.deepInclude(runtime.providerSession, { status: "error", extensionStatuses: [{ key: "watch", text: "Watching 3 files" }], diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index a82a7f45f4e47..68ac8542ad01b 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -92,6 +92,29 @@ const PI_UNSOLICITED_ACTIVITY_ERROR = const SETTLE_PROBE_MAX_ATTEMPTS = 3; const SETTLE_PROBE_RETRY_DELAY = Duration.millis(100); +const PI_USAGE_LIMIT = /usage[ _]limit/i; +// Pi rounds the server's resets_at to whole minutes; the margin keeps recovery after the real reset. +const PI_USAGE_LIMIT_RESET = /Try again in ~(\d{1,6}) min\b/; +const PI_USAGE_LIMIT_RESET_MARGIN_MS = 30_000; + +function piModelErrorFailure(message: string, receivedAt: DateTime.Utc) { + if (!PI_USAGE_LIMIT.test(message)) + return makeProviderFailure({ message, class: "provider_error" }); + const minutes = PI_USAGE_LIMIT_RESET.exec(message)?.[1]; + return makeProviderFailure({ + message, + class: "usage_limit", + resetAt: + minutes === undefined + ? null + : DateTime.formatIso( + DateTime.add(receivedAt, { + milliseconds: Number(minutes) * 60_000 + PI_USAGE_LIMIT_RESET_MARGIN_MS, + }), + ), + }); +} + const PiProviderCapabilitiesV2 = { runtimePolicy: { enforcement: "client-boundary" }, sessions: { @@ -1974,10 +1997,10 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( if (recordString(message, "role") !== "assistant") return; yield* completeOpenStreamItems(turn); if (recordString(message, "stopReason") === "error" && turn.failure === null) { - turn.failure = makeProviderFailure({ - message: recordString(message, "errorMessage") ?? "Pi reported a model error.", - class: "provider_error", - }); + turn.failure = piModelErrorFailure( + recordString(message, "errorMessage") ?? "Pi reported a model error.", + yield* DateTime.now, + ); } return; } From d38d381380050546f8cf182bbf877f09d7991e8e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 9 Nov 2026 09:01:09 +0000 Subject: [PATCH 3/7] fix(provider-pi): a Pi retry that ends on a usage limit is classified as one Co-Authored-By: Claude Opus 5.5 --- .../provider-pi/src/server/adapter.test.ts | 46 +++++++++++++++++++ packages/provider-pi/src/server/adapter.ts | 10 ++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index d68d89ced880d..7f2ed8d58c31c 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -4691,6 +4691,52 @@ describe("PiAdapterV2", () => { }).pipe(Effect.scoped, Effect.provide(layerTest)), ); + it.effect("fails the turn as a usage limit when Pi's retry hits the limit", () => + Effect.gen(function* () { + const fake = yield* makeFakePi; + const { runtime, takeEvent } = yield* openRuntime(fake); + const providerThread = yield* runtime.ensureThread({ + threadId: THREAD_ID, + modelSelection: modelSelection("default"), + runtimePolicy, + }); + yield* startTurn(runtime, providerThread); + yield* fake.takeRequest("prompt"); + yield* fake.emit({ + type: "message_end", + message: { + role: "assistant", + stopReason: "error", + errorMessage: "stream disconnected before completion", + }, + }); + yield* fake.emit({ + type: "auto_retry_start", + attempt: 1, + maxAttempts: 3, + delayMs: 2_000, + errorMessage: "stream disconnected before completion", + }); + const errorMessage = "Codex error: The usage limit has been reached"; + yield* fake.emit({ + type: "message_end", + message: { role: "assistant", stopReason: "error", errorMessage }, + }); + yield* fake.emit({ + type: "auto_retry_end", + success: false, + attempt: 1, + finalError: errorMessage, + }); + yield* fake.emit({ type: "agent_settled" }); + + const terminal = yield* takeEvent((event) => event.type === "turn.terminal"); + assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "failed"); + if (terminal.type !== "turn.terminal" || terminal.status !== "failed") return; + assert.strictEqual(terminal.failure.class, "usage_limit"); + }).pipe(Effect.scoped, Effect.provide(layerTest)), + ); + it.effect.each([ { pi: "clears its native queue", diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 68ac8542ad01b..1870786b916ad 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -2134,11 +2134,13 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( turn.failure = null; return; } - const failure = makeProviderFailure({ - message: recordString(event, "finalError") ?? "Pi auto-retry failed.", - class: "provider_error", + const failure = { + ...piModelErrorFailure( + recordString(event, "finalError") ?? "Pi auto-retry failed.", + emittedAt, + ), retryable: false, - }); + }; const attempt = Math.max(1, Math.trunc(recordNumber(event, "attempt") ?? 1)); const current = turn.activeProviderRetry; const providerRetry = { From 107c62505bbaa56b1f7d2c9cdd73a5fd66fc0831 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 9 Nov 2026 09:13:20 +0000 Subject: [PATCH 4/7] fix(provider-pi): a Pi reset that already passed arms no recovery Co-Authored-By: Claude Opus 5.5 --- packages/provider-pi/src/server/adapter.test.ts | 12 +++++++++--- packages/provider-pi/src/server/adapter.ts | 5 +++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index 7f2ed8d58c31c..d60fe0dd591e7 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -4657,6 +4657,11 @@ describe("PiAdapterV2", () => { errorMessage: "You have hit your ChatGPT usage limit (plus plan). Try again in ~137 min.", resetsInMinutes: 137, }, + { + name: "a ChatGPT plan limit whose reset already passed", + errorMessage: "You have hit your ChatGPT usage limit (plus plan). Try again in ~0 min.", + resetsInMinutes: null, + }, ])("fails the turn as a usage limit on $name", ({ errorMessage, resetsInMinutes }) => Effect.gen(function* () { yield* TestClock.setTime(Date.parse("2026-11-09T12:00:00.000Z")); @@ -4684,10 +4689,11 @@ describe("PiAdapterV2", () => { assert.isUndefined(terminal.failure.resetAt); return; } - const earliest = DateTime.toEpochMillis(receivedAt) + resetsInMinutes * 60_000; + const latestRealReset = + DateTime.toEpochMillis(receivedAt) + resetsInMinutes * 60_000 + 30_000; const resetAt = Date.parse(terminal.failure.resetAt ?? ""); - assert.isAtLeast(resetAt, earliest); - assert.isAtMost(resetAt, earliest + 60_000); + assert.isAtLeast(resetAt, latestRealReset); + assert.isAtMost(resetAt, latestRealReset + 30_000); }).pipe(Effect.scoped, Effect.provide(layerTest)), ); diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 1870786b916ad..42e3b89f019a2 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -93,8 +93,9 @@ const SETTLE_PROBE_MAX_ATTEMPTS = 3; const SETTLE_PROBE_RETRY_DELAY = Duration.millis(100); const PI_USAGE_LIMIT = /usage[ _]limit/i; -// Pi rounds the server's resets_at to whole minutes; the margin keeps recovery after the real reset. -const PI_USAGE_LIMIT_RESET = /Try again in ~(\d{1,6}) min\b/; +// Pi rounds the server's resets_at to whole minutes and clamps a past one to ~0; the margin keeps +// recovery after the real reset, and ~0 carries no usable reset. +const PI_USAGE_LIMIT_RESET = /Try again in ~([1-9]\d{0,5}) min\b/; const PI_USAGE_LIMIT_RESET_MARGIN_MS = 30_000; function piModelErrorFailure(message: string, receivedAt: DateTime.Utc) { From 6e8aaad8731c4bc2ef8441f19d941226ac861f35 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 13:02:05 +0000 Subject: [PATCH 5/7] fix(provider-pi): describe ~0 min as a reset past or under 30 s away Co-Authored-By: Claude Opus 5.5 --- packages/provider-pi/src/server/adapter.test.ts | 2 +- packages/provider-pi/src/server/adapter.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index d60fe0dd591e7..98c0eb922aba5 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -4658,7 +4658,7 @@ describe("PiAdapterV2", () => { resetsInMinutes: 137, }, { - name: "a ChatGPT plan limit whose reset already passed", + name: "a ChatGPT plan limit whose reset rounds to ~0 min", errorMessage: "You have hit your ChatGPT usage limit (plus plan). Try again in ~0 min.", resetsInMinutes: null, }, diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 42e3b89f019a2..0560383673e2d 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -93,8 +93,8 @@ const SETTLE_PROBE_MAX_ATTEMPTS = 3; const SETTLE_PROBE_RETRY_DELAY = Duration.millis(100); const PI_USAGE_LIMIT = /usage[ _]limit/i; -// Pi rounds the server's resets_at to whole minutes and clamps a past one to ~0; the margin keeps -// recovery after the real reset, and ~0 carries no usable reset. +// Pi rounds the server's resets_at to the nearest minute, so the margin keeps recovery after the +// real reset; ~0 (a reset past or under 30 s away) carries no usable reset. const PI_USAGE_LIMIT_RESET = /Try again in ~([1-9]\d{0,5}) min\b/; const PI_USAGE_LIMIT_RESET_MARGIN_MS = 30_000; From d284ac3baa93ec6d9e502baf4fccd575b8bbce28 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 20:05:33 +0000 Subject: [PATCH 6/7] refactor(provider-pi): pass retryable through the Pi failure helper AI-assisted; verified with the tests and tools listed above. --- packages/provider-pi/src/server/adapter.test.ts | 11 ++++++----- packages/provider-pi/src/server/adapter.ts | 17 ++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index 5452381fd8d4c..aa860e785664c 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -4744,11 +4744,12 @@ describe("PiAdapterV2", () => { assert.isUndefined(terminal.failure.resetAt); return; } - const latestRealReset = - DateTime.toEpochMillis(receivedAt) + resetsInMinutes * 60_000 + 30_000; - const resetAt = Date.parse(terminal.failure.resetAt ?? ""); - assert.isAtLeast(resetAt, latestRealReset); - assert.isAtMost(resetAt, latestRealReset + 30_000); + assert.strictEqual( + terminal.failure.resetAt, + new Date( + DateTime.toEpochMillis(receivedAt) + resetsInMinutes * 60_000 + 30_000, + ).toISOString(), + ); }).pipe(Effect.scoped, Effect.provide(layerTest)), ); diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 6a8d491c46055..01ec54dce2259 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -100,13 +100,14 @@ const PI_USAGE_LIMIT = /usage[ _]limit/i; const PI_USAGE_LIMIT_RESET = /Try again in ~([1-9]\d{0,5}) min\b/; const PI_USAGE_LIMIT_RESET_MARGIN_MS = 30_000; -function piModelErrorFailure(message: string, receivedAt: DateTime.Utc) { +function piModelErrorFailure(message: string, receivedAt: DateTime.Utc, retryable?: boolean) { if (!PI_USAGE_LIMIT.test(message)) - return makeProviderFailure({ message, class: "provider_error" }); + return makeProviderFailure({ message, class: "provider_error", retryable }); const minutes = PI_USAGE_LIMIT_RESET.exec(message)?.[1]; return makeProviderFailure({ message, class: "usage_limit", + retryable, resetAt: minutes === undefined ? null @@ -2163,13 +2164,11 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( turn.failure = null; return; } - const failure = { - ...piModelErrorFailure( - recordString(event, "finalError") ?? "Pi auto-retry failed.", - emittedAt, - ), - retryable: false, - }; + const failure = piModelErrorFailure( + recordString(event, "finalError") ?? "Pi auto-retry failed.", + emittedAt, + false, + ); const attempt = Math.max(1, Math.trunc(recordNumber(event, "attempt") ?? 1)); const current = turn.activeProviderRetry; const providerRetry = { From dd1c482150a4c7c7c33f8c8bc31b62a50acbff96 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 20:07:20 +0000 Subject: [PATCH 7/7] fix(provider-pi): keep Pi failure helper and its test type-clean AI-assisted; verified with the tests and tools listed above. --- packages/provider-pi/src/server/adapter.test.ts | 6 +++--- packages/provider-pi/src/server/adapter.ts | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index aa860e785664c..9c1846cc866d5 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -4746,9 +4746,9 @@ describe("PiAdapterV2", () => { } assert.strictEqual( terminal.failure.resetAt, - new Date( - DateTime.toEpochMillis(receivedAt) + resetsInMinutes * 60_000 + 30_000, - ).toISOString(), + DateTime.formatIso( + DateTime.add(receivedAt, { milliseconds: resetsInMinutes * 60_000 + 30_000 }), + ), ); }).pipe(Effect.scoped, Effect.provide(layerTest)), ); diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 01ec54dce2259..f2257671780fd 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -100,7 +100,11 @@ const PI_USAGE_LIMIT = /usage[ _]limit/i; const PI_USAGE_LIMIT_RESET = /Try again in ~([1-9]\d{0,5}) min\b/; const PI_USAGE_LIMIT_RESET_MARGIN_MS = 30_000; -function piModelErrorFailure(message: string, receivedAt: DateTime.Utc, retryable?: boolean) { +function piModelErrorFailure( + message: string, + receivedAt: DateTime.Utc, + retryable: boolean | null = null, +) { if (!PI_USAGE_LIMIT.test(message)) return makeProviderFailure({ message, class: "provider_error", retryable }); const minutes = PI_USAGE_LIMIT_RESET.exec(message)?.[1];