diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index f0c4d00c7bdf..2f391107173a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -8918,6 +8918,25 @@ describe("ClaudeAdapterV2 background wake turns", () => { { status: 503, cooldown: "999999999999999999999999h", seconds: 0 }, { status: 503, cooldown: "4m 6s", seconds: 0, errorType: "overloaded_error" }, { status: 401, cooldown: "4m 6s", seconds: 0 }, + { status: 429, cooldown: "1h2m3s", seconds: 3723, proxy: true, sdkResetSeconds: 60 }, + { status: 429, cooldown: "1h2m3s", seconds: 7200, proxy: true, sdkResetSeconds: 7200 }, + { status: 429, cooldown: "1h2m3s", seconds: 0, proxy: true, sdkResetSeconds: null }, + { + status: 429, + cooldown: "nested 1h", + seconds: 0, + proxy: true, + result: + "API Error: Request rejected (429) · All credentials for model claude-opus-5-5 are cooling down (last error: rate_limit_error (reset after 1h))", + }, + { + status: 429, + cooldown: "outer cooldown", + seconds: 60, + proxy: true, + result: + "API Error: Request rejected (429) · All credentials for model claude-opus-5-5 are cooling down (reset after 1m) (last error: All credentials for model upstream are cooling down (reset after 1h))", + }, ])("recovers wrapped gateway limits with status $status and cooldown $cooldown", (input) => Effect.gen(function* () { const harness = yield* makeWakeHarness; @@ -8933,13 +8952,35 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); yield* Queue.offerAll(harness.sdkMessages, [ + ...(input.sdkResetSeconds !== undefined + ? [ + claudeSdkFrame({ + type: "rate_limit_event", + rate_limit_info: { + status: "rejected", + rateLimitType: "five_hour", + ...(input.sdkResetSeconds === null + ? {} + : { + resetsAt: DateTime.toEpochMillis(now) / 1000 + input.sdkResetSeconds, + }), + }, + uuid: "00000000-0000-4000-8000-000000000651", + session_id: WAKE_NATIVE_SESSION, + }), + ] + : []), makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000652", - error: "server_error", + error: input.proxy ? "rate_limit" : "server_error", }), makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000653", - result: `API Error: ${input.status} [claude/claude-opus-5-5] [429]: {"type":"error","error":{"type":"${input.errorType ?? "rate_limit_error"}","message":"Please try again later."}} (reset after ${input.cooldown}).`, + result: + input.result ?? + (input.proxy + ? `API Error: Request rejected (429) · All credentials for model claude-opus-5-5 are cooling down (reset after ${input.cooldown}) (last error: rate_limit_error: Please try again later.)` + : `API Error: ${input.status} [claude/claude-opus-5-5] [429]: {"type":"error","error":{"type":"${input.errorType ?? "rate_limit_error"}","message":"Please try again later."}} (reset after ${input.cooldown}).`), terminalReason: "api_error", isError: true, apiErrorStatus: input.status, @@ -8948,7 +8989,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { const terminal = yield* Queue.take(harness.terminalReceipts); assert.equal(terminal.status, "failed"); if (terminal.status !== "failed") return; - assert.equal(terminal.failure.class, input.seconds > 0 ? "usage_limit" : "provider_error"); + assert.equal( + terminal.failure.class, + input.seconds > 0 || input.proxy ? "usage_limit" : "provider_error", + ); if (input.seconds > 0) { assert.equal(terminal.failure.retryable, true); assert.equal( @@ -8958,7 +9002,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); } else { - assert.isUndefined(terminal.failure.resetAt); + assert.equal(terminal.failure.resetAt, input.proxy ? null : undefined); } }).pipe( Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 74cdde264e7e..dd416b3f5289 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4637,7 +4637,13 @@ function providerFailureFromResult( const resetAt = apiErrorStatus === 429 && message.result.startsWith("API Error:") ? gatewayCooldownResetAt( - /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result), + // CLIProxyAPI nests upstream errors in its message; only its own cooldown counts. + (message.result.startsWith( + "API Error: Request rejected (429) · All credentials for model ", + ) + ? /^API Error: Request rejected \(429\) · All credentials for model [^\s()]+ are cooling down \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/ + : /\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/ + ).exec(message.result), receivedAt, ) : undefined; @@ -11397,18 +11403,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( usageLimited && message.subtype === "success" && message.local_command === "compact" ? context.compactionFailure : null; - const resetTimes = Array.from(context.rateLimitResetTimes.values()); - const assistantLimit = context.latestAssistantRateLimit; - if ( - assistantLimit?.type != null && - !context.rateLimitResetTimes.has(assistantLimit.type) - ) { - resetTimes.push(assistantLimit.resetAt); - } - const resetAt = - resetTimes.length > 0 && resetTimes.every((time) => time !== null) - ? resetTimes.reduce((latest, time) => (time! > latest ? time! : latest), "") - : null; const resultFailure = interrupted ? null : limitedCompaction !== null @@ -11423,9 +11417,24 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( failureHint, usageLimited, ); + const resetTimes = Array.from(context.rateLimitResetTimes.values()); + const assistantLimit = context.latestAssistantRateLimit; + if ( + assistantLimit?.type != null && + !context.rateLimitResetTimes.has(assistantLimit.type) + ) { + resetTimes.push(assistantLimit.resetAt); + } + if (resultFailure?.class === "usage_limit" && resultFailure.resetAt != null) { + resetTimes.push(resultFailure.resetAt); + } + const resetAt = + resetTimes.length > 0 && resetTimes.every((time) => time !== null) + ? resetTimes.reduce((latest, time) => (time! > latest ? time! : latest), "") + : null; const terminalFailure = resultFailure?.class === "usage_limit" - ? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null } + ? { ...resultFailure, resetAt } : resultFailure; yield* finalizeActiveTurn({ context,