From 476021044a8e24ca311e417b3924e28120a9152b Mon Sep 17 00:00:00 2001 From: surya-gorla <217902563+surya-gorla@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:43:01 +0530 Subject: [PATCH 1/7] fix(server): ignore expired buffered Claude wake limits Prevent a pre-reset wake rejection from failing a continuation replayed after the quota reset. Preserve charged token usage and keep genuinely fresh or still-active 429 errors visible. Fixes #36405. --- .../Adapters/ClaudeAdapterV2.test.ts | 116 ++++++++++++++++-- .../Adapters/ClaudeAdapterV2.ts | 102 ++++++++++++++- 2 files changed, 209 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index fc46e2b100232..2f80ecc63ab13 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15455,10 +15455,62 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect.each([true, false])( - "carries a rejected wake rate limit into the continuation failure (event=%s)", - (standaloneEvent) => + it.effect.each([ + { + standaloneEvent: true, + expiredBeforeReplay: false, + newerTurnBeforeReplay: false, + resetOffsetMs: null, + }, + { + standaloneEvent: false, + expiredBeforeReplay: false, + newerTurnBeforeReplay: false, + resetOffsetMs: null, + }, + { + standaloneEvent: true, + expiredBeforeReplay: true, + newerTurnBeforeReplay: false, + resetOffsetMs: null, + }, + { + standaloneEvent: false, + expiredBeforeReplay: true, + newerTurnBeforeReplay: false, + resetOffsetMs: null, + }, + { + standaloneEvent: true, + expiredBeforeReplay: true, + newerTurnBeforeReplay: true, + resetOffsetMs: null, + }, + { + standaloneEvent: true, + expiredBeforeReplay: true, + newerTurnBeforeReplay: false, + resetOffsetMs: 0, + }, + { + standaloneEvent: true, + expiredBeforeReplay: false, + newerTurnBeforeReplay: false, + resetOffsetMs: -1, + }, + { + standaloneEvent: true, + expiredBeforeReplay: false, + newerTurnBeforeReplay: false, + resetOffsetMs: null, + arrivedAfterReset: true, + }, + ])( + "replays a rejected wake rate limit without a stale failure (event=$standaloneEvent expired=$expiredBeforeReplay newerTurn=$newerTurnBeforeReplay offset=$resetOffsetMs arrived=$arrivedAfterReset)", + (scenario) => Effect.gen(function* () { + const { standaloneEvent, expiredBeforeReplay, newerTurnBeforeReplay, resetOffsetMs } = + scenario; const harness = yield* makeWakeHarness; const now = yield* DateTime.now; const resetsAt = 1790298600; @@ -15476,7 +15528,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); yield* Queue.offer(harness.sdkMessages, turnOneResult); yield* Queue.take(harness.terminalReceipts); - + if ("arrivedAfterReset" in scenario && scenario.arrivedAfterReset) { + yield* TestClock.setTime(resetsAt * 1_000 + 1_000); + } yield* Queue.offer(harness.sdkMessages, wakeNotification); if (standaloneEvent) { yield* harness.offerAndWait( @@ -15534,22 +15588,68 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); + if (resetOffsetMs !== null) { + yield* TestClock.setTime(resetsAt * 1_000 + resetOffsetMs); + } else if (expiredBeforeReplay) { + yield* TestClock.setTime(resetsAt * 1_000 + 60_000); + } + if (newerTurnBeforeReplay) { + // A queued continuation may run behind an ordinary turn served after reset. + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-wake-limit-intervening"), + text: "Check whether the limit cleared.", + attachments: [], + providerTurnOrdinal: 2, + }), + ); + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000636", + result: "The new turn succeeded.", + }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); + } yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ threadId: harness.threadId, providerThread: harness.providerThread, - now, + now: yield* DateTime.now, attemptId: RunAttemptId.make("attempt-claude-wake-limit-2"), text: "Background task completed.", attachments: [], - providerTurnOrdinal: 2, + providerTurnOrdinal: newerTurnBeforeReplay ? 3 : 2, messageCreatedBy: "agent", messageCreationSource: "provider", }), ); const terminal = yield* Queue.take(harness.terminalReceipts); - assert.equal(terminal?.status, "failed"); - if (terminal === undefined || terminal.status !== "failed") return; + if (expiredBeforeReplay) { + assert.equal(terminal.status, "completed"); + assert.isFalse( + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "system_notice" && + event.turnItem.message.includes("usage limit reached"), + ), + ); + const completed = harness.events.findLast( + (event) => event.type === "provider_turn.updated", + ); + assert.equal(completed?.type, "provider_turn.updated"); + if (completed?.type === "provider_turn.updated") { + assert.equal(completed.providerTurn.turnTokenUsage?.inputTokens, 1); + assert.equal(completed.providerTurn.turnTokenUsage?.outputTokens, 1); + } + return; + } + assert.equal(terminal.status, "failed"); + if (terminal.status !== "failed") return; assert.equal(terminal.failure.class, "usage_limit"); assert.equal(terminal.failure.resetAt, "2026-09-25T01:10:00.000Z"); if (standaloneEvent) { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 17a9ca8486a38..43badef5945f2 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4451,6 +4451,69 @@ const isClaudeBlockedAssistantRateLimit = Schema.is( }), ); +// An idle wake can be rejected before a reset, then replayed after a newer turn succeeds. +// Only discard the old rejection when its explicitly reported reset has already passed. +// A live 429 or one with an unknown/future reset still needs normal recovery. +function expiredBufferedWakeRateLimitFrames( + frames: ReadonlyArray, + result: SDKMessage, + receivedAtMs: number, + nowMs: number, +): ReadonlySet | null { + if ( + result.type !== "result" || + result.subtype !== "success" || + !result.is_error || + result.api_error_status !== 429 + ) { + return null; + } + const resultIndex = frames.indexOf(result); + if (resultIndex < 0) return null; + const beforeResult = frames.slice(0, resultIndex); + const lastRootStart = beforeResult.findLastIndex(isClaudeTurnStartMessage); + const lastResult = beforeResult.findLastIndex((frame) => frame.type === "result"); + const segment = frames.slice(Math.max(lastRootStart, lastResult) + 1, resultIndex); + const directResets = segment.flatMap((frame) => + frame.type === "rate_limit_event" && + frame.rate_limit_info?.status === "rejected" && + frame.rate_limit_info.overageStatus !== "allowed" && + frame.rate_limit_info.overageStatus !== "allowed_warning" && + frame.rate_limit_info.isUsingOverage !== true && + frame.rate_limit_info.overageInUse !== true + ? [frame.rate_limit_info.resetsAt] + : [], + ); + // Standalone rate-limit events are authoritative over the embedded assistant error. + const embeddedResets = segment.flatMap((frame) => + isClaudeBlockedAssistantRateLimit(frame) + ? [frame.api_error_params.rate_limit_info.resetsAt] + : [], + ); + const resets = directResets.length > 0 ? directResets : embeddedResets; + if ( + resets.length === 0 || + !resets.every( + (reset) => + typeof reset === "number" && + Number.isFinite(reset) && + reset > 0 && + reset * 1_000 > receivedAtMs && + reset * 1_000 <= nowMs, + ) + ) { + return null; + } + return new Set( + segment.filter( + (frame) => + (frame.type === "rate_limit_event" && frame.rate_limit_info?.status === "rejected") || + (frame.type === "assistant" && + (frame.error === "rate_limit" || isClaudeBlockedAssistantRateLimit(frame))), + ), + ); +} + function claudeUsageLimitResetAt(resetsAt: unknown): string | null { const resetMs = typeof resetsAt === "number" ? resetsAt * 1000 : NaN; return Number.isFinite(resetMs) && resetMs > 0 && resetMs < 8.64e15 @@ -12774,6 +12837,29 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); const lastResult = drained[replayResultIndex === -1 ? lastResultIndex : replayResultIndex]; + const lastUnsettledRootIndex = drained.findLastIndex(isClaudeTurnStartMessage); + const newerRootStillOpen = + lastResult !== undefined && + lastUnsettledRootIndex > drained.indexOf(lastResult) && + !drained + .slice(lastUnsettledRootIndex + 1) + .some((entry) => entry.type === "result"); + const bufferedResultReceivedAt = + lastResult?.type === "result" + ? bufferedResultArrivals.get(lastResult) + : undefined; + const expiredWakeFrames = + lastResult !== undefined && + bufferedResultReceivedAt !== undefined && + !emptyTurnBeforeNextTurn && + !newerRootStillOpen + ? expiredBufferedWakeRateLimitFrames( + drained, + lastResult, + DateTime.toEpochMillis(bufferedResultReceivedAt), + DateTime.toEpochMillis(yield* DateTime.now), + ) + : null; const opaqueReplayTombstones = taskIdSetForNativeThread( yield* Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread), nativeThreadId, @@ -12790,7 +12876,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ) { continue; } - if (entry.type !== "result") { + if (entry.type !== "result" && !expiredWakeFrames?.has(entry)) { yield* handleSdkMessage({ query: querySession.query, message: entry }); } yield* Ref.update(wakeBuffers, (current) => { @@ -12808,6 +12894,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return next; }); } + if (expiredWakeFrames !== null) { + // The buffered wake ended before the limit reset. It is no longer a + // rejection of this continuation, so do not fail it or show stale errors. + const completedAt = yield* DateTime.now; + yield* finalizeActiveTurn({ + context, + status: "completed", + completedAt, + // The old wake may have incurred usage even though its rejection has expired. + ...(lastResult?.type === "result" ? { result: lastResult } : {}), + }); + if (leavesLaterWake) yield* offerBufferedWake(nativeThreadId); + return; + } if (lastResult !== undefined && !emptyTurnBeforeNextTurn) { yield* handleSdkMessage({ query: querySession.query, message: lastResult }); if (leavesLaterWake) yield* offerBufferedWake(nativeThreadId); From 780e387002f5ff453a2a20c18157fe84bbe49a31 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 14 Nov 2026 08:46:48 +0000 Subject: [PATCH 2/7] test(server): trim expired wake limit replay cases Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 73 ++++--------------- 1 file changed, 13 insertions(+), 60 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 2f80ecc63ab13..767817669dc3d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15456,61 +15456,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); it.effect.each([ - { - standaloneEvent: true, - expiredBeforeReplay: false, - newerTurnBeforeReplay: false, - resetOffsetMs: null, - }, - { - standaloneEvent: false, - expiredBeforeReplay: false, - newerTurnBeforeReplay: false, - resetOffsetMs: null, - }, - { - standaloneEvent: true, - expiredBeforeReplay: true, - newerTurnBeforeReplay: false, - resetOffsetMs: null, - }, - { - standaloneEvent: false, - expiredBeforeReplay: true, - newerTurnBeforeReplay: false, - resetOffsetMs: null, - }, - { - standaloneEvent: true, - expiredBeforeReplay: true, - newerTurnBeforeReplay: true, - resetOffsetMs: null, - }, - { - standaloneEvent: true, - expiredBeforeReplay: true, - newerTurnBeforeReplay: false, - resetOffsetMs: 0, - }, - { - standaloneEvent: true, - expiredBeforeReplay: false, - newerTurnBeforeReplay: false, - resetOffsetMs: -1, - }, - { - standaloneEvent: true, - expiredBeforeReplay: false, - newerTurnBeforeReplay: false, - resetOffsetMs: null, - arrivedAfterReset: true, - }, + { standaloneEvent: true, arrivedAfterReset: false, expired: false, newerTurn: false }, + { standaloneEvent: false, arrivedAfterReset: false, expired: false, newerTurn: false }, + { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: false }, + { standaloneEvent: false, arrivedAfterReset: false, expired: true, newerTurn: false }, + { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: true }, + { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, ])( - "replays a rejected wake rate limit without a stale failure (event=$standaloneEvent expired=$expiredBeforeReplay newerTurn=$newerTurnBeforeReplay offset=$resetOffsetMs arrived=$arrivedAfterReset)", - (scenario) => + "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn)", + ({ standaloneEvent, arrivedAfterReset, expired, newerTurn }) => Effect.gen(function* () { - const { standaloneEvent, expiredBeforeReplay, newerTurnBeforeReplay, resetOffsetMs } = - scenario; const harness = yield* makeWakeHarness; const now = yield* DateTime.now; const resetsAt = 1790298600; @@ -15528,7 +15483,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); yield* Queue.offer(harness.sdkMessages, turnOneResult); yield* Queue.take(harness.terminalReceipts); - if ("arrivedAfterReset" in scenario && scenario.arrivedAfterReset) { + if (arrivedAfterReset) { yield* TestClock.setTime(resetsAt * 1_000 + 1_000); } yield* Queue.offer(harness.sdkMessages, wakeNotification); @@ -15588,12 +15543,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); - if (resetOffsetMs !== null) { - yield* TestClock.setTime(resetsAt * 1_000 + resetOffsetMs); - } else if (expiredBeforeReplay) { + if (expired) { yield* TestClock.setTime(resetsAt * 1_000 + 60_000); } - if (newerTurnBeforeReplay) { + if (newerTurn) { // A queued continuation may run behind an ordinary turn served after reset. yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ @@ -15622,13 +15575,13 @@ describe("ClaudeAdapterV2 background wake turns", () => { attemptId: RunAttemptId.make("attempt-claude-wake-limit-2"), text: "Background task completed.", attachments: [], - providerTurnOrdinal: newerTurnBeforeReplay ? 3 : 2, + providerTurnOrdinal: newerTurn ? 3 : 2, messageCreatedBy: "agent", messageCreationSource: "provider", }), ); const terminal = yield* Queue.take(harness.terminalReceipts); - if (expiredBeforeReplay) { + if (expired) { assert.equal(terminal.status, "completed"); assert.isFalse( harness.events.some( From 01e854df75d344f5f244aa62c69747b79642fd4f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 14 Nov 2026 08:58:59 +0000 Subject: [PATCH 3/7] fix(server): keep active embedded limits of other types on wake replay Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 39 ++++++++++++------- .../Adapters/ClaudeAdapterV2.ts | 22 +++++++---- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 767817669dc3d..8336cd5104dda 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15455,20 +15455,24 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect.each([ - { standaloneEvent: true, arrivedAfterReset: false, expired: false, newerTurn: false }, - { standaloneEvent: false, arrivedAfterReset: false, expired: false, newerTurn: false }, - { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: false }, - { standaloneEvent: false, arrivedAfterReset: false, expired: true, newerTurn: false }, - { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: true }, - { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, - ])( - "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn)", - ({ standaloneEvent, arrivedAfterReset, expired, newerTurn }) => + it.effect.each( + [ + { standaloneEvent: true, arrivedAfterReset: false, expired: false, newerTurn: false }, + { standaloneEvent: false, arrivedAfterReset: false, expired: false, newerTurn: false }, + { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: false }, + { standaloneEvent: false, arrivedAfterReset: false, expired: true, newerTurn: false }, + { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: true }, + { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, + { standaloneEvent: true, arrivedAfterReset: false, expired: true, weeklyStillActive: true }, + ].map((row) => ({ newerTurn: false, weeklyStillActive: false, ...row })), + )( + "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn weekly=$weeklyStillActive)", + ({ standaloneEvent, arrivedAfterReset, expired, newerTurn, weeklyStillActive }) => Effect.gen(function* () { const harness = yield* makeWakeHarness; const now = yield* DateTime.now; const resetsAt = 1790298600; + const weeklyResetsAt = resetsAt + 604_800; yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ @@ -15510,9 +15514,13 @@ describe("ClaudeAdapterV2 background wake turns", () => { apiErrorParams: { rate_limit_info: { status: "rejected", - rateLimitType: "five_hour", + rateLimitType: weeklyStillActive ? "seven_day" : "five_hour", // A conflicting embedded time must not replace the event for this window. - resetsAt: standaloneEvent ? resetsAt + 3_600 : resetsAt, + resetsAt: weeklyStillActive + ? weeklyResetsAt + : standaloneEvent + ? resetsAt + 3_600 + : resetsAt, }, }, }), @@ -15581,7 +15589,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); const terminal = yield* Queue.take(harness.terminalReceipts); - if (expired) { + if (expired && !weeklyStillActive) { assert.equal(terminal.status, "completed"); assert.isFalse( harness.events.some( @@ -15604,7 +15612,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(terminal.status, "failed"); if (terminal.status !== "failed") return; assert.equal(terminal.failure.class, "usage_limit"); - assert.equal(terminal.failure.resetAt, "2026-09-25T01:10:00.000Z"); + assert.equal( + terminal.failure.resetAt, + weeklyStillActive ? "2026-10-02T01:10:00.000Z" : "2026-09-25T01:10:00.000Z", + ); if (standaloneEvent) { const pause = yield* Queue.take(harness.systemNoticeReceipts); assert.equal(pause.turnItem.type, "system_notice"); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 43badef5945f2..6ec19276b4ba8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4474,23 +4474,29 @@ function expiredBufferedWakeRateLimitFrames( const lastRootStart = beforeResult.findLastIndex(isClaudeTurnStartMessage); const lastResult = beforeResult.findLastIndex((frame) => frame.type === "result"); const segment = frames.slice(Math.max(lastRootStart, lastResult) + 1, resultIndex); - const directResets = segment.flatMap((frame) => + const directLimits = segment.flatMap((frame) => frame.type === "rate_limit_event" && frame.rate_limit_info?.status === "rejected" && frame.rate_limit_info.overageStatus !== "allowed" && frame.rate_limit_info.overageStatus !== "allowed_warning" && frame.rate_limit_info.isUsingOverage !== true && frame.rate_limit_info.overageInUse !== true - ? [frame.rate_limit_info.resetsAt] + ? [frame.rate_limit_info] : [], ); - // Standalone rate-limit events are authoritative over the embedded assistant error. - const embeddedResets = segment.flatMap((frame) => - isClaudeBlockedAssistantRateLimit(frame) - ? [frame.api_error_params.rate_limit_info.resetsAt] - : [], + const directTypes = new Set( + directLimits.map((limit) => limit.rateLimitType ?? "unknown"), ); - const resets = directResets.length > 0 ? directResets : embeddedResets; + // Like result handling, a standalone event wins for its own limit type only. + const embeddedResets = segment.flatMap((frame) => { + if (!isClaudeBlockedAssistantRateLimit(frame)) return []; + const { rateLimitType, resetsAt } = frame.api_error_params.rate_limit_info; + return directLimits.length === 0 || + (rateLimitType != null && !directTypes.has(String(rateLimitType))) + ? [resetsAt] + : []; + }); + const resets = [...directLimits.map((limit) => limit.resetsAt), ...embeddedResets]; if ( resets.length === 0 || !resets.every( From 3dfe26cf982d7710992d7e48e3c2f1fbf9db93ff Mon Sep 17 00:00:00 2001 From: surya-gorla <217902563+surya-gorla@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:36:28 +0530 Subject: [PATCH 4/7] fix(server): ignore expired limits from every buffered wake Suppress stale rejection events from all completed wake segments, while preserving settlement ownership for the final selected result. Regress two expired wakes and a later still-limited wake. --- .../Adapters/ClaudeAdapterV2.test.ts | 84 +++++++++++++++++-- .../Adapters/ClaudeAdapterV2.ts | 42 +++++----- 2 files changed, 101 insertions(+), 25 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 8336cd5104dda..93da98b7e6b48 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15464,10 +15464,32 @@ describe("ClaudeAdapterV2 background wake turns", () => { { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: true }, { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, { standaloneEvent: true, arrivedAfterReset: false, expired: true, weeklyStillActive: true }, - ].map((row) => ({ newerTurn: false, weeklyStillActive: false, ...row })), + { standaloneEvent: true, arrivedAfterReset: false, expired: true, twoWakes: true }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + twoWakes: true, + secondWakeStillActive: true, + }, + ].map((row) => ({ + newerTurn: false, + weeklyStillActive: false, + twoWakes: false, + secondWakeStillActive: false, + ...row, + })), )( - "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn weekly=$weeklyStillActive)", - ({ standaloneEvent, arrivedAfterReset, expired, newerTurn, weeklyStillActive }) => + "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive)", + ({ + standaloneEvent, + arrivedAfterReset, + expired, + newerTurn, + weeklyStillActive, + twoWakes, + secondWakeStillActive, + }) => Effect.gen(function* () { const harness = yield* makeWakeHarness; const now = yield* DateTime.now; @@ -15538,6 +15560,49 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); assert.lengthOf(harness.continuationRequests, 1); + if (twoWakes) { + // A second rejected background wake completes while the first + // continuation remains queued. Both rejections are expired by replay. + yield* harness.offerAndWait(wakeTurnInit); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "rate_limit_event", + rate_limit_info: { + status: "rejected", + rateLimitType: secondWakeStillActive ? "seven_day" : "five_hour", + resetsAt: secondWakeStillActive ? weeklyResetsAt : resetsAt, + overageStatus: "rejected", + }, + uuid: "00000000-0000-4000-8000-000000000637", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* harness.offerAndWait( + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000638", + error: "rate_limit", + apiErrorParams: { + rate_limit_info: { + status: "rejected", + rateLimitType: secondWakeStillActive ? "seven_day" : "five_hour", + resetsAt: secondWakeStillActive ? weeklyResetsAt : resetsAt, + }, + }, + }), + ); + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000639", + result: "The second background wake also hit the limit.", + isError: true, + apiErrorStatus: 429, + terminalReason: "api_error", + origin: { kind: "task-notification" }, + }), + ); + assert.lengthOf(harness.continuationRequests, 1); + } + // An empty session-task lifecycle result arriving before T3 opens the // wake run must not replace the root's terminal usage-limit result. yield* harness.offerAndWait(wakeTurnInit); @@ -15589,7 +15654,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); const terminal = yield* Queue.take(harness.terminalReceipts); - if (expired && !weeklyStillActive) { + if (expired && !weeklyStillActive && !secondWakeStillActive) { assert.equal(terminal.status, "completed"); assert.isFalse( harness.events.some( @@ -15614,13 +15679,20 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(terminal.failure.class, "usage_limit"); assert.equal( terminal.failure.resetAt, - weeklyStillActive ? "2026-10-02T01:10:00.000Z" : "2026-09-25T01:10:00.000Z", + weeklyStillActive || secondWakeStillActive + ? "2026-10-02T01:10:00.000Z" + : "2026-09-25T01:10:00.000Z", ); if (standaloneEvent) { const pause = yield* Queue.take(harness.systemNoticeReceipts); assert.equal(pause.turnItem.type, "system_notice"); if (pause.turnItem.type !== "system_notice") return; - assert.include(pause.turnItem.message, "This turn is paused until the 5-hour limit"); + assert.include( + pause.turnItem.message, + secondWakeStillActive + ? "This turn is paused until the 7-day limit" + : "This turn is paused until the 5-hour limit", + ); } }).pipe( Effect.provide( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6ec19276b4ba8..6ca1878a2cf4c 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -12850,22 +12850,26 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( !drained .slice(lastUnsettledRootIndex + 1) .some((entry) => entry.type === "result"); - const bufferedResultReceivedAt = - lastResult?.type === "result" - ? bufferedResultArrivals.get(lastResult) - : undefined; - const expiredWakeFrames = - lastResult !== undefined && - bufferedResultReceivedAt !== undefined && - !emptyTurnBeforeNextTurn && - !newerRootStillOpen - ? expiredBufferedWakeRateLimitFrames( - drained, - lastResult, - DateTime.toEpochMillis(bufferedResultReceivedAt), - DateTime.toEpochMillis(yield* DateTime.now), - ) - : null; + // More than one rejected wake can be buffered behind a continuation. + // Suppress expired evidence from every completed wake, but only the + // selected terminal result may complete this continuation. + const expiredWakeFrames = new Set(); + let lastResultExpired = false; + const replayedAtMs = DateTime.toEpochMillis(yield* DateTime.now); + for (const entry of drained) { + if (entry.type !== "result") continue; + const receivedAt = bufferedResultArrivals.get(entry); + if (receivedAt === undefined) continue; + const expired = expiredBufferedWakeRateLimitFrames( + drained, + entry, + DateTime.toEpochMillis(receivedAt), + replayedAtMs, + ); + if (expired === null) continue; + for (const frame of expired) expiredWakeFrames.add(frame); + if (entry === lastResult) lastResultExpired = true; + } const opaqueReplayTombstones = taskIdSetForNativeThread( yield* Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread), nativeThreadId, @@ -12882,7 +12886,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ) { continue; } - if (entry.type !== "result" && !expiredWakeFrames?.has(entry)) { + if (entry.type !== "result" && !expiredWakeFrames.has(entry)) { yield* handleSdkMessage({ query: querySession.query, message: entry }); } yield* Ref.update(wakeBuffers, (current) => { @@ -12900,8 +12904,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return next; }); } - if (expiredWakeFrames !== null) { - // The buffered wake ended before the limit reset. It is no longer a + if (lastResultExpired && !emptyTurnBeforeNextTurn && !newerRootStillOpen) { + // The selected wake ended before the limit reset. It is no longer a // rejection of this continuation, so do not fail it or show stale errors. const completedAt = yield* DateTime.now; yield* finalizeActiveTurn({ From af0879d2388d2a4d63d02f69a5f41a81b938b779 Mon Sep 17 00:00:00 2001 From: surya-gorla <217902563+surya-gorla@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:40:54 +0530 Subject: [PATCH 5/7] fix(server): exclude overage-allowed wake resets from expiry Ignore overage-allowed embedded limit metadata when evaluating stale 429 wake results. Keep a 429 failure when no blocking reset remains; cover overage status and both overage-use flags. --- .../Adapters/ClaudeAdapterV2.test.ts | 58 ++++++++++++++++--- .../Adapters/ClaudeAdapterV2.ts | 12 +++- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 93da98b7e6b48..e2db2fb6ef8cb 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15464,6 +15464,33 @@ describe("ClaudeAdapterV2 background wake turns", () => { { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: true }, { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, { standaloneEvent: true, arrivedAfterReset: false, expired: true, weeklyStillActive: true }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + weeklyStillActive: true, + overageIndicator: "status", + }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + weeklyStillActive: true, + overageIndicator: "using", + }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + weeklyStillActive: true, + overageIndicator: "inUse", + }, + { + standaloneEvent: false, + arrivedAfterReset: false, + expired: true, + overageIndicator: "status", + }, { standaloneEvent: true, arrivedAfterReset: false, expired: true, twoWakes: true }, { standaloneEvent: true, @@ -15477,10 +15504,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { weeklyStillActive: false, twoWakes: false, secondWakeStillActive: false, + overageIndicator: "none", ...row, })), )( - "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive)", + "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive overage=$overageIndicator)", ({ standaloneEvent, arrivedAfterReset, @@ -15489,6 +15517,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { weeklyStillActive, twoWakes, secondWakeStillActive, + overageIndicator, }) => Effect.gen(function* () { const harness = yield* makeWakeHarness; @@ -15543,6 +15572,9 @@ describe("ClaudeAdapterV2 background wake turns", () => { : standaloneEvent ? resetsAt + 3_600 : resetsAt, + ...(overageIndicator === "status" ? { overageStatus: "allowed" } : {}), + ...(overageIndicator === "using" ? { isUsingOverage: true } : {}), + ...(overageIndicator === "inUse" ? { overageInUse: true } : {}), }, }, }), @@ -15654,7 +15686,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); const terminal = yield* Queue.take(harness.terminalReceipts); - if (expired && !weeklyStillActive && !secondWakeStillActive) { + if ( + expired && + !secondWakeStillActive && + (overageIndicator !== "none" ? standaloneEvent : !weeklyStillActive) + ) { assert.equal(terminal.status, "completed"); assert.isFalse( harness.events.some( @@ -15677,12 +15713,18 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(terminal.status, "failed"); if (terminal.status !== "failed") return; assert.equal(terminal.failure.class, "usage_limit"); - assert.equal( - terminal.failure.resetAt, - weeklyStillActive || secondWakeStillActive - ? "2026-10-02T01:10:00.000Z" - : "2026-09-25T01:10:00.000Z", - ); + if (!standaloneEvent && overageIndicator !== "none") { + // With no blocking reset evidence, preserve the actual 429 failure, + // but do not infer a reset timestamp from overage-allowed metadata. + assert.isNotOk(terminal.failure.resetAt); + } else { + assert.equal( + terminal.failure.resetAt, + weeklyStillActive || secondWakeStillActive + ? "2026-10-02T01:10:00.000Z" + : "2026-09-25T01:10:00.000Z", + ); + } if (standaloneEvent) { const pause = yield* Queue.take(harness.systemNoticeReceipts); assert.equal(pause.turnItem.type, "system_notice"); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6ca1878a2cf4c..5e7fae0a13db7 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -4490,13 +4490,23 @@ function expiredBufferedWakeRateLimitFrames( // Like result handling, a standalone event wins for its own limit type only. const embeddedResets = segment.flatMap((frame) => { if (!isClaudeBlockedAssistantRateLimit(frame)) return []; - const { rateLimitType, resetsAt } = frame.api_error_params.rate_limit_info; + const info = frame.api_error_params.rate_limit_info; + if ( + info.overageStatus === "allowed" || + info.overageStatus === "allowed_warning" || + info.isUsingOverage === true || + info.overageInUse === true + ) { + return []; + } + const { rateLimitType, resetsAt } = info; return directLimits.length === 0 || (rateLimitType != null && !directTypes.has(String(rateLimitType))) ? [resetsAt] : []; }); const resets = [...directLimits.map((limit) => limit.resetsAt), ...embeddedResets]; + // Without a blocking reset, an overage-allowed payload cannot prove a 429 expired. if ( resets.length === 0 || !resets.every( From 93442614dee8fc5b0396af1b1d2a8f99c82c4683 Mon Sep 17 00:00:00 2001 From: surya-gorla <217902563+surya-gorla@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:59:51 +0530 Subject: [PATCH 6/7] fix(server): preserve running wake during expired result replay Do not dispatch an expired buffered 429 into a newer active Claude wake. Carry its token and ledger usage into the newer root result; cover replay with and without an empty lifecycle result. --- .../Adapters/ClaudeAdapterV2.test.ts | 83 ++++++++++++++++--- .../Adapters/ClaudeAdapterV2.ts | 16 +++- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index e2db2fb6ef8cb..15c8af4531931 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15462,6 +15462,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: false }, { standaloneEvent: false, arrivedAfterReset: false, expired: true, newerTurn: false }, { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerTurn: true }, + { standaloneEvent: true, arrivedAfterReset: false, expired: true, newerRootOpen: true }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + newerRootOpen: true, + emptyBeforeNewRoot: true, + }, { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, { standaloneEvent: true, arrivedAfterReset: false, expired: true, weeklyStillActive: true }, { @@ -15501,6 +15509,8 @@ describe("ClaudeAdapterV2 background wake turns", () => { }, ].map((row) => ({ newerTurn: false, + newerRootOpen: false, + emptyBeforeNewRoot: false, weeklyStillActive: false, twoWakes: false, secondWakeStillActive: false, @@ -15508,12 +15518,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { ...row, })), )( - "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive overage=$overageIndicator)", + "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn rootOpen=$newerRootOpen emptyBeforeRoot=$emptyBeforeNewRoot weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive overage=$overageIndicator)", ({ standaloneEvent, arrivedAfterReset, expired, newerTurn, + newerRootOpen, + emptyBeforeNewRoot, weeklyStillActive, twoWakes, secondWakeStillActive, @@ -15638,19 +15650,37 @@ describe("ClaudeAdapterV2 background wake turns", () => { // An empty session-task lifecycle result arriving before T3 opens the // wake run must not replace the root's terminal usage-limit result. yield* harness.offerAndWait(wakeTurnInit); - yield* harness.offerAndWait( - makeResultFrame({ - uuid: "00000000-0000-4000-8000-000000000633", - result: "", - numTurns: 0, - origin: { kind: "task-notification" }, - terminalReason: null, - }), - ); + if (!newerRootOpen || emptyBeforeNewRoot) { + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000633", + result: "", + numTurns: 0, + origin: { kind: "task-notification" }, + terminalReason: null, + }), + ); + } if (expired) { yield* TestClock.setTime(resetsAt * 1_000 + 60_000); } + if (newerRootOpen) { + // The earlier 429 has expired, but another background wake has + // started and must be allowed to finish this same continuation. + yield* harness.offerAndWait( + claudeSdkFrame({ + ...wakeTurnInit, + uuid: "00000000-0000-4000-8000-000000000643", + }), + ); + yield* harness.offerAndWait( + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000644", + text: "A later wake is still running.", + }), + ); + } if (newerTurn) { // A queued continuation may run behind an ordinary turn served after reset. yield* harness.runtime.startTurn( @@ -15685,6 +15715,39 @@ describe("ClaudeAdapterV2 background wake turns", () => { messageCreationSource: "provider", }), ); + if (newerRootOpen) { + // The replay must neither fail nor prematurely complete the newer + // root before its own result arrives. + assert.lengthOf(harness.terminalEvents(), 1); + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000645", + result: "The newer background wake completed.", + origin: { kind: "task-notification" }, + }), + ); + const finished = yield* Queue.take(harness.terminalReceipts); + assert.equal(finished.status, "completed"); + const completed = harness.events.findLast( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "completed", + ); + assert.equal(completed?.type, "provider_turn.updated"); + if (completed?.type === "provider_turn.updated") { + // Count both the skipped older 429 result and the newer root's result. + assert.equal(completed.providerTurn.turnTokenUsage?.inputTokens, 2); + assert.equal(completed.providerTurn.turnTokenUsage?.outputTokens, 2); + } + assert.isFalse( + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "system_notice" && + event.turnItem.message.includes("usage limit reached"), + ), + ); + return; + } const terminal = yield* Queue.take(harness.terminalReceipts); if ( expired && diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 5e7fae0a13db7..1753ee63bc941 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -12914,7 +12914,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return next; }); } - if (lastResultExpired && !emptyTurnBeforeNextTurn && !newerRootStillOpen) { + if (lastResultExpired && newerRootStillOpen) { + // The expired result belongs to an earlier wake, not the + // still-running root. Keep its usage without settling that root, + // even if an empty task result was buffered after the 429. + if (lastResult?.type === "result") { + context.steeredSegmentUsage.push(lastResult.usage); + context.steeredLedgerUsage = addClaudeLedgerUsage( + context.steeredLedgerUsage, + resultLedgerUsage.get(lastResult), + ); + } + if (leavesLaterWake) yield* offerBufferedWake(nativeThreadId); + return; + } + if (lastResultExpired && !emptyTurnBeforeNextTurn) { // The selected wake ended before the limit reset. It is no longer a // rejection of this continuation, so do not fail it or show stale errors. const completedAt = yield* DateTime.now; From 850516a7c629561e198b3b51eef1ac33901a1d9f Mon Sep 17 00:00:00 2001 From: surya-gorla <217902563+surya-gorla@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:14 +0530 Subject: [PATCH 7/7] fix(server): settle absorbed wakes and account for all expired usage Let an expired buffered rejection settle when its newer root was absorbed by a user turn. Preserve usage from each expired buffered 429 once, including multiple wakes preceding an unfinished root. Exercise both absorption orders and distinct usage totals. --- .../Adapters/ClaudeAdapterV2.test.ts | 127 +++++++++++++++--- .../Adapters/ClaudeAdapterV2.ts | 32 +++-- 2 files changed, 128 insertions(+), 31 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 15c8af4531931..7c3903d8151ba 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -15470,6 +15470,28 @@ describe("ClaudeAdapterV2 background wake turns", () => { newerRootOpen: true, emptyBeforeNewRoot: true, }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + newerRootOpen: true, + absorbedByUser: true, + }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + newerRootOpen: true, + emptyBeforeNewRoot: true, + absorbedByUser: true, + }, + { + standaloneEvent: true, + arrivedAfterReset: false, + expired: true, + newerRootOpen: true, + twoWakes: true, + }, { standaloneEvent: true, arrivedAfterReset: true, expired: false, newerTurn: false }, { standaloneEvent: true, arrivedAfterReset: false, expired: true, weeklyStillActive: true }, { @@ -15511,6 +15533,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { newerTurn: false, newerRootOpen: false, emptyBeforeNewRoot: false, + absorbedByUser: false, weeklyStillActive: false, twoWakes: false, secondWakeStillActive: false, @@ -15518,7 +15541,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ...row, })), )( - "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn rootOpen=$newerRootOpen emptyBeforeRoot=$emptyBeforeNewRoot weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive overage=$overageIndicator)", + "replays a rejected wake rate limit only while it is current (event=$standaloneEvent arrivedAfterReset=$arrivedAfterReset expired=$expired newerTurn=$newerTurn rootOpen=$newerRootOpen emptyBeforeRoot=$emptyBeforeNewRoot absorbed=$absorbedByUser weekly=$weeklyStillActive twoWakes=$twoWakes secondActive=$secondWakeStillActive overage=$overageIndicator)", ({ standaloneEvent, arrivedAfterReset, @@ -15526,6 +15549,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { newerTurn, newerRootOpen, emptyBeforeNewRoot, + absorbedByUser, weeklyStillActive, twoWakes, secondWakeStillActive, @@ -15596,6 +15620,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000632", result: "You've hit your session limit ยท resets 11:10am (Australia/Sydney)", + ...(twoWakes && newerRootOpen + ? { + usage: { + input_tokens: 2, + output_tokens: 3, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + } + : {}), isError: true, apiErrorStatus: 429, terminalReason: "api_error", @@ -15638,6 +15672,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000639", result: "The second background wake also hit the limit.", + ...(newerRootOpen + ? { + usage: { + input_tokens: 4, + output_tokens: 5, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + } + : {}), isError: true, apiErrorStatus: 429, terminalReason: "api_error", @@ -15681,6 +15725,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); } + if (absorbedByUser) { + // A normal user message can consume the newer root's output, + // so no terminal SDK result will ever follow for the continuation. + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-wake-limit-absorber"), + text: "Did the new background task finish?", + attachments: [], + providerTurnOrdinal: 2, + }), + ); + yield* harness.until(() => harness.offeredMessages.length === 2); + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000646", + result: "The background task was absorbed into this answer.", + }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); + } if (newerTurn) { // A queued continuation may run behind an ordinary turn served after reset. yield* harness.runtime.startTurn( @@ -15710,33 +15777,55 @@ describe("ClaudeAdapterV2 background wake turns", () => { attemptId: RunAttemptId.make("attempt-claude-wake-limit-2"), text: "Background task completed.", attachments: [], - providerTurnOrdinal: newerTurn ? 3 : 2, + providerTurnOrdinal: newerTurn || absorbedByUser ? 3 : 2, messageCreatedBy: "agent", messageCreationSource: "provider", }), ); if (newerRootOpen) { - // The replay must neither fail nor prematurely complete the newer - // root before its own result arrives. - assert.lengthOf(harness.terminalEvents(), 1); - yield* harness.offerAndWait( - makeResultFrame({ - uuid: "00000000-0000-4000-8000-000000000645", - result: "The newer background wake completed.", - origin: { kind: "task-notification" }, - }), - ); - const finished = yield* Queue.take(harness.terminalReceipts); - assert.equal(finished.status, "completed"); + // An unabsorbed root must not settle until its own result arrives. + // An absorbed root has no result left to deliver and must settle now. + yield* yieldTicks(20); + assert.lengthOf(harness.terminalEvents(), absorbedByUser ? 3 : 1); + if (!absorbedByUser) { + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000645", + result: "The newer background wake completed.", + ...(twoWakes + ? { + usage: { + input_tokens: 7, + output_tokens: 11, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + } + : {}), + origin: { kind: "task-notification" }, + }), + ); + } + const finished = absorbedByUser + ? harness.terminalEvents().at(-1) + : yield* Queue.take(harness.terminalReceipts); + assert.equal(finished?.status, "completed"); const completed = harness.events.findLast( (event) => event.type === "provider_turn.updated" && event.providerTurn.status === "completed", ); assert.equal(completed?.type, "provider_turn.updated"); if (completed?.type === "provider_turn.updated") { - // Count both the skipped older 429 result and the newer root's result. - assert.equal(completed.providerTurn.turnTokenUsage?.inputTokens, 2); - assert.equal(completed.providerTurn.turnTokenUsage?.outputTokens, 2); + // Retain every charged rejected wake exactly once, even when a + // newer root is absorbed by a user turn and emits no own result. + assert.equal( + completed.providerTurn.turnTokenUsage?.inputTokens, + twoWakes ? 13 : absorbedByUser ? 1 : 2, + ); + assert.equal( + completed.providerTurn.turnTokenUsage?.outputTokens, + twoWakes ? 19 : absorbedByUser ? 1 : 2, + ); } assert.isFalse( harness.events.some( @@ -15768,8 +15857,8 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); assert.equal(completed?.type, "provider_turn.updated"); if (completed?.type === "provider_turn.updated") { - assert.equal(completed.providerTurn.turnTokenUsage?.inputTokens, 1); - assert.equal(completed.providerTurn.turnTokenUsage?.outputTokens, 1); + assert.equal(completed.providerTurn.turnTokenUsage?.inputTokens, twoWakes ? 2 : 1); + assert.equal(completed.providerTurn.turnTokenUsage?.outputTokens, twoWakes ? 2 : 1); } return; } diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 1753ee63bc941..fa96545ba4eda 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -12864,6 +12864,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // Suppress expired evidence from every completed wake, but only the // selected terminal result may complete this continuation. const expiredWakeFrames = new Set(); + const expiredResults: SDKResultMessage[] = []; let lastResultExpired = false; const replayedAtMs = DateTime.toEpochMillis(yield* DateTime.now); for (const entry of drained) { @@ -12878,6 +12879,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); if (expired === null) continue; for (const frame of expired) expiredWakeFrames.add(frame); + expiredResults.push(entry); if (entry === lastResult) lastResultExpired = true; } const opaqueReplayTombstones = taskIdSetForNativeThread( @@ -12914,21 +12916,27 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return next; }); } - if (lastResultExpired && newerRootStillOpen) { - // The expired result belongs to an earlier wake, not the - // still-running root. Keep its usage without settling that root, - // even if an empty task result was buffered after the 429. - if (lastResult?.type === "result") { - context.steeredSegmentUsage.push(lastResult.usage); - context.steeredLedgerUsage = addClaudeLedgerUsage( - context.steeredLedgerUsage, - resultLedgerUsage.get(lastResult), - ); - } + // Every expired result was consumed from the wake buffer. Retain its + // charged usage once, except when the selected result is about to + // supply that usage through finalizeActiveTurn itself. + const selectedResultWillFinalize = + lastResultExpired && + (wakeAbsorbed || (!emptyTurnBeforeNextTurn && !newerRootStillOpen)); + for (const entry of expiredResults) { + if (selectedResultWillFinalize && entry === lastResult) continue; + context.steeredSegmentUsage.push(entry.usage); + context.steeredLedgerUsage = addClaudeLedgerUsage( + context.steeredLedgerUsage, + resultLedgerUsage.get(entry), + ); + } + if (lastResultExpired && newerRootStillOpen && !wakeAbsorbed) { + // A newer root still owns this continuation. Do not dispatch + // the expired result or settle the root before its own result. if (leavesLaterWake) yield* offerBufferedWake(nativeThreadId); return; } - if (lastResultExpired && !emptyTurnBeforeNextTurn) { + if (lastResultExpired && (!emptyTurnBeforeNextTurn || wakeAbsorbed)) { // The selected wake ended before the limit reset. It is no longer a // rejection of this continuation, so do not fail it or show stale errors. const completedAt = yield* DateTime.now;