From 3e69b685b9cd7b708160f32f4e3f4a1c56e388bb Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 03:20:50 +0600 Subject: [PATCH 1/8] fix(server): a usage-limited thread no longer auto-settles A run that stops on a usage limit ends the activity that kept the thread active, so the settle sweep could settle it in the same second (for example when its pull request had merged). Settled threads refuse limit recovery, so Resume at reset and Snooze until reset then always failed. Settlement candidates now skip a thread whose latest run failed with a usage_limit root error, and isAutoSettlementCandidate applies the same rule to shell threads. --- .../ProjectionSettlement.test.ts | 57 ++++++++++++++++++- .../src/orchestration-v2/ProjectionStore.ts | 12 ++++ .../ThreadSettlementService.test.ts | 11 ++++ .../ThreadSettlementService.ts | 2 + 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 1352e17585bfd..442bc5e0a67b4 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -136,6 +136,44 @@ const createItem = Effect.fn(function* ( }); }); +const createFailure = Effect.fn(function* ( + threadId: ThreadId, + runId: RunId, + failureClass: "usage_limit" | "provider_error", +) { + const store = yield* ProjectionStore.ProjectionStoreV2; + yield* store.apply({ + id: EventId.make(`event:failure:${runId}`), + type: "turn-item.updated", + threadId, + occurredAt: old, + payload: { + id: TurnItemId.make(`failure:${runId}`), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + type: "error", + status: "failed", + title: failureClass, + startedAt: old, + completedAt: old, + updatedAt: old, + failure: { + class: failureClass, + message: failureClass, + code: null, + retryable: true, + resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + }, + }, + }); +}); + it.effect( "sql: discovers settlement work with the same activity and background semantics as the shell", () => @@ -210,6 +248,14 @@ it.effect( snoozedAt: DateTime.subtract(old, { days: 1 }), }); yield* createRun(woke, "failed"); + const limited = yield* createThread("usage-limited"); + yield* createFailure(limited, yield* createRun(limited, "failed"), "usage_limit"); + const providerFailed = yield* createThread("provider-failed"); + yield* createFailure( + providerFailed, + yield* createRun(providerFailed, "failed"), + "provider_error", + ); const background = yield* createThread("idle-background"); yield* createItem(background, yield* createRun(background), "idle"); const persistent = yield* createThread("persistent-monitor"); @@ -250,7 +296,16 @@ it.effect( ); assert.deepEqual( new Set(eligible.map((thread) => thread.id)), - new Set([idle, completed, queued, woke, background, persistent, rolledBack]), + new Set([ + idle, + completed, + queued, + woke, + providerFailed, + background, + persistent, + rolledBack, + ]), ); assert.deepEqual( new Set(eligible.map((thread) => thread.id)), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2e88fdc119081..1347844d037e5 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -212,6 +212,7 @@ export type ProjectionSettlementCandidate = Pick< | "latestRunCompletedAt" | "latestUserMessageAt" | "status" + | "lastErrorClass" | "activityRunStartedAt" | "activityRunStatus" | "pendingRuntimeRequest" @@ -6292,6 +6293,15 @@ export const layer: Layer.Layer = WHERE link.thread_id = t.thread_id AND link.type = 'link_request' AND link.status = 'waiting' ) + AND NOT ( + r.status IS 'failed' AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items failure + WHERE failure.thread_id = t.thread_id AND failure.run_id = r.run_id + AND failure.type = 'error' AND failure.status = 'failed' + AND failure.node_id IS json_extract(r.payload_json, '$.rootNodeId') + AND json_extract(failure.payload_json, '$.failure.class') = 'usage_limit' + ) + ) ORDER BY t.updated_at ASC, t.thread_id ASC `; if (rows.length === 0) return []; @@ -6339,6 +6349,8 @@ export const layer: Layer.Layer = activityRunStatus: null, activityRunStartedAt: null, pendingRuntimeRequest: null, + // The query above drops usage-limited threads. + lastErrorClass: null, pendingGuestMessageCount: pendingGuestMessageCount(thread.guestShare), pendingBackgroundTasks: derivePendingBackgroundWork({ latestRun: diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts index 0a0996c0ede0c..607c2986197aa 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts @@ -158,6 +158,17 @@ describe("isAutoSettlementCandidate", () => { expect(ThreadSettlementService.isSnoozed(snoozed, NOW_MS)).toBe(false); }); + it("never settles a thread stopped by a usage limit", () => { + const limited = shell({ status: "failed", lastErrorClass: "usage_limit" }); + expect(ThreadSettlementService.isAutoSettlementCandidate(limited, NOW_MS)).toBe(false); + expect( + ThreadSettlementService.isAutoSettlementCandidate( + { ...limited, lastErrorClass: "provider_error" }, + NOW_MS, + ), + ).toBe(true); + }); + it("keeps snoozed threads parked until a fresh failure or timer wake", () => { const snoozed = shell({ snoozedUntil: at(60 * 60 * 1_000), diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.ts index 6a3d4fffb0399..32879f94d4ea0 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.ts @@ -152,6 +152,8 @@ export function isAutoSettlementCandidate( // A guest is waiting on the owner; settling would hide the Request. if ((thread.pendingGuestMessageCount ?? 0) > 0) return false; if (thread.activityRunStatus != null) return false; + // A usage limit interrupts the work; settling would hide Resume at reset. + if (thread.status === "failed" && thread.lastErrorClass === "usage_limit") return false; if (backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? [])) return false; if (threadHasQueuedTurnStart(thread, nowMs)) return false; // Interrupted runs must keep the same snooze protection the sidebar shows. From 54eada734f34c3cd8342ba5a0e7998eb583a51f8 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 03:52:34 +0600 Subject: [PATCH 2/8] fix(server): match the shell's blocked run when skipping usage-limited threads A message queued behind a usage limit becomes the newest run, so checking only the newest run let the sweep settle a thread the shell still shows as limited. Use the latest executed run, as the shell and the recovery worker do, and its latest root error. --- .../ProjectionSettlement.test.ts | 8 +++++ .../src/orchestration-v2/ProjectionStore.ts | 34 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 442bc5e0a67b4..e02a657e69e75 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -250,6 +250,14 @@ it.effect( yield* createRun(woke, "failed"); const limited = yield* createThread("usage-limited"); yield* createFailure(limited, yield* createRun(limited, "failed"), "usage_limit"); + // A message queued behind the limit is the newest run, but the limit still blocks. + const queuedBehindLimit = yield* createThread("queued-behind-limit"); + yield* createFailure( + queuedBehindLimit, + yield* createRun(queuedBehindLimit, "failed"), + "usage_limit", + ); + yield* createRun(queuedBehindLimit, "queued", 2); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 1347844d037e5..2a4117e72c691 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -6293,14 +6293,42 @@ export const layer: Layer.Layer = WHERE link.thread_id = t.thread_id AND link.type = 'link_request' AND link.status = 'waiting' ) + -- A usage limit interrupts work; settling would hide its recovery controls. + -- Matches the shell's blocked run, so a message queued behind the limit + -- does not hide it. The cheap EXISTS keeps the run sort off other threads. AND NOT ( - r.status IS 'failed' AND EXISTS ( + EXISTS ( SELECT 1 FROM orchestration_v2_projection_turn_items failure - WHERE failure.thread_id = t.thread_id AND failure.run_id = r.run_id + WHERE failure.thread_id = t.thread_id AND failure.type = 'error' AND failure.status = 'failed' - AND failure.node_id IS json_extract(r.payload_json, '$.rootNodeId') AND json_extract(failure.payload_json, '$.failure.class') = 'usage_limit' ) + AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs blocked + WHERE blocked.run_id = ( + SELECT latest.run_id FROM orchestration_v2_projection_runs latest + WHERE latest.thread_id = t.thread_id + AND latest.status <> 'queued' + AND NOT ( + latest.status = 'cancelled' + AND json_extract(latest.payload_json, '$.startedAt') IS NULL + ) + -- latestExecutedRun: the run that ended last (runRanAfter). + ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC, + latest.ordinal DESC, latest.run_id DESC + LIMIT 1 + ) + AND blocked.status = 'failed' + AND ( + SELECT json_extract(error.payload_json, '$.failure.class') + FROM orchestration_v2_projection_turn_items error + WHERE error.thread_id = t.thread_id AND error.run_id = blocked.run_id + AND error.type = 'error' AND error.status = 'failed' + AND error.node_id IS json_extract(blocked.payload_json, '$.rootNodeId') + ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC + LIMIT 1 + ) = 'usage_limit' + ) ) ORDER BY t.updated_at ASC, t.thread_id ASC `; From b2c9981a7418db9e0c4476262745c99a116b3259 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 03:56:13 +0600 Subject: [PATCH 3/8] test(server): cover a usage limit that ended after a newer run --- .../orchestration-v2/ProjectionSettlement.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index e02a657e69e75..c822b42534323 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -72,6 +72,7 @@ const createRun = Effect.fn(function* ( threadId: ThreadId, status: OrchestrationV2Run["status"] = "completed", ordinal = 1, + completedAt = old, ) { const store = yield* ProjectionStore.ProjectionStoreV2; const runId = RunId.make(`run:${threadId}:${ordinal}`); @@ -93,7 +94,7 @@ const createRun = Effect.fn(function* ( status, requestedAt: old, startedAt: status === "queued" ? null : old, - completedAt: ["completed", "rolled_back", "failed"].includes(status) ? old : null, + completedAt: ["completed", "rolled_back", "failed"].includes(status) ? completedAt : null, checkpointId: null, contextHandoffId: null, }, @@ -258,6 +259,18 @@ it.effect( "usage_limit", ); yield* createRun(queuedBehindLimit, "queued", 2); + // Held queues run out of order; the run that ended last decides, not the ordinal. + const limitEndedLast = yield* createThread("limit-ended-last"); + yield* createFailure( + limitEndedLast, + yield* createRun(limitEndedLast, "failed", 1, DateTime.add(old, { hours: 1 })), + "usage_limit", + ); + yield* createFailure( + limitEndedLast, + yield* createRun(limitEndedLast, "failed", 2), + "provider_error", + ); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, From 1b7badf4fdfdd759fc639de2e63826f5161f5931 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 04:11:03 +0600 Subject: [PATCH 4/8] fix(server): a differing session error lets a usage-limited thread settle The shell drops the usage-limit class when the provider session reports a different, newer error, so it shows no recovery controls. The settlement query now applies the same rule instead of holding such a thread. --- .../ProjectionSettlement.test.ts | 30 +++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 15 ++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index c822b42534323..c0f3a703823da 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -8,6 +8,7 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, + ProviderSessionId, ProviderThreadId, RunId, RuntimeRequestId, @@ -19,6 +20,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import { isAutoSettlementCandidate, resolveAutoSettlementAt } from "./ThreadSettlementService.ts"; @@ -271,6 +273,33 @@ it.effect( yield* createRun(limitEndedLast, "failed", 2), "provider_error", ); + // A newer, different session error replaces the limit, as in threadErrorSummary. + const sessionErrorAfterLimit = yield* createThread("session-error-after-limit"); + yield* createFailure( + sessionErrorAfterLimit, + yield* createRun(sessionErrorAfterLimit, "failed"), + "usage_limit", + ); + yield* store.apply({ + id: EventId.make("event:settlement:session-error"), + type: "provider-session.attached", + threadId: sessionErrorAfterLimit, + driver: ProviderDriverKind.make("codex"), + providerInstanceId, + occurredAt: old, + payload: { + id: ProviderSessionId.make("provider-session:settlement:session-error"), + driver: ProviderDriverKind.make("codex"), + providerInstanceId, + status: "error", + cwd: "/workspace", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: old, + updatedAt: old, + lastError: "Session crashed.", + }, + }); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, @@ -323,6 +352,7 @@ it.effect( queued, woke, providerFailed, + sessionErrorAfterLimit, background, persistent, rolledBack, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2a4117e72c691..75a88ff1883a3 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -6319,15 +6319,26 @@ export const layer: Layer.Layer = LIMIT 1 ) AND blocked.status = 'failed' + -- threadErrorSummary: a differing session error replaces the failure. AND ( - SELECT json_extract(error.payload_json, '$.failure.class') + SELECT json_extract(error.payload_json, '$.failure.class') = 'usage_limit' + AND COALESCE(( + SELECT json_extract(session.payload_json, '$.lastError') + FROM orchestration_v2_projection_provider_session_bindings binding + CROSS JOIN orchestration_v2_projection_provider_sessions session + ON session.provider_session_id = binding.provider_session_id + WHERE binding.thread_id = t.thread_id + AND session.provider_instance_id = t.provider_instance_id + ORDER BY session.updated_at DESC, session.provider_session_id DESC + LIMIT 1 + ) = json_extract(error.payload_json, '$.failure.message'), 1) FROM orchestration_v2_projection_turn_items error WHERE error.thread_id = t.thread_id AND error.run_id = blocked.run_id AND error.type = 'error' AND error.status = 'failed' AND error.node_id IS json_extract(blocked.payload_json, '$.rootNodeId') ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC LIMIT 1 - ) = 'usage_limit' + ) = 1 ) ) ORDER BY t.updated_at ASC, t.thread_id ASC From 26144a20e01c7bf55e16659a4051ac8225fe55d7 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 04:23:18 +0600 Subject: [PATCH 5/8] test(server): cover a session error that matches the usage limit --- .../ProjectionSettlement.test.ts | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index c0f3a703823da..55258bf010666 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -177,6 +177,31 @@ const createFailure = Effect.fn(function* ( }); }); +const attachSessionError = Effect.fn(function* (threadId: ThreadId, lastError: string) { + const store = yield* ProjectionStore.ProjectionStoreV2; + const driver = ProviderDriverKind.make("codex"); + yield* store.apply({ + id: EventId.make(`event:session:${threadId}`), + type: "provider-session.attached", + threadId, + driver, + providerInstanceId, + occurredAt: old, + payload: { + id: ProviderSessionId.make(`provider-session:${threadId}`), + driver, + providerInstanceId, + status: "error", + cwd: "/workspace", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: old, + updatedAt: old, + lastError, + }, + }); +}); + it.effect( "sql: discovers settlement work with the same activity and background semantics as the shell", () => @@ -273,33 +298,21 @@ it.effect( yield* createRun(limitEndedLast, "failed", 2), "provider_error", ); - // A newer, different session error replaces the limit, as in threadErrorSummary. + // threadErrorSummary: a matching session error keeps the limit, a differing one replaces it. + const sessionErrorMatchingLimit = yield* createThread("session-error-matching-limit"); + yield* createFailure( + sessionErrorMatchingLimit, + yield* createRun(sessionErrorMatchingLimit, "failed"), + "usage_limit", + ); + yield* attachSessionError(sessionErrorMatchingLimit, "usage_limit"); const sessionErrorAfterLimit = yield* createThread("session-error-after-limit"); yield* createFailure( sessionErrorAfterLimit, yield* createRun(sessionErrorAfterLimit, "failed"), "usage_limit", ); - yield* store.apply({ - id: EventId.make("event:settlement:session-error"), - type: "provider-session.attached", - threadId: sessionErrorAfterLimit, - driver: ProviderDriverKind.make("codex"), - providerInstanceId, - occurredAt: old, - payload: { - id: ProviderSessionId.make("provider-session:settlement:session-error"), - driver: ProviderDriverKind.make("codex"), - providerInstanceId, - status: "error", - cwd: "/workspace", - model: modelSelection.model, - capabilities: CodexProviderCapabilitiesV2, - createdAt: old, - updatedAt: old, - lastError: "Session crashed.", - }, - }); + yield* attachSessionError(sessionErrorAfterLimit, "Session crashed."); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, From e62eae893d2d62b93e7ae460638de3ac74249d2f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 21:13:12 +0000 Subject: [PATCH 6/8] fix(server): abandoned usage-limited subagents still settle for cleanup A subagent whose parent is settled, archived or deleted settles so its terminals close and its settle script runs. The usage-limit hold now applies to such a child only while auto-resume is armed for its blocked run, which the limit recovery worker then owns. AI-assisted; verified with the tests and tools listed above. --- .../ProjectionSettlement.test.ts | 25 +++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 9 +++++++ .../ThreadSettlementService.ts | 12 +++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 53ffbeef2ec31..333dc3fd67248 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -393,6 +393,30 @@ it.effect( yield* createRun(providerFailed, "failed"), "provider_error", ); + // A subagent of a settled parent still settles for cleanup unless recovery is armed. + const putAwayParent = yield* createThread("put-away-parent", { settledOverride: "settled" }); + const childOf = () => ({ + lineage: { + parentThreadId: putAwayParent, + relationshipToParent: "subagent" as const, + rootThreadId: putAwayParent, + }, + }); + const abandonedLimited = yield* createThread("abandoned-limited", childOf()); + yield* createFailure( + abandonedLimited, + yield* createRun(abandonedLimited, "failed"), + "usage_limit", + ); + const armedChild = yield* createThread("armed-child", { + ...childOf(), + limitRecovery: { + runId: RunId.make("run:thread:settlement:armed-child:1"), + resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + autoResume: true, + }, + }); + yield* createFailure(armedChild, yield* createRun(armedChild, "failed"), "usage_limit"); const background = yield* createThread("idle-background"); yield* createItem(background, yield* createRun(background), "idle"); const persistent = yield* createThread("persistent-monitor"); @@ -440,6 +464,7 @@ it.effect( woke, providerFailed, sessionErrorAfterLimit, + abandonedLimited, background, persistent, rolledBack, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8b01e043d9b97..6250c4e7f83fb 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -227,6 +227,7 @@ export type ProjectionSettlementCandidate = Pick< | "latestUserMessageAt" | "status" | "lastErrorClass" + | "limitRecovery" | "activityRunStartedAt" | "activityRunStatus" | "pendingRuntimeRequest" @@ -7949,6 +7950,14 @@ export const layer: Layer.Layer = LIMIT 1 ) AND blocked.status = 'failed' + -- Abandoned subagents still settle for cleanup unless recovery is armed. + AND ( + json_extract(t.payload_json, '$.lineage.relationshipToParent') IS NOT 'subagent' + OR ( + json_extract(t.payload_json, '$.limitRecovery.runId') IS blocked.run_id + AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1 + ) + ) -- threadErrorSummary: a differing session error replaces the failure. AND ( SELECT json_extract(error.payload_json, '$.failure.class') = 'usage_limit' diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.ts index eaf3eeeb57265..c4bfb49d8e338 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.ts @@ -154,8 +154,16 @@ export function isAutoSettlementCandidate( // A guest is waiting on the owner; settling would hide the Request. if ((thread.pendingGuestMessageCount ?? 0) > 0) return false; if (thread.activityRunStatus != null) return false; - // A usage limit interrupts the work; settling would hide Resume at reset. - if (thread.status === "failed" && thread.lastErrorClass === "usage_limit") return false; + // A usage limit interrupts the work; settling would hide Resume at reset. Only subagents with + // armed recovery are held: the store returns them once their parent is put away. + if ( + thread.status === "failed" && + thread.lastErrorClass === "usage_limit" && + (thread.lineage.relationshipToParent !== "subagent" || + (thread.limitRecovery?.runId === thread.latestRunId && thread.limitRecovery.autoResume)) + ) { + return false; + } if (backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? [])) return false; if (threadHasQueuedTurnStart(thread, nowMs)) return false; // Interrupted runs must keep the same snooze protection the sidebar shows. From 2e962b2c5b08d2557316515b88581b168f98aaf2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 21:21:52 +0000 Subject: [PATCH 7/8] fix(server): abandoned subagents settle even with limit recovery armed Children of a settled, archived or deleted parent are abandoned work. Holding one because auto-resume was armed let the resume fire at reset and reopen the settled parent. The usage-limit hold now applies only to non-subagent threads, as main already settles abandoned children. AI-assisted; verified with the tests and tools listed above. --- .../src/orchestration-v2/ProjectionSettlement.test.ts | 3 ++- apps/server/src/orchestration-v2/ProjectionStore.ts | 11 ++--------- .../src/orchestration-v2/ThreadSettlementService.ts | 7 +++---- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 333dc3fd67248..9e30b780588c5 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -393,7 +393,7 @@ it.effect( yield* createRun(providerFailed, "failed"), "provider_error", ); - // A subagent of a settled parent still settles for cleanup unless recovery is armed. + // A subagent of a settled parent is abandoned work; it settles even with recovery armed. const putAwayParent = yield* createThread("put-away-parent", { settledOverride: "settled" }); const childOf = () => ({ lineage: { @@ -465,6 +465,7 @@ it.effect( providerFailed, sessionErrorAfterLimit, abandonedLimited, + armedChild, background, persistent, rolledBack, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 6250c4e7f83fb..2299547396d85 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -227,7 +227,6 @@ export type ProjectionSettlementCandidate = Pick< | "latestUserMessageAt" | "status" | "lastErrorClass" - | "limitRecovery" | "activityRunStartedAt" | "activityRunStatus" | "pendingRuntimeRequest" @@ -7950,14 +7949,8 @@ export const layer: Layer.Layer = LIMIT 1 ) AND blocked.status = 'failed' - -- Abandoned subagents still settle for cleanup unless recovery is armed. - AND ( - json_extract(t.payload_json, '$.lineage.relationshipToParent') IS NOT 'subagent' - OR ( - json_extract(t.payload_json, '$.limitRecovery.runId') IS blocked.run_id - AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1 - ) - ) + -- Subagents here have a put-away parent: abandoned work that still settles. + AND json_extract(t.payload_json, '$.lineage.relationshipToParent') IS NOT 'subagent' -- threadErrorSummary: a differing session error replaces the failure. AND ( SELECT json_extract(error.payload_json, '$.failure.class') = 'usage_limit' diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.ts index c4bfb49d8e338..ebbd45dc50b2f 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.ts @@ -154,13 +154,12 @@ export function isAutoSettlementCandidate( // A guest is waiting on the owner; settling would hide the Request. if ((thread.pendingGuestMessageCount ?? 0) > 0) return false; if (thread.activityRunStatus != null) return false; - // A usage limit interrupts the work; settling would hide Resume at reset. Only subagents with - // armed recovery are held: the store returns them once their parent is put away. + // A usage limit interrupts the work; settling would hide Resume at reset. Subagents reach the + // sweep only once their parent is put away, and abandoned work settles even when limited. if ( thread.status === "failed" && thread.lastErrorClass === "usage_limit" && - (thread.lineage.relationshipToParent !== "subagent" || - (thread.limitRecovery?.runId === thread.latestRunId && thread.limitRecovery.autoResume)) + thread.lineage.relationshipToParent !== "subagent" ) { return false; } From 2d062a6170b64ad610b58b3e710d86200b91eb61 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 23:59:19 +0000 Subject: [PATCH 8/8] test(server): trim usage-limit settlement cases to the SQL branches Co-Authored-By: Claude Opus 5.5 --- .../ProjectionSettlement.test.ts | 17 +++-------------- .../ThreadSettlementService.test.ts | 11 ----------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 9e30b780588c5..fc998363b6a15 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -393,30 +393,20 @@ it.effect( yield* createRun(providerFailed, "failed"), "provider_error", ); - // A subagent of a settled parent is abandoned work; it settles even with recovery armed. + // A subagent of a settled parent is abandoned work; it settles. const putAwayParent = yield* createThread("put-away-parent", { settledOverride: "settled" }); - const childOf = () => ({ + const abandonedLimited = yield* createThread("abandoned-limited", { lineage: { parentThreadId: putAwayParent, - relationshipToParent: "subagent" as const, + relationshipToParent: "subagent", rootThreadId: putAwayParent, }, }); - const abandonedLimited = yield* createThread("abandoned-limited", childOf()); yield* createFailure( abandonedLimited, yield* createRun(abandonedLimited, "failed"), "usage_limit", ); - const armedChild = yield* createThread("armed-child", { - ...childOf(), - limitRecovery: { - runId: RunId.make("run:thread:settlement:armed-child:1"), - resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })), - autoResume: true, - }, - }); - yield* createFailure(armedChild, yield* createRun(armedChild, "failed"), "usage_limit"); const background = yield* createThread("idle-background"); yield* createItem(background, yield* createRun(background), "idle"); const persistent = yield* createThread("persistent-monitor"); @@ -465,7 +455,6 @@ it.effect( providerFailed, sessionErrorAfterLimit, abandonedLimited, - armedChild, background, persistent, rolledBack, diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts index 74608ff00a718..baf51b8feffdd 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts @@ -164,17 +164,6 @@ describe("isAutoSettlementCandidate", () => { expect(ThreadSettlementService.isSnoozed(snoozed, NOW_MS)).toBe(false); }); - it("never settles a thread stopped by a usage limit", () => { - const limited = shell({ status: "failed", lastErrorClass: "usage_limit" }); - expect(ThreadSettlementService.isAutoSettlementCandidate(limited, NOW_MS)).toBe(false); - expect( - ThreadSettlementService.isAutoSettlementCandidate( - { ...limited, lastErrorClass: "provider_error" }, - NOW_MS, - ), - ).toBe(true); - }); - it("keeps snoozed threads parked until a fresh failure or timer wake", () => { const snoozed = shell({ snoozedUntil: at(60 * 60 * 1_000),