From a630ed8532e55371f275501d3058e78dd299f0bd Mon Sep 17 00:00:00 2001
From: Zenco <64717583+notzenco@users.noreply.github.com>
Date: Mon, 9 Nov 2026 04:34:49 +0800
Subject: [PATCH] fix(server): stop retrying auto-resume for a disabled
 provider every tick

When a thread armed for usage-limit auto-resume belongs to a provider
instance the user has since disabled, the resume dispatch is refused
before it reaches the orchestrator. The sweep counted that refusal as
work done, so it skipped recording its idle hint and repeated the
candidate query, thread reads, rejected dispatch and a warning every
5 s for as long as the instance stayed disabled.

Treat that refusal as a wait instead. Re-enabling the instance is a
settings change, which already wakes the sweep, and other dispatch
failures still retry on the next tick.
---
 .../UsageLimitRecoveryWorker.test.ts          | 65 ++++++++++++++++++-
 .../UsageLimitRecoveryWorker.ts               | 11 +++-
 2 files changed, 72 insertions(+), 4 deletions(-)

diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts
index 430018adcb06a..396dfc9fbc9e4 100644
--- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts
+++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts
@@ -1,5 +1,11 @@
 import { assert, describe, it } from "@effect/vitest";
-import { RunId, ThreadId, type OrchestrationV2DomainEvent } from "@t3tools/contracts";
+import {
+  ProjectId,
+  ProviderInstanceId,
+  RunId,
+  ThreadId,
+  type OrchestrationV2DomainEvent,
+} from "@t3tools/contracts";
 import * as Clock from "effect/Clock";
 import * as DateTime from "effect/DateTime";
 import * as Effect from "effect/Effect";
@@ -92,6 +98,63 @@ const failedThread = (failedAgoMs: number, resetAgoMs: number) => {
   return candidate;
 };
 
+it.effect("a resume refused for a disabled provider instance waits for a settings change", () =>
+  Effect.gen(function* () {
+    yield* TestClock.setTime(nowMs);
+    const queries = yield* Queue.unbounded<number>();
+    const settingsChanges = yield* Queue.unbounded<never>();
+    const failed = failedThread(2 * HOUR_MS, HOUR_MS);
+    const armed: ProjectionStore.ProjectionLimitRecoveryCandidate = {
+      ...failed,
+      limitRecovery: {
+        runId: RunId.make("run:limited"),
+        resetAt: failed.usageLimitResetAt ?? "",
+        autoResume: true,
+      },
+    };
+    const layer = UsageLimitRecoveryWorker.layer.pipe(
+      Layer.provide(
+        Layer.mergeAll(
+          Scheduler.layer,
+          Layer.mock(ProjectionStore.ProjectionStoreV2)({
+            getLimitRecoveryCandidates: () =>
+              Clock.currentTimeMillis.pipe(
+                Effect.flatMap((now) => Queue.offer(queries, now - nowMs)),
+                Effect.as([armed]),
+              ),
+            getLimitRecoveryWakeAt: () => Effect.succeed(null),
+          }),
+          Layer.mock(ThreadManagement.ThreadManagementService)({
+            dispatch: () =>
+              Effect.fail(
+                new ThreadManagement.ThreadManagementProviderInstanceDisabledError({
+                  projectId: ProjectId.make("project:limited"),
+                  instanceId: ProviderInstanceId.make("codex"),
+                }),
+              ),
+          }),
+          Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.never }),
+          Layer.mock(ServerSettings.ServerSettingsService)({
+            getSettings: Effect.succeed({
+              autoResumeLimitedThreads: true,
+              snoozeLimitedThreads: false,
+            } as never),
+            streamChanges: Stream.fromQueue(settingsChanges),
+          }),
+        ),
+      ),
+    );
+    yield* Effect.gen(function* () {
+      assert.equal(yield* Queue.take(queries), 0);
+      yield* TestClock.adjust("60 seconds");
+      // Re-enabling the instance is a settings change, so only then does the sweep query again.
+      yield* Queue.offer(settingsChanges, {} as never);
+      yield* TestClock.adjust("5 seconds");
+      assert.equal(yield* Queue.take(queries), 65_000);
+    }).pipe(Effect.provide(layer));
+  }),
+);
+
 describe("limitRecoveryCommand arming", () => {
   it("does not freshly arm a failure whose reset passed days ago", () => {
     const thread = failedThread(5 * 24 * HOUR_MS, 4 * 24 * HOUR_MS);
diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
index 985815bd73eb4..7f3f78f76047d 100644
--- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
+++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
@@ -119,8 +119,7 @@ const makeSweep = Effect.gen(function* () {
         preferences.snoozeLimitedThreads,
       );
       if (command === null) continue;
-      acted = true;
-      yield* Effect.gen(function* () {
+      const waitsForSettings = yield* Effect.gen(function* () {
         if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
           yield* orchestrator.autoArmLimitRecovery({
             commandId: command.commandId,
@@ -131,13 +130,19 @@ const makeSweep = Effect.gen(function* () {
           yield* threads.dispatch(command);
         }
       }).pipe(
+        Effect.as(false),
+        // Only re-enabling the instance lets it through, and that settings change wakes the sweep.
+        Effect.catchTags({
+          ThreadManagementProviderInstanceDisabledError: () => Effect.succeed(true),
+        }),
         Effect.catchCause((cause) =>
           Effect.logWarning("orchestration-v2.limit-recovery.dispatch-failed", {
             threadId: thread.id,
             cause,
-          }),
+          }).pipe(Effect.as(false)),
         ),
       );
+      if (!waitsForSettings) acted = true;
     }
     // A sweep that acted, or failed to, re-checks on the next tick.
     if (acted) return;