feat: continue on another account with usage left when one runs out

apps/server/src/orchestration-v2/CommandPolicy.ts

@@ -1,5 +1,6 @@
 import {
   type ChatAttachment,
+  type ModelSelection,
   CommandId,
   type OrchestrationV2Command,
   OrchestrationV2ProviderCapabilities,
@@ -16,6 +17,16 @@ import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Schema from "effect/Schema";
 
+// Account choice is preflighted outside the dispatch lock; a raced user action wins.
+export const AccountFallback = Context.Reference<
+  | {
+      readonly commandId: CommandId;
+      readonly threadSelection: ModelSelection;
+      readonly requestedSelection: ModelSelection | undefined;
+    }
+  | undefined
+>("t3/orchestration-v2/AccountFallback", { defaultValue: () => undefined });
+
 export const SteeringExecutionPolicyV2 = Schema.Literals(["active_steering", "interrupt_restart"]);
 export type SteeringExecutionPolicyV2 = typeof SteeringExecutionPolicyV2.Type;
 

apps/server/src/orchestration-v2/Orchestrator.ts

@@ -1,3 +1,4 @@
+import { AccountFallback } from "./CommandPolicy.ts";
 import type {
   OrchestrationV2ThreadLinkMetadata,
   GuestShareId,
@@ -6794,6 +6795,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
       });
       return;
     }
+    const fallback = yield* AccountFallback;
+    const automaticFallback = fallback?.commandId === command.commandId ? fallback : undefined;
+    if (automaticFallback && command.manualContinuationOfRunId === undefined) {
+      const mode = resolveMessageDispatchIntent(
+        projection,
+        command.dispatchMode,
+        command.deliveryIntent,
+      );
+      if (
+        mode.type === "steer_active" ||
+        mode.type === "restart_active" ||
+        !modelSelectionsEqual(projection.thread.modelSelection, automaticFallback.threadSelection)
+      ) {
+        command = {
+          ...command,
+          modelSelection: automaticFallback.requestedSelection ?? projection.thread.modelSelection,
+        };
+      }
+    }
     const activeProviderInstanceId = projection.providerThreads.find(
       (candidate) => candidate.id === projection.thread.activeProviderThreadId,
     )?.providerInstanceId;
@@ -6817,7 +6837,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
         latestExecutedRun(projection.runs)?.id !== source.id ||
         projection.thread.archivedAt !== null ||
         projection.thread.deletedAt !== null ||
-        projection.runtimeRequests.some((request) => request.status === "pending")
+        projection.runtimeRequests.some((request) => request.status === "pending") ||
+        (automaticFallback !== undefined &&
+          (projection.thread.settledOverride === "settled" ||
+            !modelSelectionsEqual(source.modelSelection, automaticFallback.threadSelection) ||
+            !modelSelectionsEqual(
+              projection.thread.modelSelection,
+              automaticFallback.threadSelection,
+            ) ||
+            (projection.thread.snoozedUntil != null &&
+              DateTime.toEpochMillis(projection.thread.snoozedUntil) >
+                DateTime.toEpochMillis(yield* DateTime.now)) ||
+            (yield* stopReachedRun(command, command.threadId, source.id))))
       ) {
         return yield* new OrchestratorDispatchError({
           commandId: command.commandId,

apps/server/src/orchestration-v2/ProjectionStore.test.ts

@@ -8307,6 +8307,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
         if (lastErrorClass === "usage_limit") {
           assert.deepEqual(candidate, {
             id: sqlShell.id,
+            projectId: sqlShell.projectId,
+            modelSelection: sqlShell.modelSelection,
             status: sqlShell.status,
             lastErrorClass: sqlShell.lastErrorClass,
             usageLimitResetAt: sqlShell.usageLimitResetAt,
@@ -8475,6 +8477,33 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
           (row) => row.id === threadId,
         ),
       );
+      for (const [since, found] of [
+        [DateTime.subtract(now, { minutes: 1 }), true],
+        [DateTime.add(now, { minutes: 1 }), false],
+      ] as const) {
+        assert.strictEqual(
+          (yield* store.getLimitRecoveryCandidates({
+            ...recoveryOptions,
+            switchAccountsSince: since,
+          })).some((row) => row.id === threadId),
+          found,
+        );
+      }
+      yield* sql`UPDATE orchestration_v2_projection_turn_items
+        SET payload_json = json_remove(payload_json, '$.failure.resetAt') WHERE turn_item_id = ${limitItem.id}`;
+      assert.isTrue(
+        (yield* store.getLimitRecoveryCandidates({
+          ...recoveryOptions,
+          switchAccountsSince: DateTime.subtract(now, { minutes: 1 }),
+        })).some((row) => row.id === threadId),
+      );
+      assert.isFalse(
+        (yield* store.getLimitRecoveryCandidates({ ...recoveryOptions, autoResume: true })).some(
+          (row) => row.id === threadId,
+        ),
+      );
+      yield* sql`UPDATE orchestration_v2_projection_turn_items
+        SET payload_json = json_set(payload_json, '$.failure.resetAt', ${limitItem.failure.resetAt}) WHERE turn_item_id = ${limitItem.id}`;
       const reset = DateTime.makeUnsafe(limitItem.failure.resetAt);
       const recovery = {
         runId: original.id,
@@ -8739,6 +8768,17 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
         deletedParentChild,
         activeParentChild,
       ];
+      assert.deepEqual(
+        (yield* store.getLimitRecoveryCandidates({
+          now,
+          autoResume: false,
+          snooze: false,
+          switchAccountsSince: DateTime.subtract(now, { minutes: 1 }),
+        }))
+          .filter((row) => children.includes(row.id))
+          .map((row) => row.id),
+        [activeParentChild],
+      );
       for (const options of [
         { now, autoResume: true, snooze: false },
         { now, autoResume: false, snooze: true },

apps/server/src/orchestration-v2/ProjectionStore.ts

@@ -184,6 +184,8 @@ export type ProjectionRecoveryKind =
 export type ProjectionLimitRecoveryCandidate = Pick<
   OrchestrationV2ThreadShell,
   | "id"
+  | "projectId"
+  | "modelSelection"
   | "status"
   | "lastErrorClass"
   | "latestRunId"
@@ -622,6 +624,8 @@ export interface ProjectionStoreV2Shape {
     readonly now: DateTime.Utc;
     readonly autoResume: boolean;
     readonly snooze: boolean;
+    /** Also returns limits hit since then whose window has not reset, to move to another account. */
+    readonly switchAccountsSince?: DateTime.Utc;
   }) => Effect.Effect<ReadonlyArray<ProjectionLimitRecoveryCandidate>, ProjectionStoreV2Error>;
   /** Epoch ms of the next armed reset or snooze end after `now`, when a waiting thread becomes a candidate. */
   readonly getLimitRecoveryWakeAt: (
@@ -4662,11 +4666,28 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
             AND json_extract(t.payload_json, '$.archivedAt') IS NULL
             AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled'
             AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit'
-            AND json_extract(item.payload_json, '$.failure.resetAt') IS NOT NULL
-            AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
             AND (
               (
-                json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
+                ${booleanInt(options.switchAccountsSince !== undefined)}
+                AND julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) >= julianday(${DateTime.formatIso(options.switchAccountsSince ?? options.now)})
+                AND (
+                  json_extract(item.payload_json, '$.failure.resetAt') IS NULL
+                  OR julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})
+                )
+                AND NOT EXISTS (
+                  SELECT 1 FROM orchestration_v2_projection_threads parent
+                  WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent'
+                    AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId')
+                    AND (
+                      json_extract(parent.payload_json, '$.settledOverride') = 'settled'
+                      OR json_extract(parent.payload_json, '$.archivedAt') IS NOT NULL
+                      OR parent.deleted_at IS NOT NULL
+                    )
+                )
+              )
+              OR (
+                julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
+                AND json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
                 AND json_extract(t.payload_json, '$.limitRecovery.resetAt') IS json_extract(item.payload_json, '$.failure.resetAt')
                 AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
                 AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) <= julianday(${DateTime.formatIso(options.now)})
@@ -4676,6 +4697,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
                 )
               )
               OR (
+                julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
+                AND
                 (
                   json_extract(t.payload_json, '$.limitRecovery.runId') IS NOT r.run_id
                   OR json_extract(t.payload_json, '$.limitRecovery.resetAt') IS NOT json_extract(item.payload_json, '$.failure.resetAt')
@@ -4714,6 +4737,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
           if (summary.lastErrorClass !== "usage_limit") continue;
           candidates.push({
             id: thread.id,
+            projectId: thread.projectId,
+            modelSelection: thread.modelSelection,
             status: "failed",
             lastErrorClass: summary.lastErrorClass,
             usageLimitResetAt: summary.usageLimitResetAt,

apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts

@@ -1,4 +1,5 @@
 import * as DocumentText from "../provider/DocumentText.ts";
+import { AccountFallback } from "./CommandPolicy.ts";
 // @effect-diagnostics nodeBuiltinImport:off
 import * as NodeFS from "node:fs";
 import * as NodePath from "node:path";
@@ -10,13 +11,17 @@ import {
   EventId,
   GuestMessageId,
   MessageId,
+  ProjectId,
+  ProviderDriverKind,
+  ProviderInstanceId,
   RunId,
   RuntimeRequestId,
   ThreadId,
   TurnItemId,
   type OrchestrationV2Command,
   type OrchestrationV2ServerCommand,
   type OrchestrationV2StoredEvent,
+  type ServerProvider,
   type UserInputAttachments,
 } from "@t3tools/contracts";
 import * as DateTime from "effect/DateTime";
@@ -28,6 +33,7 @@ import {
   resolveAttachmentPath,
 } from "../attachmentStore.ts";
 import * as ServerConfig from "../config.ts";
+import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
 import * as ServerSettings from "../serverSettings.ts";
 import {
   OrchestratorCommandPreviouslyRejectedError,
@@ -41,6 +47,9 @@ const layerIntakeTest = ServerConfig.layerTest(process.cwd(), {
 }).pipe(
   Layer.provideMerge(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
   Layer.provideMerge(ServerSettings.layerTest()),
+  Layer.provideMerge(
+    Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
+  ),
 );
 
 const layerFailingDispatchHandler =
@@ -428,6 +437,112 @@ it.effect("sends a guest message with the guest's files, never ones the client n
   }).pipe(Effect.provide(layerIntakeTest)),
 );
 
+it.effect.each([false, true])(
+  "routes an exhausted account while preserving manual resume: %s",
+  (manual) =>
+    Effect.gen(function* () {
+      const claude = (id: string, usedPercent: number): ServerProvider => ({
+        instanceId: ProviderInstanceId.make(id),
+        driver: ProviderDriverKind.make("claudeAgent"),
+        enabled: true,
+        installed: true,
+        version: null,
+        status: "ready",
+        auth: { status: "authenticated" },
+        checkedAt: "2026-09-13T00:00:00.000Z",
+        models: [
+          { slug: "claude-opus-5-5", name: "Opus 5.5", isCustom: false, capabilities: null },
+        ],
+        slashCommands: [],
+        skills: [],
+        usageLimits: {
+          checkedAt: "2026-09-13T00:00:00.000Z",
+          windows: [
+            {
+              id: "five_hour",
+              kind: "session",
+              label: "Session",
+              usedPercent,
+              resetsAt: "2099-01-01T00:00:00.000Z",
+            },
+          ],
+        },
+      });
+      const captured: OrchestrationV2ServerCommand[] = [];
+      const selection = {
+        instanceId: ProviderInstanceId.make("claude_work"),
+        model: "claude-opus-5-5",
+      };
+      yield* dispatchCommand({
+        type: "message.dispatch",
+        commandId: CommandId.make("send-limited"),
+        threadId: ThreadId.make("thread-limited"),
+        messageId: MessageId.make("message-limited"),
+        ...(manual ? { manualContinuationOfRunId: RunId.make("limited-run") } : {}),
+        text: "go",
+        attachments: [],
+        dispatchMode: { type: "start_immediately" },
+        createdBy: "user",
+        creationSource: "web",
+      }).pipe(
+        Effect.provide(
+          Layer.mergeAll(
+            Layer.mock(ThreadManagementService.ThreadManagementService)({
+              dispatch: (command) =>
+                AccountFallback.pipe(
+                  Effect.flatMap((fallback) => {
+                    expect(fallback).toEqual(
+                      manual
+                        ? undefined
+                        : {
+                            commandId: command.commandId,
+                            threadSelection: selection,
+                            requestedSelection: undefined,
+                          },
+                    );
+                    return layerFailingDispatchHandler(captured)(command);
+                  }),
+                ),
+              getThreadRecords: () =>
+                Effect.succeed({
+                  thread: {
+                    projectId: ProjectId.make("project-limited"),
+                    modelSelection: selection,
+                  },
+                } as never),
+            }),
+            Layer.mock(ProviderRegistry.ProviderRegistry)({
+              getProviders: Effect.succeed([
+                claude("claude_work", 100),
+                claude("claude_personal", 30),
+              ]),
+            }),
+            ServerSettings.layerTest({
+              fallbackToAvailableAccounts: true,
+              providerInstances: Object.fromEntries(
+                ["claude_work", "claude_personal"].map((id) => [
+                  ProviderInstanceId.make(id),
+                  { driver: ProviderDriverKind.make("claudeAgent"), enabled: true },
+                ]),
+              ),
+            }),
+          ),
+        ),
+        Effect.result,
+      );
+      if (manual) {
+        expect(captured[0]).not.toHaveProperty("modelSelection");
+        return;
+      }
+      expect(captured[0]).toMatchObject({
+        modelSelection: {
+          instanceId: "claude_personal",
+          model: "claude-opus-5-5",
+        },
+      });
+    }).pipe(Effect.provide(layerIntakeTest)),
+);
+
 it.effect("refuses a queued edit that keeps a file another edit already deleted", () =>
   Effect.gen(function* () {
     const config = yield* ServerConfig.ServerConfig;