From 8b2500577c3af73bf3a480ddd5fa9a5e68fca505 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 08:58:19 +0000 Subject: [PATCH 1/4] test(server): archived destination stops a bound schedule (repro #24519) Fails on main: a thread-bound task whose thread is archived stays enabled and keeps a next run after a dispatch fails with ThreadArchivedError. AI-assisted; verified with the tests and tools listed above. --- .../ScheduledTaskService.test.ts | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index ee4385ff2e0af..cd8b659652a06 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -663,6 +663,61 @@ it.effect("a fresh-session task launches one thread, then starts each later run }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("a bound task whose thread was archived stops scheduling instead of retrying", () => + Effect.gen(function* () { + const projectId = ProjectId.make("project:test"); + const archived = yield* Ref.make(false); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Effect.fail( + new ThreadManagementService.ThreadManagementThreadArchivedError({ + threadId: input.threadId, + }), + ), + getThreadShell: () => + Effect.map(Ref.get(archived), (isArchived) => + threadShell(projectId, isArchived ? { archivedAt: "2026-11-03T00:00:00Z" } : {}), + ), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), + ), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert({ + id: ScheduledTaskId.make("task:archived-destination"), + title: "Check progress", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId, + threadId: ThreadId.make("thread:archived"), + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }); + yield* Ref.set(archived, true); + yield* Effect.exit(service.runNow({ id: task.id })); + const [current] = (yield* service.list()).tasks; + assert.equal(current?.enabled, false); + assert.isNull(current?.nextRunAt); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a first run whose thread was deleted before it bound removes the task", () => Effect.gen(function* () { const layer = ScheduledTaskService.layer.pipe( From 92879e66f0393d967de6a6908899dfacf31abc33 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:07:32 +0000 Subject: [PATCH 2/4] fix(server): a schedule bound to an archived thread pauses instead of retrying every interval A dispatch that fails because the destination thread is archived now disables the task and clears its next run in the same completion write. The failure message stays as the last run error, so the reason is visible, and turning the task back on after unarchiving recomputes the next run. Fixes pingdotgg/t3code#24519 AI-assisted; verified with the tests and tools listed above. --- .../scheduledTasks/ScheduledTaskService.ts | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 5af0aba39e6ed..9c9a3d38c4ba3 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -777,10 +777,12 @@ export const layer = Layer.effect( readonly status: "succeeded" | "failed"; readonly error: string | null; readonly startedAtIso: string; + readonly disable: boolean; }) => sql` UPDATE scheduled_tasks SET updated_at = ${input.completedAtIso}, + enabled = CASE WHEN ${input.disable ? 1 : 0} = 1 THEN 0 ELSE enabled END, next_run_at = ${input.nextRunAtIso}, last_run_status = ${input.status}, last_run_error = ${input.error}, @@ -1100,6 +1102,12 @@ export const layer = Layer.effect( const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); const lastRunError = runSucceeded ? null : errorMessage(result.cause); const threadId = runSucceeded ? result.value : null; + // An archived destination rejects every later run too, so stop instead of retrying. + const destinationArchived = + !runSucceeded && + Cause.findErrorOption<{ readonly _tag: string }>(result.cause).pipe( + Option.exists((error) => error._tag === "ThreadManagementThreadArchivedError"), + ); // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads // files, and the transaction holds SQLite's writer lock. @@ -1110,13 +1118,16 @@ export const layer = Layer.effect( const scheduleSource = current ?? task; const completed: ScheduledTask = { ...scheduleSource, + ...(destinationArchived ? { enabled: false } : {}), updatedAt: iso(completedAt), lastRunAt: startedAtIso, // A manual run that starts before a one-time run must not use it up. - nextRunAt: nextRunAt( - scheduleSource, - scheduleSource.schedule.type === "once" ? startedAt : completedAt, - ), + nextRunAt: destinationArchived + ? null + : nextRunAt( + scheduleSource, + scheduleSource.schedule.type === "once" ? startedAt : completedAt, + ), lastRunStatus, lastRunError, runCount: scheduleSource.runCount + 1, @@ -1132,6 +1143,7 @@ export const layer = Layer.effect( status: lastRunStatus, error: lastRunError, startedAtIso, + disable: destinationArchived, }); let written = false; if (webhook === undefined) written = yield* completion; From 7b17d33a0f49dfede2154ea1b9adafb48c322546 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:20:07 +0000 Subject: [PATCH 3/4] fix(server): scheduled tasks pause after repeated failed deliveries An archived-destination failure now stops a task only when its destination is still the thread that was archived, so a task rebound mid-dispatch keeps running. Every other failed dispatch counts toward a consecutive-failure limit stored on the task; the fifth in a row disables it, clears its next run, and records the reason as the last run error. A successful run and toggling the task reset the count. AI-assisted; verified with the tests and tools listed above. --- apps/server/src/persistence/Migrations.ts | 2 + .../083_ScheduledTaskConsecutiveFailures.ts | 8 + .../reconcileV2PreviewMigration.test.ts | 2 + .../ScheduledTaskService.test.ts | 163 +++++++++++++----- .../scheduledTasks/ScheduledTaskService.ts | 27 ++- 5 files changed, 158 insertions(+), 44 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/083_ScheduledTaskConsecutiveFailures.ts diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 5e4784e26fc95..e46c5edc04889 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -85,6 +85,7 @@ import Migration0079 from "./Migrations/079_DelegatedRecoveryIndexes.ts"; import Migration0080 from "./Migrations/080_ScheduledTaskThreadMode.ts"; import Migration0081 from "./Migrations/081_ProjectionMessageSearchText.ts"; import Migration0082 from "./Migrations/082_TurnItemConversationIndex.ts"; +import Migration0083 from "./Migrations/083_ScheduledTaskConsecutiveFailures.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -170,6 +171,7 @@ const migrationEntries = [ [80, "ScheduledTaskThreadMode", Migration0080], [81, "ProjectionMessageSearchText", Migration0081], [82, "TurnItemConversationIndex", Migration0082], + [83, "ScheduledTaskConsecutiveFailures", Migration0083], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/083_ScheduledTaskConsecutiveFailures.ts b/apps/server/src/persistence/Migrations/083_ScheduledTaskConsecutiveFailures.ts new file mode 100644 index 0000000000000..fc9f8f303a262 --- /dev/null +++ b/apps/server/src/persistence/Migrations/083_ScheduledTaskConsecutiveFailures.ts @@ -0,0 +1,8 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* sql`ALTER TABLE scheduled_tasks ADD COLUMN consecutive_failures INTEGER NOT NULL DEFAULT 0`; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 88348b742b470..eca2ff46cc866 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -62,6 +62,7 @@ describe("V2 preview upgrade", () => { [80, "ScheduledTaskThreadMode"], [81, "ProjectionMessageSearchText"], [82, "TurnItemConversationIndex"], + [83, "ScheduledTaskConsecutiveFailures"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -167,6 +168,7 @@ describe("V2 preview upgrade", () => { [80, "ScheduledTaskThreadMode"], [81, "ProjectionMessageSearchText"], [82, "TurnItemConversationIndex"], + [83, "ScheduledTaskConsecutiveFailures"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index cd8b659652a06..2e0c0f1af9991 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -663,52 +663,63 @@ it.effect("a fresh-session task launches one thread, then starts each later run }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +const boundProjectId = ProjectId.make("project:test"); + +const boundTaskLayer = ( + threadManagement: Pick< + ThreadManagementService.ThreadManagementService["Service"], + "sendToThread" | "getThreadShell" + >, +) => + ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)(threadManagement), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), + ), + ); + +const boundTaskInput = (threadId: string): ScheduledTaskUpsertInput => ({ + id: ScheduledTaskId.make("task:bound"), + title: "Check progress", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: boundProjectId, + threadId: ThreadId.make(threadId), + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", +}); + it.effect("a bound task whose thread was archived stops scheduling instead of retrying", () => Effect.gen(function* () { - const projectId = ProjectId.make("project:test"); const archived = yield* Ref.make(false); - const layer = ScheduledTaskService.layer.pipe( - Layer.provide( - Layer.mergeAll( - ManagedPolicy.layerTest(), - Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), - Layer.mock(ThreadManagementService.ThreadManagementService)({ - sendToThread: (input) => - Effect.fail( - new ThreadManagementService.ThreadManagementThreadArchivedError({ - threadId: input.threadId, - }), - ), - getThreadShell: () => - Effect.map(Ref.get(archived), (isArchived) => - threadShell(projectId, isArchived ? { archivedAt: "2026-11-03T00:00:00Z" } : {}), - ), - }), - Layer.mock(SecretRequests.SecretRequests)({}), - Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ - namedProjectsRoot: "/projects", - isScratchProject: () => Effect.succeed(false), + const layer = boundTaskLayer({ + sendToThread: (input) => + Effect.fail( + new ThreadManagementService.ThreadManagementThreadArchivedError({ + threadId: input.threadId, }), - NodeCrypto.layer, - Scheduler.layer, ), - ), - ); + getThreadShell: () => + Effect.map(Ref.get(archived), (isArchived) => + threadShell(boundProjectId, isArchived ? { archivedAt: "2026-11-03T00:00:00Z" } : {}), + ), + }); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; - const { task } = yield* service.upsert({ - id: ScheduledTaskId.make("task:archived-destination"), - title: "Check progress", - prompt: "Check progress", - enabled: true, - schedule: { type: "interval", everyMs: 60_000 }, - projectId, - threadId: ThreadId.make("thread:archived"), - workspaceStrategy: { type: "root" }, - modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, - runtimeMode: "full-access", - interactionMode: "default", - }); + const { task } = yield* service.upsert(boundTaskInput("thread:archived")); yield* Ref.set(archived, true); yield* Effect.exit(service.runNow({ id: task.id })); const [current] = (yield* service.list()).tasks; @@ -718,6 +729,80 @@ it.effect("a bound task whose thread was archived stops scheduling instead of re }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("an archived error does not stop a task rebound to another thread mid-dispatch", () => + Effect.gen(function* () { + const sending = yield* Deferred.make(); + const release = yield* Deferred.make(); + const layer = boundTaskLayer({ + sendToThread: (input) => + Deferred.succeed(sending, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen( + Effect.fail( + new ThreadManagementService.ThreadManagementThreadArchivedError({ + threadId: input.threadId, + }), + ), + ), + ), + getThreadShell: () => Effect.succeed(threadShell(boundProjectId)), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(boundTaskInput("thread:a")); + const run = yield* Effect.forkChild(Effect.exit(service.runNow({ id: task.id }))); + yield* Deferred.await(sending); + yield* service.upsert(boundTaskInput("thread:b")); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(run); + const [current] = (yield* service.list()).tasks; + assert.equal(current?.threadId, "thread:b"); + assert.equal(current?.enabled, true); + assert.isNotNull(current?.nextRunAt); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("repeated delivery failures pause a task, and a success resets the count", () => + Effect.gen(function* () { + const failing = yield* Ref.make(true); + const layer = boundTaskLayer({ + sendToThread: (input) => + Effect.flatMap(Ref.get(failing), (fail) => + fail + ? Effect.fail( + new ThreadManagementService.ThreadManagementProjectionLoadError({ + projectId: input.projectId, + threadId: input.threadId, + cause: new Error("database busy"), + }), + ) + : Effect.succeed({} as never), + ), + getThreadShell: () => Effect.succeed(threadShell(boundProjectId)), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(boundTaskInput("thread:flaky")); + const runTimes = (count: number) => + Effect.repeat(Effect.exit(service.runNow({ id: task.id })), { times: count - 1 }); + yield* runTimes(4); + yield* Ref.set(failing, false); + yield* runTimes(1); + yield* Ref.set(failing, true); + yield* runTimes(4); + const [afterFour] = (yield* service.list()).tasks; + assert.equal(afterFour?.enabled, true); + assert.isNotNull(afterFour?.nextRunAt); + yield* runTimes(1); + const [paused] = (yield* service.list()).tasks; + assert.equal(paused?.enabled, false); + assert.isNull(paused?.nextRunAt); + assert.match(paused?.lastRunError ?? "", /^Paused after 5 failed runs in a row\./); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a first run whose thread was deleted before it bound removes the task", () => Effect.gen(function* () { const layer = ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 9c9a3d38c4ba3..8f07d0b98cb47 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -229,6 +229,7 @@ interface ScheduledTaskRow { readonly last_run_status: string; readonly last_run_error: string | null; readonly run_count: number; + readonly consecutive_failures: number; readonly webhook_token: string | null; readonly webhook_secret: string | null; } @@ -308,6 +309,8 @@ function iso(value: DateTime.DateTime): string { } const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000; +// Transient delivery failures retry this many times in a row before the task pauses. +const MAX_CONSECUTIVE_FAILED_RUNS = 5; const UNSUCCESSFUL_RUN_ERRORS = { failed: "The run failed.", interrupted: "The run was stopped.", @@ -576,6 +579,7 @@ export const layer = Layer.effect( last_run_status, last_run_error, run_count, + consecutive_failures, webhook_token, webhook_secret FROM scheduled_tasks @@ -778,11 +782,13 @@ export const layer = Layer.effect( readonly error: string | null; readonly startedAtIso: string; readonly disable: boolean; + readonly consecutiveFailures: number; }) => sql` UPDATE scheduled_tasks SET updated_at = ${input.completedAtIso}, enabled = CASE WHEN ${input.disable ? 1 : 0} = 1 THEN 0 ELSE enabled END, + consecutive_failures = ${input.consecutiveFailures}, next_run_at = ${input.nextRunAtIso}, last_run_status = ${input.status}, last_run_error = ${input.error}, @@ -1116,20 +1122,29 @@ export const layer = Layer.effect( const [row] = yield* getRows(task.id); const current = row === undefined ? null : yield* decodeRow(row, origin); const scheduleSource = current ?? task; + // A destination edited mid-dispatch is not the archived one. + const archived = destinationArchived && current?.threadId === active.threadId; + const consecutiveFailures = runSucceeded ? 0 : (row?.consecutive_failures ?? 0) + 1; + const exhausted = consecutiveFailures >= MAX_CONSECUTIVE_FAILED_RUNS; + const stop = archived || exhausted; + const recordedError = + exhausted && !archived + ? `Paused after ${MAX_CONSECUTIVE_FAILED_RUNS} failed runs in a row. ${lastRunError}` + : lastRunError; const completed: ScheduledTask = { ...scheduleSource, - ...(destinationArchived ? { enabled: false } : {}), + ...(stop ? { enabled: false } : {}), updatedAt: iso(completedAt), lastRunAt: startedAtIso, // A manual run that starts before a one-time run must not use it up. - nextRunAt: destinationArchived + nextRunAt: stop ? null : nextRunAt( scheduleSource, scheduleSource.schedule.type === "once" ? startedAt : completedAt, ), lastRunStatus, - lastRunError, + lastRunError: recordedError, runCount: scheduleSource.runCount + 1, }; if (current === null) return { recorded: completed, written: false }; @@ -1141,9 +1156,10 @@ export const layer = Layer.effect( completedAtIso: completed.updatedAt, nextRunAtIso: completed.nextRunAt, status: lastRunStatus, - error: lastRunError, + error: recordedError, startedAtIso, - disable: destinationArchived, + disable: stop, + consecutiveFailures, }); let written = false; if (webhook === undefined) written = yield* completion; @@ -1600,6 +1616,7 @@ export const layer = Layer.effect( const updated = yield* sql<{ task_id: string }>` UPDATE scheduled_tasks SET enabled = ${input.enabled ? 1 : 0}, + consecutive_failures = 0, next_run_at = ${next}, updated_at = ${iso(now)} WHERE task_id = ${input.id} From 1a5d0326e6eadb37ed0acad128b5fef7ad39c4f6 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 09:34:29 +0000 Subject: [PATCH 4/4] fix(server): resuming a paused task from its edit dialog resets its failure count A save that turns a disabled task back on now clears consecutive_failures, matching the list toggle, so a resumed task gets a fresh retry budget. AI-assisted; verified with the tests and tools listed above. --- apps/server/src/scheduledTasks/ScheduledTaskService.test.ts | 5 +++++ apps/server/src/scheduledTasks/ScheduledTaskService.ts | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 2e0c0f1af9991..74cd113e4d03d 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -799,6 +799,11 @@ it.effect("repeated delivery failures pause a task, and a success resets the cou assert.equal(paused?.enabled, false); assert.isNull(paused?.nextRunAt); assert.match(paused?.lastRunError ?? "", /^Paused after 5 failed runs in a row\./); + yield* service.upsert(boundTaskInput("thread:flaky")); + yield* runTimes(1); + const [resumed] = (yield* service.list()).tasks; + assert.equal(resumed?.enabled, true); + assert.isNotNull(resumed?.nextRunAt); }).pipe(Effect.provide(layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 8f07d0b98cb47..1ffa6b82111a1 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -692,6 +692,10 @@ export const layer = Layer.effect( title = excluded.title, prompt = excluded.prompt, enabled = excluded.enabled, + consecutive_failures = CASE + WHEN scheduled_tasks.enabled = 0 AND excluded.enabled = 1 THEN 0 + ELSE scheduled_tasks.consecutive_failures + END, schedule_json = excluded.schedule_json, project_id = excluded.project_id, -- A first run may bind the thread while this save is in flight; only a