diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 422d2122d03bd..17dcb7a0e9877 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -86,6 +86,7 @@ import Migration0080 from "./Migrations/080_ScheduledTaskThreadMode.ts"; import Migration0081 from "./Migrations/081_ProjectionMessageSearchText.ts"; import Migration0082 from "./Migrations/082_TurnItemConversationIndex.ts"; import Migration0083 from "./Migrations/083_CommandReadIndexes.ts"; +import Migration0084 from "./Migrations/084_ScheduledTaskConsecutiveFailures.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -172,6 +173,7 @@ const migrationEntries = [ [81, "ProjectionMessageSearchText", Migration0081], [82, "TurnItemConversationIndex", Migration0082], [83, "CommandReadIndexes", Migration0083], + [84, "ScheduledTaskConsecutiveFailures", Migration0084], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/084_ScheduledTaskConsecutiveFailures.ts b/apps/server/src/persistence/Migrations/084_ScheduledTaskConsecutiveFailures.ts new file mode 100644 index 0000000000000..fc9f8f303a262 --- /dev/null +++ b/apps/server/src/persistence/Migrations/084_ScheduledTaskConsecutiveFailures.ts @@ -0,0 +1,8 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* sql`ALTER TABLE scheduled_tasks ADD COLUMN consecutive_failures INTEGER NOT NULL DEFAULT 0`; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 05f09f8e16850..e5e3d0c9ead09 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -63,6 +63,7 @@ describe("V2 preview upgrade", () => { [81, "ProjectionMessageSearchText"], [82, "TurnItemConversationIndex"], [83, "CommandReadIndexes"], + [84, "ScheduledTaskConsecutiveFailures"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -169,6 +170,7 @@ describe("V2 preview upgrade", () => { [81, "ProjectionMessageSearchText"], [82, "TurnItemConversationIndex"], [83, "CommandReadIndexes"], + [84, "ScheduledTaskConsecutiveFailures"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index ee4385ff2e0af..74cd113e4d03d 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -663,6 +663,151 @@ it.effect("a fresh-session task launches one thread, then starts each later run }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +const boundProjectId = ProjectId.make("project:test"); + +const boundTaskLayer = ( + threadManagement: Pick< + ThreadManagementService.ThreadManagementService["Service"], + "sendToThread" | "getThreadShell" + >, +) => + ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)(threadManagement), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), + ), + ); + +const boundTaskInput = (threadId: string): ScheduledTaskUpsertInput => ({ + id: ScheduledTaskId.make("task:bound"), + title: "Check progress", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: boundProjectId, + threadId: ThreadId.make(threadId), + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", +}); + +it.effect("a bound task whose thread was archived stops scheduling instead of retrying", () => + Effect.gen(function* () { + const archived = yield* Ref.make(false); + const layer = boundTaskLayer({ + sendToThread: (input) => + Effect.fail( + new ThreadManagementService.ThreadManagementThreadArchivedError({ + threadId: input.threadId, + }), + ), + getThreadShell: () => + Effect.map(Ref.get(archived), (isArchived) => + threadShell(boundProjectId, isArchived ? { archivedAt: "2026-11-03T00:00:00Z" } : {}), + ), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(boundTaskInput("thread:archived")); + yield* Ref.set(archived, true); + yield* Effect.exit(service.runNow({ id: task.id })); + const [current] = (yield* service.list()).tasks; + assert.equal(current?.enabled, false); + assert.isNull(current?.nextRunAt); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("an archived error does not stop a task rebound to another thread mid-dispatch", () => + Effect.gen(function* () { + const sending = yield* Deferred.make(); + const release = yield* Deferred.make(); + const layer = boundTaskLayer({ + sendToThread: (input) => + Deferred.succeed(sending, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen( + Effect.fail( + new ThreadManagementService.ThreadManagementThreadArchivedError({ + threadId: input.threadId, + }), + ), + ), + ), + getThreadShell: () => Effect.succeed(threadShell(boundProjectId)), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(boundTaskInput("thread:a")); + const run = yield* Effect.forkChild(Effect.exit(service.runNow({ id: task.id }))); + yield* Deferred.await(sending); + yield* service.upsert(boundTaskInput("thread:b")); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(run); + const [current] = (yield* service.list()).tasks; + assert.equal(current?.threadId, "thread:b"); + assert.equal(current?.enabled, true); + assert.isNotNull(current?.nextRunAt); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("repeated delivery failures pause a task, and a success resets the count", () => + Effect.gen(function* () { + const failing = yield* Ref.make(true); + const layer = boundTaskLayer({ + sendToThread: (input) => + Effect.flatMap(Ref.get(failing), (fail) => + fail + ? Effect.fail( + new ThreadManagementService.ThreadManagementProjectionLoadError({ + projectId: input.projectId, + threadId: input.threadId, + cause: new Error("database busy"), + }), + ) + : Effect.succeed({} as never), + ), + getThreadShell: () => Effect.succeed(threadShell(boundProjectId)), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert(boundTaskInput("thread:flaky")); + const runTimes = (count: number) => + Effect.repeat(Effect.exit(service.runNow({ id: task.id })), { times: count - 1 }); + yield* runTimes(4); + yield* Ref.set(failing, false); + yield* runTimes(1); + yield* Ref.set(failing, true); + yield* runTimes(4); + const [afterFour] = (yield* service.list()).tasks; + assert.equal(afterFour?.enabled, true); + assert.isNotNull(afterFour?.nextRunAt); + yield* runTimes(1); + const [paused] = (yield* service.list()).tasks; + assert.equal(paused?.enabled, false); + assert.isNull(paused?.nextRunAt); + assert.match(paused?.lastRunError ?? "", /^Paused after 5 failed runs in a row\./); + yield* service.upsert(boundTaskInput("thread:flaky")); + yield* runTimes(1); + const [resumed] = (yield* service.list()).tasks; + assert.equal(resumed?.enabled, true); + assert.isNotNull(resumed?.nextRunAt); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a first run whose thread was deleted before it bound removes the task", () => Effect.gen(function* () { const layer = ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 5af0aba39e6ed..1ffa6b82111a1 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -229,6 +229,7 @@ interface ScheduledTaskRow { readonly last_run_status: string; readonly last_run_error: string | null; readonly run_count: number; + readonly consecutive_failures: number; readonly webhook_token: string | null; readonly webhook_secret: string | null; } @@ -308,6 +309,8 @@ function iso(value: DateTime.DateTime): string { } const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000; +// Transient delivery failures retry this many times in a row before the task pauses. +const MAX_CONSECUTIVE_FAILED_RUNS = 5; const UNSUCCESSFUL_RUN_ERRORS = { failed: "The run failed.", interrupted: "The run was stopped.", @@ -576,6 +579,7 @@ export const layer = Layer.effect( last_run_status, last_run_error, run_count, + consecutive_failures, webhook_token, webhook_secret FROM scheduled_tasks @@ -688,6 +692,10 @@ export const layer = Layer.effect( title = excluded.title, prompt = excluded.prompt, enabled = excluded.enabled, + consecutive_failures = CASE + WHEN scheduled_tasks.enabled = 0 AND excluded.enabled = 1 THEN 0 + ELSE scheduled_tasks.consecutive_failures + END, schedule_json = excluded.schedule_json, project_id = excluded.project_id, -- A first run may bind the thread while this save is in flight; only a @@ -777,10 +785,14 @@ export const layer = Layer.effect( readonly status: "succeeded" | "failed"; readonly error: string | null; readonly startedAtIso: string; + readonly disable: boolean; + readonly consecutiveFailures: number; }) => sql` UPDATE scheduled_tasks SET updated_at = ${input.completedAtIso}, + enabled = CASE WHEN ${input.disable ? 1 : 0} = 1 THEN 0 ELSE enabled END, + consecutive_failures = ${input.consecutiveFailures}, next_run_at = ${input.nextRunAtIso}, last_run_status = ${input.status}, last_run_error = ${input.error}, @@ -1100,6 +1112,12 @@ export const layer = Layer.effect( const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); const lastRunError = runSucceeded ? null : errorMessage(result.cause); const threadId = runSucceeded ? result.value : null; + // An archived destination rejects every later run too, so stop instead of retrying. + const destinationArchived = + !runSucceeded && + Cause.findErrorOption<{ readonly _tag: string }>(result.cause).pipe( + Option.exists((error) => error._tag === "ThreadManagementThreadArchivedError"), + ); // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads // files, and the transaction holds SQLite's writer lock. @@ -1108,17 +1126,29 @@ export const layer = Layer.effect( const [row] = yield* getRows(task.id); const current = row === undefined ? null : yield* decodeRow(row, origin); const scheduleSource = current ?? task; + // A destination edited mid-dispatch is not the archived one. + const archived = destinationArchived && current?.threadId === active.threadId; + const consecutiveFailures = runSucceeded ? 0 : (row?.consecutive_failures ?? 0) + 1; + const exhausted = consecutiveFailures >= MAX_CONSECUTIVE_FAILED_RUNS; + const stop = archived || exhausted; + const recordedError = + exhausted && !archived + ? `Paused after ${MAX_CONSECUTIVE_FAILED_RUNS} failed runs in a row. ${lastRunError}` + : lastRunError; const completed: ScheduledTask = { ...scheduleSource, + ...(stop ? { enabled: false } : {}), updatedAt: iso(completedAt), lastRunAt: startedAtIso, // A manual run that starts before a one-time run must not use it up. - nextRunAt: nextRunAt( - scheduleSource, - scheduleSource.schedule.type === "once" ? startedAt : completedAt, - ), + nextRunAt: stop + ? null + : nextRunAt( + scheduleSource, + scheduleSource.schedule.type === "once" ? startedAt : completedAt, + ), lastRunStatus, - lastRunError, + lastRunError: recordedError, runCount: scheduleSource.runCount + 1, }; if (current === null) return { recorded: completed, written: false }; @@ -1130,8 +1160,10 @@ export const layer = Layer.effect( completedAtIso: completed.updatedAt, nextRunAtIso: completed.nextRunAt, status: lastRunStatus, - error: lastRunError, + error: recordedError, startedAtIso, + disable: stop, + consecutiveFailures, }); let written = false; if (webhook === undefined) written = yield* completion; @@ -1588,6 +1620,7 @@ export const layer = Layer.effect( const updated = yield* sql<{ task_id: string }>` UPDATE scheduled_tasks SET enabled = ${input.enabled ? 1 : 0}, + consecutive_failures = 0, next_run_at = ${next}, updated_at = ${iso(now)} WHERE task_id = ${input.id}