From 404b655b0a3f523c07f9a990acd650a45400202a Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Thu, 22 Oct 2026 08:47:16 +1100 Subject: [PATCH 01/19] fix(server): scheduled runs record how their turn ended A scheduled run was recorded as succeeded as soon as its message was accepted, so a turn the provider rejected later (for example a usage limit) still showed succeeded. A dispatched run now stays running until its turn ends, then records succeeded or failed with the provider's error. The run count moves to the end of the run, so a restart that interrupts a run counts it once. The thread panel no longer disables Run now while a task is running, matching Settings and mobile. list_scheduled_tasks also returns lastRunError. Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/mcp/OrchestratorMcpService.ts | 1 + ...laudeAutomaticDelivery.integration.test.ts | 2 +- .../ThreadLaunchService.test.ts | 68 ++++++- .../ScheduledTaskService.test.ts | 189 ++++++++++++++++++ .../scheduledTasks/ScheduledTaskService.ts | 111 ++++++++-- .../ScheduledTaskService.webhook.test.ts | 22 +- .../src/scheduledTasks/TrackedRuns.test.ts | 109 ++++++++++ apps/server/src/scheduledTasks/TrackedRuns.ts | 155 ++++++++++++++ .../chat/ThreadAutomationsPanel.tsx | 4 +- packages/contracts/src/orchestratorMcp.ts | 3 + 10 files changed, 631 insertions(+), 33 deletions(-) create mode 100644 apps/server/src/scheduledTasks/TrackedRuns.test.ts create mode 100644 apps/server/src/scheduledTasks/TrackedRuns.ts diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index edaa8777b422e..a9ed38631f1c5 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -301,6 +301,7 @@ function scheduledTaskSummary(task: ScheduledTask, mayRun: boolean): Orchestrato schedule: task.schedule, nextRunAt: task.nextRunAt, lastRunStatus: task.lastRunStatus, + ...(task.lastRunError === null ? {} : { lastRunError: task.lastRunError }), ...(task.webhook?.url == null || !mayRun ? {} : { webhookUrl: task.webhook.url }), ...(task.webhook === undefined ? {} diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index cf425716bcbb8..2e0b28e6691ad 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -318,7 +318,7 @@ it.effect.each(deliveries)( }), ); const ran = yield* service.runNow({ id: task.id }); - assert.equal(ran.task.lastRunStatus, "succeeded"); + assert.equal(ran.task.lastRunStatus, "running"); }).pipe( Effect.provide( ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 0391412fe5f9b..93f8062548350 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -349,7 +349,7 @@ it.effect.each( creationSource: createdBy === "agent" ? "mcp" : "web", }); const result = yield* tasks.runNow({ id: task.id }); - assert.equal(result.task.lastRunStatus, "succeeded"); + assert.equal(result.task.lastRunStatus, "running"); const projectThreads = yield* threads.listProjectThreads({ projectId, includeSubagents: false, @@ -455,7 +455,7 @@ it.effect("runs a thread-bound automation on the provider the thread uses now", }); const result = yield* tasks.runNow({ id: task.id }); - assert.equal(result.task.lastRunStatus, "succeeded"); + assert.equal(result.task.lastRunStatus, "running"); const projection = yield* threads.getThreadProjection(launched.threadId); const message = projection.messages.find((item) => item.scheduledTaskId === task.id); const run = projection.runs.find((item) => item.id === message?.runId); @@ -501,6 +501,68 @@ it.effect("retains automation and sender attribution while a message waits in th }).pipe(Effect.provide(harness.layer)); }); +it.effect("an automation queued behind a running turn records how its own turn ends", () => { + const harness = makeHarness({ runSetup: () => Effect.never }); + const layerScheduledTasks = ScheduledTasks.layer.pipe( + Layer.provide( + Layer.mergeAll( + harness.layer, + NodeCrypto.layer, + Scheduler.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + ), + ), + ); + return Effect.gen(function* () { + const tasks = yield* ScheduledTasks.ScheduledTaskService; + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const launched = yield* launches.launch( + launchInput({ + command: "command:automation:outcome", + thread: "thread:automation:outcome", + message: "First message", + workspace: { type: "worktree", baseRef: "main" }, + }), + ); + const { task } = yield* tasks.upsert({ + id: ScheduledTaskId.make("scheduled-task:outcome"), + title: "Daily audit", + prompt: "Audit performance and crashes.", + enabled: false, + schedule: { type: "interval", everyMs: 60_000 }, + projectId, + threadId: launched.threadId, + workspaceStrategy: { type: "root" }, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + createdBy: "user", + creationSource: "web", + }); + const ran = yield* tasks.runNow({ id: task.id }); + assert.equal(ran.task.lastRunStatus, "running"); + const projection = yield* threads.getThreadProjection(launched.threadId); + const queued = projection.runs.find((run) => run.status === "queued"); + assert.isDefined(queued); + + yield* threads.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("command:automation:outcome:cancel"), + threadId: launched.threadId, + runId: queued!.id, + }); + yield* tasks.subscribeList().pipe( + Stream.filter(({ tasks: listed }) => listed[0]?.lastRunStatus === "failed"), + Stream.take(1), + Stream.runDrain, + ); + const [settled] = (yield* tasks.list()).tasks; + assert.equal(settled?.lastRunError, "The run was cancelled."); + assert.equal(settled?.runCount, 1); + }).pipe(Effect.provide(Layer.mergeAll(harness.layer, layerScheduledTasks))); +}); + it.effect("provisions independent launches concurrently instead of behind a global semaphore", () => Effect.gen(function* () { const setupCount = yield* Ref.make(0); @@ -1003,7 +1065,7 @@ it.effect("runs a Scratch scheduled task saved with a worktree in its own folder WHERE task_id = ${task.id} `; const ran = yield* tasks.runNow({ id: task.id }); - assert.equal(ran.task.lastRunStatus, "succeeded"); + assert.equal(ran.task.lastRunStatus, "running"); const [thread] = yield* threads.listProjectThreads({ projectId, includeSubagents: false }); assert.isDefined(thread); const projection = yield* preparationSettled(thread!.id); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 466af14e7f229..858201c1d9208 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -4,11 +4,17 @@ import * as NodeUtil from "node:util"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it } from "@effect/vitest"; import { + EventId, + MessageId, + type OrchestrationV2DomainEvent, + type OrchestrationV2Run, ProjectId, ProviderInstanceId, + RunId, ScheduledTaskError, ScheduledTaskId, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -19,6 +25,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as Fiber from "effect/Fiber"; import * as Stream from "effect/Stream"; import * as Deferred from "effect/Deferred"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -405,3 +412,185 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea }).pipe(Effect.provide(layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +const turnThreadId = ThreadId.make("thread:turn-outcome"); +const epoch = DateTime.makeUnsafe(0); + +const runOf = ( + runId: string, + userMessageId: MessageId, + status: OrchestrationV2Run["status"], +): OrchestrationV2Run => ({ + id: RunId.make(runId), + threadId: turnThreadId, + ordinal: 1, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + providerThreadId: null, + userMessageId, + rootNodeId: null, + activeAttemptId: null, + status, + requestedAt: epoch, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, +}); + +const runEvent = (run: OrchestrationV2Run): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${run.id}:${run.status}`), + type: "run.updated", + threadId: run.threadId, + runId: run.id, + occurredAt: epoch, + payload: run, +}); + +const providerErrorEvent = (runId: string, message: string): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${runId}:error`), + type: "turn-item.updated", + threadId: turnThreadId, + runId: RunId.make(runId), + occurredAt: epoch, + payload: { + id: TurnItemId.make(`turn-item:${runId}:error`), + threadId: turnThreadId, + runId: RunId.make(runId), + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "failed", + title: null, + startedAt: epoch, + completedAt: epoch, + updatedAt: epoch, + type: "error", + failure: { class: "usage_limit", message, code: null, retryable: false }, + }, +}); + +// Dispatches to a bound thread succeed at once; the run's events are fed by hand. +const makeTurnHarness = Effect.gen(function* () { + const events = yield* Queue.unbounded(); + const dispatched = yield* Queue.unbounded(); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Queue.offer(dispatched, input.messageId).pipe(Effect.as(undefined as never)), + streamDomainEvents: Stream.fromQueue(events), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), + ), + ); + return { events, dispatched, layer }; +}); + +const upsertBoundTask = (service: ScheduledTaskService.ScheduledTaskService["Service"]) => + service.upsert({ + id: ScheduledTaskId.make("task:turn-outcome"), + title: "Check orchestration", + prompt: "Check progress", + enabled: false, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project:test"), + threadId: turnThreadId, + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }); + +const untilStatus = ( + service: ScheduledTaskService.ScheduledTaskService["Service"], + id: ScheduledTaskId, + status: string, +) => + service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks.some((t) => t.id === id && t.lastRunStatus === status)), + Stream.take(1), + Stream.runDrain, + ); + +it.effect("a failed turn replaces the dispatched run's status with the provider error", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness; + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const ran = yield* service.runNow({ id: task.id }); + // Accepted is not finished: the turn has not ended yet. + assert.equal(ran.task.lastRunStatus, "running"); + assert.equal(ran.task.runCount, 0); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:usage-limit", messageId, "queued")), + runEvent(runOf("run:usage-limit", messageId, "running")), + providerErrorEvent("run:usage-limit", "You have reached your usage limit."), + runEvent(runOf("run:usage-limit", messageId, "failed")), + ]); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunStatus, "failed"); + assert.equal(settled?.lastRunError, "You have reached your usage limit."); + assert.equal(settled?.runCount, 1); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a completed turn records the run as succeeded", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness; + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:completes", messageId, "running")), + runEvent(runOf("run:completes", messageId, "completed")), + ]); + yield* untilStatus(service, task.id, "succeeded"); + const [settled] = (yield* service.list()).tasks; + assert.isNull(settled?.lastRunError); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("an earlier run's turn ending cannot overwrite a newer run", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness; + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const first = yield* Queue.take(harness.dispatched); + // Run now again while the first turn is still in flight. + yield* TestClock.adjust("1 second"); + yield* service.runNow({ id: task.id }); + const second = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:first", first, "failed")), + runEvent(runOf("run:second", second, "completed")), + ]); + yield* untilStatus(service, task.id, "succeeded"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunStatus, "succeeded"); + assert.isNull(settled?.lastRunError); + assert.equal(settled?.runCount, 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 374d6b38027b8..70291588cc791 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -48,7 +48,9 @@ import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; +import { forkParked } from "../serverActivation.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; +import { makeTrackedRuns, type SettledRun } from "./TrackedRuns.ts"; import { redactHeaders, redactQuery, @@ -425,6 +427,7 @@ export const layer = Layer.effect( ReadonlyMap> >(new Map()); const activeRuns = yield* Ref.make>(new Set()); + const trackedRuns = makeTrackedRuns(); // Sliding(1) coalesces the dirty-signal: every notification triggers a // full list() re-emit anyway, so a slow subscriber only ever needs the // latest signal — an unbounded backlog would just grow memory. @@ -651,6 +654,9 @@ export const layer = Layer.effect( SET updated_at = ${startedAtIso}, last_run_at = ${startedAtIso}, last_run_webhook_delivery_id = ${webhookDeliveryId ?? null}, + -- A run still 'running' here was dispatched and never reported an end. + -- This run replaces it on the row, so it is counted now. + run_count = run_count + CASE WHEN last_run_status = 'running' THEN 1 ELSE 0 END, last_run_status = 'running', last_run_error = NULL WHERE task_id = ${id} @@ -668,7 +674,10 @@ export const layer = Layer.effect( readonly id: ScheduledTaskId; readonly completedAtIso: string; readonly nextRunAtIso: string | null; - readonly status: "succeeded" | "failed"; + // A dispatched run stays 'running' until its turn ends, and counts then: + // startup recovery counts a row it finds 'running', so counting here too + // would count a restart-interrupted run twice. + readonly status: "running" | "succeeded" | "failed"; readonly error: string | null; readonly startedAtIso: string; }) => @@ -678,7 +687,7 @@ export const layer = Layer.effect( next_run_at = ${input.nextRunAtIso}, last_run_status = ${input.status}, last_run_error = ${input.error}, - run_count = run_count + 1 + run_count = run_count + ${input.status === "running" ? 0 : 1} WHERE task_id = ${input.id} AND last_run_status = 'running' AND last_run_at = ${input.startedAtIso} @@ -688,6 +697,25 @@ export const layer = Layer.effect( ), ); + // The startedAtIso guard leaves the row alone once a newer run has taken it. + const recordRunEnd = Effect.fn("ScheduledTaskService.recordRunEnd")(function* ( + run: SettledRun, + ) { + const now = yield* localNow; + const rows = yield* sql<{ task_id: string }>` + UPDATE scheduled_tasks + SET updated_at = ${iso(now)}, + last_run_status = ${run.status}, + last_run_error = ${run.error}, + run_count = run_count + 1 + WHERE task_id = ${run.taskId} + AND last_run_status = 'running' + AND last_run_at = ${run.startedAtIso} + RETURNING task_id + `; + if (rows.length > 0) yield* notifyChanged; + }); + // Best-effort escape hatch: if anything fails between markRunning and // markCompleted, write a full terminal record so runDueTasks neither skips // the task forever (it filters out 'running' rows) nor re-fires it @@ -695,7 +723,12 @@ export const layer = Layer.effect( // advance. Pending webhook attempts count only with their completion state; // other attempts count here. const releaseStuckRun = Effect.fn("ScheduledTaskService.releaseStuckRun")( - function* (task: ScheduledTask, message: string, pendingDeliveryId?: string) { + function* ( + task: ScheduledTask, + message: string, + startedAtIso: string, + pendingDeliveryId?: string, + ) { const now = yield* localNow; const reread = yield* Effect.result(findTask(task.id)); if (Result.isSuccess(reread) && reread.success === null) return; @@ -711,10 +744,11 @@ export const layer = Layer.effect( WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running' ) THEN 0 ELSE 1 END WHERE task_id = ${task.id} AND last_run_status = 'running' + AND last_run_at = ${startedAtIso} `; yield* notifyChanged; }, - (effect, task, _message, _pendingDeliveryId) => + (effect, task, _message, _startedAtIso, _pendingDeliveryId) => effect.pipe( Effect.catch((cause) => Effect.logWarning("Could not release stuck schedule task run", { @@ -747,6 +781,9 @@ export const layer = Layer.effect( return { task, threadId: null }; } + // Set once this run owns the row, so a failure before that cannot + // release a turn an earlier run left in flight. + const markedAtIso = yield* Ref.make(null); return yield* Effect.gen(function* () { const startedAt = yield* localNow; const startedAtIso = iso(startedAt); @@ -813,6 +850,7 @@ export const layer = Layer.effect( ), ); + yield* Ref.set(markedAtIso, startedAtIso); yield* notifyChanged; const fireKey = @@ -821,6 +859,7 @@ export const layer = Layer.effect( : `${active.id}:webhook:${webhook.deliveryId}`; const commandId = CommandId.make(`scheduled-task:${fireKey}`); const messageId = MessageId.make(`scheduled-task-message:${fireKey}`); + trackedRuns.track(messageId, active.id, startedAtIso); // A webhook prompt was rendered from the row when the request arrived. const prompt = webhook?.prompt ?? active.prompt; @@ -872,8 +911,12 @@ export const layer = Layer.effect( const runSucceeded = result._tag === "Success"; if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; - const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); - const lastRunError = runSucceeded ? null : errorMessage(result.cause); + // Accepted is not finished: the row stays 'running' until the turn + // ends, unless it already ended while the dispatch was returning. + const ended = runSucceeded ? trackedRuns.dispatched(messageId) : null; + if (!runSucceeded) trackedRuns.forgetTask(active.id); + const lastRunStatus = runSucceeded ? (ended?.status ?? "running") : ("failed" as const); + const lastRunError = runSucceeded ? (ended?.error ?? null) : errorMessage(result.cause); const threadId = runSucceeded ? result.value : null; const current = yield* findTask(task.id); const scheduleSource = current ?? task; @@ -884,7 +927,7 @@ export const layer = Layer.effect( nextRunAt: nextRunAt(scheduleSource, completedAt), lastRunStatus, lastRunError, - runCount: scheduleSource.runCount + 1, + runCount: scheduleSource.runCount + (lastRunStatus === "running" ? 0 : 1), }; if (current !== null) { // startedAtIso in the guard ensures this writes only to the row this @@ -920,10 +963,22 @@ export const layer = Layer.effect( ), ); yield* notifyChanged; - } + } else trackedRuns.forgetTask(task.id); return { task: completed, threadId }; }).pipe( - Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause), webhook?.deliveryId)), + Effect.onError((cause) => + Ref.get(markedAtIso).pipe( + Effect.flatMap((startedAtIso) => + startedAtIso === null + ? Effect.void + : Effect.sync(() => trackedRuns.forgetTask(task.id)).pipe( + Effect.andThen( + releaseStuckRun(task, errorMessage(cause), startedAtIso, webhook?.deliveryId), + ), + ), + ), + ), + ), Effect.ensuring( Ref.update(activeRuns, (active) => { const next = new Set(active); @@ -1037,6 +1092,26 @@ export const layer = Layer.effect( yield* scheduler.register("scheduled-tasks", runDueTasks()); + // Dispatching a run only gets it accepted; its turn ends later, so the + // outcome is read from the thread's run events. + yield* forkParked( + Stream.runForEach(threadManagement.streamDomainEvents, (event) => { + // Every domain event passes through here, so the common case stays allocation-free. + const ended = trackedRuns.observe(event); + return ended.length === 0 + ? Effect.void + : Effect.forEach(ended, recordRunEnd, { discard: true }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not record scheduled task run outcome", { cause }), + ), + ); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Scheduled task run event stream failed", { cause }), + ), + ), + ); + const list: ScheduledTaskService["Service"]["list"] = () => listRows().pipe( Effect.map((tasks) => ({ tasks })), @@ -1593,12 +1668,18 @@ export const layer = Layer.effect( const dispatched = yield* Ref.make(false); return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe( Effect.flatMap(({ task: completed }) => - completed.lastRunStatus === "failed" - ? runOutcome("failed").pipe( - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), - Effect.andThen(releaseFailedDelivery(deliveryId)), - ) - : runOutcome("started"), + // The row may already hold the turn's own failure; the delivery log + // records only whether the dispatch was accepted. + Ref.get(dispatched).pipe( + Effect.flatMap((accepted) => + completed.lastRunStatus === "failed" && !accepted + ? runOutcome("failed").pipe( + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), + Effect.andThen(releaseFailedDelivery(deliveryId)), + ) + : runOutcome("started"), + ), + ), ), Effect.catchTags({ WebhookDeliverySkipped: (skipped) => diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 65cecbcf43286..3ca9a37945a4b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -1091,7 +1091,7 @@ it.effect( launch.commandId, `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`, ); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); assert.equal( (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`) .length, @@ -1159,7 +1159,7 @@ it.effect("start failures release both replay keys and relay claims for redelive ); assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); yield* Queue.take(launches); - assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded"); + assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "running"); }), { failLaunches: { remaining: 1 } }, ), @@ -1253,7 +1253,7 @@ it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping f }), ); yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; yield* waitForSpan( service.triggerWebhook(requestFor(task)), "ScheduledTaskService.runWebhookDelivery", @@ -1352,7 +1352,7 @@ it.effect.each([1, 11])( const retry = yield* service.triggerWebhook(request(count)); assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); assert.equal(yield* Queue.size(launches), 0); }), { workers }, @@ -1403,7 +1403,7 @@ it.effect("failed windows release every member's replay key and relay claim", () yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(workers); assert.include((yield* Queue.take(launches)).initialMessage!.text, "B11"); - assert.equal((yield* service.list()).tasks[0]!.runCount, 2); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); }), { workers, failLaunches: { remaining: 1 } }, ); @@ -1465,7 +1465,7 @@ it.effect( yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(restartedWorkers); yield* Queue.take(restartedLaunches); - assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); }), { sql, workers: restartedWorkers }, ); @@ -1505,7 +1505,7 @@ it.effect("dispatches an overdue window exactly once after its deadline passed w yield* Queue.take(launches); for (let i = 0; i < 3; i++) yield* restartedWorkers.get("webhook-coalescing")!; assert.equal(yield* Queue.size(launches), 0); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); }), { sql, workers: restartedWorkers }, @@ -1582,7 +1582,7 @@ it.effect("removes a window with a render defect and dispatches the task's next assert.equal(failed.outcome, "dispatch_failed"); assert.equal(failed.error, "The pending webhook payload could not be prepared."); yield* workers.get("webhook-coalescing")!; - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); }), { workers }, ); @@ -1611,7 +1611,7 @@ it.effect.each([false, true])( requestFor(task, { headers: { group: "old" }, bodyText: '{"value":"old"}' }), ); yield* sql`CREATE TRIGGER fail_old_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(workers); yield* Queue.take(launches); @@ -1655,13 +1655,13 @@ it.effect.each([false, true])( yield* sql`UPDATE scheduled_tasks SET model_selection_json = ${before!.model_selection_json} WHERE task_id = ${task.id}`; yield* drainPendingDispatch(restartedWorkers); assert.equal(yield* Queue.size(recoveredLaunches), 0); - assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); assert.equal( (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0, ); yield* restartedWorkers.get("webhook-coalescing")!; - assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); }), { sql, workers: restartedWorkers, completedLaunches }, ); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts new file mode 100644 index 0000000000000..aefe9841c780c --- /dev/null +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -0,0 +1,109 @@ +import { assert, it } from "@effect/vitest"; +import { + EventId, + MessageId, + type OrchestrationV2DomainEvent, + ProviderInstanceId, + RunId, + ScheduledTaskId, + ThreadId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; + +import { makeTrackedRuns } from "./TrackedRuns.ts"; + +const taskId = ScheduledTaskId.make("task:tracked"); +const threadId = ThreadId.make("thread:tracked"); +const messageId = MessageId.make("message:tracked"); +const epoch = DateTime.makeUnsafe(0); + +const runEvent = ( + runId: string, + status: "queued" | "running" | "completed" | "failed" | "cancelled", + userMessageId = messageId, +): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${runId}:${status}`), + type: "run.updated", + threadId, + runId: RunId.make(runId), + occurredAt: epoch, + payload: { + id: RunId.make(runId), + threadId, + ordinal: 1, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + providerThreadId: null, + userMessageId, + rootNodeId: null, + activeAttemptId: null, + status, + requestedAt: epoch, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, +}); + +it("holds an outcome that lands during the dispatch until the dispatch returns", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + assert.deepEqual(runs.observe(runEvent("run:fast", "failed")), []); + assert.deepEqual(runs.dispatched(messageId), { + taskId, + startedAtIso: "start", + status: "failed", + error: "The run failed.", + }); + assert.isNull(runs.dispatched(messageId)); +}); + +it("ignores the end of a queued run whose message was promoted into the active run", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId); + assert.deepEqual(runs.observe(runEvent("run:queued", "queued")), []); + runs.observe({ + id: EventId.make("event:promoted"), + type: "message.updated", + threadId, + runId: RunId.make("run:active"), + occurredAt: epoch, + payload: { + id: messageId, + threadId, + runId: RunId.make("run:active"), + nodeId: null, + role: "user", + text: "Run the audit", + attachments: [], + streaming: false, + createdAt: epoch, + updatedAt: epoch, + createdBy: "user", + creationSource: "web", + }, + }); + assert.deepEqual(runs.observe(runEvent("run:queued", "cancelled")), []); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + { taskId, startedAtIso: "start", status: "succeeded", error: null }, + ]); +}); + +it("fails a run whose thread is deleted before the turn ends", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId); + runs.observe(runEvent("run:orphaned", "running")); + assert.deepEqual( + runs.observe({ + id: EventId.make("event:deleted"), + type: "thread.deleted", + threadId, + occurredAt: epoch, + payload: undefined as never, + }), + [{ taskId, startedAtIso: "start", status: "failed", error: "The thread was deleted." }], + ); +}); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts new file mode 100644 index 0000000000000..2dbc78e7b77cf --- /dev/null +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -0,0 +1,155 @@ +import type { + MessageId, + OrchestrationV2DomainEvent, + OrchestrationV2RunStatus, + RunId, + ScheduledTaskId, + ThreadId, +} from "@t3tools/contracts"; + +/** How a dispatched scheduled run ended, keyed to the task row run that started it. */ +export interface SettledRun { + readonly taskId: ScheduledTaskId; + /** The task's `last_run_at` for this run; a newer run replaces it. */ + readonly startedAtIso: string; + readonly status: "succeeded" | "failed"; + readonly error: string | null; +} + +type RunEnd = Pick; + +interface TrackedRun { + /** The user message the dispatch stamped, which the run is first found by. */ + readonly messageId: MessageId; + readonly taskId: ScheduledTaskId; + readonly startedAtIso: string; + threadId: ThreadId | null; + /** Unset until the first event for the run's user message arrives. */ + runId: RunId | null; + /** The latest provider error the run reported, used if it ends failed. */ + providerError: string | null; + /** Set once the dispatch returned, so the row is ready to take the outcome. */ + dispatched: boolean; + /** An outcome that landed before the dispatch returned. */ + ended: RunEnd | null; +} + +const ENDED_RUN_ERRORS: Partial> = { + failed: "The run failed.", + interrupted: "The run was interrupted.", + cancelled: "The run was cancelled.", + rolled_back: "The run was rolled back.", +}; + +const settled = (run: SettledRun | null) => (run === null ? [] : [run]); + +/** + * Follows dispatched scheduled runs to their end. The dispatch stamps a message + * id on the run's user message, the only handle available before the run + * exists, so a run is found by it and then followed by run id. Kept in memory: + * a restart ends every provider turn, and startup recovery records the rows it + * leaves behind as interrupted. + */ +export function makeTrackedRuns() { + const runs = new Map(); + + const forgetTask = (taskId: ScheduledTaskId) => { + for (const [messageId, run] of runs) if (run.taskId === taskId) runs.delete(messageId); + }; + + const findByRun = (runId: RunId) => { + for (const run of runs.values()) if (run.runId === runId) return run; + return undefined; + }; + + // The row's dispatch bookkeeping must be written before an outcome replaces + // it, so an outcome that lands during the dispatch waits for `dispatched`. + const end = (run: TrackedRun, outcome: RunEnd): SettledRun | null => { + if (!run.dispatched) { + run.ended = outcome; + return null; + } + runs.delete(run.messageId); + return { taskId: run.taskId, startedAtIso: run.startedAtIso, ...outcome }; + }; + + return { + /** Must run before the dispatch, whose events can arrive before it returns. */ + track: (messageId: MessageId, taskId: ScheduledTaskId, startedAtIso: string) => { + // Only a task's latest run is reflected on its row. + forgetTask(taskId); + runs.set(messageId, { + messageId, + taskId, + startedAtIso, + threadId: null, + runId: null, + providerError: null, + dispatched: false, + ended: null, + }); + }, + forgetTask, + /** Returns the outcome if the run already ended while it was being dispatched. */ + dispatched: (messageId: MessageId): SettledRun | null => { + const run = runs.get(messageId); + if (run === undefined) return null; + run.dispatched = true; + return run.ended === null ? null : end(run, run.ended); + }, + /** Folds in one domain event and returns the outcome once a tracked run ends. */ + observe: (event: OrchestrationV2DomainEvent): ReadonlyArray => { + if (runs.size === 0) return []; + switch (event.type) { + case "message.updated": { + // A queued message promoted to a steer joins the active run. + const run = runs.get(event.payload.id); + if (run !== undefined && event.payload.runId != null) { + run.runId = event.payload.runId; + run.threadId = event.payload.threadId; + } + return []; + } + case "turn-item.updated": { + const item = event.payload; + if (item.type !== "error" || item.runId === null) return []; + const run = findByRun(item.runId); + if (run !== undefined) run.providerError = item.failure.message; + return []; + } + case "run.created": + case "run.updated": { + const { id, userMessageId, status, threadId } = event.payload; + let run = findByRun(id); + if (run === undefined) { + const candidate = runs.get(userMessageId); + if (candidate === undefined || candidate.runId !== null) return []; + run = candidate; + run.runId = id; + run.threadId = threadId; + } + if (status === "completed") + return settled(end(run, { status: "succeeded", error: null })); + const failure = ENDED_RUN_ERRORS[status]; + if (failure === undefined) return []; + // Only a failed run carries a provider error; an interrupt or cancel is the user's. + return settled( + end(run, { + status: "failed", + error: status === "failed" ? (run.providerError ?? failure) : failure, + }), + ); + } + case "thread.deleted": { + return [...runs.values()] + .filter((run) => run.threadId === event.threadId) + .flatMap((run) => + settled(end(run, { status: "failed", error: "The thread was deleted." })), + ); + } + default: + return []; + } + }, + }; +} diff --git a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx index 28fb297b35265..3ba3be72bd209 100644 --- a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx +++ b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx @@ -223,9 +223,7 @@ export function ThreadAutomationsPanel(props: { variant="ghost" part="icon" aria-label={`Run ${task.title} now`} - disabled={ - !canOperate || busyTaskId !== null || task.lastRunStatus === "running" - } + disabled={!canOperate || busyTaskId !== null} onClick={() => void runNow(task)} > diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index c962c3975c17d..29c2d5afcb6f4 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -612,6 +612,9 @@ export const OrchestratorMcpScheduledTask = Schema.Struct({ schedule: ScheduledTaskSchedule, nextRunAt: Schema.NullOr(IsoDateTime), lastRunStatus: ScheduledTaskRunStatus, + lastRunError: Schema.optional(Schema.String).annotate({ + description: "Why the last run failed. Absent unless it failed.", + }), webhookUrl: Schema.optional(Schema.String).annotate({ description: "Public URL to give the sender. Absent when this environment has no T3 Connect managed tunnel; the user must enable T3 Connect remote access first.", From 0e9db7228dc301a0a401a8d71c04aec197295e98 Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Thu, 22 Oct 2026 10:19:38 +1100 Subject: [PATCH 02/19] fix(server): scheduled run outcomes survive early ends, promoted steers and a dropped event stream - Persist the dispatch bookkeeping before an outcome can replace it, so a turn that ends right after dispatch still advances next_run_at. - A queued run cancelled only because its message was promoted into the active run no longer ends the task; the message's current run is checked first. - An outcome that cannot be saved is retried, then the run is released as failed instead of staying running. - The event listener resumes with a bounded backoff, catches up runs that ended meanwhile from the thread projection, and releases the rest as failed once retries are spent. Co-Authored-By: Claude Sonnet 5.5 --- .../ScheduledTaskService.test.ts | 230 +++++++++++++++--- .../scheduledTasks/ScheduledTaskService.ts | 131 ++++++++-- .../src/scheduledTasks/TrackedRuns.test.ts | 92 ++++--- apps/server/src/scheduledTasks/TrackedRuns.ts | 79 +++++- 4 files changed, 442 insertions(+), 90 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 858201c1d9208..1c79110d1ce38 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -474,37 +474,77 @@ const providerErrorEvent = (runId: string, message: string): OrchestrationV2Doma }); // Dispatches to a bound thread succeed at once; the run's events are fed by hand. -const makeTurnHarness = Effect.gen(function* () { - const events = yield* Queue.unbounded(); - const dispatched = yield* Queue.unbounded(); - const layer = ScheduledTaskService.layer.pipe( - Layer.provide( - Layer.mergeAll( - Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), - Layer.mock(ThreadManagementService.ThreadManagementService)({ - sendToThread: (input) => - Queue.offer(dispatched, input.messageId).pipe(Effect.as(undefined as never)), - streamDomainEvents: Stream.fromQueue(events), - }), - Layer.mock(SecretRequests.SecretRequests)({}), - Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ - namedProjectsRoot: "/projects", - isScratchProject: () => Effect.succeed(false), - }), - NodeCrypto.layer, - Scheduler.layer, +const makeTurnHarness = ( + options: { + /** Runs inside the dispatch, so the turn's events can land before it returns. */ + readonly duringDispatch?: ( + events: Queue.Queue, + messageId: MessageId, + ) => Effect.Effect; + /** The run the thread's projection says the scheduled message now belongs to. */ + readonly messageRunId?: RunId; + /** The runs the thread's projection holds, as read back after the event stream drops. */ + readonly runs?: ReadonlyArray; + /** How many times the event stream fails before it starts to deliver events. */ + readonly streamFailures?: number; + } = {}, +) => + Effect.gen(function* () { + const events = yield* Queue.unbounded(); + const dispatched = yield* Queue.unbounded(); + const subscriptions = yield* Ref.make(0); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Queue.offer(dispatched, input.messageId).pipe( + Effect.andThen(options.duringDispatch?.(events, input.messageId) ?? Effect.void), + Effect.as(undefined as never), + ), + getThreadRecords: (_threadId, _fields, filter) => + Effect.succeed({ + messages: (filter?.messageIds ?? []).map((id) => ({ + id, + runId: options.messageRunId ?? null, + })), + runs: (options.runs ?? []).filter( + (run) => filter?.runIds === undefined || filter.runIds.includes(run.id), + ), + } as never), + streamDomainEvents: Stream.unwrap( + Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe( + Effect.map((attempt) => + attempt < (options.streamFailures ?? 0) + ? Stream.fail(new Error("event stream dropped") as never) + : Stream.fromQueue(events), + ), + ), + ), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), ), - ), - ); - return { events, dispatched, layer }; -}); + ); + return { events, dispatched, subscriptions, layer }; + }); -const upsertBoundTask = (service: ScheduledTaskService.ScheduledTaskService["Service"]) => +const upsertBoundTask = ( + service: ScheduledTaskService.ScheduledTaskService["Service"], + enabled = false, +) => service.upsert({ id: ScheduledTaskId.make("task:turn-outcome"), title: "Check orchestration", prompt: "Check progress", - enabled: false, + enabled, schedule: { type: "interval", everyMs: 60_000 }, projectId: ProjectId.make("project:test"), threadId: turnThreadId, @@ -527,7 +567,7 @@ const untilStatus = ( it.effect("a failed turn replaces the dispatched run's status with the provider error", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness; + const harness = yield* makeTurnHarness(); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); @@ -553,7 +593,7 @@ it.effect("a failed turn replaces the dispatched run's status with the provider it.effect("a completed turn records the run as succeeded", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness; + const harness = yield* makeTurnHarness(); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); @@ -572,7 +612,7 @@ it.effect("a completed turn records the run as succeeded", () => it.effect("an earlier run's turn ending cannot overwrite a newer run", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness; + const harness = yield* makeTurnHarness(); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); @@ -594,3 +634,137 @@ it.effect("an earlier run's turn ending cannot overwrite a newer run", () => }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +const messageMovedEvent = (messageId: MessageId, runId: string): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${messageId}:moved`), + type: "message.updated", + threadId: turnThreadId, + runId: RunId.make(runId), + occurredAt: epoch, + payload: { + id: messageId, + threadId: turnThreadId, + runId: RunId.make(runId), + nodeId: null, + role: "user", + text: "Check progress", + attachments: [], + streaming: false, + createdAt: epoch, + updatedAt: epoch, + createdBy: "user", + creationSource: "web", + }, +}); + +it.effect("a turn that ends while the dispatch is being recorded still advances the schedule", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ + duringDispatch: (events, messageId) => + Queue.offerAll(events, [ + runEvent(runOf("run:fast", messageId, "queued")), + runEvent(runOf("run:fast", messageId, "failed")), + ]), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service, true); + yield* TestClock.adjust("30 seconds"); + yield* service.runNow({ id: task.id }); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.runCount, 1); + // Dispatched 30 seconds after the task was saved, so the next run is 60 seconds on from then. + assert.equal( + settled?.nextRunAt, + DateTime.formatIso(DateTime.makeUnsafe(Date.parse(task.nextRunAt!) + 30_000)), + ); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a queued message promoted to a steer keeps its run going in the active run", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ messageRunId: RunId.make("run:active") }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:queued", messageId, "queued")), + // Promotion cancels the queued run before it moves the message into the active one. + runEvent(runOf("run:queued", messageId, "cancelled")), + messageMovedEvent(messageId, "run:active"), + runEvent(runOf("run:active", MessageId.make("message:active-turn"), "completed")), + ]); + yield* service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks.some((t) => t.lastRunStatus !== "running")), + Stream.take(1), + Stream.runDrain, + ); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunStatus, "succeeded"); + assert.isNull(settled?.lastRunError); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a run whose outcome cannot be saved is released instead of left running", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const sql = yield* SqlClient.SqlClient; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + // Only the release path may write the terminal state. + yield* sql`CREATE TRIGGER fail_outcome BEFORE UPDATE ON scheduled_tasks + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'succeeded' + BEGIN SELECT RAISE(FAIL, 'outcome unavailable'); END`; + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:unsaved", messageId, "completed")), + ]); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "Could not record how the run ended."); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a dropped event stream is resumed, and runs that ended meanwhile are caught up", () => + Effect.gen(function* () { + const runId = RunId.make("run:during-gap"); + const harness = yield* makeTurnHarness({ + streamFailures: 1, + messageRunId: runId, + runs: [runOf(runId, MessageId.make("unused"), "completed")], + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + // The run completed while no one was listening; the resumed listener finds it. + yield* TestClock.adjust("30 seconds"); + yield* untilStatus(service, task.id, "succeeded"); + assert.equal(yield* Ref.get(harness.subscriptions), 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("runs are released as failed once the event stream cannot be resumed", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ streamFailures: 1_000 }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* TestClock.adjust("10 minutes"); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "Run outcomes could no longer be followed."); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 70291588cc791..7462995082cc3 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1,5 +1,6 @@ import { CommandId, + EventId, MessageId, PROVIDER_SEND_TURN_MAX_INPUT_CHARS, ScheduledTask, @@ -37,6 +38,7 @@ import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -716,6 +718,25 @@ export const layer = Layer.effect( if (rows.length > 0) yield* notifyChanged; }); + // Promoting a queued message to a steer cancels its queued run first and + // moves the message into the active run afterwards, so a cancel only ends + // the run if the message still belongs to the cancelled run. + const settleEnded = Effect.fn("ScheduledTaskService.settleEnded")(function* (run: SettledRun) { + if (run.cancelledRunId !== null) { + const moved = + run.threadId === null + ? null + : yield* threadManagement + .getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] }) + .pipe( + Effect.map(({ messages }) => messages[0]?.runId ?? null), + Effect.orElseSucceed(() => null), + ); + if (!trackedRuns.confirmCancel(run.messageId, moved)) return; + } + yield* recordRunEnd(run); + }); + // Best-effort escape hatch: if anything fails between markRunning and // markCompleted, write a full terminal record so runDueTasks neither skips // the task forever (it filters out 'running' rows) nor re-fires it @@ -759,6 +780,28 @@ export const layer = Layer.effect( ), ); + // Each outcome stands alone: one that still cannot be saved after retries is + // released as failed so its row does not stay `running` and block the task. + const settleOutcome = (run: SettledRun) => + settleEnded(run).pipe( + Effect.retry({ times: 2 }), + Effect.catchCause((cause) => + Effect.logWarning("Could not record scheduled task run outcome", { cause }).pipe( + Effect.andThen(Effect.result(findTask(run.taskId))), + Effect.flatMap((found) => + Result.isSuccess(found) && found.success !== null + ? releaseStuckRun( + found.success, + "Could not record how the run ended.", + run.startedAtIso, + undefined, + ) + : Effect.void, + ), + ), + ), + ); + const runTask = Effect.fn("ScheduledTaskService.runTask")(function* ( task: ScheduledTask, trigger: "scheduled" | "manual" | "webhook", @@ -911,12 +954,10 @@ export const layer = Layer.effect( const runSucceeded = result._tag === "Success"; if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; - // Accepted is not finished: the row stays 'running' until the turn - // ends, unless it already ended while the dispatch was returning. - const ended = runSucceeded ? trackedRuns.dispatched(messageId) : null; + // Accepted is not finished: the row stays 'running' until the turn ends. if (!runSucceeded) trackedRuns.forgetTask(active.id); - const lastRunStatus = runSucceeded ? (ended?.status ?? "running") : ("failed" as const); - const lastRunError = runSucceeded ? (ended?.error ?? null) : errorMessage(result.cause); + const lastRunStatus = runSucceeded ? ("running" as const) : ("failed" as const); + const lastRunError = runSucceeded ? null : errorMessage(result.cause); const threadId = runSucceeded ? result.value : null; const current = yield* findTask(task.id); const scheduleSource = current ?? task; @@ -963,6 +1004,12 @@ export const layer = Layer.effect( ), ); yield* notifyChanged; + // Only now can an outcome replace the bookkeeping just written; one that + // landed since the dispatch returned is settled here. + if (runSucceeded) { + const ended = trackedRuns.dispatched(messageId, threadId); + if (ended !== null) yield* settleOutcome(ended); + } } else trackedRuns.forgetTask(task.id); return { task: completed, threadId }; }).pipe( @@ -1092,23 +1139,71 @@ export const layer = Layer.effect( yield* scheduler.register("scheduled-tasks", runDueTasks()); + // Events missed while the stream was down are caught up from the thread + // projection: the run each still-tracked message belongs to is replayed as a + // run update. Provider error text is not recovered, so a failed run reads as + // the generic failure. + const catchUpRuns = Effect.suspend(() => + Effect.forEach( + trackedRuns.pending(), + ({ messageId, threadId }) => + Effect.gen(function* () { + const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], { + messageIds: [messageId], + }); + const runId = messages[0]?.runId ?? null; + if (runId === null) return; + const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { + runIds: [runId], + }); + const run = runs[0]; + if (run === undefined) return; + trackedRuns.bindRun(messageId, runId); + const occurredAt = yield* DateTime.now; + const ended = trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${runId}`), + type: "run.updated", + threadId, + runId, + occurredAt, + payload: run, + }); + yield* Effect.forEach(ended, settleOutcome, { discard: true }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not catch up a scheduled task run", { messageId, cause }), + ), + ), + { discard: true }, + ), + ); + // Dispatching a run only gets it accepted; its turn ends later, so the - // outcome is read from the thread's run events. + // outcome is read from the thread's run events. If the stream drops it is + // resumed with a bounded backoff; once that is spent the runs still waiting + // are released as failed, because no one will report their end. yield* forkParked( - Stream.runForEach(threadManagement.streamDomainEvents, (event) => { - // Every domain event passes through here, so the common case stays allocation-free. - const ended = trackedRuns.observe(event); - return ended.length === 0 - ? Effect.void - : Effect.forEach(ended, recordRunEnd, { discard: true }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Could not record scheduled task run outcome", { cause }), - ), - ); - }).pipe( - Effect.catchCause((cause) => + catchUpRuns.pipe( + Effect.andThen( + Stream.runForEach(threadManagement.streamDomainEvents, (event) => { + // Every domain event passes through here, so the common case stays allocation-free. + const ended = trackedRuns.observe(event); + return ended.length === 0 + ? Effect.void + : Effect.forEach(ended, settleOutcome, { discard: true }); + }), + ), + Effect.tapCause((cause) => Effect.logWarning("Scheduled task run event stream failed", { cause }), ), + Effect.retry({ schedule: Schedule.exponential("1 second"), times: 5 }), + Effect.catchCause(() => + Effect.forEach( + trackedRuns.abandonAll("Run outcomes could no longer be followed."), + settleOutcome, + { discard: true }, + ), + ), ), ); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts index aefe9841c780c..e6b11bf59e8aa 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.test.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -46,55 +46,77 @@ const runEvent = ( }, }); -it("holds an outcome that lands during the dispatch until the dispatch returns", () => { +const settledFor = ( + status: "succeeded" | "failed", + error: string | null, + cancelledRunId: string | null = null, +) => ({ + taskId, + startedAtIso: "start", + messageId, + threadId, + status, + error, + cancelledRunId: cancelledRunId === null ? null : RunId.make(cancelledRunId), +}); + +it("holds an outcome that lands before the dispatch bookkeeping is saved", () => { const runs = makeTrackedRuns(); runs.track(messageId, taskId, "start"); assert.deepEqual(runs.observe(runEvent("run:fast", "failed")), []); - assert.deepEqual(runs.dispatched(messageId), { - taskId, - startedAtIso: "start", - status: "failed", - error: "The run failed.", - }); - assert.isNull(runs.dispatched(messageId)); + assert.deepEqual(runs.dispatched(messageId, threadId), settledFor("failed", "The run failed.")); + assert.isNull(runs.dispatched(messageId, threadId)); }); -it("ignores the end of a queued run whose message was promoted into the active run", () => { +it("does not end a queued run cancelled only because its message was promoted into the active run", () => { const runs = makeTrackedRuns(); runs.track(messageId, taskId, "start"); - runs.dispatched(messageId); + runs.dispatched(messageId, threadId); assert.deepEqual(runs.observe(runEvent("run:queued", "queued")), []); - runs.observe({ - id: EventId.make("event:promoted"), - type: "message.updated", - threadId, - runId: RunId.make("run:active"), - occurredAt: epoch, - payload: { - id: messageId, - threadId, - runId: RunId.make("run:active"), - nodeId: null, - role: "user", - text: "Run the audit", - attachments: [], - streaming: false, - createdAt: epoch, - updatedAt: epoch, - createdBy: "user", - creationSource: "web", - }, - }); - assert.deepEqual(runs.observe(runEvent("run:queued", "cancelled")), []); + // Promotion cancels the queued run first, then moves the message into the active run. + const [cancelled] = runs.observe(runEvent("run:queued", "cancelled")); + assert.deepEqual(cancelled, settledFor("failed", "The run was cancelled.", "run:queued")); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:active"))); assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ - { taskId, startedAtIso: "start", status: "succeeded", error: null }, + settledFor("succeeded", null), + ]); +}); + +it("ends a queued run that was cancelled outright", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:queued", "queued")); + runs.observe(runEvent("run:queued", "cancelled")); + assert.isTrue(runs.confirmCancel(messageId, RunId.make("run:queued"))); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"))); +}); + +it("lists only dispatched runs as pending, and follows a run bound from the projection", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + assert.deepEqual(runs.pending(), []); + runs.dispatched(messageId, threadId); + assert.deepEqual(runs.pending(), [{ messageId, threadId }]); + runs.bindRun(messageId, RunId.make("run:caught-up")); + assert.deepEqual(runs.observe(runEvent("run:caught-up", "completed", MessageId.make("other"))), [ + settledFor("succeeded", null), ]); + assert.deepEqual(runs.pending(), []); +}); + +it("fails every dispatched run when events can no longer be followed", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + assert.deepEqual(runs.abandonAll("gave up"), [settledFor("failed", "gave up")]); + assert.deepEqual(runs.abandonAll("gave up"), []); }); it("fails a run whose thread is deleted before the turn ends", () => { const runs = makeTrackedRuns(); runs.track(messageId, taskId, "start"); - runs.dispatched(messageId); + runs.dispatched(messageId, threadId); runs.observe(runEvent("run:orphaned", "running")); assert.deepEqual( runs.observe({ @@ -104,6 +126,6 @@ it("fails a run whose thread is deleted before the turn ends", () => { occurredAt: epoch, payload: undefined as never, }), - [{ taskId, startedAtIso: "start", status: "failed", error: "The thread was deleted." }], + [settledFor("failed", "The thread was deleted.")], ); }); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 2dbc78e7b77cf..23442a359a974 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -14,9 +14,17 @@ export interface SettledRun { readonly startedAtIso: string; readonly status: "succeeded" | "failed"; readonly error: string | null; + readonly messageId: MessageId; + readonly threadId: ThreadId | null; + /** + * Set when the run was cancelled. Promoting a queued message to a steer + * cancels its queued run before moving the message, so a cancel is only an + * end once `confirmCancel` finds the message still belongs to that run. + */ + readonly cancelledRunId: RunId | null; } -type RunEnd = Pick; +type RunEnd = Pick; interface TrackedRun { /** The user message the dispatch stamped, which the run is first found by. */ @@ -63,14 +71,22 @@ export function makeTrackedRuns() { }; // The row's dispatch bookkeeping must be written before an outcome replaces - // it, so an outcome that lands during the dispatch waits for `dispatched`. + // it, so an outcome that lands during the dispatch waits for `dispatched`, + // which the caller marks only once that bookkeeping is saved. const end = (run: TrackedRun, outcome: RunEnd): SettledRun | null => { if (!run.dispatched) { run.ended = outcome; return null; } - runs.delete(run.messageId); - return { taskId: run.taskId, startedAtIso: run.startedAtIso, ...outcome }; + // A cancelled run stays tracked until `confirmCancel` rules on it. + if (outcome.cancelledRunId === null) runs.delete(run.messageId); + return { + taskId: run.taskId, + startedAtIso: run.startedAtIso, + messageId: run.messageId, + threadId: run.threadId, + ...outcome, + }; }; return { @@ -90,12 +106,50 @@ export function makeTrackedRuns() { }); }, forgetTask, - /** Returns the outcome if the run already ended while it was being dispatched. */ - dispatched: (messageId: MessageId): SettledRun | null => { + /** + * Marks the dispatch bookkeeping as saved, and returns the outcome if the + * run already ended while that was happening. + */ + dispatched: (messageId: MessageId, threadId: ThreadId | null): SettledRun | null => { const run = runs.get(messageId); if (run === undefined) return null; run.dispatched = true; - return run.ended === null ? null : end(run, run.ended); + run.threadId ??= threadId; + const held = run.ended; + run.ended = null; + return held === null ? null : end(run, held); + }, + /** Dispatched runs still waiting for an end, for catching up after missed events. */ + pending: (): ReadonlyArray<{ readonly messageId: MessageId; readonly threadId: ThreadId }> => + [...runs.values()].flatMap((run) => + run.dispatched && run.threadId !== null + ? [{ messageId: run.messageId, threadId: run.threadId }] + : [], + ), + /** Follows a run found by reading the projection rather than by events. */ + bindRun: (messageId: MessageId, runId: RunId) => { + const run = runs.get(messageId); + if (run !== undefined) run.runId = runId; + }, + /** Ends every dispatched run as failed, for when events can no longer be followed. */ + abandonAll: (error: string): ReadonlyArray => + [...runs.values()].flatMap((run) => + run.dispatched ? settled(end(run, { status: "failed", error, cancelledRunId: null })) : [], + ), + /** + * Rules on a cancelled run, given the run its message belongs to now. + * Returns false when the message moved into another run, which is then + * followed instead. + */ + confirmCancel: (messageId: MessageId, messageRunId: RunId | null): boolean => { + const run = runs.get(messageId); + if (run === undefined) return false; + if (messageRunId !== null && messageRunId !== run.runId) { + run.runId = messageRunId; + return false; + } + runs.delete(messageId); + return true; }, /** Folds in one domain event and returns the outcome once a tracked run ends. */ observe: (event: OrchestrationV2DomainEvent): ReadonlyArray => { @@ -129,7 +183,7 @@ export function makeTrackedRuns() { run.threadId = threadId; } if (status === "completed") - return settled(end(run, { status: "succeeded", error: null })); + return settled(end(run, { status: "succeeded", error: null, cancelledRunId: null })); const failure = ENDED_RUN_ERRORS[status]; if (failure === undefined) return []; // Only a failed run carries a provider error; an interrupt or cancel is the user's. @@ -137,6 +191,7 @@ export function makeTrackedRuns() { end(run, { status: "failed", error: status === "failed" ? (run.providerError ?? failure) : failure, + cancelledRunId: status === "cancelled" ? id : null, }), ); } @@ -144,7 +199,13 @@ export function makeTrackedRuns() { return [...runs.values()] .filter((run) => run.threadId === event.threadId) .flatMap((run) => - settled(end(run, { status: "failed", error: "The thread was deleted." })), + settled( + end(run, { + status: "failed", + error: "The thread was deleted.", + cancelledRunId: null, + }), + ), ); } default: From 2cbe737350fc69242a8af3ebfb547b5ead8cc95a Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Thu, 22 Oct 2026 10:37:20 +1100 Subject: [PATCH 03/19] fix(server): scheduled run listener resumes forever and settles each outcome once - Decide a cancel once and retry only the save, so a failed first write no longer makes the retry a silent no-op that leaves the row running. - Resume the run event stream forever with a capped backoff instead of giving up after five tries, which left later dispatches with no listener. - Start the stream before the projection catch-up so a terminal event between them is delivered rather than missed. Co-Authored-By: Claude Sonnet 5.5 --- .../ScheduledTaskService.test.ts | 100 +++++++++++++++--- .../scheduledTasks/ScheduledTaskService.ts | 95 ++++++++++------- .../src/scheduledTasks/TrackedRuns.test.ts | 8 -- apps/server/src/scheduledTasks/TrackedRuns.ts | 5 - 4 files changed, 139 insertions(+), 69 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 1c79110d1ce38..4ced3ef222aac 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -487,6 +487,12 @@ const makeTurnHarness = ( readonly runs?: ReadonlyArray; /** How many times the event stream fails before it starts to deliver events. */ readonly streamFailures?: number; + /** Makes the stream a live tail: events from before it was opened are not delivered. */ + readonly liveTail?: boolean; + /** Runs when the projection's runs are read, so an event can land mid catch-up. */ + readonly duringRunsRead?: ( + events: Queue.Queue, + ) => Effect.Effect; } = {}, ) => Effect.gen(function* () { @@ -503,22 +509,27 @@ const makeTurnHarness = ( Effect.andThen(options.duringDispatch?.(events, input.messageId) ?? Effect.void), Effect.as(undefined as never), ), - getThreadRecords: (_threadId, _fields, filter) => - Effect.succeed({ - messages: (filter?.messageIds ?? []).map((id) => ({ - id, - runId: options.messageRunId ?? null, - })), - runs: (options.runs ?? []).filter( - (run) => filter?.runIds === undefined || filter.runIds.includes(run.id), - ), - } as never), + getThreadRecords: (_threadId, fields: ReadonlyArray, filter) => + Effect.gen(function* () { + if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void; + return { + messages: (filter?.messageIds ?? []).map((id) => ({ + id, + runId: options.messageRunId ?? null, + })), + runs: (options.runs ?? []).filter( + (run) => filter?.runIds === undefined || filter.runIds.includes(run.id), + ), + } as never; + }), streamDomainEvents: Stream.unwrap( Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe( - Effect.map((attempt) => + Effect.flatMap((attempt) => attempt < (options.streamFailures ?? 0) - ? Stream.fail(new Error("event stream dropped") as never) - : Stream.fromQueue(events), + ? Effect.succeed(Stream.fail(new Error("event stream dropped") as never)) + : (options.liveTail ? Queue.clear(events) : Effect.void).pipe( + Effect.as(Stream.fromQueue(events)), + ), ), ), ), @@ -754,17 +765,74 @@ it.effect("a dropped event stream is resumed, and runs that ended meanwhile are }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); -it.effect("runs are released as failed once the event stream cannot be resumed", () => +it.effect("the event stream keeps being resumed after many failures", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness({ streamFailures: 1_000 }); + const runId = RunId.make("run:late"); + const harness = yield* makeTurnHarness({ + streamFailures: 8, + messageRunId: runId, + runs: [runOf(runId, MessageId.make("unused"), "failed")], + }); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); yield* TestClock.adjust("10 minutes"); yield* untilStatus(service, task.id, "failed"); + assert.equal(yield* Ref.get(harness.subscriptions), 9); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("an event landing while the resumed listener catches up is not missed", () => + Effect.gen(function* () { + const runId = RunId.make("run:handoff"); + const harness = yield* makeTurnHarness({ + streamFailures: 1, + liveTail: true, + messageRunId: runId, + // The projection read is stale: the run ends right after it. + runs: [runOf(runId, MessageId.make("unused"), "running")], + duringRunsRead: (events) => + Queue.offer(events, runEvent(runOf(runId, MessageId.make("unused"), "completed"))).pipe( + Effect.asVoid, + ), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("30 seconds"); + yield* untilStatus(service, task.id, "succeeded"); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a cancelled run is still recorded when its first save fails", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const sql = yield* SqlClient.SqlClient; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + // Fails the first write only; the retry must still record the cancel. + yield* sql`CREATE TABLE fail_once (id INTEGER)`; + yield* sql`INSERT INTO fail_once VALUES (1)`; + yield* sql`CREATE TRIGGER fail_first_outcome BEFORE UPDATE ON scheduled_tasks + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'failed' + AND EXISTS (SELECT 1 FROM fail_once) + BEGIN DELETE FROM fail_once; SELECT RAISE(FAIL, 'outcome unavailable'); END`; + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:cancelled", messageId, "queued")), + runEvent(runOf("run:cancelled", messageId, "cancelled")), + ]); + yield* untilStatus(service, task.id, "failed"); const [settled] = (yield* service.list()).tasks; - assert.equal(settled?.lastRunError, "Run outcomes could no longer be followed."); + assert.equal(settled?.lastRunError, "The run was cancelled."); }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 7462995082cc3..17c241903bbfd 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -31,6 +31,7 @@ import * as Data from "effect/Data"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Duration from "effect/Duration"; import * as Metric from "effect/Metric"; import * as Layer from "effect/Layer"; @@ -718,25 +719,6 @@ export const layer = Layer.effect( if (rows.length > 0) yield* notifyChanged; }); - // Promoting a queued message to a steer cancels its queued run first and - // moves the message into the active run afterwards, so a cancel only ends - // the run if the message still belongs to the cancelled run. - const settleEnded = Effect.fn("ScheduledTaskService.settleEnded")(function* (run: SettledRun) { - if (run.cancelledRunId !== null) { - const moved = - run.threadId === null - ? null - : yield* threadManagement - .getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] }) - .pipe( - Effect.map(({ messages }) => messages[0]?.runId ?? null), - Effect.orElseSucceed(() => null), - ); - if (!trackedRuns.confirmCancel(run.messageId, moved)) return; - } - yield* recordRunEnd(run); - }); - // Best-effort escape hatch: if anything fails between markRunning and // markCompleted, write a full terminal record so runDueTasks neither skips // the task forever (it filters out 'running' rows) nor re-fires it @@ -782,8 +764,26 @@ export const layer = Layer.effect( // Each outcome stands alone: one that still cannot be saved after retries is // released as failed so its row does not stay `running` and block the task. - const settleOutcome = (run: SettledRun) => - settleEnded(run).pipe( + const settleOutcome = Effect.fn("ScheduledTaskService.settleOutcome")(function* ( + run: SettledRun, + ) { + // Promoting a queued message to a steer cancels its queued run first and + // moves the message into the active run afterwards, so a cancel only ends + // the run if the message still belongs to the cancelled run. Decided once: + // confirming forgets the run, so it must not be repeated by a retry. + if (run.cancelledRunId !== null) { + const moved = + run.threadId === null + ? null + : yield* threadManagement + .getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] }) + .pipe( + Effect.map(({ messages }) => messages[0]?.runId ?? null), + Effect.orElseSucceed(() => null), + ); + if (!trackedRuns.confirmCancel(run.messageId, moved)) return; + } + yield* recordRunEnd(run).pipe( Effect.retry({ times: 2 }), Effect.catchCause((cause) => Effect.logWarning("Could not record scheduled task run outcome", { cause }).pipe( @@ -801,6 +801,7 @@ export const layer = Layer.effect( ), ), ); + }); const runTask = Effect.fn("ScheduledTaskService.runTask")(function* ( task: ScheduledTask, @@ -1178,30 +1179,44 @@ export const layer = Layer.effect( ), ); + // The stream resumes after a cursor it fixes on its first pull. Starting + // that pull before the catch-up read means an event landing in between is + // delivered by the stream instead of missed, and a run seen by both is + // settled once because the tracker forgets it when it ends. + const runEvents = Stream.fromPull( + Effect.gen(function* () { + const pull = yield* Stream.toPull(threadManagement.streamDomainEvents); + const first = yield* Effect.forkScoped(pull, { startImmediately: true }); + yield* catchUpRuns; + let joined = false; + return Effect.suspend(() => { + if (joined) return pull; + joined = true; + return Fiber.join(first); + }); + }), + ); + // Dispatching a run only gets it accepted; its turn ends later, so the - // outcome is read from the thread's run events. If the stream drops it is - // resumed with a bounded backoff; once that is spent the runs still waiting - // are released as failed, because no one will report their end. + // outcome is read from the thread's run events. A dropped stream is resumed + // forever with a capped backoff: runs dispatched meanwhile stay tracked and + // are caught up on the next subscribe. yield* forkParked( - catchUpRuns.pipe( - Effect.andThen( - Stream.runForEach(threadManagement.streamDomainEvents, (event) => { - // Every domain event passes through here, so the common case stays allocation-free. - const ended = trackedRuns.observe(event); - return ended.length === 0 - ? Effect.void - : Effect.forEach(ended, settleOutcome, { discard: true }); - }), - ), + Stream.runForEach(runEvents, (event) => { + // Every domain event passes through here, so the common case stays allocation-free. + const ended = trackedRuns.observe(event); + return ended.length === 0 + ? Effect.void + : Effect.forEach(ended, settleOutcome, { discard: true }); + }).pipe( Effect.tapCause((cause) => Effect.logWarning("Scheduled task run event stream failed", { cause }), ), - Effect.retry({ schedule: Schedule.exponential("1 second"), times: 5 }), - Effect.catchCause(() => - Effect.forEach( - trackedRuns.abandonAll("Run outcomes could no longer be followed."), - settleOutcome, - { discard: true }, + Effect.retry( + Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.minutes(1))), + ), ), ), ), diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts index e6b11bf59e8aa..2ae4e83f65478 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.test.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -105,14 +105,6 @@ it("lists only dispatched runs as pending, and follows a run bound from the proj assert.deepEqual(runs.pending(), []); }); -it("fails every dispatched run when events can no longer be followed", () => { - const runs = makeTrackedRuns(); - runs.track(messageId, taskId, "start"); - runs.dispatched(messageId, threadId); - assert.deepEqual(runs.abandonAll("gave up"), [settledFor("failed", "gave up")]); - assert.deepEqual(runs.abandonAll("gave up"), []); -}); - it("fails a run whose thread is deleted before the turn ends", () => { const runs = makeTrackedRuns(); runs.track(messageId, taskId, "start"); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 23442a359a974..027eee8988286 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -131,11 +131,6 @@ export function makeTrackedRuns() { const run = runs.get(messageId); if (run !== undefined) run.runId = runId; }, - /** Ends every dispatched run as failed, for when events can no longer be followed. */ - abandonAll: (error: string): ReadonlyArray => - [...runs.values()].flatMap((run) => - run.dispatched ? settled(end(run, { status: "failed", error, cancelledRunId: null })) : [], - ), /** * Rules on a cancelled run, given the run its message belongs to now. * Returns false when the message moved into another run, which is then From 229911c90db40826b287ab6aa06914da0259c7a0 Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Thu, 22 Oct 2026 11:01:19 +1100 Subject: [PATCH 04/19] fix(server): scheduled run catch-up no longer strands a run it cannot read - Retry the projection read a few times, then release the run as failed and stop tracking it, so one unreadable thread neither leaves its row running nor makes the listener resubscribe forever. - Record a thread deleted while the stream was down as failed; deletion keeps the projection flagged rather than removing it. Co-Authored-By: Claude Sonnet 5.5 --- .../ScheduledTaskService.test.ts | 43 +++++++++++++ .../scheduledTasks/ScheduledTaskService.ts | 62 ++++++++++++++----- .../src/scheduledTasks/TrackedRuns.test.ts | 2 +- apps/server/src/scheduledTasks/TrackedRuns.ts | 13 +++- 4 files changed, 100 insertions(+), 20 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 4ced3ef222aac..b7b900e423808 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -487,6 +487,10 @@ const makeTurnHarness = ( readonly runs?: ReadonlyArray; /** How many times the event stream fails before it starts to deliver events. */ readonly streamFailures?: number; + /** Reads the thread as deleted, which the projection keeps as a flagged record. */ + readonly threadDeleted?: boolean; + /** Makes every projection read fail, as for a thread whose records are gone. */ + readonly recordsUnreadable?: boolean; /** Makes the stream a live tail: events from before it was opened are not delivered. */ readonly liveTail?: boolean; /** Runs when the projection's runs are read, so an event can land mid catch-up. */ @@ -511,8 +515,11 @@ const makeTurnHarness = ( ), getThreadRecords: (_threadId, fields: ReadonlyArray, filter) => Effect.gen(function* () { + if (options.recordsUnreadable) + return yield* Effect.fail(new Error("no records") as never); if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void; return { + thread: { deletedAt: options.threadDeleted ? epoch : null }, messages: (filter?.messageIds ?? []).map((id) => ({ id, runId: options.messageRunId ?? null, @@ -836,3 +843,39 @@ it.effect("a cancelled run is still recorded when its first save fails", () => }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect( + "a run whose thread cannot be read during catch-up is released, not retried forever", + () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ streamFailures: 1, recordsUnreadable: true }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("1 minute"); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "Could not read how the run ended."); + // The stream was resubscribed once for the drop, not again for the unreadable thread. + assert.equal(yield* Ref.get(harness.subscriptions), 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a thread deleted while the stream was down fails its run on catch-up", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ streamFailures: 1, threadDeleted: true }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("30 seconds"); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "The thread was deleted."); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 17c241903bbfd..065772f46ad3b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -762,6 +762,15 @@ export const layer = Layer.effect( ), ); + const releaseRun = (run: Pick, message: string) => + Effect.result(findTask(run.taskId)).pipe( + Effect.flatMap((found) => + Result.isSuccess(found) && found.success !== null + ? releaseStuckRun(found.success, message, run.startedAtIso, undefined) + : Effect.void, + ), + ); + // Each outcome stands alone: one that still cannot be saved after retries is // released as failed so its row does not stay `running` and block the task. const settleOutcome = Effect.fn("ScheduledTaskService.settleOutcome")(function* ( @@ -787,17 +796,7 @@ export const layer = Layer.effect( Effect.retry({ times: 2 }), Effect.catchCause((cause) => Effect.logWarning("Could not record scheduled task run outcome", { cause }).pipe( - Effect.andThen(Effect.result(findTask(run.taskId))), - Effect.flatMap((found) => - Result.isSuccess(found) && found.success !== null - ? releaseStuckRun( - found.success, - "Could not record how the run ended.", - run.startedAtIso, - undefined, - ) - : Effect.void, - ), + Effect.andThen(releaseRun(run, "Could not record how the run ended.")), ), ), ); @@ -1147,11 +1146,28 @@ export const layer = Layer.effect( const catchUpRuns = Effect.suspend(() => Effect.forEach( trackedRuns.pending(), - ({ messageId, threadId }) => + (pending) => Effect.gen(function* () { - const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], { - messageIds: [messageId], - }); + const { messageId, threadId } = pending; + const { thread, messages } = yield* threadManagement.getThreadRecords( + threadId, + ["messages"], + { messageIds: [messageId] }, + ); + const occurredAt = yield* DateTime.now; + // Deleting a thread keeps its projection, flagged, so a delete missed + // with the stream shows up here rather than as a read failure. + if (thread.deletedAt !== null) { + const ended = trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${threadId}:deleted`), + type: "thread.deleted", + threadId, + occurredAt, + payload: undefined as never, + }); + yield* Effect.forEach(ended, settleOutcome, { discard: true }); + return; + } const runId = messages[0]?.runId ?? null; if (runId === null) return; const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { @@ -1160,7 +1176,6 @@ export const layer = Layer.effect( const run = runs[0]; if (run === undefined) return; trackedRuns.bindRun(messageId, runId); - const occurredAt = yield* DateTime.now; const ended = trackedRuns.observe({ id: EventId.make(`scheduled-task-catch-up:${runId}`), type: "run.updated", @@ -1171,8 +1186,21 @@ export const layer = Layer.effect( }); yield* Effect.forEach(ended, settleOutcome, { discard: true }); }).pipe( + // A thread that cannot be read (deleted, records gone) would otherwise + // be retried by every resubscribe and nothing else would settle its + // row, so after a few tries the run is released as failed. + Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }), Effect.catchCause((cause) => - Effect.logWarning("Could not catch up a scheduled task run", { messageId, cause }), + Effect.logWarning("Could not catch up a scheduled task run", { + messageId: pending.messageId, + cause, + }).pipe( + Effect.andThen( + Effect.sync(() => trackedRuns.forgetTask(pending.taskId)).pipe( + Effect.andThen(releaseRun(pending, "Could not read how the run ended.")), + ), + ), + ), ), ), { discard: true }, diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts index 2ae4e83f65478..a1998fc8a9936 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.test.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -97,7 +97,7 @@ it("lists only dispatched runs as pending, and follows a run bound from the proj runs.track(messageId, taskId, "start"); assert.deepEqual(runs.pending(), []); runs.dispatched(messageId, threadId); - assert.deepEqual(runs.pending(), [{ messageId, threadId }]); + assert.deepEqual(runs.pending(), [{ taskId, startedAtIso: "start", messageId, threadId }]); runs.bindRun(messageId, RunId.make("run:caught-up")); assert.deepEqual(runs.observe(runEvent("run:caught-up", "completed", MessageId.make("other"))), [ settledFor("succeeded", null), diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 027eee8988286..4f49fa8a98c72 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -120,10 +120,19 @@ export function makeTrackedRuns() { return held === null ? null : end(run, held); }, /** Dispatched runs still waiting for an end, for catching up after missed events. */ - pending: (): ReadonlyArray<{ readonly messageId: MessageId; readonly threadId: ThreadId }> => + pending: (): ReadonlyArray< + Pick & { readonly threadId: ThreadId } + > => [...runs.values()].flatMap((run) => run.dispatched && run.threadId !== null - ? [{ messageId: run.messageId, threadId: run.threadId }] + ? [ + { + taskId: run.taskId, + startedAtIso: run.startedAtIso, + messageId: run.messageId, + threadId: run.threadId, + }, + ] : [], ), /** Follows a run found by reading the projection rather than by events. */ From f5ae53f9b4e16cd7a0094d35fda8c88a37e6a84e Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Thu, 22 Oct 2026 11:17:31 +1100 Subject: [PATCH 05/19] fix(server): giving up on a catch-up read forgets only that run The catch-up failure branch dropped the task's tracked run, which after a Run now during the retries is the newer run. Forget the one message instead. Co-Authored-By: Claude Sonnet 5.5 --- .../ScheduledTaskService.test.ts | 28 +++++++++++++++++-- .../scheduledTasks/ScheduledTaskService.ts | 2 +- apps/server/src/scheduledTasks/TrackedRuns.ts | 4 +++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index b7b900e423808..abd3dda55159e 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -503,6 +503,7 @@ const makeTurnHarness = ( const events = yield* Queue.unbounded(); const dispatched = yield* Queue.unbounded(); const subscriptions = yield* Ref.make(0); + const failedReads = yield* Queue.unbounded(); const layer = ScheduledTaskService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -515,8 +516,10 @@ const makeTurnHarness = ( ), getThreadRecords: (_threadId, fields: ReadonlyArray, filter) => Effect.gen(function* () { - if (options.recordsUnreadable) + if (options.recordsUnreadable) { + yield* Queue.offer(failedReads, undefined); return yield* Effect.fail(new Error("no records") as never); + } if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void; return { thread: { deletedAt: options.threadDeleted ? epoch : null }, @@ -551,7 +554,7 @@ const makeTurnHarness = ( ), ), ); - return { events, dispatched, subscriptions, layer }; + return { events, dispatched, subscriptions, failedReads, layer }; }); const upsertBoundTask = ( @@ -879,3 +882,24 @@ it.effect("a thread deleted while the stream was down fails its run on catch-up" }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect("giving up on a catch-up read leaves a newer run of the same task tracked", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ recordsUnreadable: true }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + // The listener's catch-up for that run starts, fails its first read and backs off. + yield* Queue.take(harness.failedReads); + yield* TestClock.adjust("1 second"); + yield* service.runNow({ id: task.id }); + const newer = yield* Queue.take(harness.dispatched); + // The catch-up for the first run runs out of retries and gives up. + yield* TestClock.adjust("30 seconds"); + yield* Queue.offer(harness.events, runEvent(runOf("run:newer", newer, "completed"))); + yield* untilStatus(service, task.id, "succeeded"); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 065772f46ad3b..172227348aa81 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1196,7 +1196,7 @@ export const layer = Layer.effect( cause, }).pipe( Effect.andThen( - Effect.sync(() => trackedRuns.forgetTask(pending.taskId)).pipe( + Effect.sync(() => trackedRuns.forgetMessage(pending.messageId)).pipe( Effect.andThen(releaseRun(pending, "Could not read how the run ended.")), ), ), diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 4f49fa8a98c72..1687f9f9907ad 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -106,6 +106,10 @@ export function makeTrackedRuns() { }); }, forgetTask, + /** Drops one run only, for a caller whose task may have started a newer run since. */ + forgetMessage: (messageId: MessageId) => { + runs.delete(messageId); + }, /** * Marks the dispatch bookkeeping as saved, and returns the outcome if the * run already ended while that was happening. From beceabdfb06ea15acf89afbeb714fa869913d802 Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Thu, 22 Oct 2026 11:33:10 +1100 Subject: [PATCH 06/19] test(server): make the catch-up give-up test independent of fiber ordering The listener's first catch-up may run before or after the first dispatch, so the test now drops the stream after dispatching to guarantee a resubscribe catch-up, and only the first message's reads fail so the newer run's reads and event still go through whichever catch-up reads it. Co-Authored-By: Claude Sonnet 5.5 --- .../ScheduledTaskService.test.ts | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index abd3dda55159e..d83ed249a558c 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -489,6 +489,14 @@ const makeTurnHarness = ( readonly streamFailures?: number; /** Reads the thread as deleted, which the projection keeps as a flagged record. */ readonly threadDeleted?: boolean; + /** + * The first stream subscription fails only when the test releases `dropStream`. + * The listener's first catch-up can run before or after a dispatch, so the drop + * guarantees one resubscribe catch-up that sees the dispatched run. + */ + readonly dropStreamOnCommand?: boolean; + /** Reads about the first dispatched message fail; later messages read normally. */ + readonly firstMessageUnreadable?: boolean; /** Makes every projection read fail, as for a thread whose records are gone. */ readonly recordsUnreadable?: boolean; /** Makes the stream a live tail: events from before it was opened are not delivered. */ @@ -504,19 +512,28 @@ const makeTurnHarness = ( const dispatched = yield* Queue.unbounded(); const subscriptions = yield* Ref.make(0); const failedReads = yield* Queue.unbounded(); + const dropStream = yield* Deferred.make(); + const firstMessage = yield* Ref.make(null); const layer = ScheduledTaskService.layer.pipe( Layer.provide( Layer.mergeAll( Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), Layer.mock(ThreadManagementService.ThreadManagementService)({ sendToThread: (input) => - Queue.offer(dispatched, input.messageId).pipe( + Ref.update(firstMessage, (first) => first ?? input.messageId).pipe( + Effect.andThen(Queue.offer(dispatched, input.messageId)), Effect.andThen(options.duringDispatch?.(events, input.messageId) ?? Effect.void), Effect.as(undefined as never), ), getThreadRecords: (_threadId, fields: ReadonlyArray, filter) => Effect.gen(function* () { - if (options.recordsUnreadable) { + const first = yield* Ref.get(firstMessage); + if ( + options.recordsUnreadable || + (options.firstMessageUnreadable && + first !== null && + filter?.messageIds?.includes(first)) + ) { yield* Queue.offer(failedReads, undefined); return yield* Effect.fail(new Error("no records") as never); } @@ -535,8 +552,10 @@ const makeTurnHarness = ( streamDomainEvents: Stream.unwrap( Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe( Effect.flatMap((attempt) => - attempt < (options.streamFailures ?? 0) - ? Effect.succeed(Stream.fail(new Error("event stream dropped") as never)) + attempt < (options.streamFailures ?? (options.dropStreamOnCommand ? 1 : 0)) + ? (options.dropStreamOnCommand ? Deferred.await(dropStream) : Effect.void).pipe( + Effect.as(Stream.fail(new Error("event stream dropped") as never)), + ) : (options.liveTail ? Queue.clear(events) : Effect.void).pipe( Effect.as(Stream.fromQueue(events)), ), @@ -554,7 +573,7 @@ const makeTurnHarness = ( ), ), ); - return { events, dispatched, subscriptions, failedReads, layer }; + return { events, dispatched, subscriptions, failedReads, dropStream, layer }; }); const upsertBoundTask = ( @@ -885,13 +904,19 @@ it.effect("a thread deleted while the stream was down fails its run on catch-up" it.effect("giving up on a catch-up read leaves a newer run of the same task tracked", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness({ recordsUnreadable: true }); + const harness = yield* makeTurnHarness({ + dropStreamOnCommand: true, + firstMessageUnreadable: true, + }); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); yield* service.runNow({ id: task.id }); yield* Queue.take(harness.dispatched); - // The listener's catch-up for that run starts, fails its first read and backs off. + // Dropping the stream makes the resubscribe catch up the dispatched run, whose + // first read fails and backs off. + yield* Deferred.succeed(harness.dropStream, undefined); + yield* TestClock.adjust("1 second"); yield* Queue.take(harness.failedReads); yield* TestClock.adjust("1 second"); yield* service.runNow({ id: task.id }); From 206d24fe880ba6e95bdc7699027b1467ec2deffc Mon Sep 17 00:00:00 2001 From: ravdeep kainth Date: Sun, 25 Oct 2026 14:11:12 +1100 Subject: [PATCH 07/19] fix(server): trim the TrackedRuns comment to the four-line limit Co-Authored-By: Claude Opus 5.5 --- apps/server/src/scheduledTasks/TrackedRuns.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 1687f9f9907ad..ba0e19f4dd698 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -52,11 +52,9 @@ const ENDED_RUN_ERRORS: Partial> = { const settled = (run: SettledRun | null) => (run === null ? [] : [run]); /** - * Follows dispatched scheduled runs to their end. The dispatch stamps a message - * id on the run's user message, the only handle available before the run - * exists, so a run is found by it and then followed by run id. Kept in memory: - * a restart ends every provider turn, and startup recovery records the rows it - * leaves behind as interrupted. + * A run is found by the message id its dispatch stamps (the only handle before the run + * exists), then followed by run id. In memory only: a restart ends every provider turn, + * and startup recovery records the rows it leaves behind as interrupted. */ export function makeTrackedRuns() { const runs = new Map(); From db4166bd8cb506755e33b21b1fb0a7612c3e9418 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 18:48:02 +0000 Subject: [PATCH 08/19] fix(server): settle every scheduled task sharing a run --- .../ScheduledTaskService.test.ts | 33 +++++++ .../scheduledTasks/ScheduledTaskService.ts | 2 +- .../src/scheduledTasks/TrackedRuns.test.ts | 43 ++++++++- apps/server/src/scheduledTasks/TrackedRuns.ts | 95 +++++++------------ 4 files changed, 106 insertions(+), 67 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 2fe58b2e5fb0e..14f76effa0c93 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -925,3 +925,36 @@ it.effect("giving up on a catch-up read leaves a newer run of the same task trac }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect("records the provider error for every task steered into the same run", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const { task: second } = yield* service.upsert({ + ...task, + id: ScheduledTaskId.make("task:second-shared-run"), + }); + yield* service.runNow({ id: task.id }); + const firstMessage = yield* Queue.take(harness.dispatched); + yield* service.runNow({ id: second.id }); + const secondMessage = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + messageMovedEvent(firstMessage, "run:shared"), + messageMovedEvent(secondMessage, "run:shared"), + providerErrorEvent("run:shared", "You have reached your usage limit."), + runEvent(runOf("run:shared", firstMessage, "failed")), + ]); + yield* service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks.every((t) => t.lastRunStatus === "failed")), + Stream.take(1), + Stream.runDrain, + ); + for (const settled of (yield* service.list()).tasks) { + assert.equal(settled.lastRunError, "You have reached your usage limit."); + assert.equal(settled.runCount, 1); + } + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 9b9a01872c2a8..93598ba09bfab 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -777,7 +777,7 @@ export const layer = Layer.effect( Effect.map(({ messages }) => messages[0]?.runId ?? null), Effect.orElseSucceed(() => null), ); - if (!trackedRuns.confirmCancel(run.messageId, moved)) return; + if (!trackedRuns.confirmCancel(run.messageId, run.cancelledRunId, moved)) return; } yield* recordRunEnd(run).pipe( Effect.retry({ times: 2 }), diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts index a1998fc8a9936..4d263ffcaa832 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.test.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -76,7 +76,7 @@ it("does not end a queued run cancelled only because its message was promoted in // Promotion cancels the queued run first, then moves the message into the active run. const [cancelled] = runs.observe(runEvent("run:queued", "cancelled")); assert.deepEqual(cancelled, settledFor("failed", "The run was cancelled.", "run:queued")); - assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:active"))); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:active"))); assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ settledFor("succeeded", null), ]); @@ -88,8 +88,8 @@ it("ends a queued run that was cancelled outright", () => { runs.dispatched(messageId, threadId); runs.observe(runEvent("run:queued", "queued")); runs.observe(runEvent("run:queued", "cancelled")); - assert.isTrue(runs.confirmCancel(messageId, RunId.make("run:queued"))); - assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"))); + assert.isTrue(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:queued"))); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:queued"))); }); it("lists only dispatched runs as pending, and follows a run bound from the projection", () => { @@ -121,3 +121,40 @@ it("fails a run whose thread is deleted before the turn ends", () => { [settledFor("failed", "The thread was deleted.")], ); }); + +it.each(["completed", "failed"] as const)("settles every task sharing a %s run", (status) => { + const runs = makeTrackedRuns(); + const secondMessage = MessageId.make("message:second-task"); + const secondTask = ScheduledTaskId.make("task:second"); + runs.track(messageId, taskId, "start"); + runs.track(secondMessage, secondTask, "second-start"); + runs.dispatched(messageId, threadId); + runs.dispatched(secondMessage, threadId); + runs.bindRun(messageId, RunId.make("run:shared")); + runs.bindRun(secondMessage, RunId.make("run:shared")); + const error = status === "failed" ? "The run failed." : null; + assert.deepEqual(runs.observe(runEvent("run:shared", status)), [ + settledFor(status === "completed" ? "succeeded" : "failed", error), + { + ...settledFor(status === "completed" ? "succeeded" : "failed", error), + messageId: secondMessage, + taskId: secondTask, + startedAtIso: "second-start", + }, + ]); + assert.deepEqual(runs.pending(), []); +}); + +it("keeps a promoted message tracked when cancellation arrived before dispatch returned", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.observe(runEvent("run:queued", "queued")); + runs.observe(runEvent("run:queued", "cancelled")); + runs.bindRun(messageId, RunId.make("run:active")); + const held = runs.dispatched(messageId, threadId); + assert.equal(held?.cancelledRunId, "run:queued"); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:active"))); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + settledFor("succeeded", null), + ]); +}); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index ba0e19f4dd698..8d6ba23749173 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -6,39 +6,26 @@ import type { ScheduledTaskId, ThreadId, } from "@t3tools/contracts"; - -/** How a dispatched scheduled run ended, keyed to the task row run that started it. */ export interface SettledRun { readonly taskId: ScheduledTaskId; - /** The task's `last_run_at` for this run; a newer run replaces it. */ readonly startedAtIso: string; readonly status: "succeeded" | "failed"; readonly error: string | null; readonly messageId: MessageId; readonly threadId: ThreadId | null; - /** - * Set when the run was cancelled. Promoting a queued message to a steer - * cancels its queued run before moving the message, so a cancel is only an - * end once `confirmCancel` finds the message still belongs to that run. - */ readonly cancelledRunId: RunId | null; } type RunEnd = Pick; interface TrackedRun { - /** The user message the dispatch stamped, which the run is first found by. */ readonly messageId: MessageId; readonly taskId: ScheduledTaskId; readonly startedAtIso: string; threadId: ThreadId | null; - /** Unset until the first event for the run's user message arrives. */ runId: RunId | null; - /** The latest provider error the run reported, used if it ends failed. */ providerError: string | null; - /** Set once the dispatch returned, so the row is ready to take the outcome. */ dispatched: boolean; - /** An outcome that landed before the dispatch returned. */ ended: RunEnd | null; } @@ -50,12 +37,7 @@ const ENDED_RUN_ERRORS: Partial> = { }; const settled = (run: SettledRun | null) => (run === null ? [] : [run]); - -/** - * A run is found by the message id its dispatch stamps (the only handle before the run - * exists), then followed by run id. In memory only: a restart ends every provider turn, - * and startup recovery records the rows it leaves behind as interrupted. - */ +// Tracking is in memory; startup recovery fails runs interrupted by a restart. export function makeTrackedRuns() { const runs = new Map(); @@ -63,14 +45,7 @@ export function makeTrackedRuns() { for (const [messageId, run] of runs) if (run.taskId === taskId) runs.delete(messageId); }; - const findByRun = (runId: RunId) => { - for (const run of runs.values()) if (run.runId === runId) return run; - return undefined; - }; - - // The row's dispatch bookkeeping must be written before an outcome replaces - // it, so an outcome that lands during the dispatch waits for `dispatched`, - // which the caller marks only once that bookkeeping is saved. + // Hold terminal outcomes until dispatch has saved the next scheduled time. const end = (run: TrackedRun, outcome: RunEnd): SettledRun | null => { if (!run.dispatched) { run.ended = outcome; @@ -88,7 +63,6 @@ export function makeTrackedRuns() { }; return { - /** Must run before the dispatch, whose events can arrive before it returns. */ track: (messageId: MessageId, taskId: ScheduledTaskId, startedAtIso: string) => { // Only a task's latest run is reflected on its row. forgetTask(taskId); @@ -104,14 +78,9 @@ export function makeTrackedRuns() { }); }, forgetTask, - /** Drops one run only, for a caller whose task may have started a newer run since. */ forgetMessage: (messageId: MessageId) => { runs.delete(messageId); }, - /** - * Marks the dispatch bookkeeping as saved, and returns the outcome if the - * run already ended while that was happening. - */ dispatched: (messageId: MessageId, threadId: ThreadId | null): SettledRun | null => { const run = runs.get(messageId); if (run === undefined) return null; @@ -121,7 +90,6 @@ export function makeTrackedRuns() { run.ended = null; return held === null ? null : end(run, held); }, - /** Dispatched runs still waiting for an end, for catching up after missed events. */ pending: (): ReadonlyArray< Pick & { readonly threadId: ThreadId } > => @@ -137,27 +105,24 @@ export function makeTrackedRuns() { ] : [], ), - /** Follows a run found by reading the projection rather than by events. */ bindRun: (messageId: MessageId, runId: RunId) => { const run = runs.get(messageId); if (run !== undefined) run.runId = runId; }, - /** - * Rules on a cancelled run, given the run its message belongs to now. - * Returns false when the message moved into another run, which is then - * followed instead. - */ - confirmCancel: (messageId: MessageId, messageRunId: RunId | null): boolean => { + confirmCancel: ( + messageId: MessageId, + cancelledRunId: RunId, + messageRunId: RunId | null, + ): boolean => { const run = runs.get(messageId); if (run === undefined) return false; - if (messageRunId !== null && messageRunId !== run.runId) { + if (messageRunId !== null && messageRunId !== cancelledRunId) { run.runId = messageRunId; return false; } runs.delete(messageId); return true; }, - /** Folds in one domain event and returns the outcome once a tracked run ends. */ observe: (event: OrchestrationV2DomainEvent): ReadonlyArray => { if (runs.size === 0) return []; switch (event.type) { @@ -173,33 +138,37 @@ export function makeTrackedRuns() { case "turn-item.updated": { const item = event.payload; if (item.type !== "error" || item.runId === null) return []; - const run = findByRun(item.runId); - if (run !== undefined) run.providerError = item.failure.message; + for (const run of runs.values()) { + if (run.runId === item.runId) run.providerError = item.failure.message; + } return []; } case "run.created": case "run.updated": { const { id, userMessageId, status, threadId } = event.payload; - let run = findByRun(id); - if (run === undefined) { - const candidate = runs.get(userMessageId); - if (candidate === undefined || candidate.runId !== null) return []; - run = candidate; - run.runId = id; - run.threadId = threadId; + const candidate = runs.get(userMessageId); + if (candidate !== undefined && candidate.runId === null) { + candidate.runId = id; + candidate.threadId = threadId; } - if (status === "completed") - return settled(end(run, { status: "succeeded", error: null, cancelledRunId: null })); const failure = ENDED_RUN_ERRORS[status]; - if (failure === undefined) return []; - // Only a failed run carries a provider error; an interrupt or cancel is the user's. - return settled( - end(run, { - status: "failed", - error: status === "failed" ? (run.providerError ?? failure) : failure, - cancelledRunId: status === "cancelled" ? id : null, - }), - ); + if (status !== "completed" && failure === undefined) return []; + return [...runs.values()] + .filter((run) => run.runId === id) + .flatMap((run) => + settled( + end(run, { + status: status === "completed" ? "succeeded" : "failed", + error: + status === "completed" + ? null + : status === "failed" + ? (run.providerError ?? failure!) + : failure!, + cancelledRunId: status === "cancelled" ? id : null, + }), + ), + ); } case "thread.deleted": { return [...runs.values()] From c0e79f3c82bd9791622395f069887fb91ef0882c Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 18:57:48 +0000 Subject: [PATCH 09/19] fix(server): reconcile scheduled dispatches across reconnects --- .../ScheduledTaskService.test.ts | 27 ++++ .../scheduledTasks/ScheduledTaskService.ts | 139 +++++++++--------- 2 files changed, 99 insertions(+), 67 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 14f76effa0c93..edea6a5eeeef9 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -958,3 +958,30 @@ it.effect("records the provider error for every task steered into the same run", }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect("catches an outcome missed by reconnect while dispatch is still in flight", () => + Effect.gen(function* () { + const releaseDispatch = yield* Deferred.make(); + const runId = RunId.make("run:dispatch-reconnect"); + const harness = yield* makeTurnHarness({ + dropStreamOnCommand: true, + liveTail: true, + messageRunId: runId, + runs: [runOf(runId, MessageId.make("other"), "completed")], + duringDispatch: () => Deferred.await(releaseDispatch), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const dispatch = yield* service.runNow({ id: task.id }).pipe(Effect.forkChild); + yield* Queue.take(harness.dispatched); + yield* Deferred.succeed(harness.dropStream, undefined); + yield* TestClock.adjust("1 second"); + assert.equal(yield* Ref.get(harness.subscriptions), 2); + yield* Deferred.succeed(releaseDispatch, undefined); + yield* Fiber.join(dispatch); + yield* untilStatus(service, task.id, "succeeded"); + assert.equal((yield* service.list()).tasks[0]?.runCount, 1); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 93598ba09bfab..834466ab9333f 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -789,6 +789,69 @@ export const layer = Layer.effect( ); }); + const serviceScope = yield* Effect.scope; + let runEventGeneration = 0; + + const catchUpRun = Effect.fn("ScheduledTaskService.catchUpRun")( + (pending: ReturnType[number]) => + Effect.gen(function* () { + const { messageId, threadId } = pending; + const { thread, messages } = yield* threadManagement.getThreadRecords( + threadId, + ["messages"], + { messageIds: [messageId] }, + ); + const occurredAt = yield* DateTime.now; + // Deleting a thread keeps its projection, flagged, so a delete missed + // with the stream shows up here rather than as a read failure. + if (thread.deletedAt !== null) { + const ended = trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${threadId}:deleted`), + type: "thread.deleted", + threadId, + occurredAt, + payload: undefined as never, + }); + yield* Effect.forEach(ended, settleOutcome, { discard: true }); + return; + } + const runId = messages[0]?.runId ?? null; + if (runId === null) return; + const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { + runIds: [runId], + }); + const run = runs[0]; + if (run === undefined) return; + trackedRuns.bindRun(messageId, runId); + const ended = trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${runId}`), + type: "run.updated", + threadId, + runId, + occurredAt, + payload: run, + }); + yield* Effect.forEach(ended, settleOutcome, { discard: true }); + }).pipe( + // A thread that cannot be read (deleted, records gone) would otherwise + // be retried by every resubscribe and nothing else would settle its + // row, so after a few tries the run is released as failed. + Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }), + Effect.catchCause((cause) => + Effect.logWarning("Could not catch up a scheduled task run", { + messageId: pending.messageId, + cause, + }).pipe( + Effect.andThen( + Effect.sync(() => trackedRuns.forgetMessage(pending.messageId)).pipe( + Effect.andThen(releaseRun(pending, "Could not read how the run ended.")), + ), + ), + ), + ), + ), + ); + const runTask = Effect.fn("ScheduledTaskService.runTask")(function* ( task: ScheduledTask, trigger: "scheduled" | "manual" | "webhook", @@ -798,6 +861,7 @@ export const layer = Layer.effect( readonly dispatched: Ref.Ref; }, ) { + const dispatchGeneration = runEventGeneration; const reserved = yield* Ref.modify(activeRuns, (active) => { if (active.has(task.id)) return [false, active] as const; const next = new Set(active); @@ -991,6 +1055,12 @@ export const layer = Layer.effect( if (runSucceeded) { const ended = trackedRuns.dispatched(messageId, threadId); if (ended !== null) yield* settleOutcome(ended); + // Reconnect may have skipped this message before dispatch bookkeeping finished. + if (dispatchGeneration !== runEventGeneration) { + const pending = trackedRuns.pending().find((run) => run.messageId === messageId); + if (pending !== undefined) + yield* catchUpRun(pending).pipe(Effect.forkIn(serviceScope)); + } } } else trackedRuns.forgetTask(task.id); return { task: completed, threadId }; @@ -1120,72 +1190,8 @@ export const layer = Layer.effect( yield* scheduler.register("scheduled-tasks", runDueTasks()); - // Events missed while the stream was down are caught up from the thread - // projection: the run each still-tracked message belongs to is replayed as a - // run update. Provider error text is not recovered, so a failed run reads as - // the generic failure. const catchUpRuns = Effect.suspend(() => - Effect.forEach( - trackedRuns.pending(), - (pending) => - Effect.gen(function* () { - const { messageId, threadId } = pending; - const { thread, messages } = yield* threadManagement.getThreadRecords( - threadId, - ["messages"], - { messageIds: [messageId] }, - ); - const occurredAt = yield* DateTime.now; - // Deleting a thread keeps its projection, flagged, so a delete missed - // with the stream shows up here rather than as a read failure. - if (thread.deletedAt !== null) { - const ended = trackedRuns.observe({ - id: EventId.make(`scheduled-task-catch-up:${threadId}:deleted`), - type: "thread.deleted", - threadId, - occurredAt, - payload: undefined as never, - }); - yield* Effect.forEach(ended, settleOutcome, { discard: true }); - return; - } - const runId = messages[0]?.runId ?? null; - if (runId === null) return; - const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { - runIds: [runId], - }); - const run = runs[0]; - if (run === undefined) return; - trackedRuns.bindRun(messageId, runId); - const ended = trackedRuns.observe({ - id: EventId.make(`scheduled-task-catch-up:${runId}`), - type: "run.updated", - threadId, - runId, - occurredAt, - payload: run, - }); - yield* Effect.forEach(ended, settleOutcome, { discard: true }); - }).pipe( - // A thread that cannot be read (deleted, records gone) would otherwise - // be retried by every resubscribe and nothing else would settle its - // row, so after a few tries the run is released as failed. - Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }), - Effect.catchCause((cause) => - Effect.logWarning("Could not catch up a scheduled task run", { - messageId: pending.messageId, - cause, - }).pipe( - Effect.andThen( - Effect.sync(() => trackedRuns.forgetMessage(pending.messageId)).pipe( - Effect.andThen(releaseRun(pending, "Could not read how the run ended.")), - ), - ), - ), - ), - ), - { discard: true }, - ), + Effect.forEach(trackedRuns.pending(), catchUpRun, { discard: true }), ); // The stream resumes after a cursor it fixes on its first pull. Starting @@ -1195,6 +1201,7 @@ export const layer = Layer.effect( const runEvents = Stream.fromPull( Effect.gen(function* () { const pull = yield* Stream.toPull(threadManagement.streamDomainEvents); + runEventGeneration++; const first = yield* Effect.forkScoped(pull, { startImmediately: true }); yield* catchUpRuns; let joined = false; @@ -1757,8 +1764,6 @@ export const layer = Layer.effect( }); }); - const serviceScope = yield* Effect.scope; - const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")( function* (deliveryId: string) { yield* releaseReplay(deliveryId); From b9509792fccd912f2f2155d0c700ee6d8c49584b Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 19:09:24 +0000 Subject: [PATCH 10/19] fix(server): preserve live run bindings during catch-up --- .../scheduledTasks/ScheduledTaskService.ts | 2 +- .../src/scheduledTasks/TrackedRuns.test.ts | 39 ++++++++++++++++--- apps/server/src/scheduledTasks/TrackedRuns.ts | 14 +++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 834466ab9333f..e466527bce869 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -822,7 +822,7 @@ export const layer = Layer.effect( }); const run = runs[0]; if (run === undefined) return; - trackedRuns.bindRun(messageId, runId); + if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; const ended = trackedRuns.observe({ id: EventId.make(`scheduled-task-catch-up:${runId}`), type: "run.updated", diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts index 4d263ffcaa832..53ae915377d48 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.test.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -97,8 +97,10 @@ it("lists only dispatched runs as pending, and follows a run bound from the proj runs.track(messageId, taskId, "start"); assert.deepEqual(runs.pending(), []); runs.dispatched(messageId, threadId); - assert.deepEqual(runs.pending(), [{ taskId, startedAtIso: "start", messageId, threadId }]); - runs.bindRun(messageId, RunId.make("run:caught-up")); + assert.deepEqual(runs.pending(), [ + { taskId, startedAtIso: "start", messageId, threadId, runId: null }, + ]); + runs.bindRun(messageId, RunId.make("run:caught-up"), null); assert.deepEqual(runs.observe(runEvent("run:caught-up", "completed", MessageId.make("other"))), [ settledFor("succeeded", null), ]); @@ -130,8 +132,8 @@ it.each(["completed", "failed"] as const)("settles every task sharing a %s run", runs.track(secondMessage, secondTask, "second-start"); runs.dispatched(messageId, threadId); runs.dispatched(secondMessage, threadId); - runs.bindRun(messageId, RunId.make("run:shared")); - runs.bindRun(secondMessage, RunId.make("run:shared")); + runs.bindRun(messageId, RunId.make("run:shared"), null); + runs.bindRun(secondMessage, RunId.make("run:shared"), null); const error = status === "failed" ? "The run failed." : null; assert.deepEqual(runs.observe(runEvent("run:shared", status)), [ settledFor(status === "completed" ? "succeeded" : "failed", error), @@ -150,7 +152,7 @@ it("keeps a promoted message tracked when cancellation arrived before dispatch r runs.track(messageId, taskId, "start"); runs.observe(runEvent("run:queued", "queued")); runs.observe(runEvent("run:queued", "cancelled")); - runs.bindRun(messageId, RunId.make("run:active")); + runs.bindRun(messageId, RunId.make("run:active"), RunId.make("run:queued")); const held = runs.dispatched(messageId, threadId); assert.equal(held?.cancelledRunId, "run:queued"); assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:active"))); @@ -158,3 +160,30 @@ it("keeps a promoted message tracked when cancellation arrived before dispatch r settledFor("succeeded", null), ]); }); + +it("does not let a stale catch-up overwrite a live promotion", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:queued", "queued")); + const queuedRunId = RunId.make("run:queued"); + runs.observe(runEvent("run:queued", "cancelled")); + runs.confirmCancel(messageId, queuedRunId, RunId.make("run:active")); + assert.isFalse(runs.bindRun(messageId, queuedRunId, queuedRunId)); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + settledFor("succeeded", null), + ]); +}); + +it("ignores a stale cancellation read after a live promotion", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:queued", "queued")); + const queuedRunId = RunId.make("run:queued"); + runs.confirmCancel(messageId, queuedRunId, RunId.make("run:active")); + assert.isFalse(runs.confirmCancel(messageId, queuedRunId, queuedRunId)); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + settledFor("succeeded", null), + ]); +}); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 8d6ba23749173..48115c050f266 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -91,7 +91,10 @@ export function makeTrackedRuns() { return held === null ? null : end(run, held); }, pending: (): ReadonlyArray< - Pick & { readonly threadId: ThreadId } + Pick & { + readonly threadId: ThreadId; + readonly runId: RunId | null; + } > => [...runs.values()].flatMap((run) => run.dispatched && run.threadId !== null @@ -101,13 +104,16 @@ export function makeTrackedRuns() { startedAtIso: run.startedAtIso, messageId: run.messageId, threadId: run.threadId, + runId: run.runId, }, ] : [], ), - bindRun: (messageId: MessageId, runId: RunId) => { + bindRun: (messageId: MessageId, runId: RunId, expectedRunId: RunId | null) => { const run = runs.get(messageId); - if (run !== undefined) run.runId = runId; + if (run === undefined || run.runId !== expectedRunId) return false; + run.runId = runId; + return true; }, confirmCancel: ( messageId: MessageId, @@ -115,7 +121,7 @@ export function makeTrackedRuns() { messageRunId: RunId | null, ): boolean => { const run = runs.get(messageId); - if (run === undefined) return false; + if (run === undefined || run.runId !== cancelledRunId) return false; if (messageRunId !== null && messageRunId !== cancelledRunId) { run.runId = messageRunId; return false; From da5ae45606a0e616e0ecdd7169bc5d0548e403ef Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Sun, 1 Nov 2026 08:14:54 +0000 Subject: [PATCH 11/19] fix(server): preserve scheduled run ownership on interruption --- .../scheduledTasks/ScheduledTaskService.ts | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 5c95d0cd887f9..47371a1e1bcdc 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -971,23 +971,26 @@ export const layer = Layer.effect( if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); } - if (webhook === undefined) yield* markRunning(active.id, startedAtIso); - else - yield* sql - .withTransaction( - Effect.gen(function* () { - yield* markRunning(active.id, startedAtIso, webhook.deliveryId); - // Recovery must distinguish this delivery from other windows waiting on the same task. - yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`; - }), - ) - .pipe( - Effect.mapError((cause) => - taskError("Could not start webhook run.", { taskId: task.id, cause }), - ), - ); + // Persist ownership with the row so interruption cannot skip recovery. + yield* Effect.gen(function* () { + if (webhook === undefined) yield* markRunning(active.id, startedAtIso); + else + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* markRunning(active.id, startedAtIso, webhook.deliveryId); + // Recovery must distinguish this delivery from other windows waiting on the same task. + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`; + }), + ) + .pipe( + Effect.mapError((cause) => + taskError("Could not start webhook run.", { taskId: task.id, cause }), + ), + ); - yield* Ref.set(markedAtIso, startedAtIso); + yield* Ref.set(markedAtIso, startedAtIso); + }).pipe(Effect.uninterruptible); yield* notifyChanged; const fireKey = From 7512e90a02dfdc1778c336860ddcfcee6cbe2c39 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 2 Nov 2026 01:42:55 +0000 Subject: [PATCH 12/19] test(server): provide policy to scheduled outcome integration --- apps/server/src/orchestration-v2/ThreadLaunchService.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index b8591547df4d9..5d40211bcbb02 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -429,6 +429,7 @@ it.effect("runs a thread-bound automation on the provider the thread uses now", const layerScheduledTasks = ScheduledTasks.layer.pipe( Layer.provide( Layer.mergeAll( + ManagedPolicy.layerTest(), harness.layer, NodeCrypto.layer, Scheduler.layer, From 551f9ad93aefa0a801ad893b7ad0653d72bb109d Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 2 Nov 2026 01:48:56 +0000 Subject: [PATCH 13/19] fix(server): reconcile replayed webhook run outcomes --- .../ThreadLaunchService.test.ts | 2 +- .../scheduledTasks/ScheduledTaskService.ts | 19 ++-- .../ScheduledTaskService.webhook.test.ts | 103 +++++++++++++----- 3 files changed, 89 insertions(+), 35 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 5d40211bcbb02..ee2cd5eb32b1b 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -429,7 +429,6 @@ it.effect("runs a thread-bound automation on the provider the thread uses now", const layerScheduledTasks = ScheduledTasks.layer.pipe( Layer.provide( Layer.mergeAll( - ManagedPolicy.layerTest(), harness.layer, NodeCrypto.layer, Scheduler.layer, @@ -519,6 +518,7 @@ it.effect("an automation queued behind a running turn records how its own turn e const layerScheduledTasks = ScheduledTasks.layer.pipe( Layer.provide( Layer.mergeAll( + ManagedPolicy.layerTest(), harness.layer, NodeCrypto.layer, Scheduler.layer, diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 0ba4c3bbde3eb..b0447dcaac91b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -959,12 +959,15 @@ export const layer = Layer.effect( } return { task, threadId: null, dispatchFailed: false }; } - if ( - webhook !== undefined && - (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`.pipe( - NodeSqliteClient.readOnly, - )).length > 0 - ) + const [pendingDelivery] = + webhook === undefined + ? [] + : yield* sql<{ + state: string; + }>`SELECT state FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`.pipe( + NodeSqliteClient.readOnly, + ); + if (pendingDelivery?.state === "completed") return { task: { ...active, lastRunStatus: "succeeded" as const }, threadId: null, @@ -1173,8 +1176,8 @@ export const layer = Layer.effect( // landed since the dispatch returned is settled here. const ended = trackedRuns.dispatched(messageId, threadId); if (ended !== null) yield* settleOutcome(ended); - // Reconnect may have skipped this message before dispatch bookkeeping finished. - if (dispatchGeneration !== runEventGeneration) { + // Replayed webhook dispatches and reconnects can miss the original terminal event. + if (pendingDelivery?.state === "running" || dispatchGeneration !== runEventGeneration) { const pending = trackedRuns.pending().find((run) => run.messageId === messageId); if (pending !== undefined) yield* catchUpRun(pending).pipe(Effect.forkIn(serviceScope)); } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 4750f53e4c8d4..002b9ec5d45e7 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -19,6 +19,7 @@ import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import * as Tracer from "effect/Tracer"; @@ -168,6 +169,7 @@ const withService = ( readonly started: Deferred.Deferred; readonly release: Deferred.Deferred; }; + readonly getThreadRecords?: ThreadManagementService.ThreadManagementService["Service"]["getThreadRecords"]; readonly completedLaunches?: Set; readonly failLaunches?: { remaining: number }; readonly gate?: Deferred.Deferred; @@ -238,7 +240,11 @@ const withService = ( namedProjectsRoot: "/projects", isScratchProject: () => Effect.succeed(false), }), - Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + ...(options.getThreadRecords === undefined + ? {} + : { getThreadRecords: options.getThreadRecords }), + }), Layer.mock(SecretRequests.SecretRequests)({ consume: ({ ref }) => { const value = secretsByRef.get(ref); @@ -1413,32 +1419,77 @@ it.effect( }), ); -it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () => - withService(({ service, sql, launches }) => - Effect.gen(function* () { - const { task } = yield* service.upsert( - yield* webhookTaskInput({ - schedule: { - type: "webhook", - dedupe: { replayKey: "{{body}}" }, - }, - }), - ); - yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks +it.effect.each([false, true])( + "keeps replay keys and settles replayed runs after completion bookkeeping fails (coalesced: %s)", + (coalesced) => { + const workers = new Map>(); + const completedLaunches = new Set(); + let dispatched: LaunchInput | undefined; + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + ...(coalesced + ? { coalesceKey: "{{body.pull_request.url}}", coalesceWindowSeconds: 10 } + : {}), + }, + }, + }), + ); + yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; - yield* waitForSpan( - service.triggerWebhook(requestFor(task)), - "ScheduledTaskService.runWebhookDelivery", - ); - yield* Queue.take(launches); - assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1); - yield* sql`DROP TRIGGER fail_completion`; - const retry = yield* service.triggerWebhook(requestFor(task)); - assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); - assert.equal(yield* Queue.size(launches), 0); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); - }), - ), + if (coalesced) { + yield* service.triggerWebhook(requestFor(task)); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + } else { + yield* waitForSpan( + service.triggerWebhook(requestFor(task)), + "ScheduledTaskService.runWebhookDelivery", + ); + } + dispatched = yield* Queue.take(launches); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1); + yield* sql`DROP TRIGGER fail_completion`; + const retry = yield* service.triggerWebhook(requestFor(task)); + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); + if (coalesced) { + yield* drainPendingDispatch(workers); + yield* service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks[0]?.lastRunStatus === "succeeded"), + Stream.take(1), + Stream.runDrain, + ); + } + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + }), + { + workers, + completedLaunches, + getThreadRecords: () => + Effect.succeed({ + thread: { deletedAt: null }, + messages: [ + { id: dispatched?.initialMessage?.messageId, runId: "run:webhook-replayed" }, + ], + runs: [ + { + id: "run:webhook-replayed", + userMessageId: dispatched?.initialMessage?.messageId, + threadId: "thread-1", + status: "completed", + }, + ], + } as never), + }, + ); + }, ); it.effect.each([119_000, 120_000])( From d0ea7a07acae91843fa70f4c57ff2bb33c610ff1 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 2 Nov 2026 01:51:35 +0000 Subject: [PATCH 14/19] test(server): simplify webhook projection mock --- .../scheduledTasks/ScheduledTaskService.webhook.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 002b9ec5d45e7..34e7016e257b0 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -240,11 +240,11 @@ const withService = ( namedProjectsRoot: "/projects", isScratchProject: () => Effect.succeed(false), }), - Layer.mock(ThreadManagementService.ThreadManagementService)({ - ...(options.getThreadRecords === undefined + Layer.mock(ThreadManagementService.ThreadManagementService)( + options.getThreadRecords === undefined ? {} - : { getThreadRecords: options.getThreadRecords }), - }), + : { getThreadRecords: options.getThreadRecords }, + ), Layer.mock(SecretRequests.SecretRequests)({ consume: ({ ref }) => { const value = secretsByRef.get(ref); From eedff92d78c5bd17b7835bd9895c7f61b8a89a73 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 04:18:01 +0000 Subject: [PATCH 15/19] fix(server): preserve scheduled outcomes across recovery gaps --- .../src/orchestration-v2/Orchestrator.ts | 31 ++++++------ .../ThreadLaunchService.test.ts | 6 +++ .../ThreadManagementService.ts | 2 + .../src/relay/AgentAwarenessRelay.test.ts | 1 + .../ScheduledTaskService.test.ts | 34 ++++++++----- .../scheduledTasks/ScheduledTaskService.ts | 50 +++++++++++++------ .../ScheduledTaskService.webhook.test.ts | 6 ++- 7 files changed, 88 insertions(+), 42 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 3dc0a65ae03a3..3877d6811d067 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -475,6 +475,10 @@ export interface OrchestratorV2Shape { readonly eventType?: OrchestrationV2DomainEvent["type"]; }) => Stream.Stream; readonly streamDomainEvents: Stream.Stream; + readonly openDomainEventStream: Effect.Effect< + Stream.Stream, + OrchestratorV2Error + >; } export class OrchestratorV2 extends Context.Service()( @@ -14001,6 +14005,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const shellProjectionError = (cause: unknown) => new OrchestratorProjectionError({ threadId: ThreadId.make("thread:shell"), cause }); + const openDomainEventStream = Effect.fn("OrchestratorV2.openDomainEventStream")( + function* () { + const latest = yield* eventSink.latestSequence(); + return eventSink.stream({ afterSequence: latest }).pipe( + Stream.map((stored) => stored.event), + Stream.mapError((cause) => new OrchestratorDomainEventStreamError({ cause })), + ); + }, + Effect.mapError((cause) => new OrchestratorDomainEventStreamError({ cause })), + ); + return OrchestratorV2.of({ resumeQueuedRuns, recoverDelegatedTasks, @@ -14186,20 +14201,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ), - // Start at the latest sequence; awareness needs no historical replay. - streamDomainEvents: Stream.unwrap( - eventSink - .latestSequence() - .pipe(Effect.map((latest) => eventSink.stream({ afterSequence: latest }))), - ).pipe( - Stream.map((stored) => stored.event), - Stream.mapError( - (cause) => - new OrchestratorDomainEventStreamError({ - cause, - }), - ), - ), + streamDomainEvents: Stream.unwrap(openDomainEventStream()), + openDomainEventStream: openDomainEventStream(), }); }); diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 4b0c4c5071798..7885260bf3ceb 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -558,6 +558,7 @@ it.effect("an automation queued behind a running turn records how its own turn e const projection = yield* threads.getThreadProjection(launched.threadId); const queued = projection.runs.find((run) => run.status === "queued"); assert.isDefined(queued); + const events = yield* threads.openDomainEventStream; yield* threads.dispatch({ type: "queued-run.cancel", @@ -565,6 +566,11 @@ it.effect("an automation queued behind a running turn records how its own turn e threadId: launched.threadId, runId: queued!.id, }); + const cancelled = yield* events.pipe( + Stream.filter((event) => event.type === "run.updated" && event.payload.id === queued!.id), + Stream.runHead, + ); + assert.isTrue(Option.isSome(cancelled)); yield* tasks.subscribeList().pipe( Stream.filter(({ tasks: listed }) => listed[0]?.lastRunStatus === "failed"), Stream.take(1), diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index f0346b2bec4ec..8d63e5f628e44 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -492,6 +492,7 @@ export interface ThreadManagementServiceShape { readonly streamStoredEvents: Orchestrator.OrchestratorV2["Service"]["streamStoredEvents"]; readonly streamStoredEventsFrom: Orchestrator.OrchestratorV2["Service"]["streamStoredEventsFrom"]; readonly streamDomainEvents: Orchestrator.OrchestratorV2["Service"]["streamDomainEvents"]; + readonly openDomainEventStream: Orchestrator.OrchestratorV2["Service"]["openDomainEventStream"]; } export class ThreadManagementService extends Context.Service< @@ -1553,6 +1554,7 @@ const make = Effect.gen(function* () { streamStoredEvents: orchestrator.streamStoredEvents, streamStoredEventsFrom: orchestrator.streamStoredEventsFrom, streamDomainEvents: orchestrator.streamDomainEvents, + openDomainEventStream: orchestrator.openDomainEventStream, }); }); diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 87838fd5a51ed..6fbddf784b9c2 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -229,6 +229,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( streamStoredEvents: Stream.empty, streamStoredEventsFrom: () => Stream.empty, streamDomainEvents: options.domainEvents ?? Stream.empty, + openDomainEventStream: Effect.succeed(options.domainEvents ?? Stream.empty), }); const publications: Array<{ readonly url: string; diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 018b127576f0d..3d8807a3879ca 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -562,6 +562,7 @@ const makeTurnHarness = ( readonly messageRunId?: RunId; /** The runs the thread's projection holds, as read back after the event stream drops. */ readonly runs?: ReadonlyArray; + readonly providerError?: string; /** How many times the event stream fails before it starts to deliver events. */ readonly streamFailures?: number; /** Reads the thread as deleted, which the projection keeps as a flagged record. */ @@ -618,6 +619,9 @@ const makeTurnHarness = ( if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void; return { thread: { deletedAt: options.threadDeleted ? epoch : null }, + turnItems: options.providerError + ? [providerErrorEvent(options.messageRunId!, options.providerError).payload] + : [], messages: (filter?.messageIds ?? []).map((id) => ({ id, runId: options.messageRunId ?? null, @@ -627,17 +631,15 @@ const makeTurnHarness = ( ), } as never; }), - streamDomainEvents: Stream.unwrap( - Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe( - Effect.flatMap((attempt) => - attempt < (options.streamFailures ?? (options.dropStreamOnCommand ? 1 : 0)) - ? (options.dropStreamOnCommand ? Deferred.await(dropStream) : Effect.void).pipe( - Effect.as(Stream.fail(new Error("event stream dropped") as never)), - ) - : (options.liveTail ? Queue.clear(events) : Effect.void).pipe( - Effect.as(Stream.fromQueue(events)), - ), - ), + openDomainEventStream: Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe( + Effect.flatMap((attempt) => + attempt < (options.streamFailures ?? (options.dropStreamOnCommand ? 1 : 0)) + ? (options.dropStreamOnCommand ? Deferred.await(dropStream) : Effect.void).pipe( + Effect.as(Stream.fail(new Error("event stream dropped") as never)), + ) + : (options.liveTail ? Queue.clear(events) : Effect.void).pipe( + Effect.as(Stream.fromQueue(events)), + ), ), ), }), @@ -803,7 +805,10 @@ it.effect("a turn that ends while the dispatch is being recorded still advances it.effect("a queued message promoted to a steer keeps its run going in the active run", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness({ messageRunId: RunId.make("run:active") }); + const harness = yield* makeTurnHarness({ + messageRunId: RunId.make("run:active"), + recordsUnreadable: true, + }); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); @@ -877,6 +882,7 @@ it.effect("the event stream keeps being resumed after many failures", () => const runId = RunId.make("run:late"); const harness = yield* makeTurnHarness({ streamFailures: 8, + providerError: "Provider unavailable during reconnect.", messageRunId: runId, runs: [runOf(runId, MessageId.make("unused"), "failed")], }); @@ -888,6 +894,10 @@ it.effect("the event stream keeps being resumed after many failures", () => yield* TestClock.adjust("10 minutes"); yield* untilStatus(service, task.id, "failed"); assert.equal(yield* Ref.get(harness.subscriptions), 9); + assert.equal( + (yield* service.list()).tasks[0]?.lastRunError, + "Provider unavailable during reconnect.", + ); }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 5f0b7f6d8c9e7..faf6f112cf457 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -35,7 +35,6 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Exit from "effect/Exit"; -import * as Fiber from "effect/Fiber"; import * as Duration from "effect/Duration"; import * as Metric from "effect/Metric"; import * as Layer from "effect/Layer"; @@ -853,9 +852,18 @@ export const layer = Layer.effect( .getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] }) .pipe( Effect.map(({ messages }) => messages[0]?.runId ?? null), - Effect.orElseSucceed(() => null), + Effect.retry({ times: 2 }), + Effect.catchCause((cause) => + Effect.logWarning("Could not confirm scheduled run cancellation", { + cause, + }).pipe(Effect.as(undefined)), + ), ); - if (!trackedRuns.confirmCancel(run.messageId, run.cancelledRunId, moved)) return; + if ( + moved === undefined || + !trackedRuns.confirmCancel(run.messageId, run.cancelledRunId, moved) + ) + return; } yield* recordRunEnd(run).pipe( Effect.retry({ times: 2 }), @@ -901,6 +909,26 @@ export const layer = Layer.effect( const run = runs[0]; if (run === undefined) return; if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; + if (run.status === "failed") { + const { turnItems } = yield* threadManagement.getThreadRecords( + threadId, + ["turnItems"], + { + turnItemRunId: runId, + turnItemTypes: ["error"], + }, + ); + for (const item of turnItems) { + trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${item.id}`), + type: "turn-item.updated", + threadId, + runId, + occurredAt, + payload: item, + }); + } + } const ended = trackedRuns.observe({ id: EventId.make(`scheduled-task-catch-up:${runId}`), type: "run.updated", @@ -1353,22 +1381,14 @@ export const layer = Layer.effect( Effect.forEach(trackedRuns.pending(), catchUpRun, { discard: true }), ); - // The stream resumes after a cursor it fixes on its first pull. Starting - // that pull before the catch-up read means an event landing in between is - // delivered by the stream instead of missed, and a run seen by both is - // settled once because the tracker forgets it when it ends. + // Capture the stream cursor before catch-up so an event between them is replayed. const runEvents = Stream.fromPull( Effect.gen(function* () { - const pull = yield* Stream.toPull(threadManagement.streamDomainEvents); + const stream = yield* threadManagement.openDomainEventStream; + const pull = yield* Stream.toPull(stream); runEventGeneration++; - const first = yield* Effect.forkScoped(pull, { startImmediately: true }); yield* catchUpRuns; - let joined = false; - return Effect.suspend(() => { - if (joined) return pull; - joined = true; - return Fiber.join(first); - }); + return pull; }), ); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index d196786458e50..dcc8b2089e91b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -1086,11 +1086,15 @@ it.effect("retrying a running window keeps the prompt it was dispatched with", ( yield* service.triggerWebhook(requestFor(task)); yield* save("Prompt at dispatch"); yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(workers); const launched = yield* Queue.take(launches); assert.equal(launched.initialMessage?.text, "Prompt at dispatch"); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'running'`).length, + 1, + ); yield* save("Prompt saved after dispatch"); yield* sql`DROP TRIGGER fail_completion`; yield* drainPendingDispatch(workers); From 8d5084f4d08f37b7fc17bee53241c41f990c3b91 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 04:27:51 +0000 Subject: [PATCH 16/19] fix(server): retry unreadable scheduled run outcomes --- .../ScheduledTaskService.test.ts | 28 +++++++++++++++++-- .../scheduledTasks/ScheduledTaskService.ts | 15 +++++----- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 3d8807a3879ca..7fc594e4c3afa 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -563,6 +563,8 @@ const makeTurnHarness = ( /** The runs the thread's projection holds, as read back after the event stream drops. */ readonly runs?: ReadonlyArray; readonly providerError?: string; + readonly messageReadFailures?: number; + readonly errorReadFailures?: number; /** How many times the event stream fails before it starts to deliver events. */ readonly streamFailures?: number; /** Reads the thread as deleted, which the projection keeps as a flagged record. */ @@ -592,6 +594,8 @@ const makeTurnHarness = ( const failedReads = yield* Queue.unbounded(); const dropStream = yield* Deferred.make(); const firstMessage = yield* Ref.make(null); + const messageReads = yield* Ref.make(0); + const errorReads = yield* Ref.make(0); const layer = ScheduledTaskService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -607,8 +611,18 @@ const makeTurnHarness = ( getThreadRecords: (_threadId, fields: ReadonlyArray, filter) => Effect.gen(function* () { const first = yield* Ref.get(firstMessage); + const unreadableMessage = + fields.includes("messages") && + (yield* Ref.getAndUpdate(messageReads, (n) => n + 1)) < + (options.messageReadFailures ?? 0); + const unreadableError = + fields.includes("turnItems") && + (yield* Ref.getAndUpdate(errorReads, (n) => n + 1)) < + (options.errorReadFailures ?? 0); if ( options.recordsUnreadable || + unreadableMessage || + unreadableError || (options.firstMessageUnreadable && first !== null && filter?.messageIds?.includes(first)) @@ -929,7 +943,11 @@ it.effect("an event landing while the resumed listener catches up is not missed" it.effect("a cancelled run is still recorded when its first save fails", () => Effect.gen(function* () { - const harness = yield* makeTurnHarness(); + const harness = yield* makeTurnHarness({ + messageReadFailures: 3, + messageRunId: RunId.make("run:cancelled"), + runs: [runOf("run:cancelled", MessageId.make("unused"), "cancelled")], + }); yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const sql = yield* SqlClient.SqlClient; @@ -1058,7 +1076,9 @@ it.effect("catches an outcome missed by reconnect while dispatch is still in fli dropStreamOnCommand: true, liveTail: true, messageRunId: runId, - runs: [runOf(runId, MessageId.make("other"), "completed")], + runs: [runOf(runId, MessageId.make("other"), "failed")], + providerError: "Reconnect provider failure.", + errorReadFailures: 1, duringDispatch: () => Deferred.await(releaseDispatch), }); yield* Effect.gen(function* () { @@ -1071,7 +1091,9 @@ it.effect("catches an outcome missed by reconnect while dispatch is still in fli assert.equal(yield* Ref.get(harness.subscriptions), 2); yield* Deferred.succeed(releaseDispatch, undefined); yield* Fiber.join(dispatch); - yield* untilStatus(service, task.id, "succeeded"); + yield* TestClock.adjust("5 seconds"); + yield* untilStatus(service, task.id, "failed"); + assert.equal((yield* service.list()).tasks[0]?.lastRunError, "Reconnect provider failure."); assert.equal((yield* service.list()).tasks[0]?.runCount, 1); }).pipe(Effect.provide(harness.layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index faf6f112cf457..147036d3c06d1 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -839,7 +839,7 @@ export const layer = Layer.effect( // released as failed so its row does not stay `running` and block the task. const settleOutcome = Effect.fn("ScheduledTaskService.settleOutcome")(function* ( run: SettledRun, - ) { + ): Effect.fn.Return { // Promoting a queued message to a steer cancels its queued run first and // moves the message into the active run afterwards, so a cancel only ends // the run if the message still belongs to the cancelled run. Decided once: @@ -859,11 +859,12 @@ export const layer = Layer.effect( }).pipe(Effect.as(undefined)), ), ); - if ( - moved === undefined || - !trackedRuns.confirmCancel(run.messageId, run.cancelledRunId, moved) - ) + if (moved === undefined) { + const pending = trackedRuns.pending().find((entry) => entry.messageId === run.messageId); + if (pending !== undefined) yield* catchUpRun(pending).pipe(Effect.forkIn(serviceScope)); return; + } + if (!trackedRuns.confirmCancel(run.messageId, run.cancelledRunId, moved)) return; } yield* recordRunEnd(run).pipe( Effect.retry({ times: 2 }), @@ -908,7 +909,6 @@ export const layer = Layer.effect( }); const run = runs[0]; if (run === undefined) return; - if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; if (run.status === "failed") { const { turnItems } = yield* threadManagement.getThreadRecords( threadId, @@ -918,6 +918,7 @@ export const layer = Layer.effect( turnItemTypes: ["error"], }, ); + if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; for (const item of turnItems) { trackedRuns.observe({ id: EventId.make(`scheduled-task-catch-up:${item.id}`), @@ -928,7 +929,7 @@ export const layer = Layer.effect( payload: item, }); } - } + } else if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; const ended = trackedRuns.observe({ id: EventId.make(`scheduled-task-catch-up:${runId}`), type: "run.updated", From 017184de74c118f666d8f78d314eb2a04243e05a Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 5 Nov 2026 20:10:17 +0000 Subject: [PATCH 17/19] test(server): wait for the scheduled run listener before forcing read failures --- .../scheduledTasks/ScheduledTaskService.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 5e61257eb0ff7..c5290c487db14 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -595,6 +595,7 @@ const makeTurnHarness = ( const subscriptions = yield* Ref.make(0); const failedReads = yield* Queue.unbounded(); const dropStream = yield* Deferred.make(); + const listening = yield* Deferred.make(); const firstMessage = yield* Ref.make(null); const messageReads = yield* Ref.make(0); const errorReads = yield* Ref.make(0); @@ -654,7 +655,13 @@ const makeTurnHarness = ( Effect.as(Stream.fail(new Error("event stream dropped") as never)), ) : (options.liveTail ? Queue.clear(events) : Effect.void).pipe( - Effect.as(Stream.fromQueue(events)), + // The first pull comes after the listener's catch-up. + Effect.as( + Stream.fromEffect(Deferred.succeed(listening, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.fromQueue(events)), + ), + ), ), ), ), @@ -669,7 +676,7 @@ const makeTurnHarness = ( ), ), ); - return { events, dispatched, subscriptions, failedReads, dropStream, layer }; + return { events, dispatched, subscriptions, listening, failedReads, dropStream, layer }; }); const upsertBoundTask = ( @@ -828,6 +835,8 @@ it.effect("a queued message promoted to a steer keeps its run going in the activ yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const { task } = yield* upsertBoundTask(service); + // Reads fail here, so the listener's first catch-up must run before the dispatch. + yield* Deferred.await(harness.listening); yield* service.runNow({ id: task.id }); const messageId = yield* Queue.take(harness.dispatched); yield* Queue.offerAll(harness.events, [ @@ -954,6 +963,8 @@ it.effect("a cancelled run is still recorded when its first save fails", () => const service = yield* ScheduledTaskService.ScheduledTaskService; const sql = yield* SqlClient.SqlClient; const { task } = yield* upsertBoundTask(service); + // Reads fail here, so the listener's first catch-up must run before the dispatch. + yield* Deferred.await(harness.listening); yield* service.runNow({ id: task.id }); const messageId = yield* Queue.take(harness.dispatched); // Fails the first write only; the retry must still record the cancel. From dddda26077c86117e9738eb139f98585a64e1fa3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 03:19:37 +0000 Subject: [PATCH 18/19] test(server): mock the destination thread read in scheduled run outcome tests --- apps/server/src/scheduledTasks/ScheduledTaskService.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index a85df6af430dc..39c406a16ae8d 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -611,6 +611,7 @@ const makeTurnHarness = ( ManagedPolicy.layerTest(), Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(threadShell(ProjectId.make("project:test"))), sendToThread: (input) => Ref.update(firstMessage, (first) => first ?? input.messageId).pipe( Effect.andThen(Queue.offer(dispatched, input.messageId)), From cb5ef5642bc6013dc133a28c338c5ea8a6163cd1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 03:21:29 +0000 Subject: [PATCH 19/19] refactor(server): settle deleted-thread runs without a synthetic event Trim TrackedRuns tests that repeat service-level cases. AI-assisted; verified with the tests and tools listed above. --- .../scheduledTasks/ScheduledTaskService.ts | 112 ++++++++---------- .../src/scheduledTasks/TrackedRuns.test.ts | 45 ------- apps/server/src/scheduledTasks/TrackedRuns.ts | 25 ++-- 3 files changed, 64 insertions(+), 118 deletions(-) diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index de15d49ce366d..86995940622ef 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -911,70 +911,62 @@ export const layer = Layer.effect( let runEventGeneration = 0; const catchUpRun = Effect.fn("ScheduledTaskService.catchUpRun")( - (pending: ReturnType[number]) => - Effect.gen(function* () { - const { messageId, threadId } = pending; - const { thread, messages } = yield* threadManagement.getThreadRecords( - threadId, - ["messages"], - { messageIds: [messageId] }, - ); - const occurredAt = yield* DateTime.now; - // Deleting a thread keeps its projection, flagged, so a delete missed - // with the stream shows up here rather than as a read failure. - if (thread.deletedAt !== null) { - const ended = trackedRuns.observe({ - id: EventId.make(`scheduled-task-catch-up:${threadId}:deleted`), - type: "thread.deleted", + function* (pending: ReturnType[number]) { + const { messageId, threadId } = pending; + const { thread, messages } = yield* threadManagement.getThreadRecords( + threadId, + ["messages"], + { messageIds: [messageId] }, + ); + const occurredAt = yield* DateTime.now; + // Deleting a thread keeps its projection, flagged, so a delete missed + // with the stream shows up here rather than as a read failure. + if (thread.deletedAt !== null) { + yield* Effect.forEach(trackedRuns.threadDeleted(threadId), settleOutcome, { + discard: true, + }); + return; + } + const runId = messages[0]?.runId ?? null; + if (runId === null) return; + const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { + runIds: [runId], + }); + const run = runs[0]; + if (run === undefined) return; + if (run.status === "failed") { + const { turnItems } = yield* threadManagement.getThreadRecords(threadId, ["turnItems"], { + turnItemRunId: runId, + turnItemTypes: ["error"], + }); + if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; + for (const item of turnItems) { + trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${item.id}`), + type: "turn-item.updated", threadId, + runId, occurredAt, - payload: undefined as never, + payload: item, }); - yield* Effect.forEach(ended, settleOutcome, { discard: true }); - return; } - const runId = messages[0]?.runId ?? null; - if (runId === null) return; - const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { - runIds: [runId], - }); - const run = runs[0]; - if (run === undefined) return; - if (run.status === "failed") { - const { turnItems } = yield* threadManagement.getThreadRecords( - threadId, - ["turnItems"], - { - turnItemRunId: runId, - turnItemTypes: ["error"], - }, - ); - if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; - for (const item of turnItems) { - trackedRuns.observe({ - id: EventId.make(`scheduled-task-catch-up:${item.id}`), - type: "turn-item.updated", - threadId, - runId, - occurredAt, - payload: item, - }); - } - } else if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; - const ended = trackedRuns.observe({ - id: EventId.make(`scheduled-task-catch-up:${runId}`), - type: "run.updated", - threadId, - runId, - occurredAt, - payload: run, - }); - yield* Effect.forEach(ended, settleOutcome, { discard: true }); - }).pipe( - // A thread that cannot be read (deleted, records gone) would otherwise - // be retried by every resubscribe and nothing else would settle its - // row, so after a few tries the run is released as failed. - Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }), + } else if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; + const ended = trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${runId}`), + type: "run.updated", + threadId, + runId, + occurredAt, + payload: run, + }); + yield* Effect.forEach(ended, settleOutcome, { discard: true }); + }, + // A thread that cannot be read (deleted, records gone) would otherwise + // be retried by every resubscribe and nothing else would settle its + // row, so after a few tries the run is released as failed. + Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }), + (effect, pending) => + effect.pipe( Effect.catchCause((cause) => Effect.logWarning("Could not catch up a scheduled task run", { messageId: pending.messageId, diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts index 53ae915377d48..c438314038700 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.test.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -60,28 +60,6 @@ const settledFor = ( cancelledRunId: cancelledRunId === null ? null : RunId.make(cancelledRunId), }); -it("holds an outcome that lands before the dispatch bookkeeping is saved", () => { - const runs = makeTrackedRuns(); - runs.track(messageId, taskId, "start"); - assert.deepEqual(runs.observe(runEvent("run:fast", "failed")), []); - assert.deepEqual(runs.dispatched(messageId, threadId), settledFor("failed", "The run failed.")); - assert.isNull(runs.dispatched(messageId, threadId)); -}); - -it("does not end a queued run cancelled only because its message was promoted into the active run", () => { - const runs = makeTrackedRuns(); - runs.track(messageId, taskId, "start"); - runs.dispatched(messageId, threadId); - assert.deepEqual(runs.observe(runEvent("run:queued", "queued")), []); - // Promotion cancels the queued run first, then moves the message into the active run. - const [cancelled] = runs.observe(runEvent("run:queued", "cancelled")); - assert.deepEqual(cancelled, settledFor("failed", "The run was cancelled.", "run:queued")); - assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:active"))); - assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ - settledFor("succeeded", null), - ]); -}); - it("ends a queued run that was cancelled outright", () => { const runs = makeTrackedRuns(); runs.track(messageId, taskId, "start"); @@ -124,29 +102,6 @@ it("fails a run whose thread is deleted before the turn ends", () => { ); }); -it.each(["completed", "failed"] as const)("settles every task sharing a %s run", (status) => { - const runs = makeTrackedRuns(); - const secondMessage = MessageId.make("message:second-task"); - const secondTask = ScheduledTaskId.make("task:second"); - runs.track(messageId, taskId, "start"); - runs.track(secondMessage, secondTask, "second-start"); - runs.dispatched(messageId, threadId); - runs.dispatched(secondMessage, threadId); - runs.bindRun(messageId, RunId.make("run:shared"), null); - runs.bindRun(secondMessage, RunId.make("run:shared"), null); - const error = status === "failed" ? "The run failed." : null; - assert.deepEqual(runs.observe(runEvent("run:shared", status)), [ - settledFor(status === "completed" ? "succeeded" : "failed", error), - { - ...settledFor(status === "completed" ? "succeeded" : "failed", error), - messageId: secondMessage, - taskId: secondTask, - startedAtIso: "second-start", - }, - ]); - assert.deepEqual(runs.pending(), []); -}); - it("keeps a promoted message tracked when cancellation arrived before dispatch returned", () => { const runs = makeTrackedRuns(); runs.track(messageId, taskId, "start"); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts index 48115c050f266..2d85d0636f88c 100644 --- a/apps/server/src/scheduledTasks/TrackedRuns.ts +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -62,7 +62,17 @@ export function makeTrackedRuns() { }; }; + const threadDeleted = (threadId: ThreadId): ReadonlyArray => + [...runs.values()] + .filter((run) => run.threadId === threadId) + .flatMap((run) => + settled( + end(run, { status: "failed", error: "The thread was deleted.", cancelledRunId: null }), + ), + ); + return { + threadDeleted, track: (messageId: MessageId, taskId: ScheduledTaskId, startedAtIso: string) => { // Only a task's latest run is reflected on its row. forgetTask(taskId); @@ -176,19 +186,8 @@ export function makeTrackedRuns() { ), ); } - case "thread.deleted": { - return [...runs.values()] - .filter((run) => run.threadId === event.threadId) - .flatMap((run) => - settled( - end(run, { - status: "failed", - error: "The thread was deleted.", - cancelledRunId: null, - }), - ), - ); - } + case "thread.deleted": + return threadDeleted(event.threadId); default: return []; }