diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 401b91f24e0c8..00984a3a00d60 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -272,6 +272,7 @@ function scheduledTaskSummary(task: ScheduledTask, mayRun: boolean): Orchestrato schedule: task.schedule, nextRunAt: task.nextRunAt, lastRunStatus: task.lastRunStatus, + ...(task.lastRunError === null ? {} : { lastRunError: task.lastRunError }), ...(task.webhook?.url == null || !mayRun || task.blockedByPolicy !== undefined ? {} : { webhookUrl: task.webhook.url }), diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index 46994a891f705..9304cee0e4399 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -323,7 +323,7 @@ it.effect.each(deliveries)( }), ); const ran = yield* service.runNow({ id: task.id }); - assert.equal(ran.task.lastRunStatus, "succeeded"); + assert.equal(ran.task.lastRunStatus, "running"); }).pipe( Effect.provide( ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 3157f55cacfa6..c37b522fbbd49 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -497,6 +497,10 @@ export interface OrchestratorV2Shape { readonly eventType?: OrchestrationV2DomainEvent["type"]; }) => Stream.Stream; readonly streamDomainEvents: Stream.Stream; + readonly openDomainEventStream: Effect.Effect< + Stream.Stream, + OrchestratorV2Error + >; } export class OrchestratorV2 extends Context.Service()( @@ -14484,6 +14488,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const shellProjectionError = (cause: unknown) => new OrchestratorProjectionError({ threadId: ThreadId.make("thread:shell"), cause }); + const openDomainEventStream = Effect.fn("OrchestratorV2.openDomainEventStream")( + function* () { + const latest = yield* eventSink.latestSequence(); + return eventSink.stream({ afterSequence: latest }).pipe( + Stream.map((stored) => stored.event), + Stream.mapError((cause) => new OrchestratorDomainEventStreamError({ cause })), + ); + }, + Effect.mapError((cause) => new OrchestratorDomainEventStreamError({ cause })), + ); + return OrchestratorV2.of({ resumeQueuedRuns, recoverDelegatedTasks, @@ -14689,20 +14704,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ), - // Start at the latest sequence; awareness needs no historical replay. - streamDomainEvents: Stream.unwrap( - eventSink - .latestSequence() - .pipe(Effect.map((latest) => eventSink.stream({ afterSequence: latest }))), - ).pipe( - Stream.map((stored) => stored.event), - Stream.mapError( - (cause) => - new OrchestratorDomainEventStreamError({ - cause, - }), - ), - ), + streamDomainEvents: Stream.unwrap(openDomainEventStream()), + openDomainEventStream: openDomainEventStream(), }); }); diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index bfa50b9793490..4bf39cb593132 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -371,7 +371,7 @@ it.effect.each( creationSource: createdBy === "agent" ? "mcp" : "web", }); const result = yield* tasks.runNow({ id: task.id }); - assert.equal(result.task.lastRunStatus, "succeeded"); + assert.equal(result.task.lastRunStatus, "running"); const projectThreads = yield* threads.listProjectThreads({ projectId, includeSubagents: false, @@ -488,7 +488,7 @@ it.effect("runs a thread-bound automation on the provider the thread uses now", }); const result = yield* tasks.runNow({ id: task.id }); - assert.equal(result.task.lastRunStatus, "succeeded"); + assert.equal(result.task.lastRunStatus, "running"); const projection = yield* threads.getThreadProjection(launched.threadId); const message = projection.messages.find((item) => item.scheduledTaskId === task.id); const run = projection.runs.find((item) => item.id === message?.runId); @@ -549,6 +549,75 @@ it.effect("retains automation and sender attribution while a message waits in th }).pipe(Effect.provide(harness.layer)); }); +it.effect("an automation queued behind a running turn records how its own turn ends", () => { + const harness = makeHarness({ runSetup: () => Effect.never }); + const layerScheduledTasks = ScheduledTasks.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + harness.layer, + NodeCrypto.layer, + Scheduler.layer, + Layer.mock(SecretRequests.SecretRequests)({}), + ), + ), + ); + return Effect.gen(function* () { + const tasks = yield* ScheduledTasks.ScheduledTaskService; + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const launched = yield* launches.launch( + launchInput({ + command: "command:automation:outcome", + thread: "thread:automation:outcome", + message: "First message", + workspace: { type: "worktree", baseRef: "main" }, + }), + ); + const { task } = yield* tasks.upsert({ + id: ScheduledTaskId.make("scheduled-task:outcome"), + title: "Daily audit", + prompt: "Audit performance and crashes.", + enabled: false, + schedule: { type: "interval", everyMs: 60_000 }, + projectId, + threadId: launched.threadId, + workspaceStrategy: { type: "root" }, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + createdBy: "user", + creationSource: "web", + }); + const ran = yield* tasks.runNow({ id: task.id }); + assert.equal(ran.task.lastRunStatus, "running"); + const projection = yield* threads.getThreadProjection(launched.threadId); + const queued = projection.runs.find((run) => run.status === "queued"); + assert.isDefined(queued); + const events = yield* threads.openDomainEventStream; + + yield* threads.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("command:automation:outcome:cancel"), + threadId: launched.threadId, + runId: queued!.id, + }); + const cancelled = yield* events.pipe( + Stream.filter((event) => event.type === "run.updated" && event.payload.id === queued!.id), + Stream.runHead, + ); + assert.isTrue(Option.isSome(cancelled)); + yield* tasks.subscribeList().pipe( + Stream.filter(({ tasks: listed }) => listed[0]?.lastRunStatus === "failed"), + Stream.take(1), + Stream.runDrain, + ); + const [settled] = (yield* tasks.list()).tasks; + assert.equal(settled?.lastRunError, "The run was cancelled."); + assert.equal(settled?.runCount, 1); + }).pipe(Effect.provide(Layer.mergeAll(harness.layer, layerScheduledTasks))); +}); + it.effect("provisions independent launches concurrently instead of behind a global semaphore", () => Effect.gen(function* () { const setupCount = yield* Ref.make(0); @@ -1051,7 +1120,7 @@ it.effect("runs a Scratch scheduled task saved with a worktree in its own folder WHERE task_id = ${task.id} `; const ran = yield* tasks.runNow({ id: task.id }); - assert.equal(ran.task.lastRunStatus, "succeeded"); + assert.equal(ran.task.lastRunStatus, "running"); const [thread] = yield* threads.listProjectThreads({ projectId, includeSubagents: false }); assert.isDefined(thread); const projection = yield* preparationSettled(thread!.id); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index de12768ac7b17..a26cbc256a060 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -497,6 +497,7 @@ export interface ThreadManagementServiceShape { readonly streamStoredEvents: Orchestrator.OrchestratorV2["Service"]["streamStoredEvents"]; readonly streamStoredEventsFrom: Orchestrator.OrchestratorV2["Service"]["streamStoredEventsFrom"]; readonly streamDomainEvents: Orchestrator.OrchestratorV2["Service"]["streamDomainEvents"]; + readonly openDomainEventStream: Orchestrator.OrchestratorV2["Service"]["openDomainEventStream"]; } export class ThreadManagementService extends Context.Service< @@ -1632,6 +1633,7 @@ const make = Effect.gen(function* () { streamStoredEvents: orchestrator.streamStoredEvents, streamStoredEventsFrom: orchestrator.streamStoredEventsFrom, streamDomainEvents: orchestrator.streamDomainEvents, + openDomainEventStream: orchestrator.openDomainEventStream, }); }); diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 1463a1a30270f..9be17348bdbb0 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -265,6 +265,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( streamStoredEvents: Stream.empty, streamStoredEventsFrom: () => Stream.empty, streamDomainEvents: options.domainEvents ?? Stream.empty, + openDomainEventStream: Effect.succeed(options.domainEvents ?? Stream.empty), }); const publications: Array<{ readonly url: string; diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index cbfea6b4e3135..39c406a16ae8d 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -4,13 +4,19 @@ import * as NodeUtil from "node:util"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it, vi } from "@effect/vitest"; import { + EventId, + MessageId, + type OrchestrationV2DomainEvent, + type OrchestrationV2Run, ProjectId, ProviderInstanceId, + RunId, ScheduledTaskError, ScheduledTaskId, type ScheduledTaskUpsertInput, SecretRef, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -22,6 +28,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as Fiber from "effect/Fiber"; import * as Stream from "effect/Stream"; import * as Deferred from "effect/Deferred"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -491,6 +498,665 @@ it.effect("a bound run follows its thread when promotion moves it mid-send", () }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +const turnThreadId = ThreadId.make("thread:turn-outcome"); +const epoch = DateTime.makeUnsafe(0); + +const runOf = ( + runId: string, + userMessageId: MessageId, + status: OrchestrationV2Run["status"], +): OrchestrationV2Run => ({ + id: RunId.make(runId), + threadId: turnThreadId, + ordinal: 1, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + providerThreadId: null, + userMessageId, + rootNodeId: null, + activeAttemptId: null, + status, + requestedAt: epoch, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, +}); + +const runEvent = (run: OrchestrationV2Run): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${run.id}:${run.status}`), + type: "run.updated", + threadId: run.threadId, + runId: run.id, + occurredAt: epoch, + payload: run, +}); + +const providerErrorEvent = (runId: string, message: string): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${runId}:error`), + type: "turn-item.updated", + threadId: turnThreadId, + runId: RunId.make(runId), + occurredAt: epoch, + payload: { + id: TurnItemId.make(`turn-item:${runId}:error`), + threadId: turnThreadId, + runId: RunId.make(runId), + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "failed", + title: null, + startedAt: epoch, + completedAt: epoch, + updatedAt: epoch, + type: "error", + failure: { class: "usage_limit", message, code: null, retryable: false }, + }, +}); + +// Dispatches to a bound thread succeed at once; the run's events are fed by hand. +const makeTurnHarness = ( + options: { + /** Runs inside the dispatch, so the turn's events can land before it returns. */ + readonly duringDispatch?: ( + events: Queue.Queue, + messageId: MessageId, + ) => Effect.Effect; + /** The run the thread's projection says the scheduled message now belongs to. */ + readonly messageRunId?: RunId; + /** The runs the thread's projection holds, as read back after the event stream drops. */ + readonly runs?: ReadonlyArray; + readonly providerError?: string; + readonly messageReadFailures?: number; + readonly errorReadFailures?: number; + /** How many times the event stream fails before it starts to deliver events. */ + readonly streamFailures?: number; + /** Reads the thread as deleted, which the projection keeps as a flagged record. */ + readonly threadDeleted?: boolean; + /** + * The first stream subscription fails only when the test releases `dropStream`. + * The listener's first catch-up can run before or after a dispatch, so the drop + * guarantees one resubscribe catch-up that sees the dispatched run. + */ + readonly dropStreamOnCommand?: boolean; + /** Reads about the first dispatched message fail; later messages read normally. */ + readonly firstMessageUnreadable?: boolean; + /** Makes every projection read fail, as for a thread whose records are gone. */ + readonly recordsUnreadable?: boolean; + /** Makes the stream a live tail: events from before it was opened are not delivered. */ + readonly liveTail?: boolean; + /** Runs when the projection's runs are read, so an event can land mid catch-up. */ + readonly duringRunsRead?: ( + events: Queue.Queue, + ) => Effect.Effect; + } = {}, +) => + Effect.gen(function* () { + const events = yield* Queue.unbounded(); + const dispatched = yield* Queue.unbounded(); + const subscriptions = yield* Ref.make(0); + const failedReads = yield* Queue.unbounded(); + const dropStream = yield* Deferred.make(); + const listening = yield* Deferred.make(); + const firstMessage = yield* Ref.make(null); + const messageReads = yield* Ref.make(0); + const errorReads = yield* Ref.make(0); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(threadShell(ProjectId.make("project:test"))), + sendToThread: (input) => + Ref.update(firstMessage, (first) => first ?? input.messageId).pipe( + Effect.andThen(Queue.offer(dispatched, input.messageId)), + Effect.andThen(options.duringDispatch?.(events, input.messageId) ?? Effect.void), + Effect.as(undefined as never), + ), + getThreadRecords: (_threadId, fields: ReadonlyArray, filter) => + Effect.gen(function* () { + const first = yield* Ref.get(firstMessage); + const unreadableMessage = + fields.includes("messages") && + (yield* Ref.getAndUpdate(messageReads, (n) => n + 1)) < + (options.messageReadFailures ?? 0); + const unreadableError = + fields.includes("turnItems") && + (yield* Ref.getAndUpdate(errorReads, (n) => n + 1)) < + (options.errorReadFailures ?? 0); + if ( + options.recordsUnreadable || + unreadableMessage || + unreadableError || + (options.firstMessageUnreadable && + first !== null && + filter?.messageIds?.includes(first)) + ) { + yield* Queue.offer(failedReads, undefined); + return yield* Effect.fail(new Error("no records") as never); + } + if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void; + return { + thread: { deletedAt: options.threadDeleted ? epoch : null }, + turnItems: options.providerError + ? [providerErrorEvent(options.messageRunId!, options.providerError).payload] + : [], + messages: (filter?.messageIds ?? []).map((id) => ({ + id, + runId: options.messageRunId ?? null, + })), + runs: (options.runs ?? []).filter( + (run) => filter?.runIds === undefined || filter.runIds.includes(run.id), + ), + } as never; + }), + openDomainEventStream: Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe( + Effect.flatMap((attempt) => + attempt < (options.streamFailures ?? (options.dropStreamOnCommand ? 1 : 0)) + ? (options.dropStreamOnCommand ? Deferred.await(dropStream) : Effect.void).pipe( + Effect.as(Stream.fail(new Error("event stream dropped") as never)), + ) + : (options.liveTail ? Queue.clear(events) : Effect.void).pipe( + // The first pull comes after the listener's catch-up. + Effect.as( + Stream.fromEffect(Deferred.succeed(listening, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.fromQueue(events)), + ), + ), + ), + ), + ), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), + ), + ); + return { events, dispatched, subscriptions, listening, failedReads, dropStream, layer }; + }); + +const upsertBoundTask = ( + service: ScheduledTaskService.ScheduledTaskService["Service"], + enabled = false, +) => + service.upsert({ + id: ScheduledTaskId.make("task:turn-outcome"), + title: "Check orchestration", + prompt: "Check progress", + enabled, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project:test"), + threadId: turnThreadId, + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }); + +const untilStatus = ( + service: ScheduledTaskService.ScheduledTaskService["Service"], + id: ScheduledTaskId, + status: string, +) => + service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks.some((t) => t.id === id && t.lastRunStatus === status)), + Stream.take(1), + Stream.runDrain, + ); + +it.effect("a failed turn replaces the dispatched run's status with the provider error", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const ran = yield* service.runNow({ id: task.id }); + // Accepted is not finished: the turn has not ended yet. + assert.equal(ran.task.lastRunStatus, "running"); + assert.equal(ran.task.runCount, 0); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:usage-limit", messageId, "queued")), + runEvent(runOf("run:usage-limit", messageId, "running")), + providerErrorEvent("run:usage-limit", "You have reached your usage limit."), + runEvent(runOf("run:usage-limit", messageId, "failed")), + ]); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunStatus, "failed"); + assert.equal(settled?.lastRunError, "You have reached your usage limit."); + assert.equal(settled?.runCount, 1); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a completed turn records the run as succeeded", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:completes", messageId, "running")), + runEvent(runOf("run:completes", messageId, "completed")), + ]); + yield* untilStatus(service, task.id, "succeeded"); + const [settled] = (yield* service.list()).tasks; + assert.isNull(settled?.lastRunError); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("an earlier run's turn ending cannot overwrite a newer run", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const first = yield* Queue.take(harness.dispatched); + // Run now again while the first turn is still in flight. + yield* TestClock.adjust("1 second"); + yield* service.runNow({ id: task.id }); + const second = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:first", first, "failed")), + runEvent(runOf("run:second", second, "completed")), + ]); + yield* untilStatus(service, task.id, "succeeded"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunStatus, "succeeded"); + assert.isNull(settled?.lastRunError); + assert.equal(settled?.runCount, 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +const messageMovedEvent = (messageId: MessageId, runId: string): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${messageId}:moved`), + type: "message.updated", + threadId: turnThreadId, + runId: RunId.make(runId), + occurredAt: epoch, + payload: { + id: messageId, + threadId: turnThreadId, + runId: RunId.make(runId), + nodeId: null, + role: "user", + text: "Check progress", + attachments: [], + streaming: false, + createdAt: epoch, + updatedAt: epoch, + createdBy: "user", + creationSource: "web", + }, +}); + +it.effect("a turn that ends while the dispatch is being recorded still advances the schedule", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ + duringDispatch: (events, messageId) => + Queue.offerAll(events, [ + runEvent(runOf("run:fast", messageId, "queued")), + runEvent(runOf("run:fast", messageId, "failed")), + ]), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service, true); + yield* TestClock.adjust("30 seconds"); + yield* service.runNow({ id: task.id }); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.runCount, 1); + // Dispatched 30 seconds after the task was saved, so the next run is 60 seconds on from then. + assert.equal( + settled?.nextRunAt, + DateTime.formatIso(DateTime.makeUnsafe(Date.parse(task.nextRunAt!) + 30_000)), + ); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a queued message promoted to a steer keeps its run going in the active run", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ + messageRunId: RunId.make("run:active"), + recordsUnreadable: true, + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + // Reads fail here, so the listener's first catch-up must run before the dispatch. + yield* Deferred.await(harness.listening); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:queued", messageId, "queued")), + // Promotion cancels the queued run before it moves the message into the active one. + runEvent(runOf("run:queued", messageId, "cancelled")), + messageMovedEvent(messageId, "run:active"), + runEvent(runOf("run:active", MessageId.make("message:active-turn"), "completed")), + ]); + yield* service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks.some((t) => t.lastRunStatus !== "running")), + Stream.take(1), + Stream.runDrain, + ); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunStatus, "succeeded"); + assert.isNull(settled?.lastRunError); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a run whose outcome cannot be saved is released instead of left running", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const sql = yield* SqlClient.SqlClient; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + // Only the release path may write the terminal state. + yield* sql`CREATE TRIGGER fail_outcome BEFORE UPDATE ON scheduled_tasks + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'succeeded' + BEGIN SELECT RAISE(FAIL, 'outcome unavailable'); END`; + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:unsaved", messageId, "completed")), + ]); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "Could not record how the run ended."); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a dropped event stream is resumed, and runs that ended meanwhile are caught up", () => + Effect.gen(function* () { + const runId = RunId.make("run:during-gap"); + const harness = yield* makeTurnHarness({ + streamFailures: 1, + messageRunId: runId, + runs: [runOf(runId, MessageId.make("unused"), "completed")], + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + // The run completed while no one was listening; the resumed listener finds it. + yield* TestClock.adjust("30 seconds"); + yield* untilStatus(service, task.id, "succeeded"); + assert.equal(yield* Ref.get(harness.subscriptions), 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("the event stream keeps being resumed after many failures", () => + Effect.gen(function* () { + const runId = RunId.make("run:late"); + const harness = yield* makeTurnHarness({ + streamFailures: 8, + providerError: "Provider unavailable during reconnect.", + messageRunId: runId, + runs: [runOf(runId, MessageId.make("unused"), "failed")], + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("10 minutes"); + yield* untilStatus(service, task.id, "failed"); + assert.equal(yield* Ref.get(harness.subscriptions), 9); + assert.equal( + (yield* service.list()).tasks[0]?.lastRunError, + "Provider unavailable during reconnect.", + ); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("an event landing while the resumed listener catches up is not missed", () => + Effect.gen(function* () { + const runId = RunId.make("run:handoff"); + const harness = yield* makeTurnHarness({ + streamFailures: 1, + liveTail: true, + messageRunId: runId, + // The projection read is stale: the run ends right after it. + runs: [runOf(runId, MessageId.make("unused"), "running")], + duringRunsRead: (events) => + Queue.offer(events, runEvent(runOf(runId, MessageId.make("unused"), "completed"))).pipe( + Effect.asVoid, + ), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("30 seconds"); + yield* untilStatus(service, task.id, "succeeded"); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a cancelled run is still recorded when its first save fails", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ + messageReadFailures: 3, + messageRunId: RunId.make("run:cancelled"), + runs: [runOf("run:cancelled", MessageId.make("unused"), "cancelled")], + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const sql = yield* SqlClient.SqlClient; + const { task } = yield* upsertBoundTask(service); + // Reads fail here, so the listener's first catch-up must run before the dispatch. + yield* Deferred.await(harness.listening); + yield* service.runNow({ id: task.id }); + const messageId = yield* Queue.take(harness.dispatched); + // Fails the first write only; the retry must still record the cancel. + yield* sql`CREATE TABLE fail_once (id INTEGER)`; + yield* sql`INSERT INTO fail_once VALUES (1)`; + yield* sql`CREATE TRIGGER fail_first_outcome BEFORE UPDATE ON scheduled_tasks + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'failed' + AND EXISTS (SELECT 1 FROM fail_once) + BEGIN DELETE FROM fail_once; SELECT RAISE(FAIL, 'outcome unavailable'); END`; + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:cancelled", messageId, "queued")), + runEvent(runOf("run:cancelled", messageId, "cancelled")), + ]); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "The run was cancelled."); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect( + "a run whose thread cannot be read during catch-up is released, not retried forever", + () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ streamFailures: 1, recordsUnreadable: true }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("1 minute"); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "Could not read how the run ended."); + // The stream was resubscribed once for the drop, not again for the unreadable thread. + assert.equal(yield* Ref.get(harness.subscriptions), 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("a thread deleted while the stream was down fails its run on catch-up", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ streamFailures: 1, threadDeleted: true }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + yield* TestClock.adjust("30 seconds"); + yield* untilStatus(service, task.id, "failed"); + const [settled] = (yield* service.list()).tasks; + assert.equal(settled?.lastRunError, "The thread was deleted."); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("giving up on a catch-up read leaves a newer run of the same task tracked", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness({ + dropStreamOnCommand: true, + firstMessageUnreadable: true, + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + yield* service.runNow({ id: task.id }); + yield* Queue.take(harness.dispatched); + // Dropping the stream makes the resubscribe catch up the dispatched run, whose + // first read fails and backs off. + yield* Deferred.succeed(harness.dropStream, undefined); + yield* TestClock.adjust("1 second"); + yield* Queue.take(harness.failedReads); + yield* TestClock.adjust("1 second"); + yield* service.runNow({ id: task.id }); + const newer = yield* Queue.take(harness.dispatched); + // The catch-up for the first run runs out of retries and gives up. + yield* TestClock.adjust("30 seconds"); + yield* Queue.offer(harness.events, runEvent(runOf("run:newer", newer, "completed"))); + yield* untilStatus(service, task.id, "succeeded"); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("records the provider error for every task steered into the same run", () => + Effect.gen(function* () { + const harness = yield* makeTurnHarness(); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const { task: second } = yield* service.upsert({ + ...task, + id: ScheduledTaskId.make("task:second-shared-run"), + }); + yield* service.runNow({ id: task.id }); + const firstMessage = yield* Queue.take(harness.dispatched); + yield* service.runNow({ id: second.id }); + const secondMessage = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + messageMovedEvent(firstMessage, "run:shared"), + messageMovedEvent(secondMessage, "run:shared"), + providerErrorEvent("run:shared", "You have reached your usage limit."), + runEvent(runOf("run:shared", firstMessage, "failed")), + ]); + yield* service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks.every((t) => t.lastRunStatus === "failed")), + Stream.take(1), + Stream.runDrain, + ); + for (const settled of (yield* service.list()).tasks) { + assert.equal(settled.lastRunError, "You have reached your usage limit."); + assert.equal(settled.runCount, 1); + } + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect("catches an outcome missed by reconnect while dispatch is still in flight", () => + Effect.gen(function* () { + const releaseDispatch = yield* Deferred.make(); + const runId = RunId.make("run:dispatch-reconnect"); + const harness = yield* makeTurnHarness({ + dropStreamOnCommand: true, + liveTail: true, + messageRunId: runId, + runs: [runOf(runId, MessageId.make("other"), "failed")], + providerError: "Reconnect provider failure.", + errorReadFailures: 1, + duringDispatch: () => Deferred.await(releaseDispatch), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const dispatch = yield* service.runNow({ id: task.id }).pipe(Effect.forkChild); + yield* Queue.take(harness.dispatched); + yield* Deferred.succeed(harness.dropStream, undefined); + yield* TestClock.adjust("1 second"); + assert.equal(yield* Ref.get(harness.subscriptions), 2); + yield* Deferred.succeed(releaseDispatch, undefined); + yield* Fiber.join(dispatch); + yield* TestClock.adjust("5 seconds"); + yield* untilStatus(service, task.id, "failed"); + assert.equal((yield* service.list()).tasks[0]?.lastRunError, "Reconnect provider failure."); + assert.equal((yield* service.list()).tasks[0]?.runCount, 1); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + +it.effect( + "a run interrupted mid-dispatch is released and the next run records its real outcome", + () => + Effect.gen(function* () { + const dispatches = yield* Ref.make(0); + const harness = yield* makeTurnHarness({ + // Only the first dispatch hangs, so the run can be interrupted in flight. + duringDispatch: () => + Ref.getAndUpdate(dispatches, (n) => n + 1).pipe( + Effect.flatMap((n) => (n === 0 ? Effect.never : Effect.void)), + ), + }); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* upsertBoundTask(service); + const interrupted = yield* service.runNow({ id: task.id }).pipe(Effect.forkChild); + yield* Queue.take(harness.dispatched); + yield* Fiber.interrupt(interrupted); + const [released] = (yield* service.list()).tasks; + assert.equal(released?.lastRunStatus, "failed"); + assert.equal(released?.runCount, 1); + + yield* TestClock.adjust("1 second"); + const next = yield* service.runNow({ id: task.id }); + assert.equal(next.task.lastRunStatus, "running"); + const messageId = yield* Queue.take(harness.dispatched); + yield* Queue.offerAll(harness.events, [ + runEvent(runOf("run:after-interrupt", messageId, "running")), + runEvent(runOf("run:after-interrupt", messageId, "completed")), + ]); + yield* untilStatus(service, task.id, "succeeded"); + const [settled] = (yield* service.list()).tasks; + assert.isNull(settled?.lastRunError); + assert.equal(settled?.runCount, 2); + }).pipe(Effect.provide(harness.layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a fresh-session task launches one thread, then starts each later run fresh in it", () => Effect.gen(function* () { const launches = yield* Ref.make(0); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 4953cec766210..86995940622ef 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1,5 +1,6 @@ import { CommandId, + EventId, MessageId, PROVIDER_SEND_TURN_MAX_INPUT_CHARS, type ProjectId, @@ -44,6 +45,7 @@ import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -57,7 +59,9 @@ import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; import * as DueHint from "../scheduling/DueHint.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; +import { forkParked } from "../serverActivation.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; +import { makeTrackedRuns, type SettledRun } from "./TrackedRuns.ts"; import { redactHeaders, redactQuery, @@ -491,6 +495,7 @@ export const layer = Layer.effect( ReadonlyMap> >(new Map()); const activeRuns = yield* Ref.make>(new Set()); + const trackedRuns = makeTrackedRuns(); // Notifications re-emit the full list; slow subscribers need only the newest dirty signal. const changesPubSub = yield* PubSub.sliding(1); const dueTasks = DueHint.make(); @@ -748,6 +753,9 @@ export const layer = Layer.effect( SET updated_at = ${startedAtIso}, last_run_at = ${startedAtIso}, last_run_webhook_delivery_id = ${webhookDeliveryId ?? null}, + -- A run still 'running' here was dispatched and never reported an end. + -- This run replaces it on the row, so it is counted now. + run_count = run_count + CASE WHEN last_run_status = 'running' THEN 1 ELSE 0 END, last_run_status = 'running', last_run_error = NULL WHERE task_id = ${id} @@ -765,7 +773,10 @@ export const layer = Layer.effect( readonly id: ScheduledTaskId; readonly completedAtIso: string; readonly nextRunAtIso: string | null; - readonly status: "succeeded" | "failed"; + // A dispatched run stays 'running' until its turn ends, and counts then: + // startup recovery counts a row it finds 'running', so counting here too + // would count a restart-interrupted run twice. + readonly status: "running" | "succeeded" | "failed"; readonly error: string | null; readonly startedAtIso: string; }) => @@ -775,7 +786,7 @@ export const layer = Layer.effect( next_run_at = ${input.nextRunAtIso}, last_run_status = ${input.status}, last_run_error = ${input.error}, - run_count = run_count + 1 + run_count = run_count + ${input.status === "running" ? 0 : 1} WHERE task_id = ${input.id} AND last_run_status = 'running' AND last_run_at = ${input.startedAtIso} @@ -787,6 +798,25 @@ export const layer = Layer.effect( ), ); + // The startedAtIso guard leaves the row alone once a newer run has taken it. + const recordRunEnd = Effect.fn("ScheduledTaskService.recordRunEnd")(function* ( + run: SettledRun, + ) { + const now = yield* localNow; + const rows = yield* sql<{ task_id: string }>` + UPDATE scheduled_tasks + SET updated_at = ${iso(now)}, + last_run_status = ${run.status}, + last_run_error = ${run.error}, + run_count = run_count + 1 + WHERE task_id = ${run.taskId} + AND last_run_status = 'running' + AND last_run_at = ${run.startedAtIso} + RETURNING task_id + `; + if (rows.length > 0) yield* notifyChanged; + }); + // A failure between markRunning and markCompleted must still write a terminal record: // runDueTasks skips 'running' rows forever, and the dispatch may already have gone out, // so next_run_at must advance. Pending webhook attempts count only with their completion. @@ -812,6 +842,7 @@ export const layer = Layer.effect( WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running' ) THEN 0 ELSE 1 END WHERE task_id = ${task.id} AND last_run_status = 'running' + AND last_run_at = ${startedAtIso} `; yield* notifyChanged; }, @@ -826,6 +857,131 @@ export const layer = Layer.effect( ), ); + const releaseRun = (run: Pick, message: string) => + Effect.result(findTask(run.taskId)).pipe( + Effect.flatMap((found) => + Result.isSuccess(found) && found.success !== null + ? releaseStuckRun(found.success, run.startedAtIso, message, undefined) + : Effect.void, + ), + ); + + // Each outcome stands alone: one that still cannot be saved after retries is + // released as failed so its row does not stay `running` and block the task. + const settleOutcome = Effect.fn("ScheduledTaskService.settleOutcome")(function* ( + run: SettledRun, + ): Effect.fn.Return { + // Promoting a queued message to a steer cancels its queued run first and + // moves the message into the active run afterwards, so a cancel only ends + // the run if the message still belongs to the cancelled run. Decided once: + // confirming forgets the run, so it must not be repeated by a retry. + if (run.cancelledRunId !== null) { + const moved = + run.threadId === null + ? null + : yield* threadManagement + .getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] }) + .pipe( + Effect.map(({ messages }) => messages[0]?.runId ?? null), + Effect.retry({ times: 2 }), + Effect.catchCause((cause) => + Effect.logWarning("Could not confirm scheduled run cancellation", { + cause, + }).pipe(Effect.as(undefined)), + ), + ); + if (moved === undefined) { + const pending = trackedRuns.pending().find((entry) => entry.messageId === run.messageId); + if (pending !== undefined) yield* catchUpRun(pending).pipe(Effect.forkIn(serviceScope)); + return; + } + if (!trackedRuns.confirmCancel(run.messageId, run.cancelledRunId, moved)) return; + } + yield* recordRunEnd(run).pipe( + Effect.retry({ times: 2 }), + Effect.catchCause((cause) => + Effect.logWarning("Could not record scheduled task run outcome", { cause }).pipe( + Effect.andThen(releaseRun(run, "Could not record how the run ended.")), + ), + ), + ); + }); + + const serviceScope = yield* Effect.scope; + let runEventGeneration = 0; + + const catchUpRun = Effect.fn("ScheduledTaskService.catchUpRun")( + function* (pending: ReturnType[number]) { + const { messageId, threadId } = pending; + const { thread, messages } = yield* threadManagement.getThreadRecords( + threadId, + ["messages"], + { messageIds: [messageId] }, + ); + const occurredAt = yield* DateTime.now; + // Deleting a thread keeps its projection, flagged, so a delete missed + // with the stream shows up here rather than as a read failure. + if (thread.deletedAt !== null) { + yield* Effect.forEach(trackedRuns.threadDeleted(threadId), settleOutcome, { + discard: true, + }); + return; + } + const runId = messages[0]?.runId ?? null; + if (runId === null) return; + const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], { + runIds: [runId], + }); + const run = runs[0]; + if (run === undefined) return; + if (run.status === "failed") { + const { turnItems } = yield* threadManagement.getThreadRecords(threadId, ["turnItems"], { + turnItemRunId: runId, + turnItemTypes: ["error"], + }); + if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; + for (const item of turnItems) { + trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${item.id}`), + type: "turn-item.updated", + threadId, + runId, + occurredAt, + payload: item, + }); + } + } else if (!trackedRuns.bindRun(messageId, runId, pending.runId)) return; + const ended = trackedRuns.observe({ + id: EventId.make(`scheduled-task-catch-up:${runId}`), + type: "run.updated", + threadId, + runId, + occurredAt, + payload: run, + }); + yield* Effect.forEach(ended, settleOutcome, { discard: true }); + }, + // A thread that cannot be read (deleted, records gone) would otherwise + // be retried by every resubscribe and nothing else would settle its + // row, so after a few tries the run is released as failed. + Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }), + (effect, pending) => + effect.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not catch up a scheduled task run", { + messageId: pending.messageId, + cause, + }).pipe( + Effect.andThen( + Effect.sync(() => trackedRuns.forgetMessage(pending.messageId)).pipe( + Effect.andThen(releaseRun(pending, "Could not read how the run ended.")), + ), + ), + ), + ), + ), + ); + // The run already started the thread, so a failed write only costs the binding. // A thread deleted before the binding landed already ran its task cleanup, so apply that // cleanup here; a later deletion finds the binding. An unconfirmed lookup keeps the binding. @@ -857,6 +1013,10 @@ export const layer = Layer.effect( // Read before reserving: nothing between the reservation and its release may fail. const startedAt = yield* localNow; const startedAtIso = iso(startedAt); + const dispatchGeneration = runEventGeneration; + // Set once this run owns the row, so a failure before that cannot + // release a turn an earlier run left in flight. + const marked = yield* Ref.make(false); const run = Effect.gen(function* () { // Re-read stale poll snapshots so deleted, paused or postponed tasks cannot fire. const active = yield* findTask(task.id); @@ -871,12 +1031,15 @@ export const layer = Layer.effect( } return { task, threadId: null, dispatchFailed: false }; } - if ( - webhook !== undefined && - (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`.pipe( - NodeSqliteClient.readOnly, - )).length > 0 - ) + const [pendingDelivery] = + webhook === undefined + ? [] + : yield* sql<{ + state: string; + }>`SELECT state FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`.pipe( + NodeSqliteClient.readOnly, + ); + if (pendingDelivery?.state === "completed") return { task: { ...active, lastRunStatus: "succeeded" as const }, threadId: null, @@ -907,22 +1070,26 @@ export const layer = Layer.effect( if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); } - if (webhook === undefined) yield* markRunning(active.id, startedAtIso); - else - yield* sql - .withTransaction( - Effect.gen(function* () { - yield* markRunning(active.id, startedAtIso, webhook.deliveryId); - // Recovery must distinguish this delivery from other windows waiting on the same task. - yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`; - }), - ) - .pipe( - Effect.mapError((cause) => - taskError("Could not start webhook run.", { taskId: task.id, cause }), - ), - ); + // Persist ownership with the row so interruption cannot skip recovery. + yield* Effect.gen(function* () { + if (webhook === undefined) yield* markRunning(active.id, startedAtIso); + else + yield* sql + .withTransaction( + Effect.gen(function* () { + yield* markRunning(active.id, startedAtIso, webhook.deliveryId); + // Recovery must distinguish this delivery from other windows waiting on the same task. + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`; + }), + ) + .pipe( + Effect.mapError((cause) => + taskError("Could not start webhook run.", { taskId: task.id, cause }), + ), + ); + yield* Ref.set(marked, true); + }).pipe(Effect.uninterruptible); yield* notifyChanged; const fireKey = @@ -931,6 +1098,7 @@ export const layer = Layer.effect( : `${active.id}:webhook:${webhook.deliveryId}`; const commandId = CommandId.make(`scheduled-task:${fireKey}`); const messageId = MessageId.make(`scheduled-task-message:${fireKey}`); + trackedRuns.track(messageId, active.id, startedAtIso); const prompt = webhook?.prompt ?? active.prompt; const threadMode = scheduledTaskThreadMode(active); @@ -1011,7 +1179,9 @@ export const layer = Layer.effect( const runSucceeded = result._tag === "Success"; if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; - const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); + // Accepted is not finished: the row stays 'running' until the turn ends. + if (!runSucceeded) trackedRuns.forgetTask(active.id); + const lastRunStatus = runSucceeded ? ("running" as const) : ("failed" as const); const lastRunError = runSucceeded ? null : errorMessage(result.cause); const threadId = runSucceeded ? result.value : null; // One transaction, so an edit cannot land between reading the schedule and @@ -1033,9 +1203,9 @@ export const layer = Layer.effect( ), lastRunStatus, lastRunError, - runCount: scheduleSource.runCount + 1, + runCount: scheduleSource.runCount + (lastRunStatus === "running" ? 0 : 1), }; - if (current === null) return { recorded: completed, written: false }; + if (current === null) return { recorded: completed, written: false, deleted: true }; // startedAtIso in the guard ensures this writes only to the row this // run marked as running — a task deleted mid-run and recreated with // the same id (idempotent commandId replay) must not be stamped. @@ -1061,9 +1231,9 @@ export const layer = Layer.effect( } } // A row recreated mid-run is not this run's, so report what is stored. - return { recorded: written ? completed : current, written }; + return { recorded: written ? completed : current, written, deleted: false }; }); - const { recorded, written } = yield* sql + const { recorded, written, deleted } = yield* sql .withTransaction(record) .pipe( Effect.mapError((cause) => @@ -1078,11 +1248,33 @@ export const layer = Layer.effect( ), ); if (written) yield* notifyChanged; + if (deleted) trackedRuns.forgetTask(task.id); + else if (runSucceeded) { + // Only now can an outcome replace the bookkeeping just written; one that + // landed since the dispatch returned is settled here. + const ended = trackedRuns.dispatched(messageId, threadId); + if (ended !== null) yield* settleOutcome(ended); + // Replayed webhook dispatches and reconnects can miss the original terminal event. + if (pendingDelivery?.state === "running" || dispatchGeneration !== runEventGeneration) { + const pending = trackedRuns.pending().find((run) => run.messageId === messageId); + if (pending !== undefined) yield* catchUpRun(pending).pipe(Effect.forkIn(serviceScope)); + } + } // The stored row can belong to a replacement, so report this dispatch separately. return { task: recorded, threadId, dispatchFailed: !runSucceeded }; }).pipe( Effect.onError((cause) => - releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId), + Ref.get(marked).pipe( + Effect.flatMap((owned) => + owned + ? Effect.sync(() => trackedRuns.forgetTask(task.id)).pipe( + Effect.andThen( + releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId), + ), + ) + : Effect.void, + ), + ), ), ); @@ -1235,6 +1427,46 @@ export const layer = Layer.effect( yield* scheduler.register("scheduled-tasks", runDueTasks()); + const catchUpRuns = Effect.suspend(() => + Effect.forEach(trackedRuns.pending(), catchUpRun, { discard: true }), + ); + + // Capture the stream cursor before catch-up so an event between them is replayed. + const runEvents = Stream.fromPull( + Effect.gen(function* () { + const stream = yield* threadManagement.openDomainEventStream; + const pull = yield* Stream.toPull(stream); + runEventGeneration++; + yield* catchUpRuns; + return pull; + }), + ); + + // Dispatching a run only gets it accepted; its turn ends later, so the + // outcome is read from the thread's run events. A dropped stream is resumed + // forever with a capped backoff: runs dispatched meanwhile stay tracked and + // are caught up on the next subscribe. + yield* forkParked( + Stream.runForEach(runEvents, (event) => { + // Every domain event passes through here, so the common case stays allocation-free. + const ended = trackedRuns.observe(event); + return ended.length === 0 + ? Effect.void + : Effect.forEach(ended, settleOutcome, { discard: true }); + }).pipe( + Effect.tapCause((cause) => + Effect.logWarning("Scheduled task run event stream failed", { cause }), + ), + Effect.retry( + Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.minutes(1))), + ), + ), + ), + ), + ); + const list: ScheduledTaskService["Service"]["list"] = () => listRows().pipe( Effect.map((tasks) => ({ tasks })), @@ -1879,8 +2111,6 @@ export const layer = Layer.effect( }); }); - const serviceScope = yield* Effect.scope; - const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")( function* (deliveryId: string) { yield* releaseReplay(deliveryId); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index e06ed817ac4f4..5f692d81a16c3 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -19,6 +19,7 @@ import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as EffectScheduler from "effect/Scheduler"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import * as Tracer from "effect/Tracer"; @@ -168,6 +169,7 @@ const withService = ( readonly started: Deferred.Deferred; readonly release: Deferred.Deferred; }; + readonly getThreadRecords?: ThreadManagementService.ThreadManagementService["Service"]["getThreadRecords"]; readonly completedLaunches?: Set; readonly failLaunches?: { remaining: number }; readonly gate?: Deferred.Deferred; @@ -240,7 +242,11 @@ const withService = ( namedProjectsRoot: "/projects", isScratchProject: () => Effect.succeed(false), }), - Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)( + options.getThreadRecords === undefined + ? {} + : { getThreadRecords: options.getThreadRecords }, + ), Layer.mock(SecretRequests.SecretRequests)({ consume: ({ ref }) => { const value = secretsByRef.get(ref); @@ -1082,11 +1088,15 @@ it.effect("retrying a running window keeps the prompt it was dispatched with", ( yield* service.triggerWebhook(requestFor(task)); yield* save("Prompt at dispatch"); yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(workers); const launched = yield* Queue.take(launches); assert.equal(launched.initialMessage?.text, "Prompt at dispatch"); + assert.equal( + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'running'`).length, + 1, + ); yield* save("Prompt saved after dispatch"); yield* sql`DROP TRIGGER fail_completion`; yield* drainPendingDispatch(workers); @@ -1339,7 +1349,7 @@ it.effect( launch.commandId, `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`, ); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); assert.equal( (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`) .length, @@ -1408,7 +1418,7 @@ it.effect("start failures release both replay keys and relay claims for redelive ); assert.equal(retry._tag === "accepted" && retry.outcome, "accepted"); yield* Queue.take(launches); - assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded"); + assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "running"); }), { failLaunches: { remaining: 1 } }, ), @@ -1516,32 +1526,77 @@ it.effect( }), ); -it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () => - withService(({ service, sql, launches }) => - Effect.gen(function* () { - const { task } = yield* service.upsert( - yield* webhookTaskInput({ - schedule: { - type: "webhook", - dedupe: { replayKey: "{{body}}" }, - }, +it.effect.each([false, true])( + "keeps replay keys and settles replayed runs after completion bookkeeping fails (coalesced: %s)", + (coalesced) => { + const workers = new Map>(); + const completedLaunches = new Set(); + let dispatched: LaunchInput | undefined; + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const { task } = yield* service.upsert( + yield* webhookTaskInput({ + schedule: { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + ...(coalesced + ? { coalesceKey: "{{body.pull_request.url}}", coalesceWindowSeconds: 10 } + : {}), + }, + }, + }), + ); + yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + if (coalesced) { + yield* service.triggerWebhook(requestFor(task)); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + } else { + yield* waitForSpan( + service.triggerWebhook(requestFor(task)), + "ScheduledTaskService.runWebhookDelivery", + ); + } + dispatched = yield* Queue.take(launches); + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1); + yield* sql`DROP TRIGGER fail_completion`; + const retry = yield* service.triggerWebhook(requestFor(task)); + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); + if (coalesced) { + yield* drainPendingDispatch(workers); + yield* service.subscribeList().pipe( + Stream.filter(({ tasks }) => tasks[0]?.lastRunStatus === "succeeded"), + Stream.take(1), + Stream.runDrain, + ); + } + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); }), - ); - yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; - yield* waitForSpan( - service.triggerWebhook(requestFor(task)), - "ScheduledTaskService.runWebhookDelivery", - ); - yield* Queue.take(launches); - assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1); - yield* sql`DROP TRIGGER fail_completion`; - const retry = yield* service.triggerWebhook(requestFor(task)); - assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); - assert.equal(yield* Queue.size(launches), 0); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); - }), - ), + { + workers, + completedLaunches, + getThreadRecords: () => + Effect.succeed({ + thread: { deletedAt: null }, + messages: [ + { id: dispatched?.initialMessage?.messageId, runId: "run:webhook-replayed" }, + ], + runs: [ + { + id: "run:webhook-replayed", + userMessageId: dispatched?.initialMessage?.messageId, + threadId: "thread-1", + status: "completed", + }, + ], + } as never), + }, + ); + }, ); it.effect.each([119_000, 120_000])( @@ -1661,7 +1716,7 @@ it.effect.each([1, 11])( const retry = yield* service.triggerWebhook(request(count)); assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate"); assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); assert.equal(yield* Queue.size(launches), 0); }), { workers }, @@ -1712,7 +1767,7 @@ it.effect("failed windows release every member's replay key and relay claim", () yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(workers); assert.include((yield* Queue.take(launches)).initialMessage!.text, "B11"); - assert.equal((yield* service.list()).tasks[0]!.runCount, 2); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); }), { workers, failLaunches: { remaining: 1 } }, ); @@ -1774,7 +1829,7 @@ it.effect( yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(restartedWorkers); yield* Queue.take(restartedLaunches); - assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); }), { sql, workers: restartedWorkers }, ); @@ -1814,7 +1869,7 @@ it.effect("dispatches an overdue window exactly once after its deadline passed w yield* Queue.take(launches); for (let i = 0; i < 3; i++) yield* restartedWorkers.get("webhook-coalescing")!; assert.equal(yield* Queue.size(launches), 0); - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); }), { sql, workers: restartedWorkers }, @@ -1891,7 +1946,7 @@ it.effect("removes a window with a render defect and dispatches the task's next assert.equal(failed.outcome, "dispatch_failed"); assert.equal(failed.error, "The pending webhook payload could not be prepared."); yield* workers.get("webhook-coalescing")!; - assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal((yield* service.list()).tasks[0]!.runCount, 0); }), { workers }, ); @@ -1920,7 +1975,7 @@ it.effect.each([false, true])( requestFor(task, { headers: { group: "old" }, bodyText: '{"value":"old"}' }), ); yield* sql`CREATE TRIGGER fail_old_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'running' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; yield* TestClock.adjust("10 seconds"); yield* drainPendingDispatch(workers); yield* Queue.take(launches); @@ -1964,13 +2019,13 @@ it.effect.each([false, true])( yield* sql`UPDATE scheduled_tasks SET model_selection_json = ${before!.model_selection_json} WHERE task_id = ${task.id}`; yield* drainPendingDispatch(restartedWorkers); assert.equal(yield* Queue.size(recoveredLaunches), 0); - assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); assert.equal( (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0, ); yield* restartedWorkers.get("webhook-coalescing")!; - assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2); + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1); }), { sql, workers: restartedWorkers, completedLaunches }, ); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.test.ts b/apps/server/src/scheduledTasks/TrackedRuns.test.ts new file mode 100644 index 0000000000000..c438314038700 --- /dev/null +++ b/apps/server/src/scheduledTasks/TrackedRuns.test.ts @@ -0,0 +1,144 @@ +import { assert, it } from "@effect/vitest"; +import { + EventId, + MessageId, + type OrchestrationV2DomainEvent, + ProviderInstanceId, + RunId, + ScheduledTaskId, + ThreadId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; + +import { makeTrackedRuns } from "./TrackedRuns.ts"; + +const taskId = ScheduledTaskId.make("task:tracked"); +const threadId = ThreadId.make("thread:tracked"); +const messageId = MessageId.make("message:tracked"); +const epoch = DateTime.makeUnsafe(0); + +const runEvent = ( + runId: string, + status: "queued" | "running" | "completed" | "failed" | "cancelled", + userMessageId = messageId, +): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${runId}:${status}`), + type: "run.updated", + threadId, + runId: RunId.make(runId), + occurredAt: epoch, + payload: { + id: RunId.make(runId), + threadId, + ordinal: 1, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + providerThreadId: null, + userMessageId, + rootNodeId: null, + activeAttemptId: null, + status, + requestedAt: epoch, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, +}); + +const settledFor = ( + status: "succeeded" | "failed", + error: string | null, + cancelledRunId: string | null = null, +) => ({ + taskId, + startedAtIso: "start", + messageId, + threadId, + status, + error, + cancelledRunId: cancelledRunId === null ? null : RunId.make(cancelledRunId), +}); + +it("ends a queued run that was cancelled outright", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:queued", "queued")); + runs.observe(runEvent("run:queued", "cancelled")); + assert.isTrue(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:queued"))); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:queued"))); +}); + +it("lists only dispatched runs as pending, and follows a run bound from the projection", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + assert.deepEqual(runs.pending(), []); + runs.dispatched(messageId, threadId); + assert.deepEqual(runs.pending(), [ + { taskId, startedAtIso: "start", messageId, threadId, runId: null }, + ]); + runs.bindRun(messageId, RunId.make("run:caught-up"), null); + assert.deepEqual(runs.observe(runEvent("run:caught-up", "completed", MessageId.make("other"))), [ + settledFor("succeeded", null), + ]); + assert.deepEqual(runs.pending(), []); +}); + +it("fails a run whose thread is deleted before the turn ends", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:orphaned", "running")); + assert.deepEqual( + runs.observe({ + id: EventId.make("event:deleted"), + type: "thread.deleted", + threadId, + occurredAt: epoch, + payload: undefined as never, + }), + [settledFor("failed", "The thread was deleted.")], + ); +}); + +it("keeps a promoted message tracked when cancellation arrived before dispatch returned", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.observe(runEvent("run:queued", "queued")); + runs.observe(runEvent("run:queued", "cancelled")); + runs.bindRun(messageId, RunId.make("run:active"), RunId.make("run:queued")); + const held = runs.dispatched(messageId, threadId); + assert.equal(held?.cancelledRunId, "run:queued"); + assert.isFalse(runs.confirmCancel(messageId, RunId.make("run:queued"), RunId.make("run:active"))); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + settledFor("succeeded", null), + ]); +}); + +it("does not let a stale catch-up overwrite a live promotion", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:queued", "queued")); + const queuedRunId = RunId.make("run:queued"); + runs.observe(runEvent("run:queued", "cancelled")); + runs.confirmCancel(messageId, queuedRunId, RunId.make("run:active")); + assert.isFalse(runs.bindRun(messageId, queuedRunId, queuedRunId)); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + settledFor("succeeded", null), + ]); +}); + +it("ignores a stale cancellation read after a live promotion", () => { + const runs = makeTrackedRuns(); + runs.track(messageId, taskId, "start"); + runs.dispatched(messageId, threadId); + runs.observe(runEvent("run:queued", "queued")); + const queuedRunId = RunId.make("run:queued"); + runs.confirmCancel(messageId, queuedRunId, RunId.make("run:active")); + assert.isFalse(runs.confirmCancel(messageId, queuedRunId, queuedRunId)); + assert.deepEqual(runs.observe(runEvent("run:active", "completed")), [ + settledFor("succeeded", null), + ]); +}); diff --git a/apps/server/src/scheduledTasks/TrackedRuns.ts b/apps/server/src/scheduledTasks/TrackedRuns.ts new file mode 100644 index 0000000000000..2d85d0636f88c --- /dev/null +++ b/apps/server/src/scheduledTasks/TrackedRuns.ts @@ -0,0 +1,196 @@ +import type { + MessageId, + OrchestrationV2DomainEvent, + OrchestrationV2RunStatus, + RunId, + ScheduledTaskId, + ThreadId, +} from "@t3tools/contracts"; +export interface SettledRun { + readonly taskId: ScheduledTaskId; + readonly startedAtIso: string; + readonly status: "succeeded" | "failed"; + readonly error: string | null; + readonly messageId: MessageId; + readonly threadId: ThreadId | null; + readonly cancelledRunId: RunId | null; +} + +type RunEnd = Pick; + +interface TrackedRun { + readonly messageId: MessageId; + readonly taskId: ScheduledTaskId; + readonly startedAtIso: string; + threadId: ThreadId | null; + runId: RunId | null; + providerError: string | null; + dispatched: boolean; + ended: RunEnd | null; +} + +const ENDED_RUN_ERRORS: Partial> = { + failed: "The run failed.", + interrupted: "The run was interrupted.", + cancelled: "The run was cancelled.", + rolled_back: "The run was rolled back.", +}; + +const settled = (run: SettledRun | null) => (run === null ? [] : [run]); +// Tracking is in memory; startup recovery fails runs interrupted by a restart. +export function makeTrackedRuns() { + const runs = new Map(); + + const forgetTask = (taskId: ScheduledTaskId) => { + for (const [messageId, run] of runs) if (run.taskId === taskId) runs.delete(messageId); + }; + + // Hold terminal outcomes until dispatch has saved the next scheduled time. + const end = (run: TrackedRun, outcome: RunEnd): SettledRun | null => { + if (!run.dispatched) { + run.ended = outcome; + return null; + } + // A cancelled run stays tracked until `confirmCancel` rules on it. + if (outcome.cancelledRunId === null) runs.delete(run.messageId); + return { + taskId: run.taskId, + startedAtIso: run.startedAtIso, + messageId: run.messageId, + threadId: run.threadId, + ...outcome, + }; + }; + + const threadDeleted = (threadId: ThreadId): ReadonlyArray => + [...runs.values()] + .filter((run) => run.threadId === threadId) + .flatMap((run) => + settled( + end(run, { status: "failed", error: "The thread was deleted.", cancelledRunId: null }), + ), + ); + + return { + threadDeleted, + track: (messageId: MessageId, taskId: ScheduledTaskId, startedAtIso: string) => { + // Only a task's latest run is reflected on its row. + forgetTask(taskId); + runs.set(messageId, { + messageId, + taskId, + startedAtIso, + threadId: null, + runId: null, + providerError: null, + dispatched: false, + ended: null, + }); + }, + forgetTask, + forgetMessage: (messageId: MessageId) => { + runs.delete(messageId); + }, + dispatched: (messageId: MessageId, threadId: ThreadId | null): SettledRun | null => { + const run = runs.get(messageId); + if (run === undefined) return null; + run.dispatched = true; + run.threadId ??= threadId; + const held = run.ended; + run.ended = null; + return held === null ? null : end(run, held); + }, + pending: (): ReadonlyArray< + Pick & { + readonly threadId: ThreadId; + readonly runId: RunId | null; + } + > => + [...runs.values()].flatMap((run) => + run.dispatched && run.threadId !== null + ? [ + { + taskId: run.taskId, + startedAtIso: run.startedAtIso, + messageId: run.messageId, + threadId: run.threadId, + runId: run.runId, + }, + ] + : [], + ), + bindRun: (messageId: MessageId, runId: RunId, expectedRunId: RunId | null) => { + const run = runs.get(messageId); + if (run === undefined || run.runId !== expectedRunId) return false; + run.runId = runId; + return true; + }, + confirmCancel: ( + messageId: MessageId, + cancelledRunId: RunId, + messageRunId: RunId | null, + ): boolean => { + const run = runs.get(messageId); + if (run === undefined || run.runId !== cancelledRunId) return false; + if (messageRunId !== null && messageRunId !== cancelledRunId) { + run.runId = messageRunId; + return false; + } + runs.delete(messageId); + return true; + }, + observe: (event: OrchestrationV2DomainEvent): ReadonlyArray => { + if (runs.size === 0) return []; + switch (event.type) { + case "message.updated": { + // A queued message promoted to a steer joins the active run. + const run = runs.get(event.payload.id); + if (run !== undefined && event.payload.runId != null) { + run.runId = event.payload.runId; + run.threadId = event.payload.threadId; + } + return []; + } + case "turn-item.updated": { + const item = event.payload; + if (item.type !== "error" || item.runId === null) return []; + for (const run of runs.values()) { + if (run.runId === item.runId) run.providerError = item.failure.message; + } + return []; + } + case "run.created": + case "run.updated": { + const { id, userMessageId, status, threadId } = event.payload; + const candidate = runs.get(userMessageId); + if (candidate !== undefined && candidate.runId === null) { + candidate.runId = id; + candidate.threadId = threadId; + } + const failure = ENDED_RUN_ERRORS[status]; + if (status !== "completed" && failure === undefined) return []; + return [...runs.values()] + .filter((run) => run.runId === id) + .flatMap((run) => + settled( + end(run, { + status: status === "completed" ? "succeeded" : "failed", + error: + status === "completed" + ? null + : status === "failed" + ? (run.providerError ?? failure!) + : failure!, + cancelledRunId: status === "cancelled" ? id : null, + }), + ), + ); + } + case "thread.deleted": + return threadDeleted(event.threadId); + default: + return []; + } + }, + }; +} diff --git a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx index 4c3326df8c635..e0a920c876bf7 100644 --- a/apps/web/src/components/chat/ThreadAutomationsPanel.tsx +++ b/apps/web/src/components/chat/ThreadAutomationsPanel.tsx @@ -217,9 +217,7 @@ export function ThreadAutomationsPanel(props: { variant="ghost" part="icon" aria-label={`Run ${task.title} now`} - disabled={ - !canOperate || busyTaskId !== null || task.lastRunStatus === "running" - } + disabled={!canOperate || busyTaskId !== null} onClick={() => void runNow(task)} > diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index d1bb9961a0c68..742a0517f2078 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -707,6 +707,9 @@ export const OrchestratorMcpScheduledTask = Schema.Struct({ schedule: ScheduledTaskSchedule, nextRunAt: Schema.NullOr(IsoDateTime), lastRunStatus: ScheduledTaskRunStatus, + lastRunError: Schema.optional(Schema.String).annotate({ + description: "Why the last run failed. Absent unless it failed.", + }), webhookUrl: Schema.optional(Schema.String).annotate({ description: "Public URL for the sender. Absent: no T3 Connect tunnel. The user must enable T3 Connect remote access.",