diff --git a/apps/server/src/mcp/BridgeStatusService.test.ts b/apps/server/src/mcp/BridgeStatusService.test.ts index 46e96a7229b7a..c4324f11f9d87 100644 --- a/apps/server/src/mcp/BridgeStatusService.test.ts +++ b/apps/server/src/mcp/BridgeStatusService.test.ts @@ -8,6 +8,7 @@ import * as Bridge from "./BridgeStatusService.ts"; import * as Invocation from "./McpInvocationContext.ts"; import * as Events from "./McpEvents.ts"; import * as Metrics from "./McpBridgeMetrics.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; it.effect( "reports fresh peer probe results and preserves bridge status when links are unavailable", @@ -40,6 +41,20 @@ it.effect( deliveriesExhaustedRetries: 0, }), }), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({ + reliability: () => + Effect.succeed([ + { taskId: "scheduled-task:quiet" }, + { + taskId: "scheduled-task:late", + missedRun: { + dueAt: "1969-12-31T00:00:00.000Z", + overdueMs: 86_400_000, + reason: "not_dispatched" as const, + }, + }, + ]), + }), Layer.mock(PeerForwarding.PeerForwarding)({ links: (caller) => Effect.gen(function* () { @@ -67,6 +82,13 @@ it.effect( yield* Effect.gen(function* () { const service = yield* Bridge.BridgeStatusService; const status = yield* service.status(); + expect(status.missedScheduledRuns).toEqual([ + { + scheduledTaskId: "scheduled-task:late", + dueAt: "1969-12-31T00:00:00.000Z", + overdueMs: 86_400_000, + }, + ]); expect(status.linkedEnvironments).toMatchObject({ checkedAt: "1970-01-01T00:00:00.000Z", links: [ diff --git a/apps/server/src/mcp/BridgeStatusService.ts b/apps/server/src/mcp/BridgeStatusService.ts index 8a2498f9fe152..3a1aa3a6ac1f2 100644 --- a/apps/server/src/mcp/BridgeStatusService.ts +++ b/apps/server/src/mcp/BridgeStatusService.ts @@ -2,8 +2,11 @@ import { EnvironmentId, EnvironmentConnectReachability, McpEventSubscriptionsResult, + IsoDateTime, McpEventWatch, + NonNegativeInt, OrchestratorMcpFailure, + ScheduledTaskId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Context from "effect/Context"; @@ -14,6 +17,7 @@ import * as PeerForwarding from "../peer/PeerForwarding.ts"; import * as Invocation from "./McpInvocationContext.ts"; import * as McpEvents from "./McpEvents.ts"; import * as Metrics from "./McpBridgeMetrics.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; export const BridgeStatusResult = Schema.Struct({ serviceStartedAt: Schema.String, tunnel: Schema.NullOr(EnvironmentConnectReachability), @@ -39,6 +43,13 @@ export const BridgeStatusResult = Schema.Struct({ ), }), ), + missedScheduledRuns: Schema.Array( + Schema.Struct({ + scheduledTaskId: ScheduledTaskId, + dueAt: IsoDateTime, + overdueMs: NonNegativeInt, + }), + ), toolCalls: Schema.Array( Schema.Struct({ tool: Schema.String, @@ -64,6 +75,7 @@ export const layer = Layer.effect( const events = yield* McpEvents.McpEvents; const metrics = yield* Metrics.McpBridgeMetrics; const forwarding = yield* PeerForwarding.PeerForwarding; + const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; return BridgeStatusService.of({ status: Effect.fn("BridgeStatusService.status")(function* () { const scope = yield* Invocation.McpInvocationContext; @@ -93,9 +105,26 @@ export const layer = Layer.effect( ), Effect.orElseSucceed(() => null), ); + const missedScheduledRuns = yield* scheduledTasks.reliability().pipe( + Effect.map((entries) => + entries.flatMap(({ taskId, missedRun }) => + missedRun === undefined + ? [] + : [ + { + scheduledTaskId: ScheduledTaskId.make(taskId), + dueAt: missedRun.dueAt, + overdueMs: missedRun.overdueMs, + }, + ], + ), + ), + Effect.orElseSucceed(() => []), + ); return { ...status, linkedEnvironments, + missedScheduledRuns, toolCalls: yield* metrics.read(scope.client!.sessionId), }; }), diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index e428fe9e0602b..db2e6ad290e7b 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -69,6 +69,7 @@ import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import { buildUnavailableProviderSnapshot } from "../provider/unavailableProviderSnapshot.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; +import type { ScheduledTaskReliability } from "../scheduledTasks/ScheduledTaskReliability.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; @@ -2846,6 +2847,7 @@ describe("OrchestratorMcpService provider resolution", () => { tasks: ReadonlyArray, boundThread: OrchestrationV2ThreadShell | null, upserted: Ref.Ref, + reliability: ReadonlyArray = [], ) => layerOrchestratorService.pipe( Layer.provide( @@ -2866,6 +2868,7 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({ list: () => Effect.succeed({ tasks }), + reliability: () => Effect.succeed(reliability), upsert: () => Ref.update(upserted, (count) => count + 1).pipe(Effect.as({ task: tasks[0]! })), }), @@ -2896,6 +2899,41 @@ describe("OrchestratorMcpService provider resolution", () => { }), ); + it.effect("adds recent failures and missed runs to the listed task they belong to", () => + Effect.gen(function* () { + const upserted = yield* Ref.make(0); + const failing = task({ id: ScheduledTaskId.make("scheduled-task:failing") }); + const quiet = task({ id: ScheduledTaskId.make("scheduled-task:quiet") }); + const recentFailures = { + failed: 3, + cancelled: 6, + lastAt: "2026-11-12T07:11:28.827Z", + lastStatus: "failed" as const, + lastError: "Workspace preparation failed", + }; + const missedRun = { + dueAt: "2026-11-12T01:00:00.000Z", + overdueMs: 7_200_000, + reason: "not_dispatched" as const, + }; + const listed = yield* OrchestratorMcpService.OrchestratorMcpService.pipe( + Effect.flatMap((mcp) => mcp.listScheduledTasks(supervisedClient, { projectId })), + Effect.provide( + service([failing, quiet], null, upserted, [ + { taskId: failing.id, recentFailures, missedRun }, + ]), + ), + ); + assert.deepEqual( + listed.tasks.map((summary) => [summary.recentFailures, summary.missedRun]), + [ + [recentFailures, missedRun], + [undefined, undefined], + ], + ); + }), + ); + it.effect("hides a webhook URL from a thread whose turn has ended", () => Effect.gen(function* () { const callerId = ThreadId.make("thread:scheduled-ended"); @@ -2938,6 +2976,7 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({ list: () => Effect.succeed({ tasks: [task({})] }), + reliability: () => Effect.succeed([]), }), ), ), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 8763ba7ade056..8c1efaba86c06 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -2132,10 +2132,26 @@ const make = Effect.gen(function* () { failure("orchestration_error", `Could not list scheduled tasks: ${error.message}`), ), ); + // History is extra: a failed read must not hide the tasks themselves. + const reliability = new Map( + (yield* scheduledTasks.reliability().pipe(Effect.orElseSucceed(() => []))).map( + (entry) => [entry.taskId, entry] as const, + ), + ); return { tasks: yield* Effect.forEach( tasks.filter((task) => projectId === undefined || task.projectId === projectId), - (task) => summarizeScheduledTask(scope, { parent, limits }, task), + (task) => + summarizeScheduledTask(scope, { parent, limits }, task).pipe( + Effect.map((summary) => { + const { recentFailures, missedRun } = reliability.get(task.id) ?? {}; + return { + ...summary, + ...(recentFailures === undefined ? {} : { recentFailures }), + ...(missedRun === undefined ? {} : { missedRun }), + }; + }), + ), ), }; }, diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index df7e2f6d3f2ed..5a0e5b83ca331 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -191,6 +191,7 @@ const layerUnusedScheduledTaskStub = Layer.succeed( ScheduledTaskService.ScheduledTaskService.of({ list: () => Effect.succeed({ tasks: [] }), subscribeList: () => Stream.succeed({ tasks: [] }), + reliability: () => Effect.succeed([]), upsert: () => Effect.die("ScheduledTaskService.upsert is unused in this test"), setEnabled: () => Effect.die("ScheduledTaskService.setEnabled is unused in this test"), delete: () => Effect.die("ScheduledTaskService.delete is unused in this test"), diff --git a/apps/server/src/mcp/ThreadLaunchMcpService.ts b/apps/server/src/mcp/ThreadLaunchMcpService.ts index 146d4879409f8..7fea171e74111 100644 --- a/apps/server/src/mcp/ThreadLaunchMcpService.ts +++ b/apps/server/src/mcp/ThreadLaunchMcpService.ts @@ -17,6 +17,7 @@ import { OrchestratorRuntimeModeAboveCeilingError } from "../orchestration-v2/Or import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadMessageIntake from "../orchestration-v2/ThreadMessageIntake.ts"; import * as Claims from "../orchestration-v2/AttachmentClaims.ts"; +import { temporaryFolderWarning } from "../project/temporaryFolder.ts"; import * as Project from "../project/ProjectService.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; @@ -90,6 +91,7 @@ export const launchResult = Schema.Struct({ modelSelection: ModelSelection, runId: Schema.NullOr(RunId), status: Schema.NullOr(OrchestrationV2RunStatus), + warning: Schema.optional(Schema.String), }); function worktreeCheckHint( error: @@ -370,6 +372,8 @@ const launch = Effect.fn("ThreadLaunchMcpService.launch")( .pipe(Effect.ignore({ log: true })); } const run = result.projection.runs.find((run) => run.userMessageId === messageId); + const warning = + project === undefined ? undefined : yield* temporaryFolderWarning(project.workspaceRoot); const notify = yield* ThreadNotifications.finish( notification, thread.id, @@ -384,6 +388,7 @@ const launch = Effect.fn("ThreadLaunchMcpService.launch")( modelSelection: thread.modelSelection, runId: run?.id ?? null, status: run?.status ?? null, + ...(warning === undefined ? {} : { warning }), }; }), ); diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index d7693b2d348e8..69de65f6387d8 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -424,6 +424,32 @@ it.effect("a launch with a model its provider does not advertise creates no thre }), ); +it.effect.each([ + ["/tmp/review", true], + ["/projects/client-target", false], +] as const)("a launch into a project rooted at %s warns: %s", ([workspaceRoot, warns]) => + Effect.gen(function* () { + const { projectId, dependencies } = clientLaunchHarness({ + runtimeModeCeiling: "auto", + launched: [], + workspaceRoot, + }); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(dependencies)), + ), + ); + const result = yield* toolkit.handle("t3_thread_launch", { title: "Fix", projectId }).pipe( + Stream.unwrap, + Stream.runCollect, + Effect.map((chunk) => chunk.at(-1)?.result), + Effect.provide(dependencies), + ); + expect(result).toMatchObject(warns ? { warning: expect.stringContaining(workspaceRoot) } : {}); + expect(warns || (result as { warning?: string }).warning === undefined).toBe(true); + }), +); + it.effect("a launch retried with the same clientRequestId replays the first one", () => Effect.gen(function* () { const launched: Array = []; diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 170c87b2ff9fe..d3b59aa73c0f4 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -11,6 +11,7 @@ import { mutateProjectWithClone } from "../../../project/ProjectMutation.ts"; import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts"; import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts"; +import { temporaryFolderWarning } from "../../../project/temporaryFolder.ts"; import * as McpToolAccess from "../../McpToolAccess.ts"; import { newCommandId, readCaller, unavailable } from "../../threadAccess.ts"; import { ProjectToolkit } from "./tools.ts"; @@ -104,9 +105,11 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { }; } const commandId = yield* newCommandId(); - return yield* projects + const created = yield* projects .create({ ...input, workspaceRoot, commandId, projectId: ProjectId.make(commandId) }) .pipe(Effect.mapError(projectFailure)); + const warning = yield* temporaryFolderWarning(workspaceRoot); + return warning === undefined ? created : { ...created, warning }; }), // Linked work may bring the repository it works in; an existing // registered workspace is still refused. diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index 4dc3e6b14d263..6febdcdc9f191 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -75,7 +75,11 @@ const ProjectCreateTool = Tool.make("t3_project_create", { ...ProjectCreatePayload.fields, workspaceRoot: Schema.optional(ProjectCreatePayload.fields.workspaceRoot), }), - success: Schema.Struct({ ...Project.fields, commitError: Schema.optional(Schema.String) }), + success: Schema.Struct({ + ...Project.fields, + commitError: Schema.optional(Schema.String), + warning: Schema.optional(Schema.String), + }), dependencies: [...shared.dependencies, ManagedProjectFolders.ManagedProjectFolders], }).annotate(Tool.Destructive, true); const ProjectUpdateTool = Tool.make("t3_project_update", { diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 379339e0e2aae..9861420ab55a9 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -1,5 +1,6 @@ import { CommandId, + EventId, type OrchestrationV2Run, OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, @@ -31,6 +32,12 @@ import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventS import { projectStoredEventForWire } from "./WireProjection.ts"; import { makeRunOutcomeRecorder, type RunOutcome } from "./RunMetrics.ts"; import { makeRunStopStamper, type RunStopContext } from "./RunStop.ts"; +import { + RUN_BOUND_ITEM_TYPES, + isActiveRunBoundItem, + itemStatusForEndedRun, + settleItemForEndedRun, +} from "./RunItemSettlement.ts"; import * as ScheduledTaskChanges from "../scheduledTasks/ScheduledTaskChanges.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -378,7 +385,11 @@ const layerBase: Layer.Layer< unstamped: ReadonlyArray, context: RunStopContext = {}, ) => - Effect.flatMap(stampRunStops(unstamped, context), (events) => normalizeStampedEvents(events)); + stampRunStops(unstamped, context).pipe( + Effect.flatMap(normalizeStampedEvents), + // After normalization, so the closing events keep the positions their items already have. + Effect.flatMap(closeEndedRunItems), + ); const normalizeStampedEvents = (events: ReadonlyArray) => { const runOrdinals = new Map( @@ -422,6 +433,61 @@ const layerBase: Layer.Layer< ); }; + // A todo list or compaction item never outlives its run, but providers rarely close + // them. Close what is still active here, in the transaction that ends the run. + // The enclosing write span already traces this per-batch step. + const closeEndedRunItems = Effect.fnUntraced(function* ( + events: ReadonlyArray, + ) { + const closing: Array = []; + for (const event of events) { + if (event.type !== "run.updated" || itemStatusForEndedRun(event.payload.status) === null) + continue; + const run = event.payload; + const latestInBatch = new Map(); + for (const candidate of events) { + if (candidate.type === "turn-item.updated" && candidate.payload.runId === run.id) { + latestInBatch.set(candidate.payload.id, candidate.payload); + } + } + const stored = yield* projectionStore + .getThreadRecords(run.threadId, ["turnItems"], { + turnItemRunId: run.id, + turnItemTypes: [...RUN_BOUND_ITEM_TYPES], + turnItemStatuses: ["pending", "running", "waiting"], + }) + // The thread's creation can share this batch. An old item that cannot be read must not + // fail the write that ends the run; its items then close on read instead. + .pipe( + Effect.catchTags({ + ProjectionStoreThreadNotFoundError: () => Effect.succeed({ turnItems: [] }), + }), + Effect.catch((cause) => + Effect.logWarning("Could not read run items to close", { runId: run.id, cause }).pipe( + Effect.as({ turnItems: [] }), + ), + ), + ); + for (const item of stored.turnItems) { + if (!latestInBatch.has(item.id)) latestInBatch.set(item.id, item); + } + for (const item of latestInBatch.values()) { + if (!isActiveRunBoundItem(item)) continue; + closing.push({ + id: EventId.make(`${event.id}:close:${item.id}`), + type: "turn-item.updated", + threadId: run.threadId, + runId: run.id, + ...(item.nodeId === null ? {} : { nodeId: item.nodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: event.occurredAt, + payload: settleItemForEndedRun(item, run.status), + }); + } + } + return closing.length === 0 ? events : [...events, ...closing]; + }); + const applyStoredEvents = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (storedEvents: ReadonlyArray) { diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 36d26c6e60952..c8a3123db2b90 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -16,6 +16,7 @@ import { type OrchestrationV2Run, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, + PlanId, ProjectId, ProviderDriverKind, ProviderInstanceId, @@ -2085,6 +2086,120 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(layerTest))), ); + it.effect.each(["completed", "failed", "interrupted", "rolled_back"] as const)( + "closes run-bound items still active when a run ends %s, and leaves background tools", + (endStatus) => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:run-end-items:${endStatus}`); + const runId = RunId.make(`run:run-end-items:${endStatus}`); + const run: OrchestrationV2Run = { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`message:run-end-items:${endStatus}`), + rootNodeId: null, + activeAttemptId: null, + status: "running", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + const itemBase = (id: string, ordinal: number) => ({ + id: TurnItemId.make(`turn-item:run-end-items:${endStatus}:${id}`), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal, + status: "running" as const, + title: id, + startedAt: now, + completedAt: null, + updatedAt: now, + }); + const compaction: OrchestrationV2TurnItem = { + ...itemBase("compaction", 1), + type: "compaction", + driver: null, + }; + const todo: OrchestrationV2TurnItem = { + ...itemBase("todo", 2), + type: "todo_list", + planId: PlanId.make(`plan:run-end-items:${endStatus}`), + steps: [], + }; + const background: OrchestrationV2TurnItem = { + ...itemBase("background", 3), + type: "dynamic_tool", + toolName: "mcp__background", + input: {}, + }; + const itemEvent = (payload: OrchestrationV2TurnItem): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${payload.id}`), + type: "turn-item.updated", + threadId, + runId, + occurredAt: now, + payload, + }); + yield* eventSink.write({ + events: [ + threadCreatedEvent({ + id: `event:run-end-items:${endStatus}:thread`, + thread: makeThread(threadId, now), + now, + }), + { + id: EventId.make(`event:run-end-items:${endStatus}:run`), + type: "run.created", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: run, + }, + itemEvent(compaction), + itemEvent(background), + ], + }); + // The todo list arrives in the same batch that ends the run. + yield* eventSink.write({ + events: [ + itemEvent(todo), + { + id: EventId.make(`event:run-end-items:${endStatus}:ended`), + type: "run.updated", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: { ...run, status: endStatus, completedAt: now }, + }, + ], + }); + const { turnItems } = yield* projections.getThreadRecords(threadId, ["turnItems"]); + const statusOf = (item: OrchestrationV2TurnItem) => + turnItems.find((candidate) => candidate.id === item.id)?.status; + const expected = + endStatus === "failed" || endStatus === "rolled_back" ? "cancelled" : endStatus; + assert.equal(statusOf(compaction), expected); + assert.equal(statusOf(todo), expected); + assert.equal(statusOf(background), "running"); + }).pipe(Effect.provide(Layer.fresh(layerTest))), + ); + it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5762fd2fdc97f..632f12c584ac6 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -2171,7 +2171,7 @@ export const layer: Layer.Layer = AND NOT ${settledSubagent(sql`candidate`)} `; - // A todo list is never closed when its run ends, so a long thread holds an active one per + // Todo lists stored before runs closed them are still active, so a long thread holds one per // old run. Counting those as live work drags each run's nodes and subagents into every window. const todoListOfSettledRun = (row: Statement.Fragment) => sql`( ${row}.kind = 'todo_list' AND ${row}.run_id IS NOT NULL AND NOT EXISTS ( diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 50e52151fca71..2f5a440a2f8e0 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -5145,6 +5145,8 @@ it.effect.each(["missing", "file"] as const)( assert.instanceOf(error, ProviderWorkspaceMissingError); assert.include(error.message, cwd); assert.include(error.message, "Restore the folder at this path before retrying."); + // The fixture lives in the system temp folder, which a reboot clears. + assert.include(error.message, "under a temporary folder"); assert.equal((yield* Ref.get(state)).openCount, 0); assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); assert.deepEqual( diff --git a/apps/server/src/orchestration-v2/RunItemSettlement.ts b/apps/server/src/orchestration-v2/RunItemSettlement.ts new file mode 100644 index 0000000000000..c4eb93bec8df7 --- /dev/null +++ b/apps/server/src/orchestration-v2/RunItemSettlement.ts @@ -0,0 +1,42 @@ +import type { OrchestrationV2Run, OrchestrationV2TurnItem } from "@t3tools/contracts"; + +/** + * Items that cannot outlive their run. Shell, tool and subagent items can be background work, + * so a run ending does not close them. + */ +export const RUN_BOUND_ITEM_TYPES = ["todo_list", "compaction"] as const; + +const ACTIVE_ITEM_STATUSES: ReadonlySet = new Set(["pending", "running", "waiting"]); + +export function isActiveRunBoundItem(item: OrchestrationV2TurnItem): boolean { + return ( + ACTIVE_ITEM_STATUSES.has(item.status) && RUN_BOUND_ITEM_TYPES.some((type) => type === item.type) + ); +} + +/** The status a run-bound item takes when its run ends; null while the run can still change it. */ +export function itemStatusForEndedRun( + runStatus: OrchestrationV2Run["status"], +): "completed" | "interrupted" | "cancelled" | null { + switch (runStatus) { + case "completed": + return "completed"; + case "interrupted": + return "interrupted"; + case "failed": + case "cancelled": + case "rolled_back": + return "cancelled"; + default: + return null; + } +} + +export function settleItemForEndedRun( + item: OrchestrationV2TurnItem, + runStatus: OrchestrationV2Run["status"], +): OrchestrationV2TurnItem { + const status = itemStatusForEndedRun(runStatus); + if (status === null || !isActiveRunBoundItem(item)) return item; + return { ...item, status, completedAt: item.completedAt ?? item.updatedAt }; +} diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index e60d6e6b7496d..5161015af4a2e 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -212,6 +212,7 @@ function makeHarness(options: HarnessOptions = {}) { options.fetchWorktreeBaseCommit ?? (() => Effect.succeed({ commitSha: "remote-main-sha", remoteRefName: "origin/main" })), hasCommit: options.hasCommit ?? (() => Effect.succeed(false)), + hasRefNamed: () => Effect.succeed(true), currentBranch: () => Effect.succeed(null), resolveWorktreeBaseRemote: () => Effect.succeed({ remoteName: "origin", branchName: "main" }), remoteExists: options.remoteExists ?? (() => Effect.succeed(true)), @@ -2563,6 +2564,54 @@ it.effect.each(["origin/master", ":/initial"])( }), ); +it.effect.each([ + { + name: "in a folder that is not a Git repository", + repositoryRoot: null, + hasCommit: false, + remote: true, + message: `Project "Project" (/repo) is not a Git repository.`, + }, + { + name: "whose base is on no remote and not local", + repositoryRoot: "/repo", + hasCommit: false, + remote: false, + message: `Base ref "main" does not resolve to a commit in project "Project" (/repo).`, + }, +] as const)("a launched worktree $name fails with a clear message", (testCase) => + Effect.gen(function* () { + const harness = makeHarness({ + hasCommit: ({ refName }) => Effect.succeed(refName === "HEAD" || testCase.hasCommit), + git: { + repositoryRoot: () => Effect.succeed(testCase.repositoryRoot), + resolveWorktreeBaseRemote: () => + Effect.succeed(testCase.remote ? { remoteName: "origin", branchName: "main" } : null), + hasRefNamed: () => Effect.succeed(false), + }, + }); + yield* Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const launched = yield* launches.launch( + launchInput({ + command: `command:launch:clear-base-${testCase.repositoryRoot === null ? "none" : "ref"}`, + thread: `thread:launch:clear-base-${testCase.repositoryRoot === null ? "none" : "ref"}`, + message: "First message", + workspace: { type: "worktree", baseRef: "main", startFromOrigin: true }, + }), + ); + const projection = yield* preparationSettled(launched.threadId); + assert.equal(projection.runs[0]?.status, "failed"); + const failure = projection.turnItems.find((item) => item.type === "error"); + assert.equal( + failure?.type === "error" ? failure.failure.message : undefined, + `Workspace preparation failed during provision worktree: ${testCase.message}`, + ); + assert.equal(harness.createWorktree.mock.calls.length, 0); + }).pipe(Effect.provide(harness.layer)); + }), +); + it.effect("leaves uncertain Git detection failures to workspace preparation", () => Effect.gen(function* () { const harness = makeHarness({ diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 570e6a14a8cd2..c2a25f6b24e91 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -515,6 +515,16 @@ const make = Effect.gen(function* () { }) .pipe(Effect.mapError(mapError(input, "update-thread", threadId))); } + // Scheduled tasks and retries skip the entry points' check; without it, a folder that is + // not a repository, or a base found nowhere, reaches Git as an opaque failure. + yield* checkWorktreeBase({ + projectId: input.projectId, + workspaceStrategy: input.workspaceStrategy, + }).pipe( + // Provisioning flattens a temporary branch name that collides. + Effect.catchTags({ ThreadLaunchBranchConflictError: () => Effect.void }), + Effect.mapError(mapError(input, "provision-worktree", threadId)), + ); let startRef = input.workspaceStrategy.baseRef; let baseRefName = input.workspaceStrategy.baseRef; const remote = diff --git a/apps/server/src/orchestration-v2/WireProjection.test.ts b/apps/server/src/orchestration-v2/WireProjection.test.ts index 471697f10dab6..a026c1f9833c0 100644 --- a/apps/server/src/orchestration-v2/WireProjection.test.ts +++ b/apps/server/src/orchestration-v2/WireProjection.test.ts @@ -31,6 +31,7 @@ import { projectTurnItemForDetail, projectDomainEventForWire, projectStoredEventForWire, + projectThreadProjectionForWire, } from "./WireProjection.ts"; import { threadShellFromProjection } from "./ProjectionStore.ts"; import { MAX_TOOL_OUTPUT_IMAGES, toolOutputImages } from "@t3tools/shared/toolOutput"; @@ -494,3 +495,34 @@ describe("live body deduper", () => { expect([first, message(2, "a")].map(makeLiveBodyDeduper())[0]).toBe(first); }); }); + +describe("projectThreadProjectionForWire", () => { + it("shows run-bound items stuck on ended runs as finished and keeps background tools running", () => { + const item = (id: string, type: "todo_list" | "dynamic_tool", runId: string | null) => + ({ + ...base, + id: TurnItemId.make(id), + type, + runId, + status: "running", + ...(type === "todo_list" ? { planId: "plan-1", steps: [] } : {}), + }) as unknown as OrchestrationV2TurnItem; + const todoDone = item("todo-done", "todo_list", "run-done"); + const todoLive = item("todo-live", "todo_list", "run-live"); + const toolDone = item("tool-done", "dynamic_tool", "run-done"); + const projection = { + runs: [ + { id: "run-done", status: "completed" }, + { id: "run-live", status: "running" }, + ], + checkpoints: [], + contextHandoffs: [], + turnItems: [todoDone, todoLive, toolDone], + visibleTurnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + + const projected = projectThreadProjectionForWire(projection).turnItems; + + expect(projected.map((entry) => entry.status)).toEqual(["completed", "running", "running"]); + }); +}); diff --git a/apps/server/src/orchestration-v2/WireProjection.ts b/apps/server/src/orchestration-v2/WireProjection.ts index 7d4805103c372..f96a4c2c6d798 100644 --- a/apps/server/src/orchestration-v2/WireProjection.ts +++ b/apps/server/src/orchestration-v2/WireProjection.ts @@ -21,6 +21,7 @@ import { toolOutputImages, toolOutputIndicatesFailure, } from "@t3tools/shared/toolOutput"; +import { settleItemForEndedRun } from "./RunItemSettlement.ts"; // A checkpoint over vendored or generated trees can list 15k+ files; the card needs a prefix. const MAX_WIRE_CHECKPOINT_FILES = 200; @@ -294,11 +295,16 @@ export function projectThreadProjectionForWire( projection: OrchestrationV2ThreadProjection, ): OrchestrationV2ThreadProjection { const projectedById = new Map(); + const runStatuses = new Map(projection.runs.map((run) => [run.id, run.status])); const project = (item: OrchestrationV2TurnItem) => { const key = `${item.threadId}:${item.id}`; const existing = projectedById.get(key); if (existing !== undefined) return existing; - const projected = projectTurnItemForWire(item); + const runStatus = item.runId === null ? undefined : runStatuses.get(item.runId); + // Items stored before runs settled their own items still read as running. + const projected = projectTurnItemForWire( + runStatus === undefined ? item : settleItemForEndedRun(item, runStatus), + ); projectedById.set(key, projected); return projected; }; diff --git a/apps/server/src/persistence/ForkMigrations.ts b/apps/server/src/persistence/ForkMigrations.ts index 7361ae531ac2f..835366ba9f03a 100644 --- a/apps/server/src/persistence/ForkMigrations.ts +++ b/apps/server/src/persistence/ForkMigrations.ts @@ -16,6 +16,7 @@ import ForkMigration0011 from "./ForkMigrations/011_ForkRunExclusions.ts"; import ForkMigration0012 from "./ForkMigrations/012_WorkLedgerApproval.ts"; import ForkMigration0013 from "./ForkMigrations/013_WorkLedgerHygiene.ts"; import ForkMigration0014 from "./ForkMigrations/014_McpRequestLogConnection.ts"; +import ForkMigration0015 from "./ForkMigrations/015_ScheduledRunHistoryIndex.ts"; // Fork-only migrations, recorded in their own table so upstream keeps every id in // effect_sql_migrations. They run after all upstream migrations. Add new ones here, never in // Migrations.ts, and never rename one: the repair below finds old rows by name. See FORK.md. @@ -34,6 +35,7 @@ const forkMigrationEntries = [ [12, "WorkLedgerApproval", ForkMigration0012], [13, "WorkLedgerHygiene", ForkMigration0013], [14, "McpRequestLogConnection", ForkMigration0014], + [15, "ScheduledRunHistoryIndex", ForkMigration0015], ] as const; export const FORK_MIGRATIONS_TABLE = "t3_fork_migrations"; diff --git a/apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts b/apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts new file mode 100644 index 0000000000000..f7c700e902731 --- /dev/null +++ b/apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts @@ -0,0 +1,10 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // list_scheduled_tasks and bridge_status read the last 7 days of failed and cancelled runs. + yield* sql`CREATE INDEX IF NOT EXISTS fork_runs_failed_recent_idx + ON orchestration_v2_projection_runs(requested_at) + WHERE status IN ('failed', 'cancelled')`; +}); diff --git a/apps/server/src/project/temporaryFolder.test.ts b/apps/server/src/project/temporaryFolder.test.ts new file mode 100644 index 0000000000000..6b983de659f4d --- /dev/null +++ b/apps/server/src/project/temporaryFolder.test.ts @@ -0,0 +1,72 @@ +// @effect-diagnostics nodeBuiltinImport:off - builds a symlink fixture with node:fs. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; + +import { isTemporaryFolder, isTemporaryPath, temporaryFolderWarning } from "./temporaryFolder.ts"; + +it.effect.each([ + ["/tmp/review", true], + ["/var/tmp/review", true], + ["/dev/shm/work", true], + ["/run/user/1000/work", true], + ["/private/tmp/review", true], + ["/tmp/../home/user/app", false], + ["/home/user/app", false], + ["/tmpfoo/app", false], + ["/run/media/user/disk/app", false], + ["/run/user/1000/gvfs/sftp:host=nas/app", false], + ["/run/user/1000/doc/by-app/app", false], +] as const)("%s is temporary: %s", ([path, temporary]) => + Effect.gen(function* () { + assert.equal(yield* isTemporaryFolder(path), temporary); + assert.equal((yield* temporaryFolderWarning(path)) !== undefined, temporary); + assert.equal(isTemporaryPath(path), temporary); + }), +); + +it.effect("never probes the filesystem under a network or document mount", () => + Effect.gen(function* () { + const probed: string[] = []; + const probe = { + inspect: (path: string) => { + probed.push(path); + return Promise.resolve({ resolved: path, tmpfs: true }); + }, + }; + assert.isFalse(yield* isTemporaryFolder("/run/user/1000/gvfs/smb-share/x", probe)); + assert.deepEqual(probed, []); + }), +); + +it.effect("falls back to the name when the filesystem does not answer in time", () => + Effect.gen(function* () { + const hung = { inspect: () => new Promise(() => {}) }; + const byName = (path: string) => + Effect.forkChild(isTemporaryFolder(path, hung)).pipe( + Effect.tap(() => TestClock.adjust("500 millis")), + Effect.flatMap(Fiber.join), + ); + assert.isTrue(yield* byName("/tmp/review")); + assert.isFalse(yield* byName("/home/user/app")); + }), +); + +it.effect("resolves symlinks, and ignores a persistent TMPDIR", () => + Effect.gen(function* () { + const sandbox = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "temporary-folder-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(sandbox, { recursive: true, force: true })), + ); + const link = NodePath.join(sandbox, "link"); + NodeFS.symlinkSync(process.cwd(), link); + // Under /tmp by name, but it leads to a folder that is not temporary. + assert.isFalse(yield* isTemporaryFolder(NodePath.join(link, "missing"))); + assert.isTrue(yield* isTemporaryFolder(NodePath.join(sandbox, "missing"))); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/project/temporaryFolder.ts b/apps/server/src/project/temporaryFolder.ts new file mode 100644 index 0000000000000..790f081af0464 --- /dev/null +++ b/apps/server/src/project/temporaryFolder.ts @@ -0,0 +1,106 @@ +// @effect-diagnostics nodeBuiltinImport:off - probes the filesystem with node:fs/promises. +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +const TEMPORARY_PREFIXES = [ + "/tmp", + "/var/tmp", + "/dev/shm", + // macOS resolves /tmp and /var/tmp here, and keeps per-user temp folders in /var/folders. + "/private/tmp", + "/private/var/tmp", + "/private/var/folders", +]; +// Per-user runtime directory, except its network and document mounts, which persist. +const USER_RUNTIME_DIRECTORY = /^\/run\/user\/\d+(?:\/|$)/; +const PERSISTENT_MOUNTS = /^\/run\/user\/\d+\/(?:gvfs|doc)(?:\/|$)|^\/run\/media\//; +const LINUX_TMPFS_MAGIC = 0x01021994; +/** A dead network mount can hang a filesystem call, so the answer falls back after this. */ +const PROBE_TIMEOUT = Duration.millis(500); + +/** `~` and relative paths expanded and `..` removed, without touching the filesystem. */ +function lexicalPath(path: string): string { + const expanded = + path === "~" || path.startsWith("~/") ? NodePath.join(NodeOS.homedir(), path.slice(1)) : path; + return NodePath.resolve(expanded); +} + +function underTemporaryName(path: string): boolean { + return ( + !PERSISTENT_MOUNTS.test(path) && + (TEMPORARY_PREFIXES.some((root) => path === root || path.startsWith(`${root}/`)) || + USER_RUNTIME_DIRECTORY.test(path)) + ); +} + +/** By name only, for callers that cannot wait on the filesystem. */ +export function isTemporaryPath(path: string): boolean { + return underTemporaryName(lexicalPath(path)); +} + +export interface FolderProbe { + /** The path with symlinks resolved, and whether its filesystem is a tmpfs. */ + readonly inspect: ( + path: string, + ) => Promise<{ readonly resolved: string; readonly tmpfs: boolean }>; +} + +const exists = (path: string) => + NodeFSP.access(path).then( + () => true, + () => false, + ); + +const nodeProbe: FolderProbe = { + inspect: async (path) => { + // A missing folder is the usual case here, so inspect its nearest existing ancestor. + let existing = path; + while (!(await exists(existing)) && NodePath.dirname(existing) !== existing) { + existing = NodePath.dirname(existing); + } + try { + const real = await NodeFSP.realpath(existing); + return { + resolved: NodePath.join(real, NodePath.relative(existing, path)), + tmpfs: (await NodeFSP.statfs(real)).type === LINUX_TMPFS_MAGIC, + }; + } catch { + return { resolved: path, tmpfs: false }; + } + }, +}; + +/** + * A folder the system clears at reboot: a tmpfs, or under a well-known temporary directory. + * `TMPDIR` is not trusted by itself. A path under a network or document mount is never + * temporary, and is not probed. A probe that outlasts the timeout leaves the name check. + */ +export const isTemporaryFolder = Effect.fn("isTemporaryFolder")(function* ( + path: string, + probe: FolderProbe = nodeProbe, +) { + const lexical = lexicalPath(path); + if (PERSISTENT_MOUNTS.test(lexical)) return false; + const byName = underTemporaryName(lexical); + // A probe that fails answers like one that times out: by name. + const inspected = yield* Effect.promise(() => probe.inspect(lexical).catch(() => undefined)).pipe( + Effect.timeoutOption(PROBE_TIMEOUT), + Effect.map(Option.flatMapNullishOr((value) => value)), + ); + if (Option.isNone(inspected)) return byName; + const { resolved, tmpfs } = inspected.value; + return !PERSISTENT_MOUNTS.test(resolved) && (underTemporaryName(resolved) || tmpfs); +}); + +export const temporaryFolderWarning = Effect.fn("temporaryFolderWarning")(function* ( + path: string, + probe?: FolderProbe, +) { + return (yield* isTemporaryFolder(path, probe)) + ? `${path} is under a temporary folder, which the system clears at reboot; the project's threads then fail to start. Use a folder that persists.` + : undefined; +}); diff --git a/apps/server/src/provider/Errors.ts b/apps/server/src/provider/Errors.ts index b1ef6192faf3b..b2f544767790b 100644 --- a/apps/server/src/provider/Errors.ts +++ b/apps/server/src/provider/Errors.ts @@ -1,5 +1,7 @@ import * as Schema from "effect/Schema"; +import { isTemporaryPath } from "../project/temporaryFolder.ts"; + export class ProviderInstanceNotFoundError extends Schema.TaggedError()( "ProviderInstanceNotFoundError", { @@ -22,6 +24,9 @@ export class ProviderWorkspaceMissingError extends Schema.TaggedError { it("skips to the next matching fixed-time weekday", () => { @@ -83,3 +83,75 @@ describe("scheduled task schedule calculation", () => { ).toBe(false); }); }); + +describe("missed scheduled runs", () => { + const at = (iso: string) => DateTime.makeUnsafe(iso); + const interval = { type: "interval", everyMs: 3_600_000 } as const; + const task = (nextRunAt: string | null, lastRunStatus = "succeeded") => ({ + enabled: true, + schedule: interval, + nextRunAt, + lastRunStatus, + }); + + it("flags a slot overdue by more than one interval and separates a stuck running task", () => { + const now = at("2026-11-13T05:00:00.000Z"); + expect(missedScheduleRun(task("2026-11-13T03:30:00.000Z"), now)).toEqual({ + dueAt: "2026-11-13T03:30:00.000Z", + overdueMs: 90 * 60_000, + reason: "not_dispatched", + }); + expect(missedScheduleRun(task("2026-11-13T03:30:00.000Z", "running"), now)?.reason).toBe( + "run_in_progress", + ); + }); + + it.each([ + ["within one interval", task("2026-11-13T04:30:00.000Z")], + ["disabled", { ...task("2026-11-13T00:00:00.000Z"), enabled: false }], + ["no next run", task(null)], + [ + "webhook", + { + ...task("2026-11-13T00:00:00.000Z"), + schedule: { type: "webhook", signature: null } as const, + }, + ], + ])("does not flag a task that is %s", (_name, candidate) => { + expect(missedScheduleRun(candidate, at("2026-11-13T05:00:00.000Z"))).toBeNull(); + }); + + it("measures the gap between fixed-time slots in the scheduler's zone", () => { + // Monday 22:00 in Los Angeles is Tuesday 06:00 UTC, so a UTC reading would see 6d16h, not 7d. + const weekly = { + enabled: true, + schedule: { type: "fixed_time", timeOfDay: "22:00", weekdays: [1] } as const, + nextRunAt: "2026-11-10T06:00:00.000Z", + lastRunStatus: "succeeded", + }; + const inLosAngeles = (iso: string) => + DateTime.setZone(at(iso), DateTime.zoneMakeNamedUnsafe("America/Los_Angeles")); + expect(missedScheduleRun(weekly, inLosAngeles("2026-11-17T02:00:00.000Z"))).toBeNull(); + expect(missedScheduleRun(weekly, inLosAngeles("2026-11-17T08:00:00.000Z"))).not.toBeNull(); + }); + + it("uses the gap to the following slot for fixed-time tasks and 10 minutes for one-time tasks", () => { + const daily = { + enabled: true, + schedule: { type: "fixed_time", timeOfDay: "09:00" } as const, + nextRunAt: "2026-11-12T09:00:00.000Z", + lastRunStatus: "succeeded", + }; + const utc = (iso: string) => DateTime.setZone(at(iso), DateTime.zoneMakeNamedUnsafe("UTC")); + expect(missedScheduleRun(daily, utc("2026-11-13T08:00:00.000Z"))).toBeNull(); + expect(missedScheduleRun(daily, utc("2026-11-13T10:00:00.000Z"))).not.toBeNull(); + const once = { + enabled: true, + schedule: { type: "once", runAt: "2026-11-13T04:00:00.000Z" } as const, + nextRunAt: "2026-11-13T04:00:00.000Z", + lastRunStatus: "never", + }; + expect(missedScheduleRun(once, at("2026-11-13T04:09:00.000Z"))).toBeNull(); + expect(missedScheduleRun(once, at("2026-11-13T04:11:00.000Z"))).not.toBeNull(); + }); +}); diff --git a/apps/server/src/scheduledTasks/Schedule.ts b/apps/server/src/scheduledTasks/Schedule.ts index f8fb648aaaaa9..9578b39f8c7cc 100644 --- a/apps/server/src/scheduledTasks/Schedule.ts +++ b/apps/server/src/scheduledTasks/Schedule.ts @@ -80,3 +80,51 @@ export function isMissedFixedTimeRun( if (schedule.type !== "fixed_time") return false; return DateTime.toEpochMillis(now) - DateTime.toEpochMillis(dueAt) > MISSED_FIXED_TIME_GRACE_MS; } + +export interface MissedScheduleRun { + readonly dueAt: string; + readonly overdueMs: number; + /** The run that should have fired later is still going, or nothing started it. */ + readonly reason: "run_in_progress" | "not_dispatched"; +} + +const DAY_MS = 24 * 60 * MINUTE_MS; + +/** + * An enabled task whose `nextRunAt` passed by more than one interval without a new run. + * A task stuck in `running` counts: its later slots are skipped while the run lasts. + */ +export function missedScheduleRun( + task: { + readonly enabled: boolean; + readonly schedule: ScheduledTaskSchedule; + readonly nextRunAt: string | null; + readonly lastRunStatus: string; + }, + now: DateTime.DateTime, +): MissedScheduleRun | null { + if (!task.enabled || task.nextRunAt === null || task.schedule.type === "webhook") return null; + const dueAt = DateTime.make(task.nextRunAt); + if (Option.isNone(dueAt)) return null; + const dueMs = DateTime.toEpochMillis(dueAt.value); + const overdueMs = DateTime.toEpochMillis(now) - dueMs; + let intervalMs: number; + if (task.schedule.type === "interval") { + intervalMs = Math.max(task.schedule.everyMs, MIN_SCHEDULED_TASK_INTERVAL_MS); + } else if (task.schedule.type === "once") { + intervalMs = MISSED_FIXED_TIME_GRACE_MS; + } else { + // Slots are wall-clock times in the scheduler's zone, so the gap to the next one is too. + const zone = DateTime.isZoned(now) ? now.zone : DateTime.zoneMakeLocal(); + const following = nextScheduledRunAt(task.schedule, DateTime.setZone(dueAt.value, zone)); + intervalMs = + following === null ? DAY_MS : Math.max(DateTime.toEpochMillis(following) - dueMs, MINUTE_MS); + } + return overdueMs > intervalMs + ? { + dueAt: DateTime.formatIso(dueAt.value), + overdueMs, + reason: task.lastRunStatus === "running" ? "run_in_progress" : "not_dispatched", + } + : null; +} diff --git a/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts b/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts new file mode 100644 index 0000000000000..7adff8d9068a7 --- /dev/null +++ b/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts @@ -0,0 +1,148 @@ +import type { ScheduledTask } from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { missedScheduleRun, type MissedScheduleRun } from "./Schedule.ts"; + +const RECENT_RUN_WINDOW_DAYS = 7; +const RECENT_RUN_WINDOW_MS = RECENT_RUN_WINDOW_DAYS * 24 * 60 * 60 * 1000; +const MESSAGE_PREFIX = "scheduled-task-message:"; +const ERROR_TEXT_LIMIT = 500; + +export interface RecentRunFailures { + readonly failed: number; + readonly cancelled: number; + readonly lastAt: string; + readonly lastStatus: "failed" | "cancelled"; + /** The error of the latest failed run, which may be older than `lastAt`. */ + readonly lastError: string | null; +} + +export interface ScheduledTaskReliability { + readonly taskId: string; + readonly recentFailures?: RecentRunFailures; + readonly missedRun?: MissedScheduleRun; +} + +/** A scheduled run's message id is `scheduled-task-message:::`. */ +function parseScheduledRunMessageId( + messageId: string, +): { readonly taskId: string; readonly startedAtMs: number | null } | null { + if (!messageId.startsWith(MESSAGE_PREFIX)) return null; + const key = messageId.slice(MESSAGE_PREFIX.length); + const timed = /^(.+):(\d+):(?:scheduled|manual)$/.exec(key); + if (timed !== null) return { taskId: timed[1]!, startedAtMs: Number(timed[2]) }; + // Delivery ids are `delivery:`, or `delivery:relay:` for relayed requests. + const webhook = /^(.+?):webhook:delivery:.+$/.exec(key); + return webhook === null ? null : { taskId: webhook[1]!, startedAtMs: null }; +} + +interface FailedRun { + readonly status: "failed" | "cancelled"; + readonly at: string; + readonly threadId: string; + readonly runId: string; + readonly startedAtMs: number | null; +} + +/** + * Failures and cancels of the last 7 days per task, derived from the runs themselves, so no + * second record exists to drift. A failure that never created a run comes from the task row. + */ +export const readTaskReliability = Effect.fn("ScheduledTaskReliability.read")(function* ( + tasks: ReadonlyArray, + now: DateTime.DateTime, +) { + const sql = yield* SqlClient.SqlClient; + const sinceMs = DateTime.toEpochMillis(now) - RECENT_RUN_WINDOW_MS; + const sinceIso = DateTime.formatIso(DateTime.makeUnsafe(sinceMs)); + const rows = yield* sql<{ + readonly status: "failed" | "cancelled"; + readonly requested_at: string; + readonly thread_id: string; + readonly run_id: string; + readonly message_id: string; + }>` + SELECT status, requested_at, thread_id, run_id, + json_extract(payload_json, '$.userMessageId') AS message_id + FROM orchestration_v2_projection_runs + WHERE status IN ('failed', 'cancelled') AND requested_at >= ${sinceIso} + AND substr(json_extract(payload_json, '$.userMessageId'), 1, ${MESSAGE_PREFIX.length}) = ${MESSAGE_PREFIX} + `.pipe(NodeSqliteClient.readOnly); + + const runsByTask = new Map(); + for (const row of rows) { + const parsed = parseScheduledRunMessageId(row.message_id); + if (parsed === null) continue; + runsByTask.set(parsed.taskId, [ + ...(runsByTask.get(parsed.taskId) ?? []), + { + status: row.status, + at: row.requested_at, + threadId: row.thread_id, + runId: row.run_id, + startedAtMs: parsed.startedAtMs, + }, + ]); + } + + const reliability: ScheduledTaskReliability[] = []; + for (const task of tasks) { + const runs = [...(runsByTask.get(task.id) ?? [])]; + let failed = runs.filter((run) => run.status === "failed").length; + const cancelled = runs.filter((run) => run.status === "cancelled").length; + const byLatest = runs.toSorted((x, y) => y.at.localeCompare(x.at)); + let latest: { at: string; status: "failed" | "cancelled" } | null = + byLatest[0] === undefined ? null : { at: byLatest[0].at, status: byLatest[0].status }; + const latestFailedRun = byLatest.find((run) => run.status === "failed"); + let latestFailed: { at: string; error: string | null } | null = null; + if (latestFailedRun !== undefined) { + const [errorRow] = yield* sql<{ readonly message: string | null }>` + SELECT json_extract(payload_json, '$.failure.message') AS message + FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${latestFailedRun.threadId} AND run_id = ${latestFailedRun.runId} + AND type = 'error' AND status = 'failed' + ORDER BY updated_at DESC, ordinal DESC + LIMIT 1 + `.pipe(NodeSqliteClient.readOnly); + latestFailed = { at: latestFailedRun.at, error: errorRow?.message ?? null }; + } + // A dispatch that failed before a run existed leaves only the task row. + const lastStartedMs = task.lastRunAt === null ? Number.NaN : Date.parse(task.lastRunAt); + if ( + task.lastRunStatus === "failed" && + task.schedule.type !== "webhook" && + lastStartedMs >= sinceMs && + !runs.some((run) => run.startedAtMs === lastStartedMs) + ) { + failed += 1; + if (latest === null || task.lastRunAt! > latest.at) { + latest = { at: task.lastRunAt!, status: "failed" }; + } + if (latestFailed === null || task.lastRunAt! > latestFailed.at) { + latestFailed = { at: task.lastRunAt!, error: task.lastRunError }; + } + } + + const missedRun = missedScheduleRun(task, now); + if (latest === null && missedRun === null) continue; + reliability.push({ + taskId: task.id, + ...(latest === null + ? {} + : { + recentFailures: { + failed, + cancelled, + lastAt: latest.at, + lastStatus: latest.status, + lastError: latestFailed?.error?.slice(0, ERROR_TEXT_LIMIT) ?? null, + }, + }), + ...(missedRun === null ? {} : { missedRun }), + }); + } + return reliability; +}); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 342688b025959..59285d84031b3 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -1530,3 +1530,123 @@ it.effect("a run interrupted right after it reserves the task does not block the }).pipe(Effect.provide(layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect( + "reliability counts the last 7 days of failed and cancelled runs and the latest error", + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-11-13T02:00:00.000Z"; + yield* TestClock.setTime(Date.parse(now)); + const threadId = "thread:history"; + const run = ( + task: string, + at: string, + status: string, + ordinal: number, + trigger = "scheduled", + ) => + sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES (${`run:${task}:${at}`}, ${threadId}, ${ordinal}, 'codex', ${status}, ${at}, + ${JSON.stringify({ + userMessageId: `scheduled-task-message:${task}:${Date.parse(at)}:${trigger}`, + })}) + `; + const errorItem = (task: string, at: string, message: string) => + sql` + INSERT INTO orchestration_v2_projection_turn_items + (turn_item_id, thread_id, run_id, ordinal, type, status, updated_at, payload_json) + VALUES (${`item:${task}:${at}`}, ${threadId}, ${`run:${task}:${at}`}, 1, 'error', 'failed', + ${at}, ${JSON.stringify({ failure: { message } })}) + `; + for (const id of [ + "task:a", + "task:folded", + "task:matched", + "task:clean", + "task:hook", + "task:relay", + ]) { + yield* insertRow(sql, { id, next: null, enabled: 1, status: "succeeded" }, now); + } + // 10 days old, outside the window. + yield* run("task:a", "2026-11-03T02:00:00.000Z", "failed", 1); + yield* run("task:a", "2026-11-11T02:00:00.000Z", "failed", 2); + yield* errorItem("task:a", "2026-11-11T02:00:00.000Z", "Earlier error"); + yield* run("task:a", "2026-11-12T05:00:00.000Z", "failed", 3); + yield* errorItem("task:a", "2026-11-12T05:00:00.000Z", "Workspace preparation failed"); + yield* run("task:a", "2026-11-12T06:00:00.000Z", "cancelled", 4, "manual"); + yield* run("task:a", "2026-11-12T07:00:00.000Z", "completed", 5); + // A webhook run's message id ends `:webhook:delivery:`. + yield* sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES ('run:hook', ${threadId}, 7, 'codex', 'failed', '2026-11-12T22:00:00.000Z', + ${JSON.stringify({ userMessageId: "scheduled-task-message:task:hook:webhook:delivery:3f2a-91" })}) + `; + // A relayed webhook's delivery id is `delivery:relay:`. + yield* sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES ('run:relay', ${threadId}, 8, 'codex', 'cancelled', '2026-11-12T23:00:00.000Z', + ${JSON.stringify({ userMessageId: "scheduled-task-message:task:relay:webhook:delivery:relay:7c1d" })}) + `; + // A failed dispatch that never created a run leaves only the task row. + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed', last_run_error = 'No such folder', + last_run_at = '2026-11-12T20:00:00.000Z' WHERE task_id = 'task:folded'`; + // The same failure is already counted from its run, so the row adds nothing. + yield* run("task:matched", "2026-11-12T21:00:00.000Z", "failed", 6); + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed', last_run_error = 'Boom', + last_run_at = '2026-11-12T21:00:00.000Z' WHERE task_id = 'task:matched'`; + + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const history = Object.fromEntries( + (yield* service.reliability()).map((entry) => [entry.taskId, entry.recentFailures]), + ); + assert.deepEqual(history, { + "task:a": { + failed: 2, + cancelled: 1, + lastAt: "2026-11-12T06:00:00.000Z", + lastStatus: "cancelled", + lastError: "Workspace preparation failed", + }, + "task:folded": { + failed: 1, + cancelled: 0, + lastAt: "2026-11-12T20:00:00.000Z", + lastStatus: "failed", + lastError: "No such folder", + }, + "task:hook": { + failed: 1, + cancelled: 0, + lastAt: "2026-11-12T22:00:00.000Z", + lastStatus: "failed", + lastError: null, + }, + "task:relay": { + failed: 0, + cancelled: 1, + lastAt: "2026-11-12T23:00:00.000Z", + lastStatus: "cancelled", + lastError: null, + }, + "task:matched": { + failed: 1, + cancelled: 0, + lastAt: "2026-11-12T21:00:00.000Z", + lastStatus: "failed", + lastError: null, + }, + }); + }).pipe( + Effect.provide( + followerLayer(Layer.mock(ThreadManagementService.ThreadManagementService)({})), + ), + ); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 206cf4c3ce113..56abe37014227 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -64,6 +64,7 @@ import * as Scheduler from "../scheduling/Scheduler.ts"; import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError"; import { findGitEntry } from "../mcp/gitEntryProbe.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; +import { readTaskReliability, type ScheduledTaskReliability } from "./ScheduledTaskReliability.ts"; import { redactHeaders, redactQuery, @@ -269,6 +270,11 @@ export class ScheduledTaskService extends Context.Service< { readonly list: () => Effect.Effect; readonly subscribeList: () => Stream.Stream; + /** Recent failures and missed runs, for tasks that have any. Read on demand, never streamed. */ + readonly reliability: () => Effect.Effect< + ReadonlyArray, + ScheduledTaskError + >; readonly upsert: ( input: ScheduledTaskUpsertInput, ) => Effect.Effect; @@ -1578,6 +1584,21 @@ export const layer = Layer.effect( Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), ); + const reliability: ScheduledTaskService["Service"]["reliability"] = Effect.fn( + "ScheduledTaskService.reliability", + )( + function* () { + const tasks = yield* listRows(); + return yield* readTaskReliability(tasks, yield* localNow); + }, + Effect.provideService(SqlClient.SqlClient, sql), + Effect.mapError((cause) => + isScheduledTaskError(cause) + ? cause + : taskError("Could not read schedule task history.", { cause }), + ), + ); + const subscribeList: ScheduledTaskService["Service"]["subscribeList"] = () => Stream.unwrap( Effect.gen(function* () { @@ -2859,6 +2880,7 @@ export const layer = Layer.effect( return ScheduledTaskService.of({ list, subscribeList, + reliability, upsert, setEnabled, delete: deleteTask, diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 1a906ea560381..1c7bcdbc09328 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -820,6 +820,22 @@ export const OrchestratorMcpScheduledTask = Schema.Struct({ blockedByPolicy: Schema.optional(Schema.String).annotate({ description: "Present when an administrator's policy stops this task from running.", }), + recentFailures: Schema.optional( + Schema.Struct({ + failed: NonNegativeInt, + cancelled: NonNegativeInt, + lastAt: IsoDateTime, + lastStatus: Schema.Literals(["failed", "cancelled"]), + lastError: Schema.NullOr(Schema.String), + }), + ), + missedRun: Schema.optional( + Schema.Struct({ + dueAt: IsoDateTime, + overdueMs: NonNegativeInt, + reason: Schema.Literals(["not_dispatched", "run_in_progress"]), + }), + ), }); export type OrchestratorMcpScheduledTask = typeof OrchestratorMcpScheduledTask.Type;