From 5b26670da9381399c9d62c63869ff2de4fec95b1 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 19:22:15 -0700 Subject: [PATCH 1/9] fix(server): a run ending closes its todo list, compaction and reasoning items Todo lists, compactions and reasoning items are run-bound but providers rarely close them, so 940 items stayed running on finished runs. Close them in the transaction that ends the run, and show already stuck ones as finished on read. Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/orchestration-v2/EventSink.ts | 55 ++++++++- .../FoundationPersistence.test.ts | 113 ++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 2 +- .../src/orchestration-v2/RunItemSettlement.ts | 42 +++++++ .../orchestration-v2/WireProjection.test.ts | 32 +++++ .../src/orchestration-v2/WireProjection.ts | 8 +- 6 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 apps/server/src/orchestration-v2/RunItemSettlement.ts diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 379339e0e2aae..e614cd7852955 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -1,5 +1,6 @@ import { CommandId, + EventId, type OrchestrationV2Run, OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, @@ -31,6 +32,12 @@ import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventS import { projectStoredEventForWire } from "./WireProjection.ts"; import { makeRunOutcomeRecorder, type RunOutcome } from "./RunMetrics.ts"; import { makeRunStopStamper, type RunStopContext } from "./RunStop.ts"; +import { + RUN_BOUND_ITEM_TYPES, + isActiveRunBoundItem, + itemStatusForEndedRun, + settleItemForEndedRun, +} from "./RunItemSettlement.ts"; import * as ScheduledTaskChanges from "../scheduledTasks/ScheduledTaskChanges.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -378,7 +385,11 @@ const layerBase: Layer.Layer< unstamped: ReadonlyArray, context: RunStopContext = {}, ) => - Effect.flatMap(stampRunStops(unstamped, context), (events) => normalizeStampedEvents(events)); + stampRunStops(unstamped, context).pipe( + Effect.flatMap(normalizeStampedEvents), + // After normalization, so the closing events keep the positions their items already have. + Effect.flatMap(closeEndedRunItems), + ); const normalizeStampedEvents = (events: ReadonlyArray) => { const runOrdinals = new Map( @@ -422,6 +433,48 @@ const layerBase: Layer.Layer< ); }; + // A todo list, compaction or reasoning item never outlives its run, but providers rarely close + // them. Close what is still active here, in the transaction that ends the run. + // The enclosing write span already traces this per-batch step. + const closeEndedRunItems = Effect.fnUntraced(function* ( + events: ReadonlyArray, + ) { + const closing: Array = []; + for (const event of events) { + if (event.type !== "run.updated" || itemStatusForEndedRun(event.payload.status) === null) + continue; + const run = event.payload; + const latestInBatch = new Map(); + for (const candidate of events) { + if (candidate.type === "turn-item.updated" && candidate.payload.runId === run.id) { + latestInBatch.set(candidate.payload.id, candidate.payload); + } + } + const stored = yield* projectionStore.getThreadRecords(run.threadId, ["turnItems"], { + turnItemRunId: run.id, + turnItemTypes: [...RUN_BOUND_ITEM_TYPES], + turnItemStatuses: ["pending", "running", "waiting"], + }); + for (const item of stored.turnItems) { + if (!latestInBatch.has(item.id)) latestInBatch.set(item.id, item); + } + for (const item of latestInBatch.values()) { + if (!isActiveRunBoundItem(item)) continue; + closing.push({ + id: EventId.make(`${event.id}:close:${item.id}`), + type: "turn-item.updated", + threadId: run.threadId, + runId: run.id, + ...(item.nodeId === null ? {} : { nodeId: item.nodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: event.occurredAt, + payload: settleItemForEndedRun(item, run.status), + }); + } + } + return closing.length === 0 ? events : [...events, ...closing]; + }); + const applyStoredEvents = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (storedEvents: ReadonlyArray) { diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 36d26c6e60952..b0210aa915fb3 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -2085,6 +2085,119 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(layerTest))), ); + it.effect.each(["completed", "failed", "interrupted"] as const)( + "closes run-bound items still active when a run ends %s, and leaves background tools", + (endStatus) => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:run-end-items:${endStatus}`); + const runId = RunId.make(`run:run-end-items:${endStatus}`); + const run: OrchestrationV2Run = { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`message:run-end-items:${endStatus}`), + rootNodeId: null, + activeAttemptId: null, + status: "running", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + const itemBase = (id: string, ordinal: number) => ({ + id: TurnItemId.make(`turn-item:run-end-items:${endStatus}:${id}`), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal, + status: "running" as const, + title: id, + startedAt: now, + completedAt: null, + updatedAt: now, + }); + const compaction: OrchestrationV2TurnItem = { + ...itemBase("compaction", 1), + type: "compaction", + driver: null, + }; + const reasoning: OrchestrationV2TurnItem = { + ...itemBase("reasoning", 2), + type: "reasoning", + text: "thinking", + streaming: true, + }; + const background: OrchestrationV2TurnItem = { + ...itemBase("background", 3), + type: "dynamic_tool", + toolName: "mcp__background", + input: {}, + }; + const itemEvent = (payload: OrchestrationV2TurnItem): OrchestrationV2DomainEvent => ({ + id: EventId.make(`event:${payload.id}`), + type: "turn-item.updated", + threadId, + runId, + occurredAt: now, + payload, + }); + yield* eventSink.write({ + events: [ + threadCreatedEvent({ + id: `event:run-end-items:${endStatus}:thread`, + thread: makeThread(threadId, now), + now, + }), + { + id: EventId.make(`event:run-end-items:${endStatus}:run`), + type: "run.created", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: run, + }, + itemEvent(compaction), + itemEvent(background), + ], + }); + // The reasoning item arrives in the same batch that ends the run. + yield* eventSink.write({ + events: [ + itemEvent(reasoning), + { + id: EventId.make(`event:run-end-items:${endStatus}:ended`), + type: "run.updated", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: { ...run, status: endStatus, completedAt: now }, + }, + ], + }); + const { turnItems } = yield* projections.getThreadRecords(threadId, ["turnItems"]); + const statusOf = (item: OrchestrationV2TurnItem) => + turnItems.find((candidate) => candidate.id === item.id)?.status; + const expected = endStatus === "failed" ? "cancelled" : endStatus; + assert.equal(statusOf(compaction), expected); + assert.equal(statusOf(reasoning), expected); + assert.equal(statusOf(background), "running"); + }).pipe(Effect.provide(Layer.fresh(layerTest))), + ); + it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5762fd2fdc97f..632f12c584ac6 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -2171,7 +2171,7 @@ export const layer: Layer.Layer = AND NOT ${settledSubagent(sql`candidate`)} `; - // A todo list is never closed when its run ends, so a long thread holds an active one per + // Todo lists stored before runs closed them are still active, so a long thread holds one per // old run. Counting those as live work drags each run's nodes and subagents into every window. const todoListOfSettledRun = (row: Statement.Fragment) => sql`( ${row}.kind = 'todo_list' AND ${row}.run_id IS NOT NULL AND NOT EXISTS ( diff --git a/apps/server/src/orchestration-v2/RunItemSettlement.ts b/apps/server/src/orchestration-v2/RunItemSettlement.ts new file mode 100644 index 0000000000000..1d33815aea65a --- /dev/null +++ b/apps/server/src/orchestration-v2/RunItemSettlement.ts @@ -0,0 +1,42 @@ +import type { OrchestrationV2Run, OrchestrationV2TurnItem } from "@t3tools/contracts"; + +/** + * Items that cannot outlive their run. Shell, tool and subagent items can be background work, + * so a run ending does not close them. + */ +export const RUN_BOUND_ITEM_TYPES = ["todo_list", "compaction", "reasoning"] as const; + +const ACTIVE_ITEM_STATUSES: ReadonlySet = new Set(["pending", "running", "waiting"]); + +export function isActiveRunBoundItem(item: OrchestrationV2TurnItem): boolean { + return ( + ACTIVE_ITEM_STATUSES.has(item.status) && RUN_BOUND_ITEM_TYPES.some((type) => type === item.type) + ); +} + +/** The status a run-bound item takes when its run ends; null while the run can still change it. */ +export function itemStatusForEndedRun( + runStatus: OrchestrationV2Run["status"], +): "completed" | "interrupted" | "cancelled" | null { + switch (runStatus) { + case "completed": + return "completed"; + case "interrupted": + return "interrupted"; + case "failed": + case "cancelled": + return "cancelled"; + default: + return null; + } +} + +export function settleItemForEndedRun( + item: OrchestrationV2TurnItem, + runStatus: OrchestrationV2Run["status"], +): OrchestrationV2TurnItem { + const status = itemStatusForEndedRun(runStatus); + if (status === null || !isActiveRunBoundItem(item)) return item; + const settled = { ...item, status, completedAt: item.completedAt ?? item.updatedAt }; + return settled.type === "reasoning" ? { ...settled, streaming: false } : settled; +} diff --git a/apps/server/src/orchestration-v2/WireProjection.test.ts b/apps/server/src/orchestration-v2/WireProjection.test.ts index 471697f10dab6..a026c1f9833c0 100644 --- a/apps/server/src/orchestration-v2/WireProjection.test.ts +++ b/apps/server/src/orchestration-v2/WireProjection.test.ts @@ -31,6 +31,7 @@ import { projectTurnItemForDetail, projectDomainEventForWire, projectStoredEventForWire, + projectThreadProjectionForWire, } from "./WireProjection.ts"; import { threadShellFromProjection } from "./ProjectionStore.ts"; import { MAX_TOOL_OUTPUT_IMAGES, toolOutputImages } from "@t3tools/shared/toolOutput"; @@ -494,3 +495,34 @@ describe("live body deduper", () => { expect([first, message(2, "a")].map(makeLiveBodyDeduper())[0]).toBe(first); }); }); + +describe("projectThreadProjectionForWire", () => { + it("shows run-bound items stuck on ended runs as finished and keeps background tools running", () => { + const item = (id: string, type: "todo_list" | "dynamic_tool", runId: string | null) => + ({ + ...base, + id: TurnItemId.make(id), + type, + runId, + status: "running", + ...(type === "todo_list" ? { planId: "plan-1", steps: [] } : {}), + }) as unknown as OrchestrationV2TurnItem; + const todoDone = item("todo-done", "todo_list", "run-done"); + const todoLive = item("todo-live", "todo_list", "run-live"); + const toolDone = item("tool-done", "dynamic_tool", "run-done"); + const projection = { + runs: [ + { id: "run-done", status: "completed" }, + { id: "run-live", status: "running" }, + ], + checkpoints: [], + contextHandoffs: [], + turnItems: [todoDone, todoLive, toolDone], + visibleTurnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + + const projected = projectThreadProjectionForWire(projection).turnItems; + + expect(projected.map((entry) => entry.status)).toEqual(["completed", "running", "running"]); + }); +}); diff --git a/apps/server/src/orchestration-v2/WireProjection.ts b/apps/server/src/orchestration-v2/WireProjection.ts index 7d4805103c372..f96a4c2c6d798 100644 --- a/apps/server/src/orchestration-v2/WireProjection.ts +++ b/apps/server/src/orchestration-v2/WireProjection.ts @@ -21,6 +21,7 @@ import { toolOutputImages, toolOutputIndicatesFailure, } from "@t3tools/shared/toolOutput"; +import { settleItemForEndedRun } from "./RunItemSettlement.ts"; // A checkpoint over vendored or generated trees can list 15k+ files; the card needs a prefix. const MAX_WIRE_CHECKPOINT_FILES = 200; @@ -294,11 +295,16 @@ export function projectThreadProjectionForWire( projection: OrchestrationV2ThreadProjection, ): OrchestrationV2ThreadProjection { const projectedById = new Map(); + const runStatuses = new Map(projection.runs.map((run) => [run.id, run.status])); const project = (item: OrchestrationV2TurnItem) => { const key = `${item.threadId}:${item.id}`; const existing = projectedById.get(key); if (existing !== undefined) return existing; - const projected = projectTurnItemForWire(item); + const runStatus = item.runId === null ? undefined : runStatuses.get(item.runId); + // Items stored before runs settled their own items still read as running. + const projected = projectTurnItemForWire( + runStatus === undefined ? item : settleItemForEndedRun(item, runStatus), + ); projectedById.set(key, projected); return projected; }; From 8dbf7b1dae0acbae98a8214bf734815db72af086 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 19:28:24 -0700 Subject: [PATCH 2/9] feat(mcp): list_scheduled_tasks shows recent failures; list and bridge_status report missed runs Co-Authored-By: Claude Sonnet 5.5 --- .../src/mcp/BridgeStatusService.test.ts | 23 +++ apps/server/src/mcp/BridgeStatusService.ts | 24 +++ .../src/mcp/OrchestratorMcpService.test.ts | 39 +++++ apps/server/src/mcp/OrchestratorMcpService.ts | 18 ++- ...OrchestratorMcpToolkit.integration.test.ts | 1 + apps/server/src/mcp/toolkits/bridge/tools.ts | 2 +- .../src/scheduledTasks/Schedule.test.ts | 59 ++++++- apps/server/src/scheduledTasks/Schedule.ts | 46 ++++++ .../ScheduledTaskReliability.ts | 147 ++++++++++++++++++ .../ScheduledTaskService.test.ts | 85 ++++++++++ .../scheduledTasks/ScheduledTaskService.ts | 22 +++ packages/contracts/src/orchestratorMcp.ts | 22 +++ 12 files changed, 485 insertions(+), 3 deletions(-) create mode 100644 apps/server/src/scheduledTasks/ScheduledTaskReliability.ts diff --git a/apps/server/src/mcp/BridgeStatusService.test.ts b/apps/server/src/mcp/BridgeStatusService.test.ts index 46e96a7229b7a..7c21be29f5ebe 100644 --- a/apps/server/src/mcp/BridgeStatusService.test.ts +++ b/apps/server/src/mcp/BridgeStatusService.test.ts @@ -8,6 +8,7 @@ import * as Bridge from "./BridgeStatusService.ts"; import * as Invocation from "./McpInvocationContext.ts"; import * as Events from "./McpEvents.ts"; import * as Metrics from "./McpBridgeMetrics.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; it.effect( "reports fresh peer probe results and preserves bridge status when links are unavailable", @@ -40,6 +41,20 @@ it.effect( deliveriesExhaustedRetries: 0, }), }), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({ + reliability: () => + Effect.succeed([ + { taskId: "scheduled-task:quiet" }, + { + taskId: "scheduled-task:late", + missedRun: { + dueAt: "1969-12-31T00:00:00.000Z", + overdueMs: 86_400_000, + reason: "not_dispatched" as const, + }, + }, + ]), + }), Layer.mock(PeerForwarding.PeerForwarding)({ links: (caller) => Effect.gen(function* () { @@ -67,6 +82,14 @@ it.effect( yield* Effect.gen(function* () { const service = yield* Bridge.BridgeStatusService; const status = yield* service.status(); + expect(status.missedScheduledRuns).toEqual([ + { + scheduledTaskId: "scheduled-task:late", + dueAt: "1969-12-31T00:00:00.000Z", + overdueMs: 86_400_000, + reason: "not_dispatched", + }, + ]); expect(status.linkedEnvironments).toMatchObject({ checkedAt: "1970-01-01T00:00:00.000Z", links: [ diff --git a/apps/server/src/mcp/BridgeStatusService.ts b/apps/server/src/mcp/BridgeStatusService.ts index 8a2498f9fe152..27d3d71c71cf0 100644 --- a/apps/server/src/mcp/BridgeStatusService.ts +++ b/apps/server/src/mcp/BridgeStatusService.ts @@ -2,8 +2,11 @@ import { EnvironmentId, EnvironmentConnectReachability, McpEventSubscriptionsResult, + IsoDateTime, McpEventWatch, + NonNegativeInt, OrchestratorMcpFailure, + ScheduledTaskId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Context from "effect/Context"; @@ -14,6 +17,7 @@ import * as PeerForwarding from "../peer/PeerForwarding.ts"; import * as Invocation from "./McpInvocationContext.ts"; import * as McpEvents from "./McpEvents.ts"; import * as Metrics from "./McpBridgeMetrics.ts"; +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; export const BridgeStatusResult = Schema.Struct({ serviceStartedAt: Schema.String, tunnel: Schema.NullOr(EnvironmentConnectReachability), @@ -39,6 +43,14 @@ export const BridgeStatusResult = Schema.Struct({ ), }), ), + missedScheduledRuns: Schema.Array( + Schema.Struct({ + scheduledTaskId: ScheduledTaskId, + dueAt: IsoDateTime, + overdueMs: NonNegativeInt, + reason: Schema.Literals(["not_dispatched", "run_in_progress"]), + }), + ), toolCalls: Schema.Array( Schema.Struct({ tool: Schema.String, @@ -64,6 +76,7 @@ export const layer = Layer.effect( const events = yield* McpEvents.McpEvents; const metrics = yield* Metrics.McpBridgeMetrics; const forwarding = yield* PeerForwarding.PeerForwarding; + const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; return BridgeStatusService.of({ status: Effect.fn("BridgeStatusService.status")(function* () { const scope = yield* Invocation.McpInvocationContext; @@ -93,9 +106,20 @@ export const layer = Layer.effect( ), Effect.orElseSucceed(() => null), ); + const missedScheduledRuns = yield* scheduledTasks.reliability().pipe( + Effect.map((entries) => + entries.flatMap(({ taskId, missedRun }) => + missedRun === undefined + ? [] + : [{ scheduledTaskId: ScheduledTaskId.make(taskId), ...missedRun }], + ), + ), + Effect.orElseSucceed(() => []), + ); return { ...status, linkedEnvironments, + missedScheduledRuns, toolCalls: yield* metrics.read(scope.client!.sessionId), }; }), diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index e428fe9e0602b..db2e6ad290e7b 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -69,6 +69,7 @@ import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import { buildUnavailableProviderSnapshot } from "../provider/unavailableProviderSnapshot.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; +import type { ScheduledTaskReliability } from "../scheduledTasks/ScheduledTaskReliability.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; @@ -2846,6 +2847,7 @@ describe("OrchestratorMcpService provider resolution", () => { tasks: ReadonlyArray, boundThread: OrchestrationV2ThreadShell | null, upserted: Ref.Ref, + reliability: ReadonlyArray = [], ) => layerOrchestratorService.pipe( Layer.provide( @@ -2866,6 +2868,7 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({ list: () => Effect.succeed({ tasks }), + reliability: () => Effect.succeed(reliability), upsert: () => Ref.update(upserted, (count) => count + 1).pipe(Effect.as({ task: tasks[0]! })), }), @@ -2896,6 +2899,41 @@ describe("OrchestratorMcpService provider resolution", () => { }), ); + it.effect("adds recent failures and missed runs to the listed task they belong to", () => + Effect.gen(function* () { + const upserted = yield* Ref.make(0); + const failing = task({ id: ScheduledTaskId.make("scheduled-task:failing") }); + const quiet = task({ id: ScheduledTaskId.make("scheduled-task:quiet") }); + const recentFailures = { + failed: 3, + cancelled: 6, + lastAt: "2026-11-12T07:11:28.827Z", + lastStatus: "failed" as const, + lastError: "Workspace preparation failed", + }; + const missedRun = { + dueAt: "2026-11-12T01:00:00.000Z", + overdueMs: 7_200_000, + reason: "not_dispatched" as const, + }; + const listed = yield* OrchestratorMcpService.OrchestratorMcpService.pipe( + Effect.flatMap((mcp) => mcp.listScheduledTasks(supervisedClient, { projectId })), + Effect.provide( + service([failing, quiet], null, upserted, [ + { taskId: failing.id, recentFailures, missedRun }, + ]), + ), + ); + assert.deepEqual( + listed.tasks.map((summary) => [summary.recentFailures, summary.missedRun]), + [ + [recentFailures, missedRun], + [undefined, undefined], + ], + ); + }), + ); + it.effect("hides a webhook URL from a thread whose turn has ended", () => Effect.gen(function* () { const callerId = ThreadId.make("thread:scheduled-ended"); @@ -2938,6 +2976,7 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(RemoteDelegation.RemoteDelegation)({}), Layer.mock(ScheduledTaskService.ScheduledTaskService)({ list: () => Effect.succeed({ tasks: [task({})] }), + reliability: () => Effect.succeed([]), }), ), ), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 8763ba7ade056..8c1efaba86c06 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -2132,10 +2132,26 @@ const make = Effect.gen(function* () { failure("orchestration_error", `Could not list scheduled tasks: ${error.message}`), ), ); + // History is extra: a failed read must not hide the tasks themselves. + const reliability = new Map( + (yield* scheduledTasks.reliability().pipe(Effect.orElseSucceed(() => []))).map( + (entry) => [entry.taskId, entry] as const, + ), + ); return { tasks: yield* Effect.forEach( tasks.filter((task) => projectId === undefined || task.projectId === projectId), - (task) => summarizeScheduledTask(scope, { parent, limits }, task), + (task) => + summarizeScheduledTask(scope, { parent, limits }, task).pipe( + Effect.map((summary) => { + const { recentFailures, missedRun } = reliability.get(task.id) ?? {}; + return { + ...summary, + ...(recentFailures === undefined ? {} : { recentFailures }), + ...(missedRun === undefined ? {} : { missedRun }), + }; + }), + ), ), }; }, diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index df7e2f6d3f2ed..5a0e5b83ca331 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -191,6 +191,7 @@ const layerUnusedScheduledTaskStub = Layer.succeed( ScheduledTaskService.ScheduledTaskService.of({ list: () => Effect.succeed({ tasks: [] }), subscribeList: () => Stream.succeed({ tasks: [] }), + reliability: () => Effect.succeed([]), upsert: () => Effect.die("ScheduledTaskService.upsert is unused in this test"), setEnabled: () => Effect.die("ScheduledTaskService.setEnabled is unused in this test"), delete: () => Effect.die("ScheduledTaskService.delete is unused in this test"), diff --git a/apps/server/src/mcp/toolkits/bridge/tools.ts b/apps/server/src/mcp/toolkits/bridge/tools.ts index 31b96f3082b2c..544e9b0f35027 100644 --- a/apps/server/src/mcp/toolkits/bridge/tools.ts +++ b/apps/server/src/mcp/toolkits/bridge/tools.ts @@ -29,7 +29,7 @@ export const BridgeToolkit = Toolkit.make( .annotate(Tool.Destructive, false), Tool.make("bridge_status", { description: - "Read this outside MCP sign-in's expiry, event subscriptions, last successful webhook and exhausted retries. Per-tool calls, errors and response bytes count since this server started. Never returns callback secrets.", + "Read this outside MCP sign-in's expiry, event subscriptions, last successful webhook, exhausted retries and scheduled runs that missed their slot. Per-tool calls, errors and response bytes count since this server started. Never returns callback secrets.", parameters: Tool.EmptyParams, success: Bridge.BridgeStatusResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/scheduledTasks/Schedule.test.ts b/apps/server/src/scheduledTasks/Schedule.test.ts index 0941c7392ff94..313685ac09896 100644 --- a/apps/server/src/scheduledTasks/Schedule.test.ts +++ b/apps/server/src/scheduledTasks/Schedule.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as DateTime from "effect/DateTime"; -import { isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; +import { isSameSchedule, missedScheduleRun, nextScheduledRunAt } from "./Schedule.ts"; describe("scheduled task schedule calculation", () => { it("skips to the next matching fixed-time weekday", () => { @@ -83,3 +83,60 @@ describe("scheduled task schedule calculation", () => { ).toBe(false); }); }); + +describe("missed scheduled runs", () => { + const at = (iso: string) => DateTime.makeUnsafe(iso); + const interval = { type: "interval", everyMs: 3_600_000 } as const; + const task = (nextRunAt: string | null, lastRunStatus = "succeeded") => ({ + enabled: true, + schedule: interval, + nextRunAt, + lastRunStatus, + }); + + it("flags a slot overdue by more than one interval and separates a stuck running task", () => { + const now = at("2026-11-13T05:00:00.000Z"); + expect(missedScheduleRun(task("2026-11-13T03:30:00.000Z"), now)).toEqual({ + dueAt: "2026-11-13T03:30:00.000Z", + overdueMs: 90 * 60_000, + reason: "not_dispatched", + }); + expect(missedScheduleRun(task("2026-11-13T03:30:00.000Z", "running"), now)?.reason).toBe( + "run_in_progress", + ); + }); + + it.each([ + ["within one interval", task("2026-11-13T04:30:00.000Z")], + ["disabled", { ...task("2026-11-13T00:00:00.000Z"), enabled: false }], + ["no next run", task(null)], + [ + "webhook", + { + ...task("2026-11-13T00:00:00.000Z"), + schedule: { type: "webhook", signature: null } as const, + }, + ], + ])("does not flag a task that is %s", (_name, candidate) => { + expect(missedScheduleRun(candidate, at("2026-11-13T05:00:00.000Z"))).toBeNull(); + }); + + it("uses the gap to the following slot for fixed-time tasks and 10 minutes for one-time tasks", () => { + const daily = { + enabled: true, + schedule: { type: "fixed_time", timeOfDay: "09:00" } as const, + nextRunAt: "2026-11-12T09:00:00.000Z", + lastRunStatus: "succeeded", + }; + expect(missedScheduleRun(daily, at("2026-11-13T08:00:00.000Z"))).toBeNull(); + expect(missedScheduleRun(daily, at("2026-11-13T10:00:00.000Z"))).not.toBeNull(); + const once = { + enabled: true, + schedule: { type: "once", runAt: "2026-11-13T04:00:00.000Z" } as const, + nextRunAt: "2026-11-13T04:00:00.000Z", + lastRunStatus: "never", + }; + expect(missedScheduleRun(once, at("2026-11-13T04:09:00.000Z"))).toBeNull(); + expect(missedScheduleRun(once, at("2026-11-13T04:11:00.000Z"))).not.toBeNull(); + }); +}); diff --git a/apps/server/src/scheduledTasks/Schedule.ts b/apps/server/src/scheduledTasks/Schedule.ts index f8fb648aaaaa9..5c12117467e5a 100644 --- a/apps/server/src/scheduledTasks/Schedule.ts +++ b/apps/server/src/scheduledTasks/Schedule.ts @@ -80,3 +80,49 @@ export function isMissedFixedTimeRun( if (schedule.type !== "fixed_time") return false; return DateTime.toEpochMillis(now) - DateTime.toEpochMillis(dueAt) > MISSED_FIXED_TIME_GRACE_MS; } + +export interface MissedScheduleRun { + readonly dueAt: string; + readonly overdueMs: number; + /** The run that should have fired later is still going, or nothing started it. */ + readonly reason: "run_in_progress" | "not_dispatched"; +} + +const DAY_MS = 24 * 60 * MINUTE_MS; + +/** + * An enabled task whose `nextRunAt` passed by more than one interval without a new run. + * A task stuck in `running` counts: its later slots are skipped while the run lasts. + */ +export function missedScheduleRun( + task: { + readonly enabled: boolean; + readonly schedule: ScheduledTaskSchedule; + readonly nextRunAt: string | null; + readonly lastRunStatus: string; + }, + now: DateTime.DateTime, +): MissedScheduleRun | null { + if (!task.enabled || task.nextRunAt === null || task.schedule.type === "webhook") return null; + const dueAt = DateTime.make(task.nextRunAt); + if (Option.isNone(dueAt)) return null; + const dueMs = DateTime.toEpochMillis(dueAt.value); + const overdueMs = DateTime.toEpochMillis(now) - dueMs; + let intervalMs: number; + if (task.schedule.type === "interval") { + intervalMs = Math.max(task.schedule.everyMs, MIN_SCHEDULED_TASK_INTERVAL_MS); + } else if (task.schedule.type === "once") { + intervalMs = MISSED_FIXED_TIME_GRACE_MS; + } else { + const following = nextScheduledRunAt(task.schedule, dueAt.value); + intervalMs = + following === null ? DAY_MS : Math.max(DateTime.toEpochMillis(following) - dueMs, MINUTE_MS); + } + return overdueMs > intervalMs + ? { + dueAt: DateTime.formatIso(dueAt.value), + overdueMs, + reason: task.lastRunStatus === "running" ? "run_in_progress" : "not_dispatched", + } + : null; +} diff --git a/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts b/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts new file mode 100644 index 0000000000000..440fd2d3d863b --- /dev/null +++ b/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts @@ -0,0 +1,147 @@ +import type { ScheduledTask } from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { missedScheduleRun, type MissedScheduleRun } from "./Schedule.ts"; + +const RECENT_RUN_WINDOW_DAYS = 7; +const RECENT_RUN_WINDOW_MS = RECENT_RUN_WINDOW_DAYS * 24 * 60 * 60 * 1000; +const MESSAGE_PREFIX = "scheduled-task-message:"; +const ERROR_TEXT_LIMIT = 500; + +export interface RecentRunFailures { + readonly failed: number; + readonly cancelled: number; + readonly lastAt: string; + readonly lastStatus: "failed" | "cancelled"; + /** The error of the latest failed run, which may be older than `lastAt`. */ + readonly lastError: string | null; +} + +export interface ScheduledTaskReliability { + readonly taskId: string; + readonly recentFailures?: RecentRunFailures; + readonly missedRun?: MissedScheduleRun; +} + +/** A scheduled run's message id is `scheduled-task-message:::`. */ +function parseScheduledRunMessageId( + messageId: string, +): { readonly taskId: string; readonly startedAtMs: number | null } | null { + if (!messageId.startsWith(MESSAGE_PREFIX)) return null; + const key = messageId.slice(MESSAGE_PREFIX.length); + const timed = /^(.+):(\d+):(?:scheduled|manual)$/.exec(key); + if (timed !== null) return { taskId: timed[1]!, startedAtMs: Number(timed[2]) }; + const webhook = /^(.+):webhook:[^:]+$/.exec(key); + return webhook === null ? null : { taskId: webhook[1]!, startedAtMs: null }; +} + +interface FailedRun { + readonly status: "failed" | "cancelled"; + readonly at: string; + readonly threadId: string; + readonly runId: string; + readonly startedAtMs: number | null; +} + +/** + * Failures and cancels of the last 7 days per task, derived from the runs themselves, so no + * second record exists to drift. A failure that never created a run comes from the task row. + */ +export const readTaskReliability = Effect.fn("ScheduledTaskReliability.read")(function* ( + tasks: ReadonlyArray, + now: DateTime.DateTime, +) { + const sql = yield* SqlClient.SqlClient; + const sinceMs = DateTime.toEpochMillis(now) - RECENT_RUN_WINDOW_MS; + const sinceIso = DateTime.formatIso(DateTime.makeUnsafe(sinceMs)); + const rows = yield* sql<{ + readonly status: "failed" | "cancelled"; + readonly requested_at: string; + readonly thread_id: string; + readonly run_id: string; + readonly message_id: string; + }>` + SELECT status, requested_at, thread_id, run_id, + json_extract(payload_json, '$.userMessageId') AS message_id + FROM orchestration_v2_projection_runs + WHERE status IN ('failed', 'cancelled') AND requested_at >= ${sinceIso} + AND substr(json_extract(payload_json, '$.userMessageId'), 1, ${MESSAGE_PREFIX.length}) = ${MESSAGE_PREFIX} + `.pipe(NodeSqliteClient.readOnly); + + const runsByTask = new Map(); + for (const row of rows) { + const parsed = parseScheduledRunMessageId(row.message_id); + if (parsed === null) continue; + runsByTask.set(parsed.taskId, [ + ...(runsByTask.get(parsed.taskId) ?? []), + { + status: row.status, + at: row.requested_at, + threadId: row.thread_id, + runId: row.run_id, + startedAtMs: parsed.startedAtMs, + }, + ]); + } + + const reliability: ScheduledTaskReliability[] = []; + for (const task of tasks) { + const runs = [...(runsByTask.get(task.id) ?? [])]; + let failed = runs.filter((run) => run.status === "failed").length; + const cancelled = runs.filter((run) => run.status === "cancelled").length; + const byLatest = runs.toSorted((x, y) => y.at.localeCompare(x.at)); + let latest: { at: string; status: "failed" | "cancelled" } | null = + byLatest[0] === undefined ? null : { at: byLatest[0].at, status: byLatest[0].status }; + const latestFailedRun = byLatest.find((run) => run.status === "failed"); + let latestFailed: { at: string; error: string | null } | null = null; + if (latestFailedRun !== undefined) { + const [errorRow] = yield* sql<{ readonly message: string | null }>` + SELECT json_extract(payload_json, '$.failure.message') AS message + FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${latestFailedRun.threadId} AND run_id = ${latestFailedRun.runId} + AND type = 'error' AND status = 'failed' + ORDER BY updated_at DESC, ordinal DESC + LIMIT 1 + `.pipe(NodeSqliteClient.readOnly); + latestFailed = { at: latestFailedRun.at, error: errorRow?.message ?? null }; + } + // A dispatch that failed before a run existed leaves only the task row. + const lastStartedMs = task.lastRunAt === null ? Number.NaN : Date.parse(task.lastRunAt); + if ( + task.lastRunStatus === "failed" && + task.schedule.type !== "webhook" && + lastStartedMs >= sinceMs && + !runs.some((run) => run.startedAtMs === lastStartedMs) + ) { + failed += 1; + if (latest === null || task.lastRunAt! > latest.at) { + latest = { at: task.lastRunAt!, status: "failed" }; + } + if (latestFailed === null || task.lastRunAt! > latestFailed.at) { + latestFailed = { at: task.lastRunAt!, error: task.lastRunError }; + } + } + + const missedRun = missedScheduleRun(task, now); + if (latest === null && missedRun === null) continue; + reliability.push({ + taskId: task.id, + ...(latest === null + ? {} + : { + recentFailures: { + failed, + cancelled, + lastAt: latest.at, + lastStatus: latest.status, + lastError: latestFailed?.error?.slice(0, ERROR_TEXT_LIMIT) ?? null, + }, + }), + ...(missedRun === null ? {} : { missedRun }), + }); + } + return reliability; +}); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 342688b025959..b5a089995b930 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -1530,3 +1530,88 @@ it.effect("a run interrupted right after it reserves the task does not block the }).pipe(Effect.provide(layer)); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it.effect( + "reliability counts the last 7 days of failed and cancelled runs and the latest error", + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-11-13T02:00:00.000Z"; + yield* TestClock.setTime(Date.parse(now)); + const threadId = "thread:history"; + const run = ( + task: string, + at: string, + status: string, + ordinal: number, + trigger = "scheduled", + ) => + sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES (${`run:${task}:${at}`}, ${threadId}, ${ordinal}, 'codex', ${status}, ${at}, + ${JSON.stringify({ + userMessageId: `scheduled-task-message:${task}:${Date.parse(at)}:${trigger}`, + })}) + `; + const errorItem = (task: string, at: string, message: string) => + sql` + INSERT INTO orchestration_v2_projection_turn_items + (turn_item_id, thread_id, run_id, ordinal, type, status, updated_at, payload_json) + VALUES (${`item:${task}:${at}`}, ${threadId}, ${`run:${task}:${at}`}, 1, 'error', 'failed', + ${at}, ${JSON.stringify({ failure: { message } })}) + `; + for (const id of ["task:a", "task:folded", "task:matched", "task:clean"]) { + yield* insertRow(sql, { id, next: null, enabled: 1, status: "succeeded" }, now); + } + // 10 days old, outside the window. + yield* run("task:a", "2026-11-03T02:00:00.000Z", "failed", 1); + yield* run("task:a", "2026-11-11T02:00:00.000Z", "failed", 2); + yield* errorItem("task:a", "2026-11-11T02:00:00.000Z", "Earlier error"); + yield* run("task:a", "2026-11-12T05:00:00.000Z", "failed", 3); + yield* errorItem("task:a", "2026-11-12T05:00:00.000Z", "Workspace preparation failed"); + yield* run("task:a", "2026-11-12T06:00:00.000Z", "cancelled", 4, "manual"); + yield* run("task:a", "2026-11-12T07:00:00.000Z", "completed", 5); + // A failed dispatch that never created a run leaves only the task row. + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed', last_run_error = 'No such folder', + last_run_at = '2026-11-12T20:00:00.000Z' WHERE task_id = 'task:folded'`; + // The same failure is already counted from its run, so the row adds nothing. + yield* run("task:matched", "2026-11-12T21:00:00.000Z", "failed", 6); + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed', last_run_error = 'Boom', + last_run_at = '2026-11-12T21:00:00.000Z' WHERE task_id = 'task:matched'`; + + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const history = Object.fromEntries( + (yield* service.reliability()).map((entry) => [entry.taskId, entry.recentFailures]), + ); + assert.deepEqual(history, { + "task:a": { + failed: 2, + cancelled: 1, + lastAt: "2026-11-12T06:00:00.000Z", + lastStatus: "cancelled", + lastError: "Workspace preparation failed", + }, + "task:folded": { + failed: 1, + cancelled: 0, + lastAt: "2026-11-12T20:00:00.000Z", + lastStatus: "failed", + lastError: "No such folder", + }, + "task:matched": { + failed: 1, + cancelled: 0, + lastAt: "2026-11-12T21:00:00.000Z", + lastStatus: "failed", + lastError: null, + }, + }); + }).pipe( + Effect.provide( + followerLayer(Layer.mock(ThreadManagementService.ThreadManagementService)({})), + ), + ); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 206cf4c3ce113..56abe37014227 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -64,6 +64,7 @@ import * as Scheduler from "../scheduling/Scheduler.ts"; import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError"; import { findGitEntry } from "../mcp/gitEntryProbe.ts"; import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; +import { readTaskReliability, type ScheduledTaskReliability } from "./ScheduledTaskReliability.ts"; import { redactHeaders, redactQuery, @@ -269,6 +270,11 @@ export class ScheduledTaskService extends Context.Service< { readonly list: () => Effect.Effect; readonly subscribeList: () => Stream.Stream; + /** Recent failures and missed runs, for tasks that have any. Read on demand, never streamed. */ + readonly reliability: () => Effect.Effect< + ReadonlyArray, + ScheduledTaskError + >; readonly upsert: ( input: ScheduledTaskUpsertInput, ) => Effect.Effect; @@ -1578,6 +1584,21 @@ export const layer = Layer.effect( Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), ); + const reliability: ScheduledTaskService["Service"]["reliability"] = Effect.fn( + "ScheduledTaskService.reliability", + )( + function* () { + const tasks = yield* listRows(); + return yield* readTaskReliability(tasks, yield* localNow); + }, + Effect.provideService(SqlClient.SqlClient, sql), + Effect.mapError((cause) => + isScheduledTaskError(cause) + ? cause + : taskError("Could not read schedule task history.", { cause }), + ), + ); + const subscribeList: ScheduledTaskService["Service"]["subscribeList"] = () => Stream.unwrap( Effect.gen(function* () { @@ -2859,6 +2880,7 @@ export const layer = Layer.effect( return ScheduledTaskService.of({ list, subscribeList, + reliability, upsert, setEnabled, delete: deleteTask, diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 1a906ea560381..e966545f994f0 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -820,6 +820,28 @@ export const OrchestratorMcpScheduledTask = Schema.Struct({ blockedByPolicy: Schema.optional(Schema.String).annotate({ description: "Present when an administrator's policy stops this task from running.", }), + recentFailures: Schema.optional( + Schema.Struct({ + failed: NonNegativeInt, + cancelled: NonNegativeInt, + lastAt: IsoDateTime, + lastStatus: Schema.Literals(["failed", "cancelled"]), + lastError: Schema.NullOr(Schema.String), + }), + ).annotate({ + description: + "Only `list_scheduled_tasks`, when the last 7 days had a failed or cancelled run. `lastError` is the latest failed run's error.", + }), + missedRun: Schema.optional( + Schema.Struct({ + dueAt: IsoDateTime, + overdueMs: NonNegativeInt, + reason: Schema.Literals(["not_dispatched", "run_in_progress"]), + }), + ).annotate({ + description: + "Only `list_scheduled_tasks`: `nextRunAt` passed by more than one interval with no new run.", + }), }); export type OrchestratorMcpScheduledTask = typeof OrchestratorMcpScheduledTask.Type; From 8ef5bd6fe2afe9877bf89792845313c673de1979 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 19:41:41 -0700 Subject: [PATCH 3/9] fix(server): a launched worktree without a repository or base fails with a clear message Scheduled tasks and retries skip the entry points' base check, so a folder that is not a repository reached Git as 'Failed to resolve the VCS driver'. Run the same check in preparation, and name a temporary folder when a workspace folder is missing. Co-Authored-By: Claude Sonnet 5.5 --- .../ProviderSessionManager.test.ts | 2 + .../ThreadLaunchService.test.ts | 49 +++++++++++++++++++ .../orchestration-v2/ThreadLaunchService.ts | 10 ++++ apps/server/src/provider/Errors.ts | 9 +++- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 50e52151fca71..2f5a440a2f8e0 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -5145,6 +5145,8 @@ it.effect.each(["missing", "file"] as const)( assert.instanceOf(error, ProviderWorkspaceMissingError); assert.include(error.message, cwd); assert.include(error.message, "Restore the folder at this path before retrying."); + // The fixture lives in the system temp folder, which a reboot clears. + assert.include(error.message, "under a temporary folder"); assert.equal((yield* Ref.get(state)).openCount, 0); assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); assert.deepEqual( diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index e60d6e6b7496d..5161015af4a2e 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -212,6 +212,7 @@ function makeHarness(options: HarnessOptions = {}) { options.fetchWorktreeBaseCommit ?? (() => Effect.succeed({ commitSha: "remote-main-sha", remoteRefName: "origin/main" })), hasCommit: options.hasCommit ?? (() => Effect.succeed(false)), + hasRefNamed: () => Effect.succeed(true), currentBranch: () => Effect.succeed(null), resolveWorktreeBaseRemote: () => Effect.succeed({ remoteName: "origin", branchName: "main" }), remoteExists: options.remoteExists ?? (() => Effect.succeed(true)), @@ -2563,6 +2564,54 @@ it.effect.each(["origin/master", ":/initial"])( }), ); +it.effect.each([ + { + name: "in a folder that is not a Git repository", + repositoryRoot: null, + hasCommit: false, + remote: true, + message: `Project "Project" (/repo) is not a Git repository.`, + }, + { + name: "whose base is on no remote and not local", + repositoryRoot: "/repo", + hasCommit: false, + remote: false, + message: `Base ref "main" does not resolve to a commit in project "Project" (/repo).`, + }, +] as const)("a launched worktree $name fails with a clear message", (testCase) => + Effect.gen(function* () { + const harness = makeHarness({ + hasCommit: ({ refName }) => Effect.succeed(refName === "HEAD" || testCase.hasCommit), + git: { + repositoryRoot: () => Effect.succeed(testCase.repositoryRoot), + resolveWorktreeBaseRemote: () => + Effect.succeed(testCase.remote ? { remoteName: "origin", branchName: "main" } : null), + hasRefNamed: () => Effect.succeed(false), + }, + }); + yield* Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const launched = yield* launches.launch( + launchInput({ + command: `command:launch:clear-base-${testCase.repositoryRoot === null ? "none" : "ref"}`, + thread: `thread:launch:clear-base-${testCase.repositoryRoot === null ? "none" : "ref"}`, + message: "First message", + workspace: { type: "worktree", baseRef: "main", startFromOrigin: true }, + }), + ); + const projection = yield* preparationSettled(launched.threadId); + assert.equal(projection.runs[0]?.status, "failed"); + const failure = projection.turnItems.find((item) => item.type === "error"); + assert.equal( + failure?.type === "error" ? failure.failure.message : undefined, + `Workspace preparation failed during provision worktree: ${testCase.message}`, + ); + assert.equal(harness.createWorktree.mock.calls.length, 0); + }).pipe(Effect.provide(harness.layer)); + }), +); + it.effect("leaves uncertain Git detection failures to workspace preparation", () => Effect.gen(function* () { const harness = makeHarness({ diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 570e6a14a8cd2..c2a25f6b24e91 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -515,6 +515,16 @@ const make = Effect.gen(function* () { }) .pipe(Effect.mapError(mapError(input, "update-thread", threadId))); } + // Scheduled tasks and retries skip the entry points' check; without it, a folder that is + // not a repository, or a base found nowhere, reaches Git as an opaque failure. + yield* checkWorktreeBase({ + projectId: input.projectId, + workspaceStrategy: input.workspaceStrategy, + }).pipe( + // Provisioning flattens a temporary branch name that collides. + Effect.catchTags({ ThreadLaunchBranchConflictError: () => Effect.void }), + Effect.mapError(mapError(input, "provision-worktree", threadId)), + ); let startRef = input.workspaceStrategy.baseRef; let baseRefName = input.workspaceStrategy.baseRef; const remote = diff --git a/apps/server/src/provider/Errors.ts b/apps/server/src/provider/Errors.ts index b1ef6192faf3b..2c15a73882fac 100644 --- a/apps/server/src/provider/Errors.ts +++ b/apps/server/src/provider/Errors.ts @@ -1,3 +1,4 @@ +import * as NodeOS from "node:os"; import * as Schema from "effect/Schema"; export class ProviderInstanceNotFoundError extends Schema.TaggedError()( @@ -22,6 +23,12 @@ export class ProviderWorkspaceMissingError extends Schema.TaggedError + this.cwd.startsWith(`${directory}/`), + ) + ? "It is under a temporary folder, which the system clears at reboot. " + : ""; + return `This thread's workspace folder no longer exists or is not a directory: ${this.cwd}. ${temporary}Restore the folder at this path before retrying.`; } } From 4e416761ea84e9054735bf4b5a9e7472fbdc9798 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 19:48:48 -0700 Subject: [PATCH 4/9] fix(server): tolerate a thread created in the batch that ends its run; update the context budget Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/orchestration-v2/EventSink.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index e614cd7852955..96d259cfc5072 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -450,11 +450,18 @@ const layerBase: Layer.Layer< latestInBatch.set(candidate.payload.id, candidate.payload); } } - const stored = yield* projectionStore.getThreadRecords(run.threadId, ["turnItems"], { - turnItemRunId: run.id, - turnItemTypes: [...RUN_BOUND_ITEM_TYPES], - turnItemStatuses: ["pending", "running", "waiting"], - }); + const stored = yield* projectionStore + .getThreadRecords(run.threadId, ["turnItems"], { + turnItemRunId: run.id, + turnItemTypes: [...RUN_BOUND_ITEM_TYPES], + turnItemStatuses: ["pending", "running", "waiting"], + }) + // The thread's creation can share this batch. + .pipe( + Effect.catchTags({ + ProjectionStoreThreadNotFoundError: () => Effect.succeed({ turnItems: [] }), + }), + ); for (const item of stored.turnItems) { if (!latestInBatch.has(item.id)) latestInBatch.set(item.id, item); } From d4129effbb2933ac7d485c318f300dfd79278b51 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 19:58:02 -0700 Subject: [PATCH 5/9] feat(mcp): warn when a project root is under a temporary folder; settle only todo lists and compactions Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/mcp/ThreadLaunchMcpService.ts | 5 ++++ .../src/mcp/toolkits/project/handlers.test.ts | 26 +++++++++++++++++++ .../src/mcp/toolkits/project/handlers.ts | 5 +++- apps/server/src/mcp/toolkits/project/tools.ts | 6 ++++- apps/server/src/orchestration-v2/EventSink.ts | 2 +- .../FoundationPersistence.test.ts | 17 ++++++------ .../src/orchestration-v2/RunItemSettlement.ts | 5 ++-- .../src/project/temporaryFolder.test.ts | 14 ++++++++++ apps/server/src/project/temporaryFolder.ts | 14 ++++++++++ apps/server/src/provider/Errors.ts | 8 +++--- 10 files changed, 83 insertions(+), 19 deletions(-) create mode 100644 apps/server/src/project/temporaryFolder.test.ts create mode 100644 apps/server/src/project/temporaryFolder.ts diff --git a/apps/server/src/mcp/ThreadLaunchMcpService.ts b/apps/server/src/mcp/ThreadLaunchMcpService.ts index 146d4879409f8..de0239170add2 100644 --- a/apps/server/src/mcp/ThreadLaunchMcpService.ts +++ b/apps/server/src/mcp/ThreadLaunchMcpService.ts @@ -17,6 +17,7 @@ import { OrchestratorRuntimeModeAboveCeilingError } from "../orchestration-v2/Or import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadMessageIntake from "../orchestration-v2/ThreadMessageIntake.ts"; import * as Claims from "../orchestration-v2/AttachmentClaims.ts"; +import { temporaryFolderWarning } from "../project/temporaryFolder.ts"; import * as Project from "../project/ProjectService.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; @@ -90,6 +91,7 @@ export const launchResult = Schema.Struct({ modelSelection: ModelSelection, runId: Schema.NullOr(RunId), status: Schema.NullOr(OrchestrationV2RunStatus), + warning: Schema.optional(Schema.String), }); function worktreeCheckHint( error: @@ -370,6 +372,8 @@ const launch = Effect.fn("ThreadLaunchMcpService.launch")( .pipe(Effect.ignore({ log: true })); } const run = result.projection.runs.find((run) => run.userMessageId === messageId); + const warning = + project === undefined ? undefined : temporaryFolderWarning(project.workspaceRoot); const notify = yield* ThreadNotifications.finish( notification, thread.id, @@ -384,6 +388,7 @@ const launch = Effect.fn("ThreadLaunchMcpService.launch")( modelSelection: thread.modelSelection, runId: run?.id ?? null, status: run?.status ?? null, + ...(warning === undefined ? {} : { warning }), }; }), ); diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index d7693b2d348e8..69de65f6387d8 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -424,6 +424,32 @@ it.effect("a launch with a model its provider does not advertise creates no thre }), ); +it.effect.each([ + ["/tmp/review", true], + ["/projects/client-target", false], +] as const)("a launch into a project rooted at %s warns: %s", ([workspaceRoot, warns]) => + Effect.gen(function* () { + const { projectId, dependencies } = clientLaunchHarness({ + runtimeModeCeiling: "auto", + launched: [], + workspaceRoot, + }); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(dependencies)), + ), + ); + const result = yield* toolkit.handle("t3_thread_launch", { title: "Fix", projectId }).pipe( + Stream.unwrap, + Stream.runCollect, + Effect.map((chunk) => chunk.at(-1)?.result), + Effect.provide(dependencies), + ); + expect(result).toMatchObject(warns ? { warning: expect.stringContaining(workspaceRoot) } : {}); + expect(warns || (result as { warning?: string }).warning === undefined).toBe(true); + }), +); + it.effect("a launch retried with the same clientRequestId replays the first one", () => Effect.gen(function* () { const launched: Array = []; diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 170c87b2ff9fe..7de33246349c8 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -11,6 +11,7 @@ import { mutateProjectWithClone } from "../../../project/ProjectMutation.ts"; import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts"; import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts"; +import { temporaryFolderWarning } from "../../../project/temporaryFolder.ts"; import * as McpToolAccess from "../../McpToolAccess.ts"; import { newCommandId, readCaller, unavailable } from "../../threadAccess.ts"; import { ProjectToolkit } from "./tools.ts"; @@ -104,9 +105,11 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { }; } const commandId = yield* newCommandId(); - return yield* projects + const created = yield* projects .create({ ...input, workspaceRoot, commandId, projectId: ProjectId.make(commandId) }) .pipe(Effect.mapError(projectFailure)); + const warning = temporaryFolderWarning(workspaceRoot); + return warning === undefined ? created : { ...created, warning }; }), // Linked work may bring the repository it works in; an existing // registered workspace is still refused. diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index 4dc3e6b14d263..6febdcdc9f191 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -75,7 +75,11 @@ const ProjectCreateTool = Tool.make("t3_project_create", { ...ProjectCreatePayload.fields, workspaceRoot: Schema.optional(ProjectCreatePayload.fields.workspaceRoot), }), - success: Schema.Struct({ ...Project.fields, commitError: Schema.optional(Schema.String) }), + success: Schema.Struct({ + ...Project.fields, + commitError: Schema.optional(Schema.String), + warning: Schema.optional(Schema.String), + }), dependencies: [...shared.dependencies, ManagedProjectFolders.ManagedProjectFolders], }).annotate(Tool.Destructive, true); const ProjectUpdateTool = Tool.make("t3_project_update", { diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 96d259cfc5072..2bb1ad93cae81 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -433,7 +433,7 @@ const layerBase: Layer.Layer< ); }; - // A todo list, compaction or reasoning item never outlives its run, but providers rarely close + // A todo list or compaction item never outlives its run, but providers rarely close // them. Close what is still active here, in the transaction that ends the run. // The enclosing write span already traces this per-batch step. const closeEndedRunItems = Effect.fnUntraced(function* ( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index b0210aa915fb3..dc8d0d7baaf58 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -16,6 +16,7 @@ import { type OrchestrationV2Run, type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, + PlanId, ProjectId, ProviderDriverKind, ProviderInstanceId, @@ -2133,11 +2134,11 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { type: "compaction", driver: null, }; - const reasoning: OrchestrationV2TurnItem = { - ...itemBase("reasoning", 2), - type: "reasoning", - text: "thinking", - streaming: true, + const todo: OrchestrationV2TurnItem = { + ...itemBase("todo", 2), + type: "todo_list", + planId: PlanId.make(`plan:run-end-items:${endStatus}`), + steps: [], }; const background: OrchestrationV2TurnItem = { ...itemBase("background", 3), @@ -2173,10 +2174,10 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { itemEvent(background), ], }); - // The reasoning item arrives in the same batch that ends the run. + // The todo list arrives in the same batch that ends the run. yield* eventSink.write({ events: [ - itemEvent(reasoning), + itemEvent(todo), { id: EventId.make(`event:run-end-items:${endStatus}:ended`), type: "run.updated", @@ -2193,7 +2194,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { turnItems.find((candidate) => candidate.id === item.id)?.status; const expected = endStatus === "failed" ? "cancelled" : endStatus; assert.equal(statusOf(compaction), expected); - assert.equal(statusOf(reasoning), expected); + assert.equal(statusOf(todo), expected); assert.equal(statusOf(background), "running"); }).pipe(Effect.provide(Layer.fresh(layerTest))), ); diff --git a/apps/server/src/orchestration-v2/RunItemSettlement.ts b/apps/server/src/orchestration-v2/RunItemSettlement.ts index 1d33815aea65a..7aebd1acd6f1b 100644 --- a/apps/server/src/orchestration-v2/RunItemSettlement.ts +++ b/apps/server/src/orchestration-v2/RunItemSettlement.ts @@ -4,7 +4,7 @@ import type { OrchestrationV2Run, OrchestrationV2TurnItem } from "@t3tools/contr * Items that cannot outlive their run. Shell, tool and subagent items can be background work, * so a run ending does not close them. */ -export const RUN_BOUND_ITEM_TYPES = ["todo_list", "compaction", "reasoning"] as const; +export const RUN_BOUND_ITEM_TYPES = ["todo_list", "compaction"] as const; const ACTIVE_ITEM_STATUSES: ReadonlySet = new Set(["pending", "running", "waiting"]); @@ -37,6 +37,5 @@ export function settleItemForEndedRun( ): OrchestrationV2TurnItem { const status = itemStatusForEndedRun(runStatus); if (status === null || !isActiveRunBoundItem(item)) return item; - const settled = { ...item, status, completedAt: item.completedAt ?? item.updatedAt }; - return settled.type === "reasoning" ? { ...settled, streaming: false } : settled; + return { ...item, status, completedAt: item.completedAt ?? item.updatedAt }; } diff --git a/apps/server/src/project/temporaryFolder.test.ts b/apps/server/src/project/temporaryFolder.test.ts new file mode 100644 index 0000000000000..017ae6566db4d --- /dev/null +++ b/apps/server/src/project/temporaryFolder.test.ts @@ -0,0 +1,14 @@ +import { expect, it } from "@effect/vitest"; + +import { isTemporaryFolder, temporaryFolderWarning } from "./temporaryFolder.ts"; + +it.each([ + ["/tmp/review", true], + ["/var/tmp/review", true], + ["/dev/shm/work", true], + ["/home/user/app", false], + ["/tmpfoo/app", false], +])("%s is temporary: %s", (path, temporary) => { + expect(isTemporaryFolder(path)).toBe(temporary); + expect(temporaryFolderWarning(path) !== undefined).toBe(temporary); +}); diff --git a/apps/server/src/project/temporaryFolder.ts b/apps/server/src/project/temporaryFolder.ts new file mode 100644 index 0000000000000..adf7143f61657 --- /dev/null +++ b/apps/server/src/project/temporaryFolder.ts @@ -0,0 +1,14 @@ +import * as NodeOS from "node:os"; + +const TEMPORARY_ROOTS = ["/tmp", "/var/tmp", "/dev/shm", "/run"]; + +/** Folders the system clears at reboot (tmpfs, or a temp cleaner). */ +export function isTemporaryFolder(path: string): boolean { + return [NodeOS.tmpdir(), ...TEMPORARY_ROOTS].some((root) => path.startsWith(`${root}/`)); +} + +export function temporaryFolderWarning(path: string): string | undefined { + return isTemporaryFolder(path) + ? `${path} is under a temporary folder, which the system clears at reboot; the project's threads then fail to start. Use a folder that persists.` + : undefined; +} diff --git a/apps/server/src/provider/Errors.ts b/apps/server/src/provider/Errors.ts index 2c15a73882fac..ae146bc70a0e0 100644 --- a/apps/server/src/provider/Errors.ts +++ b/apps/server/src/provider/Errors.ts @@ -1,6 +1,7 @@ -import * as NodeOS from "node:os"; import * as Schema from "effect/Schema"; +import { isTemporaryFolder } from "../project/temporaryFolder.ts"; + export class ProviderInstanceNotFoundError extends Schema.TaggedError()( "ProviderInstanceNotFoundError", { @@ -23,10 +24,7 @@ export class ProviderWorkspaceMissingError extends Schema.TaggedError - this.cwd.startsWith(`${directory}/`), - ) + const temporary = isTemporaryFolder(this.cwd) ? "It is under a temporary folder, which the system clears at reboot. " : ""; return `This thread's workspace folder no longer exists or is not a directory: ${this.cwd}. ${temporary}Restore the folder at this path before retrying.`; From cce5cb13d5718861e5b12fd1626e20abf0e3f980 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 20:00:41 -0700 Subject: [PATCH 6/9] fix(mcp): keep the agent context budget at the base size The context-budget CI check refuses growth. Drop the new field descriptions and the bridge_status description change, so the tool list is byte-identical in size to the base. Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/mcp/BridgeStatusService.test.ts | 1 - apps/server/src/mcp/BridgeStatusService.ts | 9 +++++++-- apps/server/src/mcp/toolkits/bridge/tools.ts | 2 +- packages/contracts/src/orchestratorMcp.ts | 10 ++-------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/apps/server/src/mcp/BridgeStatusService.test.ts b/apps/server/src/mcp/BridgeStatusService.test.ts index 7c21be29f5ebe..c4324f11f9d87 100644 --- a/apps/server/src/mcp/BridgeStatusService.test.ts +++ b/apps/server/src/mcp/BridgeStatusService.test.ts @@ -87,7 +87,6 @@ it.effect( scheduledTaskId: "scheduled-task:late", dueAt: "1969-12-31T00:00:00.000Z", overdueMs: 86_400_000, - reason: "not_dispatched", }, ]); expect(status.linkedEnvironments).toMatchObject({ diff --git a/apps/server/src/mcp/BridgeStatusService.ts b/apps/server/src/mcp/BridgeStatusService.ts index 27d3d71c71cf0..3a1aa3a6ac1f2 100644 --- a/apps/server/src/mcp/BridgeStatusService.ts +++ b/apps/server/src/mcp/BridgeStatusService.ts @@ -48,7 +48,6 @@ export const BridgeStatusResult = Schema.Struct({ scheduledTaskId: ScheduledTaskId, dueAt: IsoDateTime, overdueMs: NonNegativeInt, - reason: Schema.Literals(["not_dispatched", "run_in_progress"]), }), ), toolCalls: Schema.Array( @@ -111,7 +110,13 @@ export const layer = Layer.effect( entries.flatMap(({ taskId, missedRun }) => missedRun === undefined ? [] - : [{ scheduledTaskId: ScheduledTaskId.make(taskId), ...missedRun }], + : [ + { + scheduledTaskId: ScheduledTaskId.make(taskId), + dueAt: missedRun.dueAt, + overdueMs: missedRun.overdueMs, + }, + ], ), ), Effect.orElseSucceed(() => []), diff --git a/apps/server/src/mcp/toolkits/bridge/tools.ts b/apps/server/src/mcp/toolkits/bridge/tools.ts index 544e9b0f35027..31b96f3082b2c 100644 --- a/apps/server/src/mcp/toolkits/bridge/tools.ts +++ b/apps/server/src/mcp/toolkits/bridge/tools.ts @@ -29,7 +29,7 @@ export const BridgeToolkit = Toolkit.make( .annotate(Tool.Destructive, false), Tool.make("bridge_status", { description: - "Read this outside MCP sign-in's expiry, event subscriptions, last successful webhook, exhausted retries and scheduled runs that missed their slot. Per-tool calls, errors and response bytes count since this server started. Never returns callback secrets.", + "Read this outside MCP sign-in's expiry, event subscriptions, last successful webhook and exhausted retries. Per-tool calls, errors and response bytes count since this server started. Never returns callback secrets.", parameters: Tool.EmptyParams, success: Bridge.BridgeStatusResult, failure: OrchestratorMcpFailure, diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index e966545f994f0..1c7bcdbc09328 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -828,20 +828,14 @@ export const OrchestratorMcpScheduledTask = Schema.Struct({ lastStatus: Schema.Literals(["failed", "cancelled"]), lastError: Schema.NullOr(Schema.String), }), - ).annotate({ - description: - "Only `list_scheduled_tasks`, when the last 7 days had a failed or cancelled run. `lastError` is the latest failed run's error.", - }), + ), missedRun: Schema.optional( Schema.Struct({ dueAt: IsoDateTime, overdueMs: NonNegativeInt, reason: Schema.Literals(["not_dispatched", "run_in_progress"]), }), - ).annotate({ - description: - "Only `list_scheduled_tasks`: `nextRunAt` passed by more than one interval with no new run.", - }), + ), }); export type OrchestratorMcpScheduledTask = typeof OrchestratorMcpScheduledTask.Type; From 4bc1d2d78ab36c82501f1b1048936e0302f9a3c4 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 20:32:58 -0700 Subject: [PATCH 7/9] fix(server): review fixes for run history, missed-run zones, temp folders and run-end item closing Count webhook failures, index the 7-day failed-run read, measure fixed-time gaps in the scheduler's zone, resolve paths and ask the filesystem before calling a folder temporary, close items on rolled-back runs, and keep an unreadable old item from failing the run-ending write. Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/orchestration-v2/EventSink.ts | 8 ++- .../FoundationPersistence.test.ts | 5 +- .../src/orchestration-v2/RunItemSettlement.ts | 1 + apps/server/src/persistence/ForkMigrations.ts | 2 + .../015_ScheduledRunHistoryIndex.ts | 10 +++ .../src/project/temporaryFolder.test.ts | 27 ++++++++ apps/server/src/project/temporaryFolder.ts | 61 ++++++++++++++++++- .../src/scheduledTasks/Schedule.test.ts | 19 +++++- apps/server/src/scheduledTasks/Schedule.ts | 4 +- .../ScheduledTaskReliability.ts | 3 +- .../ScheduledTaskService.test.ts | 16 ++++- 11 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 2bb1ad93cae81..9861420ab55a9 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -456,11 +456,17 @@ const layerBase: Layer.Layer< turnItemTypes: [...RUN_BOUND_ITEM_TYPES], turnItemStatuses: ["pending", "running", "waiting"], }) - // The thread's creation can share this batch. + // The thread's creation can share this batch. An old item that cannot be read must not + // fail the write that ends the run; its items then close on read instead. .pipe( Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed({ turnItems: [] }), }), + Effect.catch((cause) => + Effect.logWarning("Could not read run items to close", { runId: run.id, cause }).pipe( + Effect.as({ turnItems: [] }), + ), + ), ); for (const item of stored.turnItems) { if (!latestInBatch.has(item.id)) latestInBatch.set(item.id, item); diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index dc8d0d7baaf58..c8a3123db2b90 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -2086,7 +2086,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(layerTest))), ); - it.effect.each(["completed", "failed", "interrupted"] as const)( + it.effect.each(["completed", "failed", "interrupted", "rolled_back"] as const)( "closes run-bound items still active when a run ends %s, and leaves background tools", (endStatus) => Effect.gen(function* () { @@ -2192,7 +2192,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { const { turnItems } = yield* projections.getThreadRecords(threadId, ["turnItems"]); const statusOf = (item: OrchestrationV2TurnItem) => turnItems.find((candidate) => candidate.id === item.id)?.status; - const expected = endStatus === "failed" ? "cancelled" : endStatus; + const expected = + endStatus === "failed" || endStatus === "rolled_back" ? "cancelled" : endStatus; assert.equal(statusOf(compaction), expected); assert.equal(statusOf(todo), expected); assert.equal(statusOf(background), "running"); diff --git a/apps/server/src/orchestration-v2/RunItemSettlement.ts b/apps/server/src/orchestration-v2/RunItemSettlement.ts index 7aebd1acd6f1b..c4eb93bec8df7 100644 --- a/apps/server/src/orchestration-v2/RunItemSettlement.ts +++ b/apps/server/src/orchestration-v2/RunItemSettlement.ts @@ -25,6 +25,7 @@ export function itemStatusForEndedRun( return "interrupted"; case "failed": case "cancelled": + case "rolled_back": return "cancelled"; default: return null; diff --git a/apps/server/src/persistence/ForkMigrations.ts b/apps/server/src/persistence/ForkMigrations.ts index 7361ae531ac2f..835366ba9f03a 100644 --- a/apps/server/src/persistence/ForkMigrations.ts +++ b/apps/server/src/persistence/ForkMigrations.ts @@ -16,6 +16,7 @@ import ForkMigration0011 from "./ForkMigrations/011_ForkRunExclusions.ts"; import ForkMigration0012 from "./ForkMigrations/012_WorkLedgerApproval.ts"; import ForkMigration0013 from "./ForkMigrations/013_WorkLedgerHygiene.ts"; import ForkMigration0014 from "./ForkMigrations/014_McpRequestLogConnection.ts"; +import ForkMigration0015 from "./ForkMigrations/015_ScheduledRunHistoryIndex.ts"; // Fork-only migrations, recorded in their own table so upstream keeps every id in // effect_sql_migrations. They run after all upstream migrations. Add new ones here, never in // Migrations.ts, and never rename one: the repair below finds old rows by name. See FORK.md. @@ -34,6 +35,7 @@ const forkMigrationEntries = [ [12, "WorkLedgerApproval", ForkMigration0012], [13, "WorkLedgerHygiene", ForkMigration0013], [14, "McpRequestLogConnection", ForkMigration0014], + [15, "ScheduledRunHistoryIndex", ForkMigration0015], ] as const; export const FORK_MIGRATIONS_TABLE = "t3_fork_migrations"; diff --git a/apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts b/apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts new file mode 100644 index 0000000000000..f7c700e902731 --- /dev/null +++ b/apps/server/src/persistence/ForkMigrations/015_ScheduledRunHistoryIndex.ts @@ -0,0 +1,10 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // list_scheduled_tasks and bridge_status read the last 7 days of failed and cancelled runs. + yield* sql`CREATE INDEX IF NOT EXISTS fork_runs_failed_recent_idx + ON orchestration_v2_projection_runs(requested_at) + WHERE status IN ('failed', 'cancelled')`; +}); diff --git a/apps/server/src/project/temporaryFolder.test.ts b/apps/server/src/project/temporaryFolder.test.ts index 017ae6566db4d..715a5e0a1a58a 100644 --- a/apps/server/src/project/temporaryFolder.test.ts +++ b/apps/server/src/project/temporaryFolder.test.ts @@ -1,3 +1,8 @@ +// @effect-diagnostics nodeBuiltinImport:off - builds a symlink fixture with node:fs. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + import { expect, it } from "@effect/vitest"; import { isTemporaryFolder, temporaryFolderWarning } from "./temporaryFolder.ts"; @@ -6,9 +11,31 @@ it.each([ ["/tmp/review", true], ["/var/tmp/review", true], ["/dev/shm/work", true], + ["/run/user/1000/work", true], + ["/private/tmp/review", true], + ["/tmp/../home/user/app", false], ["/home/user/app", false], ["/tmpfoo/app", false], + ["/run/media/user/disk/app", false], ])("%s is temporary: %s", (path, temporary) => { expect(isTemporaryFolder(path)).toBe(temporary); expect(temporaryFolderWarning(path) !== undefined).toBe(temporary); }); + +it("resolves symlinks and a persistent TMPDIR", () => { + const sandbox = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "temporary-folder-")); + try { + const persistent = NodePath.join(NodeOS.homedir(), ".t3"); + const link = NodePath.join(sandbox, "link"); + NodeFS.symlinkSync(process.cwd(), link); + // Under /tmp by name, but it leads to a folder that is not temporary. + expect(isTemporaryFolder(NodePath.join(link, "missing"))).toBe(false); + expect(isTemporaryFolder(NodePath.join(sandbox, "missing"))).toBe(true); + // Relative paths resolve against the working directory, which is not temporary here. + expect(isTemporaryFolder(NodePath.relative(process.cwd(), persistent))).toBe( + isTemporaryFolder(persistent), + ); + } finally { + NodeFS.rmSync(sandbox, { recursive: true, force: true }); + } +}); diff --git a/apps/server/src/project/temporaryFolder.ts b/apps/server/src/project/temporaryFolder.ts index adf7143f61657..c53c7d37ac4a1 100644 --- a/apps/server/src/project/temporaryFolder.ts +++ b/apps/server/src/project/temporaryFolder.ts @@ -1,10 +1,65 @@ +// @effect-diagnostics nodeBuiltinImport:off - a synchronous check that backs an error message getter. +import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; -const TEMPORARY_ROOTS = ["/tmp", "/var/tmp", "/dev/shm", "/run"]; +const TEMPORARY_PREFIXES = [ + "/tmp", + "/var/tmp", + "/dev/shm", + // macOS resolves /tmp and /var/tmp here, and keeps per-user temp folders in /var/folders. + "/private/tmp", + "/private/var/tmp", + "/private/var/folders", +]; +// Per-user runtime directory; /run/media (external drives) is persistent and must not match. +const USER_RUNTIME_DIRECTORY = /^\/run\/user\/\d+(?:\/|$)/; +// Removable drives mount here; a missing path under it must not inherit /run's tmpfs. +const REMOVABLE_MEDIA = "/run/media/"; +const LINUX_TMPFS_MAGIC = 0x01021994; -/** Folders the system clears at reboot (tmpfs, or a temp cleaner). */ +/** The path as the filesystem sees it: `~` and relative paths expanded, `..` and symlinks resolved. */ +function resolveExisting(path: string): string { + const expanded = + path === "~" || path.startsWith("~/") ? NodePath.join(NodeOS.homedir(), path.slice(1)) : path; + const absolute = NodePath.resolve(expanded); + // A missing folder is the usual case here, so resolve its nearest existing ancestor. + let existing = absolute; + while (!NodeFS.existsSync(existing) && NodePath.dirname(existing) !== existing) { + existing = NodePath.dirname(existing); + } + try { + return NodePath.join(NodeFS.realpathSync(existing), NodePath.relative(existing, absolute)); + } catch { + return absolute; + } +} + +function onTmpfs(path: string): boolean { + let existing = path; + while (!NodeFS.existsSync(existing) && NodePath.dirname(existing) !== existing) { + existing = NodePath.dirname(existing); + } + try { + // Only Linux reports this magic number; elsewhere the prefixes decide. + return NodeFS.statfsSync(existing).type === LINUX_TMPFS_MAGIC; + } catch { + return false; + } +} + +/** + * A folder the system clears at reboot: a tmpfs, or under a well-known temporary directory. + * `TMPDIR` is not trusted by itself; a persistent one is not temporary. + */ export function isTemporaryFolder(path: string): boolean { - return [NodeOS.tmpdir(), ...TEMPORARY_ROOTS].some((root) => path.startsWith(`${root}/`)); + const resolved = resolveExisting(path); + if (resolved.startsWith(REMOVABLE_MEDIA)) return false; + return ( + TEMPORARY_PREFIXES.some((root) => resolved === root || resolved.startsWith(`${root}/`)) || + USER_RUNTIME_DIRECTORY.test(resolved) || + onTmpfs(resolved) + ); } export function temporaryFolderWarning(path: string): string | undefined { diff --git a/apps/server/src/scheduledTasks/Schedule.test.ts b/apps/server/src/scheduledTasks/Schedule.test.ts index 313685ac09896..39e09a36383f7 100644 --- a/apps/server/src/scheduledTasks/Schedule.test.ts +++ b/apps/server/src/scheduledTasks/Schedule.test.ts @@ -121,6 +121,20 @@ describe("missed scheduled runs", () => { expect(missedScheduleRun(candidate, at("2026-11-13T05:00:00.000Z"))).toBeNull(); }); + it("measures the gap between fixed-time slots in the scheduler's zone", () => { + // Monday 22:00 in Los Angeles is Tuesday 06:00 UTC, so a UTC reading would see 6d16h, not 7d. + const weekly = { + enabled: true, + schedule: { type: "fixed_time", timeOfDay: "22:00", weekdays: [1] } as const, + nextRunAt: "2026-11-10T06:00:00.000Z", + lastRunStatus: "succeeded", + }; + const inLosAngeles = (iso: string) => + DateTime.setZone(at(iso), DateTime.zoneMakeNamedUnsafe("America/Los_Angeles")); + expect(missedScheduleRun(weekly, inLosAngeles("2026-11-17T02:00:00.000Z"))).toBeNull(); + expect(missedScheduleRun(weekly, inLosAngeles("2026-11-17T08:00:00.000Z"))).not.toBeNull(); + }); + it("uses the gap to the following slot for fixed-time tasks and 10 minutes for one-time tasks", () => { const daily = { enabled: true, @@ -128,8 +142,9 @@ describe("missed scheduled runs", () => { nextRunAt: "2026-11-12T09:00:00.000Z", lastRunStatus: "succeeded", }; - expect(missedScheduleRun(daily, at("2026-11-13T08:00:00.000Z"))).toBeNull(); - expect(missedScheduleRun(daily, at("2026-11-13T10:00:00.000Z"))).not.toBeNull(); + const utc = (iso: string) => DateTime.setZone(at(iso), DateTime.zoneMakeNamedUnsafe("UTC")); + expect(missedScheduleRun(daily, utc("2026-11-13T08:00:00.000Z"))).toBeNull(); + expect(missedScheduleRun(daily, utc("2026-11-13T10:00:00.000Z"))).not.toBeNull(); const once = { enabled: true, schedule: { type: "once", runAt: "2026-11-13T04:00:00.000Z" } as const, diff --git a/apps/server/src/scheduledTasks/Schedule.ts b/apps/server/src/scheduledTasks/Schedule.ts index 5c12117467e5a..9578b39f8c7cc 100644 --- a/apps/server/src/scheduledTasks/Schedule.ts +++ b/apps/server/src/scheduledTasks/Schedule.ts @@ -114,7 +114,9 @@ export function missedScheduleRun( } else if (task.schedule.type === "once") { intervalMs = MISSED_FIXED_TIME_GRACE_MS; } else { - const following = nextScheduledRunAt(task.schedule, dueAt.value); + // Slots are wall-clock times in the scheduler's zone, so the gap to the next one is too. + const zone = DateTime.isZoned(now) ? now.zone : DateTime.zoneMakeLocal(); + const following = nextScheduledRunAt(task.schedule, DateTime.setZone(dueAt.value, zone)); intervalMs = following === null ? DAY_MS : Math.max(DateTime.toEpochMillis(following) - dueMs, MINUTE_MS); } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts b/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts index 440fd2d3d863b..9a7ec3e5eb453 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskReliability.ts @@ -34,7 +34,8 @@ function parseScheduledRunMessageId( const key = messageId.slice(MESSAGE_PREFIX.length); const timed = /^(.+):(\d+):(?:scheduled|manual)$/.exec(key); if (timed !== null) return { taskId: timed[1]!, startedAtMs: Number(timed[2]) }; - const webhook = /^(.+):webhook:[^:]+$/.exec(key); + // Delivery ids look like `delivery:`, so the id itself contains a colon. + const webhook = /^(.+):webhook:(?:delivery:)?[^:]+$/.exec(key); return webhook === null ? null : { taskId: webhook[1]!, startedAtMs: null }; } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index b5a089995b930..5583279d247b6 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -1561,7 +1561,7 @@ it.effect( VALUES (${`item:${task}:${at}`}, ${threadId}, ${`run:${task}:${at}`}, 1, 'error', 'failed', ${at}, ${JSON.stringify({ failure: { message } })}) `; - for (const id of ["task:a", "task:folded", "task:matched", "task:clean"]) { + for (const id of ["task:a", "task:folded", "task:matched", "task:clean", "task:hook"]) { yield* insertRow(sql, { id, next: null, enabled: 1, status: "succeeded" }, now); } // 10 days old, outside the window. @@ -1572,6 +1572,13 @@ it.effect( yield* errorItem("task:a", "2026-11-12T05:00:00.000Z", "Workspace preparation failed"); yield* run("task:a", "2026-11-12T06:00:00.000Z", "cancelled", 4, "manual"); yield* run("task:a", "2026-11-12T07:00:00.000Z", "completed", 5); + // A webhook run's message id ends `:webhook:delivery:`. + yield* sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES ('run:hook', ${threadId}, 7, 'codex', 'failed', '2026-11-12T22:00:00.000Z', + ${JSON.stringify({ userMessageId: "scheduled-task-message:task:hook:webhook:delivery:3f2a-91" })}) + `; // A failed dispatch that never created a run leaves only the task row. yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed', last_run_error = 'No such folder', last_run_at = '2026-11-12T20:00:00.000Z' WHERE task_id = 'task:folded'`; @@ -1600,6 +1607,13 @@ it.effect( lastStatus: "failed", lastError: "No such folder", }, + "task:hook": { + failed: 1, + cancelled: 0, + lastAt: "2026-11-12T22:00:00.000Z", + lastStatus: "failed", + lastError: null, + }, "task:matched": { failed: 1, cancelled: 0, From 2e9464977c37be699f075ed2ed2d29af10f4b458 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 21:36:58 -0700 Subject: [PATCH 8/9] fix(server): probe temporary folders asynchronously with a timeout; skip network mounts; count relay webhooks Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/mcp/ThreadLaunchMcpService.ts | 2 +- .../src/mcp/toolkits/project/handlers.ts | 2 +- .../src/project/temporaryFolder.test.ts | 73 ++++++++--- apps/server/src/project/temporaryFolder.ts | 121 ++++++++++++------ apps/server/src/provider/Errors.ts | 4 +- .../ScheduledTaskReliability.ts | 4 +- .../ScheduledTaskService.test.ts | 23 +++- 7 files changed, 160 insertions(+), 69 deletions(-) diff --git a/apps/server/src/mcp/ThreadLaunchMcpService.ts b/apps/server/src/mcp/ThreadLaunchMcpService.ts index de0239170add2..7fea171e74111 100644 --- a/apps/server/src/mcp/ThreadLaunchMcpService.ts +++ b/apps/server/src/mcp/ThreadLaunchMcpService.ts @@ -373,7 +373,7 @@ const launch = Effect.fn("ThreadLaunchMcpService.launch")( } const run = result.projection.runs.find((run) => run.userMessageId === messageId); const warning = - project === undefined ? undefined : temporaryFolderWarning(project.workspaceRoot); + project === undefined ? undefined : yield* temporaryFolderWarning(project.workspaceRoot); const notify = yield* ThreadNotifications.finish( notification, thread.id, diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 7de33246349c8..d3b59aa73c0f4 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -108,7 +108,7 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { const created = yield* projects .create({ ...input, workspaceRoot, commandId, projectId: ProjectId.make(commandId) }) .pipe(Effect.mapError(projectFailure)); - const warning = temporaryFolderWarning(workspaceRoot); + const warning = yield* temporaryFolderWarning(workspaceRoot); return warning === undefined ? created : { ...created, warning }; }), // Linked work may bring the repository it works in; an existing diff --git a/apps/server/src/project/temporaryFolder.test.ts b/apps/server/src/project/temporaryFolder.test.ts index 715a5e0a1a58a..6b983de659f4d 100644 --- a/apps/server/src/project/temporaryFolder.test.ts +++ b/apps/server/src/project/temporaryFolder.test.ts @@ -3,11 +3,14 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; -import { expect, it } from "@effect/vitest"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; -import { isTemporaryFolder, temporaryFolderWarning } from "./temporaryFolder.ts"; +import { isTemporaryFolder, isTemporaryPath, temporaryFolderWarning } from "./temporaryFolder.ts"; -it.each([ +it.effect.each([ ["/tmp/review", true], ["/var/tmp/review", true], ["/dev/shm/work", true], @@ -17,25 +20,53 @@ it.each([ ["/home/user/app", false], ["/tmpfoo/app", false], ["/run/media/user/disk/app", false], -])("%s is temporary: %s", (path, temporary) => { - expect(isTemporaryFolder(path)).toBe(temporary); - expect(temporaryFolderWarning(path) !== undefined).toBe(temporary); -}); + ["/run/user/1000/gvfs/sftp:host=nas/app", false], + ["/run/user/1000/doc/by-app/app", false], +] as const)("%s is temporary: %s", ([path, temporary]) => + Effect.gen(function* () { + assert.equal(yield* isTemporaryFolder(path), temporary); + assert.equal((yield* temporaryFolderWarning(path)) !== undefined, temporary); + assert.equal(isTemporaryPath(path), temporary); + }), +); -it("resolves symlinks and a persistent TMPDIR", () => { - const sandbox = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "temporary-folder-")); - try { - const persistent = NodePath.join(NodeOS.homedir(), ".t3"); +it.effect("never probes the filesystem under a network or document mount", () => + Effect.gen(function* () { + const probed: string[] = []; + const probe = { + inspect: (path: string) => { + probed.push(path); + return Promise.resolve({ resolved: path, tmpfs: true }); + }, + }; + assert.isFalse(yield* isTemporaryFolder("/run/user/1000/gvfs/smb-share/x", probe)); + assert.deepEqual(probed, []); + }), +); + +it.effect("falls back to the name when the filesystem does not answer in time", () => + Effect.gen(function* () { + const hung = { inspect: () => new Promise(() => {}) }; + const byName = (path: string) => + Effect.forkChild(isTemporaryFolder(path, hung)).pipe( + Effect.tap(() => TestClock.adjust("500 millis")), + Effect.flatMap(Fiber.join), + ); + assert.isTrue(yield* byName("/tmp/review")); + assert.isFalse(yield* byName("/home/user/app")); + }), +); + +it.effect("resolves symlinks, and ignores a persistent TMPDIR", () => + Effect.gen(function* () { + const sandbox = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "temporary-folder-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(sandbox, { recursive: true, force: true })), + ); const link = NodePath.join(sandbox, "link"); NodeFS.symlinkSync(process.cwd(), link); // Under /tmp by name, but it leads to a folder that is not temporary. - expect(isTemporaryFolder(NodePath.join(link, "missing"))).toBe(false); - expect(isTemporaryFolder(NodePath.join(sandbox, "missing"))).toBe(true); - // Relative paths resolve against the working directory, which is not temporary here. - expect(isTemporaryFolder(NodePath.relative(process.cwd(), persistent))).toBe( - isTemporaryFolder(persistent), - ); - } finally { - NodeFS.rmSync(sandbox, { recursive: true, force: true }); - } -}); + assert.isFalse(yield* isTemporaryFolder(NodePath.join(link, "missing"))); + assert.isTrue(yield* isTemporaryFolder(NodePath.join(sandbox, "missing"))); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/project/temporaryFolder.ts b/apps/server/src/project/temporaryFolder.ts index c53c7d37ac4a1..0aa109fcea7f4 100644 --- a/apps/server/src/project/temporaryFolder.ts +++ b/apps/server/src/project/temporaryFolder.ts @@ -1,7 +1,10 @@ -// @effect-diagnostics nodeBuiltinImport:off - a synchronous check that backs an error message getter. -import * as NodeFS from "node:fs"; +// @effect-diagnostics nodeBuiltinImport:off - probes the filesystem with node:fs/promises. +import * as NodeFSP from "node:fs/promises"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; const TEMPORARY_PREFIXES = [ "/tmp", @@ -12,58 +15,94 @@ const TEMPORARY_PREFIXES = [ "/private/var/tmp", "/private/var/folders", ]; -// Per-user runtime directory; /run/media (external drives) is persistent and must not match. +// Per-user runtime directory, except its network and document mounts, which persist. const USER_RUNTIME_DIRECTORY = /^\/run\/user\/\d+(?:\/|$)/; -// Removable drives mount here; a missing path under it must not inherit /run's tmpfs. -const REMOVABLE_MEDIA = "/run/media/"; +const PERSISTENT_MOUNTS = /^\/run\/user\/\d+\/(?:gvfs|doc)(?:\/|$)|^\/run\/media\//; const LINUX_TMPFS_MAGIC = 0x01021994; +/** A dead network mount can hang a filesystem call, so the answer falls back after this. */ +const PROBE_TIMEOUT = Duration.millis(500); -/** The path as the filesystem sees it: `~` and relative paths expanded, `..` and symlinks resolved. */ -function resolveExisting(path: string): string { +/** `~` and relative paths expanded and `..` removed, without touching the filesystem. */ +function lexicalPath(path: string): string { const expanded = path === "~" || path.startsWith("~/") ? NodePath.join(NodeOS.homedir(), path.slice(1)) : path; - const absolute = NodePath.resolve(expanded); - // A missing folder is the usual case here, so resolve its nearest existing ancestor. - let existing = absolute; - while (!NodeFS.existsSync(existing) && NodePath.dirname(existing) !== existing) { - existing = NodePath.dirname(existing); - } - try { - return NodePath.join(NodeFS.realpathSync(existing), NodePath.relative(existing, absolute)); - } catch { - return absolute; - } + return NodePath.resolve(expanded); } -function onTmpfs(path: string): boolean { - let existing = path; - while (!NodeFS.existsSync(existing) && NodePath.dirname(existing) !== existing) { - existing = NodePath.dirname(existing); - } - try { - // Only Linux reports this magic number; elsewhere the prefixes decide. - return NodeFS.statfsSync(existing).type === LINUX_TMPFS_MAGIC; - } catch { - return false; - } +function underTemporaryName(path: string): boolean { + return ( + !PERSISTENT_MOUNTS.test(path) && + (TEMPORARY_PREFIXES.some((root) => path === root || path.startsWith(`${root}/`)) || + USER_RUNTIME_DIRECTORY.test(path)) + ); +} + +/** By name only, for callers that cannot wait on the filesystem. */ +export function isTemporaryPath(path: string): boolean { + return underTemporaryName(lexicalPath(path)); +} + +export interface FolderProbe { + /** The path with symlinks resolved, and whether its filesystem is a tmpfs. */ + readonly inspect: ( + path: string, + ) => Promise<{ readonly resolved: string; readonly tmpfs: boolean }>; } +const exists = (path: string) => + NodeFSP.access(path).then( + () => true, + () => false, + ); + +const nodeProbe: FolderProbe = { + inspect: async (path) => { + // A missing folder is the usual case here, so inspect its nearest existing ancestor. + let existing = path; + while (!(await exists(existing)) && NodePath.dirname(existing) !== existing) { + existing = NodePath.dirname(existing); + } + try { + const real = await NodeFSP.realpath(existing); + return { + resolved: NodePath.join(real, NodePath.relative(existing, path)), + tmpfs: (await NodeFSP.statfs(real)).type === LINUX_TMPFS_MAGIC, + }; + } catch { + return { resolved: path, tmpfs: false }; + } + }, +}; + /** * A folder the system clears at reboot: a tmpfs, or under a well-known temporary directory. - * `TMPDIR` is not trusted by itself; a persistent one is not temporary. + * `TMPDIR` is not trusted by itself. A path under a network or document mount is never + * temporary, and is not probed. A probe that outlasts the timeout leaves the name check. */ -export function isTemporaryFolder(path: string): boolean { - const resolved = resolveExisting(path); - if (resolved.startsWith(REMOVABLE_MEDIA)) return false; - return ( - TEMPORARY_PREFIXES.some((root) => resolved === root || resolved.startsWith(`${root}/`)) || - USER_RUNTIME_DIRECTORY.test(resolved) || - onTmpfs(resolved) +export const isTemporaryFolder = Effect.fn("isTemporaryFolder")(function* ( + path: string, + probe: FolderProbe = nodeProbe, +) { + const lexical = lexicalPath(path); + if (PERSISTENT_MOUNTS.test(lexical)) return false; + const byName = underTemporaryName(lexical); + const inspected = yield* Effect.tryPromise({ + try: () => probe.inspect(lexical), + catch: (cause) => cause, + }).pipe( + Effect.timeoutOption(PROBE_TIMEOUT), + Effect.orElseSucceed(() => Option.none()), ); -} + if (Option.isNone(inspected)) return byName; + const { resolved, tmpfs } = inspected.value; + return !PERSISTENT_MOUNTS.test(resolved) && (underTemporaryName(resolved) || tmpfs); +}); -export function temporaryFolderWarning(path: string): string | undefined { - return isTemporaryFolder(path) +export const temporaryFolderWarning = Effect.fn("temporaryFolderWarning")(function* ( + path: string, + probe?: FolderProbe, +) { + return (yield* isTemporaryFolder(path, probe)) ? `${path} is under a temporary folder, which the system clears at reboot; the project's threads then fail to start. Use a folder that persists.` : undefined; -} +}); diff --git a/apps/server/src/provider/Errors.ts b/apps/server/src/provider/Errors.ts index ae146bc70a0e0..b2f544767790b 100644 --- a/apps/server/src/provider/Errors.ts +++ b/apps/server/src/provider/Errors.ts @@ -1,6 +1,6 @@ import * as Schema from "effect/Schema"; -import { isTemporaryFolder } from "../project/temporaryFolder.ts"; +import { isTemporaryPath } from "../project/temporaryFolder.ts"; export class ProviderInstanceNotFoundError extends Schema.TaggedError()( "ProviderInstanceNotFoundError", @@ -24,7 +24,7 @@ export class ProviderWorkspaceMissingError extends Schema.TaggedError`, so the id itself contains a colon. - const webhook = /^(.+):webhook:(?:delivery:)?[^:]+$/.exec(key); + // Delivery ids are `delivery:`, or `delivery:relay:` for relayed requests. + const webhook = /^(.+?):webhook:delivery:.+$/.exec(key); return webhook === null ? null : { taskId: webhook[1]!, startedAtMs: null }; } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 5583279d247b6..59285d84031b3 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -1561,7 +1561,14 @@ it.effect( VALUES (${`item:${task}:${at}`}, ${threadId}, ${`run:${task}:${at}`}, 1, 'error', 'failed', ${at}, ${JSON.stringify({ failure: { message } })}) `; - for (const id of ["task:a", "task:folded", "task:matched", "task:clean", "task:hook"]) { + for (const id of [ + "task:a", + "task:folded", + "task:matched", + "task:clean", + "task:hook", + "task:relay", + ]) { yield* insertRow(sql, { id, next: null, enabled: 1, status: "succeeded" }, now); } // 10 days old, outside the window. @@ -1579,6 +1586,13 @@ it.effect( VALUES ('run:hook', ${threadId}, 7, 'codex', 'failed', '2026-11-12T22:00:00.000Z', ${JSON.stringify({ userMessageId: "scheduled-task-message:task:hook:webhook:delivery:3f2a-91" })}) `; + // A relayed webhook's delivery id is `delivery:relay:`. + yield* sql` + INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json) + VALUES ('run:relay', ${threadId}, 8, 'codex', 'cancelled', '2026-11-12T23:00:00.000Z', + ${JSON.stringify({ userMessageId: "scheduled-task-message:task:relay:webhook:delivery:relay:7c1d" })}) + `; // A failed dispatch that never created a run leaves only the task row. yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed', last_run_error = 'No such folder', last_run_at = '2026-11-12T20:00:00.000Z' WHERE task_id = 'task:folded'`; @@ -1614,6 +1628,13 @@ it.effect( lastStatus: "failed", lastError: null, }, + "task:relay": { + failed: 0, + cancelled: 1, + lastAt: "2026-11-12T23:00:00.000Z", + lastStatus: "cancelled", + lastError: null, + }, "task:matched": { failed: 1, cancelled: 0, From 6ee73805c13cb8f14e55f965b534fe03c432dd74 Mon Sep 17 00:00:00 2001 From: Chris OHearn Date: Thu, 12 Nov 2026 21:39:28 -0700 Subject: [PATCH 9/9] fix(server): keep the temporary-folder probe's error channel typed Co-Authored-By: Claude Sonnet 5.5 --- apps/server/src/project/temporaryFolder.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/server/src/project/temporaryFolder.ts b/apps/server/src/project/temporaryFolder.ts index 0aa109fcea7f4..790f081af0464 100644 --- a/apps/server/src/project/temporaryFolder.ts +++ b/apps/server/src/project/temporaryFolder.ts @@ -86,12 +86,10 @@ export const isTemporaryFolder = Effect.fn("isTemporaryFolder")(function* ( const lexical = lexicalPath(path); if (PERSISTENT_MOUNTS.test(lexical)) return false; const byName = underTemporaryName(lexical); - const inspected = yield* Effect.tryPromise({ - try: () => probe.inspect(lexical), - catch: (cause) => cause, - }).pipe( + // A probe that fails answers like one that times out: by name. + const inspected = yield* Effect.promise(() => probe.inspect(lexical).catch(() => undefined)).pipe( Effect.timeoutOption(PROBE_TIMEOUT), - Effect.orElseSucceed(() => Option.none()), + Effect.map(Option.flatMapNullishOr((value) => value)), ); if (Option.isNone(inspected)) return byName; const { resolved, tmpfs } = inspected.value;