From bdb4c89ebbd17ea125c1e251ef5afc9a93870822 Mon Sep 17 00:00:00 2001 From: George Nijo Date: Wed, 28 Oct 2026 22:19:41 -0400 Subject: [PATCH 1/2] fix(server): show the database error instead of "Provider turn failed." when event ingestion can't write When the server could not write provider events to its own SQLite database, the failed turn only said "Provider turn failed.", blaming the provider for a local storage failure. The ingestion failure path now uses the existing curated SQLite recovery messages (for example SQLITE_CANTOPEN or SQLITE_FULL), checking knownProviderFailureMessage first as the other callers that pass a message do. Co-Authored-By: Claude Opus 5.5 --- .../RunExecutionService.test.ts | 17 ++++++++- .../orchestration-v2/RunExecutionService.ts | 11 +++++- .../orchestration-v2/UserFacingErrors.test.ts | 37 ++++++++++++++++++- .../src/orchestration-v2/UserFacingErrors.ts | 18 +++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 394cc1527cb7d..dc6fad1fa53c0 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -36,7 +36,9 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import { SqlError, classifySqliteError } from "effect/sql/SqlError"; +import { toPersistenceSqlError } from "../persistence/Errors.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as CheckpointService from "./CheckpointService.ts"; import type { PendingOrchestrationEffectV2 } from "./EffectOutbox.ts"; @@ -2585,7 +2587,7 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c ReadonlyArray >([]); const ids = backgroundScenarioIds(`ingestion:${failureMode}`); - const { observed, finalEvents } = yield* captureRootRunTermination({ + const { written, observed, finalEvents } = yield* captureRootRunTermination({ key: `ingestion:${failureMode}`, initialProviderTurnId: null, rejectTerminalWrite: failureMode === "ownership-lost", @@ -2618,7 +2620,16 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c const error = new ProviderEventIngestor.ProviderEventPublishError({ providerSessionId: ProviderSessionId.make(`session:${failureMode}`), eventCount: 1, - cause: new Error("SQLITE_FULL: database or disk is full"), + cause: new EventSink.EventSinkWriteError({ + eventCount: 1, + cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")( + new SqlError({ + reason: classifySqliteError( + Object.assign(new Error("unable to open database file"), { errcode: 14 }), + ), + }), + ), + }), }); return failureMode === "defect" ? Effect.die(error) : Effect.fail(error); }, @@ -2662,6 +2673,8 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c failureMode === "interrupt-failure", ); assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]); + const error = written.find((item) => item.type === "error"); + assert.include(error?.failure.message ?? "", "(SQLITE_CANTOPEN)"); const repaired = finalEvents.find((event) => event.type === "provider-turn.updated"); if (failureMode === "interrupt-failure") { assert.isUndefined(repaired); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 3e3f37a15a936..ba3de522ce5b0 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -48,11 +48,13 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; import { + knownProviderFailureMessage, makeProviderFailure, makeProviderFailureTurnItem, } from "@t3tools/provider-core/server/failure"; import * as RunFinalizationService from "./RunFinalizationService.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import { userFacingDatabaseErrorMessage } from "./UserFacingErrors.ts"; export interface ProviderEventRoutingState { readonly ownedThreadIds: ReadonlySet; @@ -1536,6 +1538,7 @@ export const layer: Layer.Layer< : null; const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal); const openSubagents = yield* Ref.get(openRunOwnedSubagents); + const squashed = Cause.squash(cause); let nextRetryWarningAt = completedAt; yield* writeFinalRunEvents({ run: input.run, @@ -1551,7 +1554,13 @@ export const layer: Layer.Layer< ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), completedAt, terminal: makeFailedTerminalEvent( - makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }), + makeProviderFailure({ + cause: squashed, + message: + knownProviderFailureMessage(squashed) ?? + userFacingDatabaseErrorMessage(squashed), + class: "unknown", + }), latestItemOrdinal + 1, ), failureItemPersisted: false, diff --git a/apps/server/src/orchestration-v2/UserFacingErrors.test.ts b/apps/server/src/orchestration-v2/UserFacingErrors.test.ts index 1d69d1c88a3ae..3d7454d7539d2 100644 --- a/apps/server/src/orchestration-v2/UserFacingErrors.test.ts +++ b/apps/server/src/orchestration-v2/UserFacingErrors.test.ts @@ -1,6 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; @@ -8,7 +9,10 @@ import * as SqlClient from "effect/sql/SqlClient"; import { SqlError, classifySqliteError } from "effect/sql/SqlError"; import { toPersistenceSqlError } from "../persistence/Errors.ts"; -import { userFacingDispatchErrorMessage } from "./UserFacingErrors.ts"; +import { + userFacingDatabaseErrorMessage, + userFacingDispatchErrorMessage, +} from "./UserFacingErrors.ts"; describe("userFacingDispatchErrorMessage", () => { it.effect("explains a real database open failure through dispatch and persistence wrappers", () => @@ -81,3 +85,34 @@ describe("userFacingDispatchErrorMessage", () => { assert.notInclude(message, "private-query-data"); }); }); + +describe("userFacingDatabaseErrorMessage", () => { + it.each([ + { errcode: 14, expected: "server could not open a database file (SQLITE_CANTOPEN)" }, + { errcode: 13, expected: "database or disk is full (SQLITE_FULL)" }, + ])("explains SQLite error $errcode behind event ingestion wrappers", ({ errcode, expected }) => { + const sqlError = new SqlError({ + reason: classifySqliteError(Object.assign(new Error("private-driver-text"), { errcode })), + }); + const message = userFacingDatabaseErrorMessage( + Cause.fail( + new Error("Failed to publish 1 normalized provider event(s).", { + cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")( + sqlError, + ), + }), + ), + ); + + assert.include(message, expected); + assert.notInclude(message, "private-driver-text"); + }); + + it("ignores failures without a SQL error", () => { + assert.isUndefined( + userFacingDatabaseErrorMessage( + new Error("provider-text", { cause: Object.assign(new Error("nested"), { errcode: 14 }) }), + ), + ); + }); +}); diff --git a/apps/server/src/orchestration-v2/UserFacingErrors.ts b/apps/server/src/orchestration-v2/UserFacingErrors.ts index c47f1c2227d62..3c075c6660511 100644 --- a/apps/server/src/orchestration-v2/UserFacingErrors.ts +++ b/apps/server/src/orchestration-v2/UserFacingErrors.ts @@ -1,4 +1,5 @@ import { PROVIDER_DISPLAY_NAMES, type ProviderDriverKind } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Predicate from "effect/Predicate"; import * as SqlError from "effect/sql/SqlError"; @@ -37,6 +38,23 @@ function databaseErrorMessage(value: unknown): string | undefined { return "The T3 Code server could not complete this database request. Retry once; if it still fails, check the server logs for the database error."; } +/** Curated text for the first SQL error in a cause chain; never the error's own text. */ +export function userFacingDatabaseErrorMessage(cause: unknown): string | undefined { + const seen = new Set(); + let value = cause; + while (Predicate.isObject(value) && !seen.has(value)) { + seen.add(value); + if (Cause.isCause(value)) { + value = Cause.squash(value); + continue; + } + const message = databaseErrorMessage(value); + if (message !== undefined) return message; + value = value.cause; + } + return undefined; +} + const CAPABILITY_REJECTION_MESSAGES: Record string> = { queued_messages: (p) => `${p} cannot queue messages behind an active run.`, active_steering: (p) => From 81e8c2146d63c41df1e03d621ae00639e1a7a593 Mon Sep 17 00:00:00 2001 From: George Nijo Date: Wed, 28 Oct 2026 22:36:37 -0400 Subject: [PATCH 2/2] test(server): keep the database message proof at the ingestion handler Drop the helper tests that repeated the RunExecutionService case, and the unused Effect Cause branch: the only caller already squashes the cause. Co-Authored-By: Claude Opus 5.5 --- .../RunExecutionService.test.ts | 17 ++++----- .../orchestration-v2/UserFacingErrors.test.ts | 37 +------------------ .../src/orchestration-v2/UserFacingErrors.ts | 6 --- 3 files changed, 8 insertions(+), 52 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index dc6fad1fa53c0..697d2cbf33397 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2620,16 +2620,13 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c const error = new ProviderEventIngestor.ProviderEventPublishError({ providerSessionId: ProviderSessionId.make(`session:${failureMode}`), eventCount: 1, - cause: new EventSink.EventSinkWriteError({ - eventCount: 1, - cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")( - new SqlError({ - reason: classifySqliteError( - Object.assign(new Error("unable to open database file"), { errcode: 14 }), - ), - }), - ), - }), + cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")( + new SqlError({ + reason: classifySqliteError( + Object.assign(new Error("unable to open database file"), { errcode: 14 }), + ), + }), + ), }); return failureMode === "defect" ? Effect.die(error) : Effect.fail(error); }, diff --git a/apps/server/src/orchestration-v2/UserFacingErrors.test.ts b/apps/server/src/orchestration-v2/UserFacingErrors.test.ts index 3d7454d7539d2..1d69d1c88a3ae 100644 --- a/apps/server/src/orchestration-v2/UserFacingErrors.test.ts +++ b/apps/server/src/orchestration-v2/UserFacingErrors.test.ts @@ -1,7 +1,6 @@ import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; -import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; @@ -9,10 +8,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import { SqlError, classifySqliteError } from "effect/sql/SqlError"; import { toPersistenceSqlError } from "../persistence/Errors.ts"; -import { - userFacingDatabaseErrorMessage, - userFacingDispatchErrorMessage, -} from "./UserFacingErrors.ts"; +import { userFacingDispatchErrorMessage } from "./UserFacingErrors.ts"; describe("userFacingDispatchErrorMessage", () => { it.effect("explains a real database open failure through dispatch and persistence wrappers", () => @@ -85,34 +81,3 @@ describe("userFacingDispatchErrorMessage", () => { assert.notInclude(message, "private-query-data"); }); }); - -describe("userFacingDatabaseErrorMessage", () => { - it.each([ - { errcode: 14, expected: "server could not open a database file (SQLITE_CANTOPEN)" }, - { errcode: 13, expected: "database or disk is full (SQLITE_FULL)" }, - ])("explains SQLite error $errcode behind event ingestion wrappers", ({ errcode, expected }) => { - const sqlError = new SqlError({ - reason: classifySqliteError(Object.assign(new Error("private-driver-text"), { errcode })), - }); - const message = userFacingDatabaseErrorMessage( - Cause.fail( - new Error("Failed to publish 1 normalized provider event(s).", { - cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")( - sqlError, - ), - }), - ), - ); - - assert.include(message, expected); - assert.notInclude(message, "private-driver-text"); - }); - - it("ignores failures without a SQL error", () => { - assert.isUndefined( - userFacingDatabaseErrorMessage( - new Error("provider-text", { cause: Object.assign(new Error("nested"), { errcode: 14 }) }), - ), - ); - }); -}); diff --git a/apps/server/src/orchestration-v2/UserFacingErrors.ts b/apps/server/src/orchestration-v2/UserFacingErrors.ts index 3c075c6660511..7d47dffd1a762 100644 --- a/apps/server/src/orchestration-v2/UserFacingErrors.ts +++ b/apps/server/src/orchestration-v2/UserFacingErrors.ts @@ -1,5 +1,4 @@ import { PROVIDER_DISPLAY_NAMES, type ProviderDriverKind } from "@t3tools/contracts"; -import * as Cause from "effect/Cause"; import * as Predicate from "effect/Predicate"; import * as SqlError from "effect/sql/SqlError"; @@ -38,16 +37,11 @@ function databaseErrorMessage(value: unknown): string | undefined { return "The T3 Code server could not complete this database request. Retry once; if it still fails, check the server logs for the database error."; } -/** Curated text for the first SQL error in a cause chain; never the error's own text. */ export function userFacingDatabaseErrorMessage(cause: unknown): string | undefined { const seen = new Set(); let value = cause; while (Predicate.isObject(value) && !seen.has(value)) { seen.add(value); - if (Cause.isCause(value)) { - value = Cause.squash(value); - continue; - } const message = databaseErrorMessage(value); if (message !== undefined) return message; value = value.cause;