diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 394cc1527cb7d..697d2cbf33397 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -36,7 +36,9 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import { SqlError, classifySqliteError } from "effect/sql/SqlError"; +import { toPersistenceSqlError } from "../persistence/Errors.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as CheckpointService from "./CheckpointService.ts"; import type { PendingOrchestrationEffectV2 } from "./EffectOutbox.ts"; @@ -2585,7 +2587,7 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c ReadonlyArray >([]); const ids = backgroundScenarioIds(`ingestion:${failureMode}`); - const { observed, finalEvents } = yield* captureRootRunTermination({ + const { written, observed, finalEvents } = yield* captureRootRunTermination({ key: `ingestion:${failureMode}`, initialProviderTurnId: null, rejectTerminalWrite: failureMode === "ownership-lost", @@ -2618,7 +2620,13 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c const error = new ProviderEventIngestor.ProviderEventPublishError({ providerSessionId: ProviderSessionId.make(`session:${failureMode}`), eventCount: 1, - cause: new Error("SQLITE_FULL: database or disk is full"), + cause: toPersistenceSqlError("OrchestrationEventStore.appendAgentEvents:insert")( + new SqlError({ + reason: classifySqliteError( + Object.assign(new Error("unable to open database file"), { errcode: 14 }), + ), + }), + ), }); return failureMode === "defect" ? Effect.die(error) : Effect.fail(error); }, @@ -2662,6 +2670,8 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c failureMode === "interrupt-failure", ); assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]); + const error = written.find((item) => item.type === "error"); + assert.include(error?.failure.message ?? "", "(SQLITE_CANTOPEN)"); const repaired = finalEvents.find((event) => event.type === "provider-turn.updated"); if (failureMode === "interrupt-failure") { assert.isUndefined(repaired); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 3e3f37a15a936..ba3de522ce5b0 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -48,11 +48,13 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; import { + knownProviderFailureMessage, makeProviderFailure, makeProviderFailureTurnItem, } from "@t3tools/provider-core/server/failure"; import * as RunFinalizationService from "./RunFinalizationService.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; +import { userFacingDatabaseErrorMessage } from "./UserFacingErrors.ts"; export interface ProviderEventRoutingState { readonly ownedThreadIds: ReadonlySet; @@ -1536,6 +1538,7 @@ export const layer: Layer.Layer< : null; const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal); const openSubagents = yield* Ref.get(openRunOwnedSubagents); + const squashed = Cause.squash(cause); let nextRetryWarningAt = completedAt; yield* writeFinalRunEvents({ run: input.run, @@ -1551,7 +1554,13 @@ export const layer: Layer.Layer< ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), completedAt, terminal: makeFailedTerminalEvent( - makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }), + makeProviderFailure({ + cause: squashed, + message: + knownProviderFailureMessage(squashed) ?? + userFacingDatabaseErrorMessage(squashed), + class: "unknown", + }), latestItemOrdinal + 1, ), failureItemPersisted: false, diff --git a/apps/server/src/orchestration-v2/UserFacingErrors.ts b/apps/server/src/orchestration-v2/UserFacingErrors.ts index c47f1c2227d62..7d47dffd1a762 100644 --- a/apps/server/src/orchestration-v2/UserFacingErrors.ts +++ b/apps/server/src/orchestration-v2/UserFacingErrors.ts @@ -37,6 +37,18 @@ function databaseErrorMessage(value: unknown): string | undefined { return "The T3 Code server could not complete this database request. Retry once; if it still fails, check the server logs for the database error."; } +export function userFacingDatabaseErrorMessage(cause: unknown): string | undefined { + const seen = new Set(); + let value = cause; + while (Predicate.isObject(value) && !seen.has(value)) { + seen.add(value); + const message = databaseErrorMessage(value); + if (message !== undefined) return message; + value = value.cause; + } + return undefined; +} + const CAPABILITY_REJECTION_MESSAGES: Record string> = { queued_messages: (p) => `${p} cannot queue messages behind an active run.`, active_steering: (p) =>