From d7ed1da76f566ac7343ded05f823c595d28b7c06 Mon Sep 17 00:00:00 2001 From: TheSalaty Date: Sat, 7 Nov 2026 22:12:31 +0100 Subject: [PATCH 1/3] fix(speech): dictation no longer loses the recording when the GPU fails A GPU out-of-memory, driver error or repeating output during local transcription failed the whole recording, and live streaming kept no copy of the audio, so everything that was said was gone. - The worker retries a failed piece in halves for memory, backend and repetition errors, not only truncation. - A GPU or CPU the person chose can retry on the other hardware through a new setting, off by default. Auto keeps its existing CPU fallback. - A failed stream frees the model before the client hears of it. - The browser keeps the streamed audio until the transcript is inserted. A failed live stream gets one pass over that recording, and if that fails too the recording stays for a Retry button instead of being dropped. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/speech/SpeechService.test.ts | 77 ++++++++++++------- apps/server/src/speech/SpeechService.ts | 71 ++++++++++------- apps/server/src/speech/http.ts | 4 +- apps/server/src/speech/native.test.ts | 24 ++++++ apps/server/src/speech/native.ts | 10 ++- .../components/chat/VoiceInputErrorPill.tsx | 18 +++-- .../settings/VoiceSettingsPanel.tsx | 24 ++++++ .../src/components/settings/settingsSearch.ts | 2 +- apps/web/src/speech/browserPcmCapture.ts | 7 +- apps/web/src/speech/browserVoiceInput.ts | 5 +- .../src/voice-input/controller.test.ts | 36 +++++++++ .../src/voice-input/controller.ts | 37 +++++---- .../src/voice-input/environment.ts | 7 +- .../client-runtime/src/voice-input/index.ts | 1 + packages/contracts/src/settings.ts | 5 ++ packages/contracts/src/speech.ts | 2 + 16 files changed, 248 insertions(+), 82 deletions(-) diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 41db47f0376a4..4644d1dd08070 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -742,24 +742,34 @@ it.effect( }), ); -it.effect("reports a failure on a GPU the person chose instead of switching to CPU", () => - Effect.gen(function* () { - const failure = new Error("GPU failed"); - native.transcribe.mockRejectedValueOnce(failure); - const speech = yield* SpeechService.SpeechService; - const acceleration = 'gpu:["vulkan","gpu-1"]'; - yield* speech.updateAcceleration(acceleration); - const result = yield* Effect.result( - speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), - ); - expect(Result.isFailure(result) && result.failure).toMatchObject({ - _tag: "SpeechOperationError", - operation: "inference", - cause: failure, - }); - expect(loadNative).toHaveBeenCalledTimes(1); - expect(loadNative.mock.calls[0]?.[3]).toBe(acceleration); - }).pipe(Effect.provide(layer)), +it.effect.each([ + ['gpu:["vulkan","gpu-1"]', "cpu"], + ["cpu", "auto"], +] as const)( + "retries a recording from a chosen %s on %s only when fallback is on", + ([acceleration, other]) => + Effect.gen(function* () { + const speech = yield* SpeechService.SpeechService; + for (const fallback of [false, true]) { + loadNative.mockClear(); + const failure = new Error("device failed"); + native.transcribe.mockRejectedValueOnce(failure); + yield* speech.updateAcceleration(acceleration, fallback); + const result = yield* Effect.result( + speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + ); + if (fallback) expect(Result.isSuccess(result) && result.success).toBe("hello"); + else + expect(Result.isFailure(result) && result.failure).toMatchObject({ + _tag: "SpeechOperationError", + operation: "inference", + cause: failure, + }); + expect(loadNative.mock.calls.map((call) => call[3])).toEqual( + fallback ? [acceleration, other] : [acceleration], + ); + } + }).pipe(Effect.provide(layer)), ); it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once", () => @@ -874,9 +884,15 @@ it.effect.each([false, true])( }).pipe(Effect.provide(layer)), ); -it.effect.each(["auto", 'gpu:["vulkan","gpu-1"]', "cpu"] as const)( - "reports finalization failure without repeated or explicit-backend fallback: %s", - (acceleration) => +it.effect.each([ + ["auto", false], + ['gpu:["vulkan","gpu-1"]', false], + ["cpu", false], + ['gpu:["vulkan","gpu-1"]', true], + ["cpu", true], +] as const)( + "reports finalization failure after at most one fallback: %s, fallback %s", + ([acceleration, fallback]) => Effect.gen(function* () { const failure = new Error("finish failed"); const worker = { @@ -886,15 +902,24 @@ it.effect.each(["auto", 'gpu:["vulkan","gpu-1"]', "cpu"] as const)( throw failure; }), }; - const cpu = { ...worker, backend: "CPU", dispose: vi.fn() }; + const other = { + ...worker, + backend: acceleration === "cpu" ? "Vulkan0" : "CPU", + dispose: vi.fn(), + }; + const retries = acceleration === "auto" || fallback; loadNative.mockResolvedValueOnce(worker); - if (acceleration === "auto") loadNative.mockResolvedValueOnce(cpu); + if (retries) loadNative.mockResolvedValueOnce(other); const speech = yield* SpeechService.SpeechService; - yield* speech.updateAcceleration(acceleration); + yield* speech.updateAcceleration(acceleration, fallback); const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS); expect(Result.isFailure(yield* Effect.result(stream.finish))).toBe(true); - expect(worker.finish).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1); - expect(loadNative).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1); + // The failed stream frees the model before its socket closes, so a retry is not busy. + expect(yield* speech.status).toMatchObject({ state: "ready" }); + expect(worker.finish).toHaveBeenCalledTimes(retries ? 2 : 1); + expect(loadNative.mock.calls.map((call) => call[3])).toEqual( + retries ? [acceleration, acceleration === "cpu" ? "auto" : "cpu"] : [acceleration], + ); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts index c5f0faebe18f4..e979ddaca13d5 100644 --- a/apps/server/src/speech/SpeechService.ts +++ b/apps/server/src/speech/SpeechService.ts @@ -148,6 +148,7 @@ export class SpeechService extends Context.Service< ) => Effect.Effect; readonly updateAcceleration: ( acceleration: SpeechAcceleration, + fallback?: boolean, ) => Effect.Effect; readonly updateModelUnloadTimeout: ( timeout: SpeechModelUnloadTimeout, @@ -236,9 +237,16 @@ const make = Effect.gen(function* () { }; // Auto placement may land on a GPU the native engine cannot actually run; one retry on - // this host's CPU recovers it. A GPU the person chose, and CPU itself, report the error. - const canRetryOnCpu = (acceleration: SpeechAcceleration, loaded: LoadedModel) => - acceleration === "auto" && loaded.backend.toLowerCase() !== "cpu"; + // this host's CPU recovers it. A device the person chose retries on the other only when allowed. + const fallbackFor = ( + settings: { speechAcceleration: SpeechAcceleration; speechAccelerationFallback: boolean }, + loaded: LoadedModel, + ) => { + const onCpu = loaded.backend.toLowerCase() === "cpu"; + if (settings.speechAcceleration === "auto") return onCpu ? undefined : "cpu"; + if (!settings.speechAccelerationFallback) return undefined; + return onCpu ? "auto" : "cpu"; + }; yield* Effect.addFinalizer(() => Effect.promise(async () => { @@ -471,6 +479,7 @@ const make = Effect.gen(function* () { languages: definition.languages, supportsLanguageDetection: definition.supportsLanguageDetection, acceleration: settings.speechAcceleration, + accelerationFallback: settings.speechAccelerationFallback, benchmarkTier: settings.speechAcceleration === "cpu" || (settings.speechAcceleration === "auto" && autoUsesCpu) @@ -649,21 +658,22 @@ const make = Effect.gen(function* () { let inference = loaded; const result = yield* transcribeWith(loaded).pipe( Effect.catch((error) => { - if (!canRetryOnCpu(settings.speechAcceleration, loaded)) return Effect.fail(error); + const fallback = fallbackFor(settings, loaded); + if (!fallback) return Effect.fail(error); return Effect.tryPromise({ try: (signal) => loadModel( modelPath, definition, AbortSignal.any([signal, lifetime.signal]), - "cpu", + fallback, ), catch: (cause) => speechError("model preparation", cause), }).pipe( - Effect.flatMap((cpu) => { - autoCpuModel = definition.id; - inference = cpu; - return transcribeWith(cpu); + Effect.flatMap((other) => { + if (settings.speechAcceleration === "auto") autoCpuModel = definition.id; + inference = other; + return transcribeWith(other); }), ); }), @@ -777,19 +787,19 @@ const make = Effect.gen(function* () { correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary); started = true; }; - // Audio already fed is kept so a CPU restart can replay it; the limit caps it at one recording. + // Audio already fed is kept so a fallback restart can replay it; the limit caps it at one recording. const received: Float32Array[] = []; - let retriedOnCpu = false; - const recoverOnCpu = async (cause: unknown) => { + let recovered = false; + const recover = async (cause: unknown) => { started = false; - if (closed || signal.aborted || retriedOnCpu) throw cause; - if (!canRetryOnCpu(settings.speechAcceleration, streamModel)) throw cause; - retriedOnCpu = true; + const fallback = fallbackFor(settings, streamModel); + if (closed || signal.aborted || recovered || !fallback) throw cause; + recovered = true; forgetModel(streamModel); await streamModel.dispose(); if (closed || signal.aborted) throw cause; - streamModel = await loadModel(modelPath, definition, signal, "cpu"); - autoCpuModel = definition.id; + streamModel = await loadModel(modelPath, definition, signal, fallback); + if (settings.speechAcceleration === "auto") autoCpuModel = definition.id; loaded = streamModel; await beginStream(); let update: Awaited> | undefined; @@ -803,7 +813,7 @@ const make = Effect.gen(function* () { try { await beginStream(); } catch (cause) { - await recoverOnCpu(cause); + await recover(cause); } }); let byteLength = 0; @@ -840,7 +850,7 @@ const make = Effect.gen(function* () { try { update = await streamModel.feed(pcm); } catch (cause) { - update = await recoverOnCpu(cause); + update = await recover(cause); revisionBase += lastNativeRevision; lastNativeRevision = 0; } @@ -859,7 +869,7 @@ const make = Effect.gen(function* () { let text: string; try { text = await streamModel.finish().catch(async (cause: unknown) => { - await recoverOnCpu(cause); + await recover(cause); if (closed || signal.aborted) throw cause; return streamModel.finish(); }); @@ -884,10 +894,14 @@ const make = Effect.gen(function* () { transcribe, startStream: Effect.fn("SpeechService.startStream")(function* (options) { const session = yield* Scope.fork(yield* Effect.scope); - return yield* openStream(options).pipe( - Scope.provide(session), - Effect.onError((cause) => Scope.close(session, Exit.failCause(cause))), - ); + const closeOnError = (effect: Effect.Effect) => + effect.pipe(Effect.onError((cause) => Scope.close(session, Exit.failCause(cause)))); + const stream = yield* closeOnError(openStream(options).pipe(Scope.provide(session))); + // A failed stream frees the model before the client hears of it, so its retry is not busy. + return { + feed: (pcm: Uint8Array) => closeOnError(stream.feed(pcm)), + finish: closeOnError(stream.finish), + }; }), downloadModel: Effect.fn("SpeechService.downloadModel")((modelId) => requireSupported.pipe( @@ -951,10 +965,11 @@ const make = Effect.gen(function* () { ), ), ), - updateAcceleration: Effect.fn("SpeechService.updateAcceleration")((acceleration) => - writeSettings("acceleration update", { speechAcceleration: acceleration }).pipe( - Effect.andThen(freshStatus("acceleration update")), - ), + updateAcceleration: Effect.fn("SpeechService.updateAcceleration")((acceleration, fallback) => + writeSettings("acceleration update", { + speechAcceleration: acceleration, + ...(fallback === undefined ? {} : { speechAccelerationFallback: fallback }), + }).pipe(Effect.andThen(freshStatus("acceleration update"))), ), updateModelUnloadTimeout: Effect.fn("SpeechService.updateModelUnloadTimeout")((timeout) => writeSettings("model unload timeout update", { speechModelUnloadTimeout: timeout }).pipe( diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index eb4a7a01b88fb..2136993513c63 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -168,7 +168,9 @@ export const speechHttpApiLayer = HttpApiBuilder.group( "updateAcceleration", Effect.fn("environment.voice.updateAcceleration")(function* (args) { yield* guard(args.endpoint.name, AuthSettingsWriteScope); - return yield* speech.updateAcceleration(args.payload.acceleration).pipe(mapSpeechError); + return yield* speech + .updateAcceleration(args.payload.acceleration, args.payload.fallback) + .pipe(mapSpeechError); }), ) .handle( diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts index 8712c63ed347d..a3ee311c7e535 100644 --- a/apps/server/src/speech/native.test.ts +++ b/apps/server/src/speech/native.test.ts @@ -51,6 +51,30 @@ it("returns transcription from an isolated process and disposes it", async () => } }); +it("splits a piece the engine could not finish instead of failing the recording", async () => { + for (const error of ["OutputRepetition", "InputTooLong", "OutOfMemory", "BackendError"]) { + const model = await loadNativeSpeechModel( + "unused.gguf", + new AbortController().signal, + fixture(`async (pcm) => { + class OutputTruncated extends Error {} + const errors = { OutputRepetition: class OutputRepetition extends OutputTruncated {}, + InputTooLong: class InputTooLong extends Error {}, OutOfMemory: class OutOfMemory extends Error {}, + BackendError: class BackendError extends Error {} }; + if (pcm.length > 5000) throw new errors.${error}("too much"); + return { text: String(pcm.length) }; + }`), + ); + try { + expect( + await model.transcribe(new Float32Array(16000), { timestamps: "none" }, false), + ).toEqual({ text: "4000 4000 4000 4000" }); + } finally { + await model.dispose(); + } + } +}); + it("probes generic vocabulary support and delivers ordered terms to offline inference", async () => { for (const supported of [false, true]) { const model = await loadNativeSpeechModel( diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts index 8331e1b5e0ec1..21b577870ee14 100644 --- a/apps/server/src/speech/native.ts +++ b/apps/server/src/speech/native.ts @@ -31,6 +31,13 @@ const newAudio = (filterSilence) => createSpeechAudio({ frameSamples: 512, filterSilence, nativeStreaming: model.capabilities.supportsStreaming, longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs, }); +// Matched by class name: OutputRepetition extends OutputTruncated, and smaller input needs less memory. +const splittable = new Set(["OutputTruncated", "InputTooLong", "OutOfMemory", "BackendError"]); +const canSplit = (error) => { + for (let proto = error && Object.getPrototypeOf(error); proto; proto = Object.getPrototypeOf(proto)) + if (splittable.has(proto.constructor?.name)) return true; + return false; +}; const openAsr = async () => { if (nativeStreaming) { session = model.createSession(); @@ -71,8 +78,7 @@ const transcribe = async (pcm) => { const result = await model.transcribe(pcm, options); texts.push(result.text); } catch (error) { - if ((error?.constructor?.name !== "OutputTruncated" && - error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error; + if (!canSplit(error) || pcm.length <= 4000) throw error; const middle = Math.floor(pcm.length / 2); await transcribe(pcm.subarray(0, middle)); await transcribe(pcm.subarray(middle)); diff --git a/apps/web/src/components/chat/VoiceInputErrorPill.tsx b/apps/web/src/components/chat/VoiceInputErrorPill.tsx index 26f4895e1b3f9..7d48a832b6ea6 100644 --- a/apps/web/src/components/chat/VoiceInputErrorPill.tsx +++ b/apps/web/src/components/chat/VoiceInputErrorPill.tsx @@ -1,4 +1,7 @@ -import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input"; +import { + MICROPHONE_PERMISSION_ERROR, + RETAINED_RECORDING_ERROR, +} from "@t3tools/client-runtime/voice-input"; import { useNavigate } from "@tanstack/react-router"; import { RotateCcwIcon, SettingsIcon, XIcon } from "lucide-react"; @@ -18,7 +21,7 @@ const VOICE_ERROR_LABELS: Record = { "Could not start voice recording.": "Couldn’t start recording", "Voice recording was interrupted.": "Recording interrupted", "Could not finish voice recording.": "Couldn’t finish recording", - "Could not transcribe this recording.": "Transcription failed", + [RETAINED_RECORDING_ERROR]: "Transcription failed", "Voice transcription on this environment failed.": "Transcription failed", "The draft changed while voice input was running. The transcript was not added.": "Draft changed", "No speech was detected.": "No speech detected", @@ -40,10 +43,12 @@ export function VoiceInputErrorPill(props: { }) { const navigate = useNavigate(); const openSettings = props.errorAction === "settings"; + const retryTranscription = + props.error === RETAINED_RECORDING_ERROR || + props.error === "Voice transcription on this environment failed."; const actionLabel = openSettings ? "Open voice settings" - : props.error === "Could not transcribe this recording." || - props.error === "Voice transcription on this environment failed." + : retryTranscription ? "Retry transcription" : "Record again"; return ( @@ -89,8 +94,8 @@ export function VoiceInputErrorPill(props: { render={ } /> diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 495abdefb6b6e..7db6c057abd48 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -729,6 +729,30 @@ function AdvancedSpeechSettings({ } /> + {supported.acceleration !== "auto" && supported.accelerationFallback !== undefined ? ( + + void run("Could not change the hardware fallback", () => + runEnvironmentSpeech( + updateEnvironmentSpeechAcceleration(prepared, supported.acceleration, checked), + ), + ) + } + /> + } + /> + ) : null} { stopped?.reject(new Error("Microphone processing failed.")); @@ -149,7 +150,7 @@ export function createBrowserPcmCapture( worklet.port.postMessage("stop"); try { await pending; - if (!feed && chunks.length) { + if (chunks.length) { const blob = new Blob(chunks, { type: "application/octet-stream" }); recordingUri = URL.createObjectURL(blob); } diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index 345ebf78f2bb4..c879a4db37631 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -120,7 +120,10 @@ export function createBrowserVoiceInputPlatform(input: { options: recordingOptions, signal: options.signal, onText: input.onText, - onError: (error) => input.onError(error.message), + // Recording continues after a stream failure; finishing retries the retained audio. + onError: (error) => { + if (!input.hasAudioConsent()) input.onError(error.message); + }, }), }; }); diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 41e56933ce104..55241221fd081 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -113,6 +113,42 @@ describe("environment voice input", () => { expect(harness.deleted).toEqual(["file:///voice.m4a"]); }); + it("falls back to the retained recording when a live stream fails, and keeps it for retry", async () => { + const transcribe = vi + .fn() + .mockResolvedValueOnce("recovered text") + .mockRejectedValueOnce(new Error("inference failed")) + .mockResolvedValueOnce("retried text"); + const finish = vi.fn(async () => { + throw new Error("stream failed"); + }); + const harness = createHarness({ + retryFailedRecording: true, + getTranscriber: () => ({ + prepare: async () => ({ ...preparedTranscription(transcribe), streaming: { finish } }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.commits.map(({ text }) => text)).toEqual(["hello recovered text"]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + + harness.deleted.length = 0; + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + expect(harness.deleted).toEqual([]); + await harness.controller.start(); + expect(finish).toHaveBeenCalledTimes(2); + expect(transcribe).toHaveBeenCalledTimes(3); + expect(harness.recorder.record).toHaveBeenCalledTimes(2); + expect(harness.commits.map(({ text }) => text)).toEqual([ + "hello recovered text", + "hello retried text", + ]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + }); + it("deletes retained failed audio when the user cancels instead of retrying", async () => { const harness = createHarness({ retryFailedRecording: true, diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 668705546a489..86cfb285d71fa 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -11,6 +11,9 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60; +/** The recording is kept, so retrying transcribes it again without recording. */ +export const RETAINED_RECORDING_ERROR = "Could not transcribe this recording."; + export const MICROPHONE_PERMISSION_ERROR = "Microphone access is required for voice input. Allow access in your browser or system settings and try again."; @@ -503,31 +506,39 @@ export class VoiceInputController { const signal = this.transcriptionAbortController.signal; const capturedDraft = this.capturedDraft; let transcript: string; + const transcribeRecording = () => + runTranscriptionOperation(() => transcription.transcribe(recordingUri!, { signal })); try { const live = this.liveTranscription ? await this.liveTranscription : null; if (live && "error" in live) throw live.error; + const streaming = transcription.streaming; transcript = live?.text ?? - (await runTranscriptionOperation(() => - transcription.streaming - ? transcription.streaming.finish({ signal }) - : transcription.transcribe(recordingUri!, { signal }), - )); + (streaming + ? await runTranscriptionOperation(() => streaming.finish({ signal })).catch( + (error: unknown) => { + // A failed live stream gets one pass over the retained recording. + if (!recordingUri || signal.aborted || !this.isCurrent(operationToken)) + throw error; + return transcribeRecording(); + }, + ) + : await transcribeRecording()); } catch (error) { if (this.isCurrent(operationToken)) { - if ( - this.dependencies.retryFailedRecording && - !this.liveTranscription && - !transcription.streaming && - recordingUri - ) { + const retained = + this.dependencies.retryFailedRecording && !this.liveTranscription && recordingUri; + if (retained) { this.retryRecording = { uri: recordingUri, ownerKey: capturedDraft.ownerKey, - transcription, + transcription: { locale: transcription.locale, transcribe: transcription.transcribe }, }; } - this.setError(transcriptionErrorMessage(error), "retry"); + this.setError( + retained ? RETAINED_RECORDING_ERROR : transcriptionErrorMessage(error), + "retry", + ); } return; } diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 52fa5a0d1fe5b..d9a4d38ab96a5 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -270,13 +270,18 @@ export const postProcessEnvironmentTranscript = ( export const updateEnvironmentSpeechAcceleration = ( prepared: PreparedConnection, acceleration: SpeechAcceleration, + fallback?: boolean, ) => request({ group: "voice", prepared, method: "POST", path: (urls) => urls.updateAcceleration(), - run: ({ client, headers }) => client.updateAcceleration({ headers, payload: { acceleration } }), + run: ({ client, headers }) => + client.updateAcceleration({ + headers, + payload: { acceleration, ...(fallback === undefined ? {} : { fallback }) }, + }), }); export const updateEnvironmentSpeechModelUnloadTimeout = ( diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts index 5843267800f67..5d96a1d715dbb 100644 --- a/packages/client-runtime/src/voice-input/index.ts +++ b/packages/client-runtime/src/voice-input/index.ts @@ -2,6 +2,7 @@ export { VoiceInputController, VOICE_RECORDING_LIMIT_SECONDS, MICROPHONE_PERMISSION_ERROR, + RETAINED_RECORDING_ERROR, voiceInputBlocksSubmission, type VoiceDraftSnapshot, type VoiceInputControllerDependencies, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index cd112d84bf9f4..5850af8c79216 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1364,6 +1364,10 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), ), speechAcceleration: SpeechAcceleration.pipe(Schema.withDecodingDefault(Effect.succeed("auto"))), + // A CPU or GPU the person chose may retry a failed recording on the other one. Auto always does. + speechAccelerationFallback: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + ), speechModelUnloadTimeout: SpeechModelUnloadTimeout.pipe( Schema.withDecodingDefault(Effect.succeed("min_15")), ), @@ -1812,6 +1816,7 @@ export const ServerSettingsPatch = Schema.Struct({ ), speechModelId: Schema.optionalKey(Schema.String), speechAcceleration: Schema.optionalKey(SpeechAcceleration), + speechAccelerationFallback: Schema.optionalKey(Schema.Boolean), speechModelUnloadTimeout: Schema.optionalKey(SpeechModelUnloadTimeout), speechProjectCustomWords: Schema.optionalKey(SpeechCustomWords), speechPostProcessingModelSelection: Schema.optionalKey(Schema.NullOr(ModelSelection)), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index ac87096a24cce..4ff226e36269f 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -219,6 +219,7 @@ const SupportedSpeechStatus = Schema.Struct({ languages: Schema.optionalKey(Schema.Array(Schema.String)), supportsLanguageDetection: Schema.optionalKey(Schema.Boolean), acceleration: SpeechAcceleration, + accelerationFallback: Schema.optionalKey(Schema.Boolean), benchmarkTier: SpeechBenchmarkTier, modelUnloadTimeout: SpeechModelUnloadTimeout, gpuDevices: Schema.Array(SpeechGpuDevice), @@ -260,6 +261,7 @@ export const EnvironmentSpeechTranscriptionResult = Schema.Struct({ text: Schema export const EnvironmentSpeechAccelerationRequest = Schema.Struct({ acceleration: SpeechAcceleration, + fallback: Schema.optionalKey(Schema.Boolean), }); export const EnvironmentSpeechModelUnloadTimeoutRequest = Schema.Struct({ timeout: SpeechModelUnloadTimeout, From c46e761fdce5b4c22e25a85071bc2fbe924eebf2 Mon Sep 17 00:00:00 2001 From: TheSalaty Date: Sat, 7 Nov 2026 22:43:47 +0100 Subject: [PATCH 2/3] fix(web): offer a transcription retry only when the recording was kept The kept-recording message matched the generic transcription error, so a failure that kept no audio still offered "Retry transcription", which records again. It now has its own text, and only it gets the labeled Retry button. The split test also covers OutputTruncated directly. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/speech/native.test.ts | 10 +++++++-- .../components/chat/VoiceInputErrorPill.tsx | 11 +++++----- .../src/voice-input/controller.test.ts | 21 +++++++++++++++++++ .../src/voice-input/controller.ts | 3 ++- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts index a3ee311c7e535..7eb45772d5d05 100644 --- a/apps/server/src/speech/native.test.ts +++ b/apps/server/src/speech/native.test.ts @@ -52,13 +52,19 @@ it("returns transcription from an isolated process and disposes it", async () => }); it("splits a piece the engine could not finish instead of failing the recording", async () => { - for (const error of ["OutputRepetition", "InputTooLong", "OutOfMemory", "BackendError"]) { + for (const error of [ + "OutputTruncated", + "OutputRepetition", + "InputTooLong", + "OutOfMemory", + "BackendError", + ]) { const model = await loadNativeSpeechModel( "unused.gguf", new AbortController().signal, fixture(`async (pcm) => { class OutputTruncated extends Error {} - const errors = { OutputRepetition: class OutputRepetition extends OutputTruncated {}, + const errors = { OutputTruncated, OutputRepetition: class OutputRepetition extends OutputTruncated {}, InputTooLong: class InputTooLong extends Error {}, OutOfMemory: class OutOfMemory extends Error {}, BackendError: class BackendError extends Error {} }; if (pcm.length > 5000) throw new errors.${error}("too much"); diff --git a/apps/web/src/components/chat/VoiceInputErrorPill.tsx b/apps/web/src/components/chat/VoiceInputErrorPill.tsx index 7d48a832b6ea6..99159aa682b09 100644 --- a/apps/web/src/components/chat/VoiceInputErrorPill.tsx +++ b/apps/web/src/components/chat/VoiceInputErrorPill.tsx @@ -21,6 +21,7 @@ const VOICE_ERROR_LABELS: Record = { "Could not start voice recording.": "Couldn’t start recording", "Voice recording was interrupted.": "Recording interrupted", "Could not finish voice recording.": "Couldn’t finish recording", + "Could not transcribe this recording.": "Transcription failed", [RETAINED_RECORDING_ERROR]: "Transcription failed", "Voice transcription on this environment failed.": "Transcription failed", "The draft changed while voice input was running. The transcript was not added.": "Draft changed", @@ -43,9 +44,9 @@ export function VoiceInputErrorPill(props: { }) { const navigate = useNavigate(); const openSettings = props.errorAction === "settings"; + const retained = props.error === RETAINED_RECORDING_ERROR; const retryTranscription = - props.error === RETAINED_RECORDING_ERROR || - props.error === "Voice transcription on this environment failed."; + retained || props.error === "Voice transcription on this environment failed."; const actionLabel = openSettings ? "Open voice settings" : retryTranscription @@ -94,8 +95,8 @@ export function VoiceInputErrorPill(props: { render={ } /> diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 55241221fd081..10c63711d5770 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { + RETAINED_RECORDING_ERROR, resetVoiceInputGlobalsForTests, resolveTranscriptCommit, VoiceInputController, @@ -149,6 +150,26 @@ describe("environment voice input", () => { expect(harness.deleted).toEqual(["file:///voice.m4a"]); }); + it("offers a transcription retry only when the failed recording was kept", async () => { + for (const retryFailedRecording of [true, false]) { + resetVoiceInputGlobalsForTests(); + const fail = async () => { + throw new Error("failed"); + }; + const harness = createHarness({ + retryFailedRecording, + getTranscriber: () => ({ + prepare: async () => ({ ...preparedTranscription(fail), streaming: { finish: fail } }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState.error === RETAINED_RECORDING_ERROR).toBe( + retryFailedRecording, + ); + } + }); + it("deletes retained failed audio when the user cancels instead of retrying", async () => { const harness = createHarness({ retryFailedRecording: true, diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 86cfb285d71fa..5f0db7687e854 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -12,7 +12,8 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60; /** The recording is kept, so retrying transcribes it again without recording. */ -export const RETAINED_RECORDING_ERROR = "Could not transcribe this recording."; +export const RETAINED_RECORDING_ERROR = + "Could not transcribe this recording. It was kept for a retry."; export const MICROPHONE_PERMISSION_ERROR = "Microphone access is required for voice input. Allow access in your browser or system settings and try again."; From 3b257107291e624eb434817199ebc91b3c5543b7 Mon Sep 17 00:00:00 2001 From: Salaty Date: Wed, 11 Nov 2026 12:56:31 +0100 Subject: [PATCH 3/3] fix(speech): fall back to the other hardware when the chosen device fails to load With fallback on, a chosen GPU or CPU only recovered from failures during inference. A device that failed to load still surfaced the error, unlike Auto, which the loader already retries on the CPU. Model preparation, batch transcription and streaming now load through one helper that retries on the other hardware when the person allowed it. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/speech/SpeechService.test.ts | 40 ++++++++++---------- apps/server/src/speech/SpeechService.ts | 32 +++++++++++++--- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 78bc6227b7cca..2398962845e40 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -752,25 +752,27 @@ it.effect.each([ ([acceleration, other]) => Effect.gen(function* () { const speech = yield* SpeechService.SpeechService; - for (const fallback of [false, true]) { - loadNative.mockClear(); - const failure = new Error("device failed"); - native.transcribe.mockRejectedValueOnce(failure); - yield* speech.updateAcceleration(acceleration, fallback); - const result = yield* Effect.result( - speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), - ); - if (fallback) expect(Result.isSuccess(result) && result.success).toBe("hello"); - else - expect(Result.isFailure(result) && result.failure).toMatchObject({ - _tag: "SpeechOperationError", - operation: "inference", - cause: failure, - }); - expect(loadNative.mock.calls.map((call) => call[3])).toEqual( - fallback ? [acceleration, other] : [acceleration], - ); - } + for (const failing of ["model preparation", "inference"] as const) + for (const fallback of [false, true]) { + loadNative.mockClear(); + const failure = new Error("device failed"); + if (failing === "inference") native.transcribe.mockRejectedValueOnce(failure); + else loadNative.mockRejectedValueOnce(failure); + yield* speech.updateAcceleration(acceleration, fallback); + const result = yield* Effect.result( + speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + ); + if (fallback) expect(Result.isSuccess(result) && result.success).toBe("hello"); + else + expect(Result.isFailure(result) && result.failure).toMatchObject({ + _tag: "SpeechOperationError", + operation: failing, + cause: failure, + }); + expect(loadNative.mock.calls.map((call) => call[3])).toEqual( + fallback ? [acceleration, other] : [acceleration], + ); + } }).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts index 7d263ccb03b4f..c3f32f3ddcaee 100644 --- a/apps/server/src/speech/SpeechService.ts +++ b/apps/server/src/speech/SpeechService.ts @@ -405,6 +405,28 @@ const make = Effect.gen(function* () { } }; + // The loader already retries "auto" on the CPU; a chosen device needs the person's fallback. + const loadSelected = async ( + modelPath: string, + definition: SpeechModel, + signal: AbortSignal, + settings: { speechAcceleration: SpeechAcceleration; speechAccelerationFallback: boolean }, + ) => { + try { + return await loadModel(modelPath, definition, signal, settings.speechAcceleration); + } catch (cause) { + const { speechAcceleration, speechAccelerationFallback } = settings; + if (speechAcceleration === "auto" || !speechAccelerationFallback) throw cause; + if (signal.aborted || isSpeechError(cause)) throw cause; + return await loadModel( + modelPath, + definition, + signal, + speechAcceleration === "cpu" ? "auto" : "cpu", + ); + } + }; + const speechError = (operation: string, cause: unknown): SpeechError => isSpeechError(cause) ? cause : new SpeechOperationError({ operation, cause }); @@ -569,11 +591,11 @@ const make = Effect.gen(function* () { Effect.andThen(installedModelPath(selectedModel(settings))), Effect.flatMap((modelPath) => attempt("model preparation", (signal) => - loadModel( + loadSelected( modelPath, selectedModel(settings), AbortSignal.any([signal, lifetime.signal]), - settings.speechAcceleration, + settings, ), ).pipe( Effect.timeout("2 minutes"), @@ -620,11 +642,11 @@ const make = Effect.gen(function* () { const modelPath = yield* installedModelPath(definition); const loaded = yield* Effect.tryPromise({ try: (signal) => - loadModel( + loadSelected( modelPath, definition, AbortSignal.any([signal, lifetime.signal]), - settings.speechAcceleration, + settings, ), catch: (cause) => speechError("model preparation", cause), }); @@ -762,7 +784,7 @@ const make = Effect.gen(function* () { let correct = makeSpeechTextCorrector(customWords, dictionary); const modelPath = yield* installedModelPath(definition); let streamModel = yield* attempt(operation, () => - loadModel(modelPath, definition, signal, settings.speechAcceleration), + loadSelected(modelPath, definition, signal, settings), ); loaded = streamModel; const vadPath = yield* silenceModelPath(streamModel, options);