diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 2e05132cd6eea..da8df58fd3cc6 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -763,24 +763,36 @@ it.effect( }), ); -it.effect("reports a failure on a GPU the person chose instead of switching to CPU", () => - Effect.gen(function* () { - const failure = new Error("GPU failed"); - native.transcribe.mockRejectedValueOnce(failure); - const speech = yield* SpeechService.SpeechService; - const acceleration = 'gpu:["vulkan","gpu-1"]'; - yield* speech.updateAcceleration(acceleration); - const result = yield* Effect.result( - speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), - ); - expect(Result.isFailure(result) && result.failure).toMatchObject({ - _tag: "SpeechOperationError", - operation: "inference", - cause: failure, - }); - expect(loadNative).toHaveBeenCalledTimes(1); - expect(loadNative.mock.calls[0]?.[3]).toBe(acceleration); - }).pipe(Effect.provide(layer)), +it.effect.each([ + ['gpu:["vulkan","gpu-1"]', "cpu"], + ["cpu", "auto"], +] as const)( + "retries a recording from a chosen %s on %s only when fallback is on", + ([acceleration, other]) => + Effect.gen(function* () { + const speech = yield* SpeechService.SpeechService; + for (const failing of ["model preparation", "inference"] as const) + for (const fallback of [false, true]) { + loadNative.mockClear(); + const failure = new Error("device failed"); + if (failing === "inference") native.transcribe.mockRejectedValueOnce(failure); + else loadNative.mockRejectedValueOnce(failure); + yield* speech.updateAcceleration(acceleration, fallback); + const result = yield* Effect.result( + speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + ); + if (fallback) expect(Result.isSuccess(result) && result.success).toBe("hello"); + else + expect(Result.isFailure(result) && result.failure).toMatchObject({ + _tag: "SpeechOperationError", + operation: failing, + cause: failure, + }); + expect(loadNative.mock.calls.map((call) => call[3])).toEqual( + fallback ? [acceleration, other] : [acceleration], + ); + } + }).pipe(Effect.provide(layer)), ); it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once", () => @@ -895,9 +907,15 @@ it.effect.each([false, true])( }).pipe(Effect.provide(layer)), ); -it.effect.each(["auto", 'gpu:["vulkan","gpu-1"]', "cpu"] as const)( - "reports finalization failure without repeated or explicit-backend fallback: %s", - (acceleration) => +it.effect.each([ + ["auto", false], + ['gpu:["vulkan","gpu-1"]', false], + ["cpu", false], + ['gpu:["vulkan","gpu-1"]', true], + ["cpu", true], +] as const)( + "reports finalization failure after at most one fallback: %s, fallback %s", + ([acceleration, fallback]) => Effect.gen(function* () { const failure = new Error("finish failed"); const worker = { @@ -907,15 +925,24 @@ it.effect.each(["auto", 'gpu:["vulkan","gpu-1"]', "cpu"] as const)( throw failure; }), }; - const cpu = { ...worker, backend: "CPU", dispose: vi.fn() }; + const other = { + ...worker, + backend: acceleration === "cpu" ? "Vulkan0" : "CPU", + dispose: vi.fn(), + }; + const retries = acceleration === "auto" || fallback; loadNative.mockResolvedValueOnce(worker); - if (acceleration === "auto") loadNative.mockResolvedValueOnce(cpu); + if (retries) loadNative.mockResolvedValueOnce(other); const speech = yield* SpeechService.SpeechService; - yield* speech.updateAcceleration(acceleration); + yield* speech.updateAcceleration(acceleration, fallback); const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS); expect(Result.isFailure(yield* Effect.result(stream.finish))).toBe(true); - expect(worker.finish).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1); - expect(loadNative).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1); + // The failed stream frees the model before its socket closes, so a retry is not busy. + expect(yield* speech.status).toMatchObject({ state: "ready" }); + expect(worker.finish).toHaveBeenCalledTimes(retries ? 2 : 1); + expect(loadNative.mock.calls.map((call) => call[3])).toEqual( + retries ? [acceleration, acceleration === "cpu" ? "auto" : "cpu"] : [acceleration], + ); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts index b4ae198b9447d..c3f32f3ddcaee 100644 --- a/apps/server/src/speech/SpeechService.ts +++ b/apps/server/src/speech/SpeechService.ts @@ -150,6 +150,7 @@ export class SpeechService extends Context.Service< ) => Effect.Effect; readonly updateAcceleration: ( acceleration: SpeechAcceleration, + fallback?: boolean, ) => Effect.Effect; readonly updateModelUnloadTimeout: ( timeout: SpeechModelUnloadTimeout, @@ -238,9 +239,16 @@ const make = Effect.gen(function* () { }; // Auto placement may land on a GPU the native engine cannot actually run; one retry on - // this host's CPU recovers it. A GPU the person chose, and CPU itself, report the error. - const canRetryOnCpu = (acceleration: SpeechAcceleration, loaded: LoadedModel) => - acceleration === "auto" && loaded.backend.toLowerCase() !== "cpu"; + // this host's CPU recovers it. A device the person chose retries on the other only when allowed. + const fallbackFor = ( + settings: { speechAcceleration: SpeechAcceleration; speechAccelerationFallback: boolean }, + loaded: LoadedModel, + ) => { + const onCpu = loaded.backend.toLowerCase() === "cpu"; + if (settings.speechAcceleration === "auto") return onCpu ? undefined : "cpu"; + if (!settings.speechAccelerationFallback) return undefined; + return onCpu ? "auto" : "cpu"; + }; yield* Effect.addFinalizer(() => Effect.promise(async () => { @@ -397,6 +405,28 @@ const make = Effect.gen(function* () { } }; + // The loader already retries "auto" on the CPU; a chosen device needs the person's fallback. + const loadSelected = async ( + modelPath: string, + definition: SpeechModel, + signal: AbortSignal, + settings: { speechAcceleration: SpeechAcceleration; speechAccelerationFallback: boolean }, + ) => { + try { + return await loadModel(modelPath, definition, signal, settings.speechAcceleration); + } catch (cause) { + const { speechAcceleration, speechAccelerationFallback } = settings; + if (speechAcceleration === "auto" || !speechAccelerationFallback) throw cause; + if (signal.aborted || isSpeechError(cause)) throw cause; + return await loadModel( + modelPath, + definition, + signal, + speechAcceleration === "cpu" ? "auto" : "cpu", + ); + } + }; + const speechError = (operation: string, cause: unknown): SpeechError => isSpeechError(cause) ? cause : new SpeechOperationError({ operation, cause }); @@ -474,6 +504,7 @@ const make = Effect.gen(function* () { languages: definition.languages, supportsLanguageDetection: definition.supportsLanguageDetection, acceleration: settings.speechAcceleration, + accelerationFallback: settings.speechAccelerationFallback, benchmarkTier: settings.speechAcceleration === "cpu" || (settings.speechAcceleration === "auto" && autoUsesCpu) @@ -560,11 +591,11 @@ const make = Effect.gen(function* () { Effect.andThen(installedModelPath(selectedModel(settings))), Effect.flatMap((modelPath) => attempt("model preparation", (signal) => - loadModel( + loadSelected( modelPath, selectedModel(settings), AbortSignal.any([signal, lifetime.signal]), - settings.speechAcceleration, + settings, ), ).pipe( Effect.timeout("2 minutes"), @@ -611,11 +642,11 @@ const make = Effect.gen(function* () { const modelPath = yield* installedModelPath(definition); const loaded = yield* Effect.tryPromise({ try: (signal) => - loadModel( + loadSelected( modelPath, definition, AbortSignal.any([signal, lifetime.signal]), - settings.speechAcceleration, + settings, ), catch: (cause) => speechError("model preparation", cause), }); @@ -653,21 +684,22 @@ const make = Effect.gen(function* () { let inference = loaded; const result = yield* transcribeWith(loaded).pipe( Effect.catch((error) => { - if (!canRetryOnCpu(settings.speechAcceleration, loaded)) return Effect.fail(error); + const fallback = fallbackFor(settings, loaded); + if (!fallback) return Effect.fail(error); return Effect.tryPromise({ try: (signal) => loadModel( modelPath, definition, AbortSignal.any([signal, lifetime.signal]), - "cpu", + fallback, ), catch: (cause) => speechError("model preparation", cause), }).pipe( - Effect.flatMap((cpu) => { - autoCpuModel = definition.id; - inference = cpu; - return transcribeWith(cpu); + Effect.flatMap((other) => { + if (settings.speechAcceleration === "auto") autoCpuModel = definition.id; + inference = other; + return transcribeWith(other); }), ); }), @@ -752,7 +784,7 @@ const make = Effect.gen(function* () { let correct = makeSpeechTextCorrector(customWords, dictionary); const modelPath = yield* installedModelPath(definition); let streamModel = yield* attempt(operation, () => - loadModel(modelPath, definition, signal, settings.speechAcceleration), + loadSelected(modelPath, definition, signal, settings), ); loaded = streamModel; const vadPath = yield* silenceModelPath(streamModel, options); @@ -781,19 +813,19 @@ const make = Effect.gen(function* () { correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary); started = true; }; - // Audio already fed is kept so a CPU restart can replay it; the limit caps it at one recording. + // Audio already fed is kept so a fallback restart can replay it; the limit caps it at one recording. const received: Float32Array[] = []; - let retriedOnCpu = false; - const recoverOnCpu = async (cause: unknown) => { + let recovered = false; + const recover = async (cause: unknown) => { started = false; - if (closed || signal.aborted || retriedOnCpu) throw cause; - if (!canRetryOnCpu(settings.speechAcceleration, streamModel)) throw cause; - retriedOnCpu = true; + const fallback = fallbackFor(settings, streamModel); + if (closed || signal.aborted || recovered || !fallback) throw cause; + recovered = true; forgetModel(streamModel); await streamModel.dispose(); if (closed || signal.aborted) throw cause; - streamModel = await loadModel(modelPath, definition, signal, "cpu"); - autoCpuModel = definition.id; + streamModel = await loadModel(modelPath, definition, signal, fallback); + if (settings.speechAcceleration === "auto") autoCpuModel = definition.id; loaded = streamModel; await beginStream(); let update: Awaited> | undefined; @@ -807,7 +839,7 @@ const make = Effect.gen(function* () { try { await beginStream(); } catch (cause) { - await recoverOnCpu(cause); + await recover(cause); } }); let byteLength = 0; @@ -844,7 +876,7 @@ const make = Effect.gen(function* () { try { update = await streamModel.feed(pcm); } catch (cause) { - update = await recoverOnCpu(cause); + update = await recover(cause); revisionBase += lastNativeRevision; lastNativeRevision = 0; } @@ -863,7 +895,7 @@ const make = Effect.gen(function* () { let text: string; try { text = await streamModel.finish().catch(async (cause: unknown) => { - await recoverOnCpu(cause); + await recover(cause); if (closed || signal.aborted) throw cause; return streamModel.finish(); }); @@ -888,10 +920,14 @@ const make = Effect.gen(function* () { transcribe, startStream: Effect.fn("SpeechService.startStream")(function* (options) { const session = yield* Scope.fork(yield* Effect.scope); - return yield* openStream(options).pipe( - Scope.provide(session), - Effect.onError((cause) => Scope.close(session, Exit.failCause(cause))), - ); + const closeOnError = (effect: Effect.Effect) => + effect.pipe(Effect.onError((cause) => Scope.close(session, Exit.failCause(cause)))); + const stream = yield* closeOnError(openStream(options).pipe(Scope.provide(session))); + // A failed stream frees the model before the client hears of it, so its retry is not busy. + return { + feed: (pcm: Uint8Array) => closeOnError(stream.feed(pcm)), + finish: closeOnError(stream.finish), + }; }), downloadModel: Effect.fn("SpeechService.downloadModel")((modelId) => requireSupported.pipe( @@ -970,10 +1006,11 @@ const make = Effect.gen(function* () { yield* writeSettings("model linking", { speechModelId: first }); return yield* modelsEffect; }), - updateAcceleration: Effect.fn("SpeechService.updateAcceleration")((acceleration) => - writeSettings("acceleration update", { speechAcceleration: acceleration }).pipe( - Effect.andThen(freshStatus("acceleration update")), - ), + updateAcceleration: Effect.fn("SpeechService.updateAcceleration")((acceleration, fallback) => + writeSettings("acceleration update", { + speechAcceleration: acceleration, + ...(fallback === undefined ? {} : { speechAccelerationFallback: fallback }), + }).pipe(Effect.andThen(freshStatus("acceleration update"))), ), updateModelUnloadTimeout: Effect.fn("SpeechService.updateModelUnloadTimeout")((timeout) => writeSettings("model unload timeout update", { speechModelUnloadTimeout: timeout }).pipe( diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index bf91b4c1ef3a6..adb7c1ecb0801 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -198,7 +198,9 @@ export const speechHttpApiLayer = HttpApiBuilder.group( "updateAcceleration", Effect.fn("environment.voice.updateAcceleration")(function* (args) { yield* guard(args.endpoint.name, AuthSettingsWriteScope); - return yield* speech.updateAcceleration(args.payload.acceleration).pipe(mapSpeechError); + return yield* speech + .updateAcceleration(args.payload.acceleration, args.payload.fallback) + .pipe(mapSpeechError); }), ) .handle( diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts index a41f917b2a682..cffd65f18edf2 100644 --- a/apps/server/src/speech/native.test.ts +++ b/apps/server/src/speech/native.test.ts @@ -55,6 +55,36 @@ it("returns transcription from an isolated process and disposes it", async () => } }); +it("splits a piece the engine could not finish instead of failing the recording", async () => { + for (const error of [ + "OutputTruncated", + "OutputRepetition", + "InputTooLong", + "OutOfMemory", + "BackendError", + ]) { + const model = await loadNativeSpeechModel( + "unused.gguf", + new AbortController().signal, + fixture(`async (pcm) => { + class OutputTruncated extends Error {} + const errors = { OutputTruncated, OutputRepetition: class OutputRepetition extends OutputTruncated {}, + InputTooLong: class InputTooLong extends Error {}, OutOfMemory: class OutOfMemory extends Error {}, + BackendError: class BackendError extends Error {} }; + if (pcm.length > 5000) throw new errors.${error}("too much"); + return { text: String(pcm.length) }; + }`), + ); + try { + expect( + await model.transcribe(new Float32Array(16000), { timestamps: "none" }, false), + ).toEqual({ text: "4000 4000 4000 4000" }); + } finally { + await model.dispose(); + } + } +}); + it("probes generic vocabulary support and delivers ordered terms to offline inference", async () => { for (const supported of [false, true]) { const model = await loadNativeSpeechModel( diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts index 70c5d21c6038d..5b19f64954f3f 100644 --- a/apps/server/src/speech/native.ts +++ b/apps/server/src/speech/native.ts @@ -31,6 +31,13 @@ const newAudio = (filterSilence) => createSpeechAudio({ frameSamples: 512, filterSilence, nativeStreaming: model.capabilities.supportsStreaming, longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs, }); +// Matched by class name: OutputRepetition extends OutputTruncated, and smaller input needs less memory. +const splittable = new Set(["OutputTruncated", "InputTooLong", "OutOfMemory", "BackendError"]); +const canSplit = (error) => { + for (let proto = error && Object.getPrototypeOf(error); proto; proto = Object.getPrototypeOf(proto)) + if (splittable.has(proto.constructor?.name)) return true; + return false; +}; const openAsr = async () => { if (nativeStreaming) { session = model.createSession(); @@ -71,8 +78,7 @@ const transcribe = async (pcm) => { const result = await model.transcribe(pcm, options); texts.push(result.text); } catch (error) { - if ((error?.constructor?.name !== "OutputTruncated" && - error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error; + if (!canSplit(error) || pcm.length <= 4000) throw error; const middle = Math.floor(pcm.length / 2); await transcribe(pcm.subarray(0, middle)); await transcribe(pcm.subarray(middle)); diff --git a/apps/web/src/components/chat/VoiceInputErrorPill.tsx b/apps/web/src/components/chat/VoiceInputErrorPill.tsx index ad05ed1558980..e666d06aa31a0 100644 --- a/apps/web/src/components/chat/VoiceInputErrorPill.tsx +++ b/apps/web/src/components/chat/VoiceInputErrorPill.tsx @@ -1,4 +1,7 @@ -import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input"; +import { + MICROPHONE_PERMISSION_ERROR, + RETAINED_RECORDING_ERROR, +} from "@t3tools/client-runtime/voice-input"; import { useNavigate } from "@tanstack/react-router"; import { CircleAlertIcon, XIcon } from "lucide-react"; @@ -19,6 +22,7 @@ const VOICE_ERROR_LABELS: Record = { "Voice recording was interrupted.": "Recording interrupted", "Could not finish voice recording.": "Couldn’t finish", "Could not transcribe this recording.": "Transcription failed", + [RETAINED_RECORDING_ERROR]: "Transcription failed", "Voice transcription on this environment failed.": "Transcription failed", "The draft changed while voice input was running. The transcript was not added.": "Draft changed", "No speech was detected.": "No speech heard", @@ -27,7 +31,7 @@ const VOICE_ERROR_LABELS: Record = { }; const RETRYABLE_TRANSCRIPTIONS = new Set([ - "Could not transcribe this recording.", + RETAINED_RECORDING_ERROR, "Voice transcription on this environment failed.", ]); diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 22ae59e55d31e..da08b3037570e 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -773,6 +773,30 @@ function AdvancedSpeechSettings({ } /> + {supported.acceleration !== "auto" && supported.accelerationFallback !== undefined ? ( + + void run("Could not change the hardware fallback", () => + runEnvironmentSpeech( + updateEnvironmentSpeechAcceleration(prepared, supported.acceleration, checked), + ), + ) + } + /> + } + /> + ) : null} { stopped?.reject(new Error("Microphone processing failed.")); @@ -162,7 +163,7 @@ export function createBrowserPcmCapture( worklet.port.postMessage("stop"); try { await pending; - if (!feed && chunks.length) { + if (chunks.length) { const blob = new Blob(chunks, { type: "application/octet-stream" }); recordingUri = URL.createObjectURL(blob); } diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index e6d164db59f3c..2b21e78128786 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -120,7 +120,10 @@ export function createBrowserVoiceInputPlatform(input: { options: recordingOptions, signal: options.signal, onText: input.onText, - onError: (error) => input.onError(error.message), + // Recording continues after a stream failure; finishing retries the retained audio. + onError: (error) => { + if (!input.hasAudioConsent()) input.onError(error.message); + }, }), }; }); diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 15f5fea3aac76..a77cd670e294e 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { + RETAINED_RECORDING_ERROR, resetVoiceInputGlobalsForTests, resolveTranscriptCommit, VoiceInputController, @@ -112,6 +113,62 @@ describe("environment voice input", () => { expect(harness.deleted).toEqual(["file:///voice.m4a"]); }); + it("falls back to the retained recording when a live stream fails, and keeps it for retry", async () => { + const transcribe = vi + .fn() + .mockResolvedValueOnce("recovered text") + .mockRejectedValueOnce(new Error("inference failed")) + .mockResolvedValueOnce("retried text"); + const finish = vi.fn(async () => { + throw new Error("stream failed"); + }); + const harness = createHarness({ + retryFailedRecording: true, + getTranscriber: () => ({ + prepare: async () => ({ ...preparedTranscription(transcribe), streaming: { finish } }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.commits.map(({ text }) => text)).toEqual(["hello recovered text"]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + + harness.deleted.length = 0; + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + expect(harness.deleted).toEqual([]); + await harness.controller.start(); + expect(finish).toHaveBeenCalledTimes(2); + expect(transcribe).toHaveBeenCalledTimes(3); + expect(harness.recorder.record).toHaveBeenCalledTimes(2); + expect(harness.commits.map(({ text }) => text)).toEqual([ + "hello recovered text", + "hello retried text", + ]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + }); + + it("offers a transcription retry only when the failed recording was kept", async () => { + for (const retryFailedRecording of [true, false]) { + resetVoiceInputGlobalsForTests(); + const fail = async () => { + throw new Error("failed"); + }; + const harness = createHarness({ + retryFailedRecording, + getTranscriber: () => ({ + prepare: async () => ({ ...preparedTranscription(fail), streaming: { finish: fail } }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState.error === RETAINED_RECORDING_ERROR).toBe( + retryFailedRecording, + ); + } + }); + it("deletes retained failed audio when the user cancels instead of retrying", async () => { const harness = createHarness({ retryFailedRecording: true, diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index d093b30a6497a..a745e612196d9 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -11,6 +11,10 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60; +/** The recording is kept, so retrying transcribes it again without recording. */ +export const RETAINED_RECORDING_ERROR = + "Could not transcribe this recording. It was kept for a retry."; + export const MICROPHONE_PERMISSION_ERROR = "Microphone access is required for voice input. Allow access in your browser or system settings and try again."; @@ -526,31 +530,39 @@ export class VoiceInputController { const signal = this.transcriptionAbortController.signal; const capturedDraft = this.capturedDraft; let transcript: string; + const transcribeRecording = () => + runTranscriptionOperation(() => transcription.transcribe(recordingUri!, { signal })); try { const live = this.liveTranscription ? await this.liveTranscription : null; if (live && "error" in live) throw live.error; + const streaming = transcription.streaming; transcript = live?.text ?? - (await runTranscriptionOperation(() => - transcription.streaming - ? transcription.streaming.finish({ signal }) - : transcription.transcribe(recordingUri!, { signal }), - )); + (streaming + ? await runTranscriptionOperation(() => streaming.finish({ signal })).catch( + (error: unknown) => { + // A failed live stream gets one pass over the retained recording. + if (!recordingUri || signal.aborted || !this.isCurrent(operationToken)) + throw error; + return transcribeRecording(); + }, + ) + : await transcribeRecording()); } catch (error) { if (this.isCurrent(operationToken)) { - if ( - this.dependencies.retryFailedRecording && - !this.liveTranscription && - !transcription.streaming && - recordingUri - ) { + const retained = + this.dependencies.retryFailedRecording && !this.liveTranscription && recordingUri; + if (retained) { this.retryRecording = { uri: recordingUri, ownerKey: capturedDraft.ownerKey, - transcription, + transcription: { locale: transcription.locale, transcribe: transcription.transcribe }, }; } - this.setError(transcriptionErrorMessage(error), "retry"); + this.setError( + retained ? RETAINED_RECORDING_ERROR : transcriptionErrorMessage(error), + "retry", + ); } return; } diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index a8ea541f75133..064e5690598f1 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -270,13 +270,18 @@ export const postProcessEnvironmentTranscript = ( export const updateEnvironmentSpeechAcceleration = ( prepared: PreparedConnection, acceleration: SpeechAcceleration, + fallback?: boolean, ) => request({ group: "voice", prepared, method: "POST", path: (urls) => urls.updateAcceleration(), - run: ({ client, headers }) => client.updateAcceleration({ headers, payload: { acceleration } }), + run: ({ client, headers }) => + client.updateAcceleration({ + headers, + payload: { acceleration, ...(fallback === undefined ? {} : { fallback }) }, + }), }); export const updateEnvironmentSpeechModelUnloadTimeout = ( diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts index 5283d64eb0dc3..7f620068ee301 100644 --- a/packages/client-runtime/src/voice-input/index.ts +++ b/packages/client-runtime/src/voice-input/index.ts @@ -2,6 +2,7 @@ export { VoiceInputController, VOICE_RECORDING_LIMIT_SECONDS, MICROPHONE_PERMISSION_ERROR, + RETAINED_RECORDING_ERROR, voiceInputBlocksSubmission, type VoiceDraftSnapshot, type VoiceInputControllerDependencies, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 49470541d90be..2eb8deac87770 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1386,6 +1386,10 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), ), speechAcceleration: SpeechAcceleration.pipe(Schema.withDecodingDefault(Effect.succeed("auto"))), + // A CPU or GPU the person chose may retry a failed recording on the other one. Auto always does. + speechAccelerationFallback: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + ), speechModelUnloadTimeout: SpeechModelUnloadTimeout.pipe( Schema.withDecodingDefault(Effect.succeed("min_15")), ), @@ -1839,6 +1843,7 @@ export const ServerSettingsPatch = Schema.Struct({ ), speechModelId: Schema.optionalKey(Schema.String), speechAcceleration: Schema.optionalKey(SpeechAcceleration), + speechAccelerationFallback: Schema.optionalKey(Schema.Boolean), speechModelUnloadTimeout: Schema.optionalKey(SpeechModelUnloadTimeout), speechProjectCustomWords: Schema.optionalKey(SpeechCustomWords), speechPostProcessingModelSelection: Schema.optionalKey(Schema.NullOr(ModelSelection)), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index ff3362f67fc0c..368f044960538 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -226,6 +226,7 @@ const SupportedSpeechStatus = Schema.Struct({ languages: Schema.optionalKey(Schema.Array(Schema.String)), supportsLanguageDetection: Schema.optionalKey(Schema.Boolean), acceleration: SpeechAcceleration, + accelerationFallback: Schema.optionalKey(Schema.Boolean), benchmarkTier: SpeechBenchmarkTier, modelUnloadTimeout: SpeechModelUnloadTimeout, gpuDevices: Schema.Array(SpeechGpuDevice), @@ -267,6 +268,7 @@ export const EnvironmentSpeechTranscriptionResult = Schema.Struct({ text: Schema export const EnvironmentSpeechAccelerationRequest = Schema.Struct({ acceleration: SpeechAcceleration, + fallback: Schema.optionalKey(Schema.Boolean), }); export const EnvironmentSpeechModelUnloadTimeoutRequest = Schema.Struct({ timeout: SpeechModelUnloadTimeout,