From 298298d118a57171d2e6ae481423ebfadbc48d16 Mon Sep 17 00:00:00 2001 From: wojas Date: Fri, 6 Nov 2026 08:49:33 +0800 Subject: [PATCH 1/7] fix(mobile): typing after in-app dictation reaches the draft again on iOS (#30139) Co-authored-by: Claude Opus 5.5 Co-authored-by: maria-rcks (cherry picked from commit bd3081711d7b568848b7928724222f2d4f8be459) --- .../t3-composer-editor/ios/T3ComposerEditorView.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift index 48eb6fdb2d9ec..c5c490a8a8253 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift @@ -445,6 +445,7 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro private var isReadOnly = false private var isApplyingControlledValue = false private var nativeEventCount = 0 + private var hasControlledDocument = false private var resetGeneration = 0 private var lastContentSize = CGSize.zero private var iconImages: [String: UIImage] = [:] @@ -666,12 +667,15 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro } if isReset { resetGeneration = document.resetGeneration - nativeEventCount = document.mostRecentEventCount endComposition() } - if document.isNativeEcho && !isReset && textView.serializedText() != document.value { + // iOS destroys this view under display: none and mounts a fresh one while JS keeps its + // revision and draft. Continue from them, or JS drops every event the new view sends. + nativeEventCount = document.mostRecentEventCount + if document.isNativeEcho && !isReset && hasControlledDocument && textView.serializedText() != document.value { return } + hasControlledDocument = true if tokensJson != document.tokensJson { tokensJson = document.tokensJson tokens = decode([ComposerTokenPayload].self, from: document.tokensJson) ?? [] From 8437c5199a3ca5c8cf72501b53496a059d0a1891 Mon Sep 17 00:00:00 2001 From: pseudo <27822378+pseudoseed@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:18:05 -0700 Subject: [PATCH 2/7] test(mobile): a remounted composer editor continues from the draft JS holds Co-Authored-By: Claude Code --- .../t3-composer-editor/tests/ios/main.swift | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift b/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift index 2266f1a451cc3..d898f18ba68f5 100644 --- a/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift +++ b/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift @@ -91,6 +91,20 @@ do { check(view.text == "abc", "A stale controlled value overwrote newer native typing") } +// iOS mounts a new view after display: none (dictation) while JS keeps its revision and draft. +do { + let view = T3ComposerEditorView() + view.frame = CGRect(x: 0, y: 0, width: 320, height: 200) + view.setControlledDocumentJson(document("dictated", revision: 40, isNativeEcho: true)) + check(view.text == "dictated", "A remounted editor did not take the draft JS holds: \(view.text)") + view.editor.insertText("!") + let typed = view.onComposerChange.payloads.last + check( + (typed?["eventCount"] as? Int ?? 0) > 40, + "Typing in a remounted editor must be newer than the parent's revision, or JS drops it" + ) +} + // A send clear during an IME composition must not come back when the composition commits. do { let view = makeEditor("hello ") From 61846530a072a29a2afbc1dcb38ba8d3bf810387 Mon Sep 17 00:00:00 2001 From: pseudo <27822378+pseudoseed@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:45:45 -0700 Subject: [PATCH 3/7] fix(mobile): the voice filter keeps your own words in noise and over a headset The filter judged every stretch of speech against the voice profile alone. The profile is recorded once, somewhere quiet, so noise or another microphone pushed the user's own speech past the limit and it was cut as someone else's. A short opening stretch has a looser limit, so in the worst case only the first words of a dictation survived. A stretch the profile rejects is now also compared with the stretches it accepted in the same recording, and kept when it matches them. Measured on 8 Mini LibriSpeech speakers, half-minute recordings, own words lost with the filter on: noise at 5 dB 9.2% before, 1.8% after; narrowband 3.4% to 0%; echo with noise 2.7% to 0%. Strangers' words kept are unchanged (11.5%, 1.0%, 6.4%), and the clean mixed-voice numbers are unchanged (2.4% own words lost, 4.8% strangers' words kept, 19 speakers). Also adds a test that half a minute of speech in the phone's recording format (AAC, 44.1 kHz, stereo) is transcribed in full by all 3 models. Co-Authored-By: Claude Code --- .../RealVoiceTests.swift | 136 +++++++++++++++++- .../SpeakerFilterTests.swift | 26 ++++ .../PseudoDictationCoreTests/TestAudio.swift | 24 ++++ .../ios/Core/DictationEngine.swift | 11 +- .../ios/Core/SpeakerFilter.swift | 24 ++++ 5 files changed, 212 insertions(+), 9 deletions(-) diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift index 3859bbf1e6469..3e84cdf6f6976 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift @@ -92,9 +92,16 @@ final class RealVoiceTests: XCTestCase { return url } - private func words(_ samples: [Float], onlyMyVoice: Bool) async throws -> (Set, FilterOutcome?) { + private func words( + _ samples: [Float], onlyMyVoice: Bool, model: DictationModel = .fast, asPhoneRecording: Bool = false + ) async throws -> (Set, FilterOutcome?) { + var url = try write(samples) + if asPhoneRecording { + url = url.deletingPathExtension().appendingPathExtension("m4a") + try writePhoneRecording(samples, to: url) + } let transcript = try await engine.transcribe( - requestId: UUID().uuidString, url: try write(samples), model: .fast, processing: .auto, + requestId: UUID().uuidString, url: url, model: model, processing: .auto, onlyMyVoice: onlyMyVoice, language: "en-US") let words = transcript.text.lowercased().split(whereSeparator: { !$0.isLetter }).map(String.init) return (Set(words), transcript.filter) @@ -139,6 +146,131 @@ final class RealVoiceTests: XCTestCase { XCTAssertLessThanOrEqual(Double(mixAccepted.count) / Double(mixes), 0.05) } + // MARK: Long recordings + + /// People dictate for longer than any other clip here, and the phone hands over AAC, not WAV. + func testAHalfMinuteOfSpeechFromThePhoneIsTranscribedInFull() async throws { + try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in }) + var lost: [String] = [] + for model in [DictationModel.fast, .balanced, .accurate] { + try await engine.download(model, processing: .auto, onProgress: { _ in }) + for speaker in speakers.prefix(4) { + try await engine.saveVoiceProfile(try speaker.enrollment.map(write)) + var recording: [Float] = [] + var expected: Set = [] + for utterance in speaker.held where seconds(recording) < 30 { + recording += utterance + expected.formUnion(try await words(utterance, onlyMyVoice: false, model: model).0) + } + XCTAssertGreaterThanOrEqual(seconds(recording), 30) + for onlyMyVoice in [false, true] { + let (heard, outcome) = try await words( + recording, onlyMyVoice: onlyMyVoice, model: model, asPhoneRecording: true) + let missing = expected.subtracting(heard).count + print( + "MEASURE long, \(model.rawValue) \(speaker.name) \(Int(seconds(recording))) s, filter " + + "\(onlyMyVoice): lost \(percent(missing, expected.count)), " + + "cut \(outcome?.droppedSeconds ?? 0) s") + if Double(missing) / Double(expected.count) > 0.05 { + lost.append("\(model.rawValue) \(speaker.name) filter \(onlyMyVoice): \(percent(missing, expected.count))") + } + } + } + } + XCTAssertEqual(lost, []) + } + + // MARK: Sound unlike enrollment + + /// White noise under the speech, `snr` decibels below it. The seed is fixed so runs agree. + private static func noisy(_ samples: [Float], snr: Float) -> [Float] { + let power = samples.reduce(Float(0)) { $0 + $1 * $1 } / Float(samples.count) + let peak = (3 * power).squareRoot() / pow(10, snr / 20) + var state: UInt64 = 0x9E37_79B9_7F4A_7C15 + return samples.map { sample in + state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 + return sample + peak * (Float(state >> 40) / Float(1 << 24) * 2 - 1) + } + } + + /// Roughly a telephone headset: the top and the bottom of the voice are gone. + private static func narrowband(_ samples: [Float]) -> [Float] { + var output = samples + var low: Float = 0 + var lower: Float = 0 + var rumble: Float = 0 + for index in samples.indices { + low += 0.72 * (samples[index] - low) + lower += 0.72 * (low - lower) + rumble = 0.89 * rumble + 0.11 * lower + output[index] = index % 2 == 0 ? lower - rumble : output[index - 1] + } + return output + } + + /// A small room with hard walls. + private static func echoing(_ samples: [Float]) -> [Float] { + var output = samples + for (delay, gain) in [(464, Float(0.55)), (592, 0.5), (688, 0.45), (1104, 0.4)] { + var echo = [Float](repeating: 0, count: samples.count) + for index in samples.indices.dropFirst(delay) { + echo[index] = gain * (samples[index - delay] + echo[index - delay]) + output[index] += 0.35 * echo[index] + } + } + return output + } + + /// The profile is recorded once, somewhere quiet. People then dictate in a car, in a kitchen or + /// over a headset, where their voice sits further from it, and that must not cost them words. + func testTheFilterKeepsTheSpeakersWordsWhereTheSoundIsUnlikeEnrollment() async throws { + try await engine.download(.fast, processing: .auto, onProgress: { _ in }) + try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in }) + let conditions: [(name: String, change: ([Float]) -> [Float])] = [ + ("noise at 5 dB", { Self.noisy($0, snr: 5) }), + ("narrowband", Self.narrowband), + ("echo with noise at 10 dB", { Self.noisy(Self.echoing($0), snr: 10) }), + ] + var tallies = conditions.map { _ in (own: 0, ownLost: 0, stranger: 0, strangerKept: 0) } + + for (index, speaker) in speakers.prefix(8).enumerated() { + try await engine.saveVoiceProfile(try speaker.enrollment.map(write)) + let stranger = speakers[(index + speakers.count / 2) % speakers.count] + var alone: [Float] = [] + for utterance in speaker.held where seconds(alone) < 30 { alone += utterance } + + for (position, condition) in conditions.enumerated() { + let recording = condition.change(alone) + let spoken = try await words(recording, onlyMyVoice: false).0 + let (kept, outcome) = try await words(recording, onlyMyVoice: true) + tallies[position].own += spoken.count + tallies[position].ownLost += spoken.subtracting(kept).count + print( + "MEASURE unlike enrollment, \(condition.name), \(speaker.name) alone for " + + "\(Int(seconds(alone))) s: lost \(percent(spoken.subtracting(kept).count, spoken.count)), " + + "cut \(outcome?.droppedSeconds ?? 0) s") + + let own = condition.change(Self.cut(speaker.held[0], 5)) + let theirs = condition.change(Self.cut(stranger.held[0], 5)) + let unwanted = try await words(theirs, onlyMyVoice: false).0 + .subtracting(try await words(own, onlyMyVoice: false).0) + let heard = try await words(own + theirs, onlyMyVoice: true).0 + tallies[position].stranger += unwanted.count + tallies[position].strangerKept += unwanted.intersection(heard).count + } + } + + for (condition, tally) in zip(conditions, tallies) { + print( + "MEASURE unlike enrollment, \(condition.name): speaker's words lost " + + "\(percent(tally.ownLost, tally.own)), stranger's words kept " + + "\(percent(tally.strangerKept, tally.stranger))") + XCTAssertLessThanOrEqual(Double(tally.ownLost) / Double(tally.own), 0.05, condition.name) + XCTAssertLessThanOrEqual( + Double(tally.strangerKept) / Double(tally.stranger), 0.15, condition.name) + } + } + // MARK: Voice filter func testTheFilterKeepsTheSpeakersWordsAndDropsTheStrangers() async throws { diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift index d7efa65802abe..fbc6efb8ff71c 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift @@ -15,6 +15,32 @@ final class SpeakerFilterTests: XCTestCase { XCTAssertEqual(SpeakerFilter.judge(duration: 0.9, distance: 0.1), .unjudged) } + /// A unit vector at `distance` from the profile `[1, 0, 0]`, leaning towards the second or third axis. + private func voiceprint(_ distance: Float, stranger: Bool = false) -> [Float] { + let along = 1 - distance + let across = (1 - along * along).squareRoot() + return stranger ? [along, 0, across] : [along, across, 0] + } + + func testATurnTheProfileRejectsIsTheUsersWhenItSoundsLikeTheirOtherTurns() { + let profile: [Float] = [1, 0, 0] + // Noise or another microphone moves every turn away from the profile. The opening words pass + // as a short turn; the half minute after them is the same voice and must not be cut. + let turns = [turn("a", 0, 1.5), turn("a", 2, 30), turn("b", 30, 35), turn("b", 35, 35.5)] + let voices = SpeakerFilter.voices( + turns: turns, + embeddings: [voiceprint(0.7), voiceprint(0.65), voiceprint(0.65, stranger: true), nil], + profile: profile) + XCTAssertEqual(voices, [.mine, .mine, .other, .unjudged]) + XCTAssertEqual(SpeakerFilter.decide(turns: turns, voices: voices), .drop([30...35.5])) + + XCTAssertEqual( + SpeakerFilter.voices( + turns: [turn("a", 0, 5), turn("b", 5, 10)], + embeddings: [voiceprint(0.9), voiceprint(0.9)], profile: profile), + [.other, .other]) + } + func testNothingIsCutUnlessTheUserAndSomeoneElseWereBothHeard() { let two = [turn("a", 0, 3), turn("b", 3, 6)] XCTAssertEqual(SpeakerFilter.decide(turns: [], voices: []), .unfiltered(.noSpeech)) diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift index 9ced2e26e3b10..284d886b0ef08 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift @@ -22,3 +22,27 @@ func writeWav(_ samples: [Float], to url: URL) throws { ]) try file.write(from: buffer) } + +/// What the app's recorder writes: AAC at 44.1 kHz in stereo. +func writePhoneRecording(_ samples: [Float], to url: URL) throws { + let rate = 44_100.0 + let file = try AVAudioFile( + forWriting: url, + settings: [ + AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: rate, AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: 128_000, + ]) + let count = Int(Double(samples.count) * rate / DictationEngine.sampleRate) + let buffer = try XCTUnwrap( + AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(count))) + buffer.frameLength = AVAudioFrameCount(count) + for frame in 0.. TurnVoice in - guard let embedding = turn.embedding else { return .unjudged } - return SpeakerFilter.judge( - duration: turn.turn.duration, - distance: SpeakerFilter.cosineDistance(embedding, profile.embedding)) - } - switch SpeakerFilter.decide(turns: heard.map(\.turn), voices: voices) { + let turns = heard.map(\.turn) + let voices = SpeakerFilter.voices( + turns: turns, embeddings: heard.map(\.embedding), profile: profile.embedding) + switch SpeakerFilter.decide(turns: turns, voices: voices) { case .unfiltered(let reason): return (samples, .skipped(reason)) case .drop(let ranges): diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift index 79626d73289bc..07c9d086ff20e 100644 --- a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift @@ -92,6 +92,30 @@ enum SpeakerFilter { return distance <= limit ? .mine : .other } + /// Whose voice each turn is. A turn too short for a voiceprint has nil in `embeddings`. + /// + /// Noise or another microphone moves the user's own turns away from the profile, which was + /// recorded somewhere else. So a turn the profile rejects is held against the turns it accepted + /// in this recording, and is the user's when it matches them. + static func voices(turns: [SpeakerTurn], embeddings: [[Float]?], profile: [Float]) -> [TurnVoice] { + precondition(turns.count == embeddings.count) + var voices = zip(turns, embeddings).map { turn, embedding -> TurnVoice in + guard let embedding else { return .unjudged } + return judge(duration: turn.duration, distance: cosineDistance(embedding, profile)) + } + let mine = voices.indices.filter { voices[$0] == .mine } + guard + let heard = meanEmbedding( + mine.compactMap { embeddings[$0] }, weights: mine.map { Float(turns[$0].duration) }) + else { return voices } + for index in voices.indices where voices[index] == .other { + if let embedding = embeddings[index], cosineDistance(embedding, heard) <= matchDistance { + voices[index] = .mine + } + } + return voices + } + /// The windows that scan a turn, evenly spaced, the last one ending where the turn does. static func scanWindows(_ turn: SpeakerTurn) -> [Range] { guard turn.duration >= scanSeconds else { return [] } From 0bb178d94b97fab0fbd33cbab7ebd2e70231573f Mon Sep 17 00:00:00 2001 From: pseudo <27822378+pseudoseed@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:06:40 -0700 Subject: [PATCH 4/7] fix(mobile): the voice filter no longer cuts the speaker's own words In a long dictation the filter cut stretches of the user's own speech whenever their voiceprint drifted from the profile. A voiceprint alone now never cuts anything. A stretch goes only when it also is more than 6 dB quieter than the user (further from the phone) and the user was heard for at least 1.25 times as long as everything cut. A voice as loud as the user's stays. Measured on 19 LibriSpeech readers: the speaker's words lost when speaking alone are 0 in noise, narrowband and echo (was up to 41.5% for one reader), and a stranger 12 dB under the speaker keeps 4.9% of their words. Co-Authored-By: Claude Code --- .../RealModelTests.swift | 6 +- .../RealVoiceTests.swift | 27 ++++-- .../SpeakerFilterTests.swift | 93 +++++++++++++------ .../ios/Core/DictationEngine.swift | 11 ++- .../ios/Core/SpeakerFilter.swift | 65 +++++++++++-- .../SettingsDictationVoiceRouteScreen.tsx | 7 +- .../src/features/dictation/dictationNative.ts | 2 + .../src/features/dictation/runSummary.ts | 4 + 8 files changed, 162 insertions(+), 53 deletions(-) diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift index 1b91c8e913e1e..d9dee71a8ad34 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift @@ -193,8 +193,12 @@ final class RealModelTests: XCTestCase { func testOnlyMyVoiceDropsTheOtherSpeaker() async throws { try await download(.fast) _ = try await enroll(voice: Self.me) - let mine = try DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine")) + // The user does most of the talking, and the other voice is further from the phone. + let mine = + try DictationEngine.decode(speak(Self.enrollment[1], voice: Self.me, name: "opening")) + + silence(0.6) + DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine")) let theirs = try DictationEngine.decode(speak(Self.theirs, voice: Self.other, name: "theirs")) + .map { $0 * 0.25 } let url = try write(mine + silence(0.6) + theirs, name: "both") let unfiltered = try await engine.transcribe( diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift index 3e84cdf6f6976..490126211d6df 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift @@ -85,6 +85,14 @@ final class RealVoiceTests: XCTestCase { Array(samples.prefix(Int(seconds * DictationEngine.sampleRate))) } + /// The voice 12 dB under `own`: about four times further from the phone than the person holding + /// it. The corpus records each reader at a different level, so the gap is set, not assumed. + private static func distant(_ samples: [Float], from own: [Float]) -> [Float] { + func power(_ clip: [Float]) -> Float { clip.reduce(Float(0)) { $0 + $1 * $1 } / Float(clip.count) } + let gain = 0.25 * (power(own) / power(samples)).squareRoot() + return samples.map { $0 * gain } + } + private func write(_ samples: [Float]) throws -> URL { written += 1 let url = clips.appendingPathComponent("clip-\(written).wav") @@ -229,6 +237,7 @@ final class RealVoiceTests: XCTestCase { let conditions: [(name: String, change: ([Float]) -> [Float])] = [ ("noise at 5 dB", { Self.noisy($0, snr: 5) }), ("narrowband", Self.narrowband), + ("narrowband with noise at 5 dB", { Self.noisy(Self.narrowband($0), snr: 5) }), ("echo with noise at 10 dB", { Self.noisy(Self.echoing($0), snr: 10) }), ] var tallies = conditions.map { _ in (own: 0, ownLost: 0, stranger: 0, strangerKept: 0) } @@ -250,11 +259,12 @@ final class RealVoiceTests: XCTestCase { + "\(Int(seconds(alone))) s: lost \(percent(spoken.subtracting(kept).count, spoken.count)), " + "cut \(outcome?.droppedSeconds ?? 0) s") - let own = condition.change(Self.cut(speaker.held[0], 5)) - let theirs = condition.change(Self.cut(stranger.held[0], 5)) - let unwanted = try await words(theirs, onlyMyVoice: false).0 - .subtracting(try await words(own, onlyMyVoice: false).0) - let heard = try await words(own + theirs, onlyMyVoice: true).0 + // The other voice is across the room, under the same noise or through the same headset. + let own = Self.cut(speaker.held[0], 8) + let theirs = Self.distant(Self.cut(stranger.held[0], 4), from: own) + let unwanted = try await words(condition.change(own + theirs), onlyMyVoice: false).0 + .subtracting(try await words(condition.change(own), onlyMyVoice: false).0) + let heard = try await words(condition.change(own + theirs), onlyMyVoice: true).0 tallies[position].stranger += unwanted.count tallies[position].strangerKept += unwanted.intersection(heard).count } @@ -307,10 +317,11 @@ final class RealVoiceTests: XCTestCase { let (_, soloOutcome) = try await words(Self.cut(stranger.held[2], 6), onlyMyVoice: true) if soloOutcome?.applied == true { strangerSoloCut.append("\(stranger.name) as \(speaker.name)") } - let own = Self.cut(speaker.held[0], 5) + // A dictation is mostly its owner, and anyone else is further from the phone. + let own = Self.cut(speaker.held[0], 8) let ownSecond = Self.cut(speaker.held[1], 4) - let theirs = Self.cut(stranger.held[0], 5) - let theirsShort = Self.cut(stranger.held[1], 2) + let theirs = Self.distant(Self.cut(stranger.held[0], 4), from: own) + let theirsShort = Self.distant(Self.cut(stranger.held[1], 2), from: own) let cases: [(name: String, mine: [[Float]], theirs: [Float], samples: [Float])] = [ ("mine then theirs", [own], theirs, own + theirs), ("theirs then mine", [own], theirs, theirs + own), diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift index fbc6efb8ff71c..4a00857032198 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift @@ -7,6 +7,14 @@ final class SpeakerFilterTests: XCTestCase { SpeakerTurn(speaker: speaker, start: start, end: end) } + /// Unless a test says otherwise the user speaks at -20 dB and everyone else 15 dB under them. + private func decide( + _ turns: [SpeakerTurn], _ voices: [TurnVoice], levels: [Float?]? = nil + ) -> SpeakerDecision { + SpeakerFilter.decide( + turns: turns, voices: voices, levels: levels ?? voices.map { $0 == .mine ? -20 : -35 }) + } + func testAShortTurnMaySitFurtherFromTheProfile() { XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.5), .mine) XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.7), .other) @@ -32,7 +40,7 @@ final class SpeakerFilterTests: XCTestCase { embeddings: [voiceprint(0.7), voiceprint(0.65), voiceprint(0.65, stranger: true), nil], profile: profile) XCTAssertEqual(voices, [.mine, .mine, .other, .unjudged]) - XCTAssertEqual(SpeakerFilter.decide(turns: turns, voices: voices), .drop([30...35.5])) + XCTAssertEqual(decide(turns, voices), .drop([30...35.5])) XCTAssertEqual( SpeakerFilter.voices( @@ -43,34 +51,68 @@ final class SpeakerFilterTests: XCTestCase { func testNothingIsCutUnlessTheUserAndSomeoneElseWereBothHeard() { let two = [turn("a", 0, 3), turn("b", 3, 6)] - XCTAssertEqual(SpeakerFilter.decide(turns: [], voices: []), .unfiltered(.noSpeech)) + XCTAssertEqual(decide([], []), .unfiltered(.noSpeech)) + XCTAssertEqual(decide(two, [.unjudged, .unjudged]), .unfiltered(.tooShort)) + XCTAssertEqual(decide(two, [.other, .other]), .unfiltered(.noMatch)) + XCTAssertEqual(decide(two, [.other, .unjudged]), .unfiltered(.noMatch)) + XCTAssertEqual(decide(two, [.mine, .mine]), .unfiltered(.onlyYou)) + XCTAssertEqual(decide(two, [.mine, .unjudged]), .unfiltered(.onlyYou)) + } + + func testNothingIsCutWhenTheUserWasHeardForLessThanTheSpeechToCut() { + // The profile took three words of a half-minute dictation for the user and the rest for + // someone else. That is a missed match on their own voice, not a stranger talking for 28 s. XCTAssertEqual( - SpeakerFilter.decide(turns: two, voices: [.unjudged, .unjudged]), .unfiltered(.tooShort)) - XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.other, .other]), .unfiltered(.noMatch)) + decide([turn("a", 0, 1.5), turn("a", 2, 30)], [.mine, .other]), .unfiltered(.mostlyOther)) XCTAssertEqual( - SpeakerFilter.decide(turns: two, voices: [.other, .unjudged]), .unfiltered(.noMatch)) - XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.mine, .mine]), .unfiltered(.onlyYou)) + decide([turn("a", 0, 5), turn("b", 5, 9.2)], [.mine, .other]), .unfiltered(.mostlyOther)) + XCTAssertEqual(decide([turn("a", 0, 5), turn("b", 5, 9)], [.mine, .other]), .drop([5...9])) + } + + func testAVoiceIsCutOnlyWhenItIsClearlyQuieterThanTheUsers() { + let turns = [turn("a", 0, 10), turn("a", 10, 14), turn("b", 14, 17)] + let voices: [TurnVoice] = [.mine, .other, .other] + // A stretch as loud as the user is as close to the phone as they are, so it may be them. + XCTAssertEqual(decide(turns, voices, levels: [-20, -22, -26]), .unfiltered(.similarLevel)) + XCTAssertEqual(decide(turns, voices, levels: [-20, -22, -27]), .drop([14...17])) + XCTAssertEqual(decide(turns, voices, levels: [-20, -27, -27]), .drop([10...17])) + XCTAssertEqual(decide(turns, voices, levels: [-20, nil, nil]), .unfiltered(.similarLevel)) + XCTAssertEqual(decide(turns, voices, levels: [nil, -40, -40]), .unfiltered(.similarLevel)) + } + + func testLevelIsHowLoudAStretchGetsNotItsMean() throws { + let rate = 1000.0 + let loud = [Float](repeating: 0.1, count: 20) + let pause = [Float](repeating: 0.001, count: 20) + XCTAssertNil(SpeakerFilter.level([], sampleRate: rate)) + XCTAssertNil(SpeakerFilter.level(loud[..<19], sampleRate: rate)) + XCTAssertNil(SpeakerFilter.level([Float](repeating: 0, count: 40)[...], sampleRate: rate)) + // Two frames in ten are speech: the pauses between them do not pull the level down. + let halting = (0..<10).flatMap { $0 % 5 == 0 ? loud : pause } + XCTAssertEqual(try XCTUnwrap(SpeakerFilter.level(halting[...], sampleRate: rate)), -20, accuracy: 0.001) + // One loud frame in twenty is a click, not the voice. + let click = (0..<20).flatMap { $0 == 7 ? loud : pause } + XCTAssertEqual(try XCTUnwrap(SpeakerFilter.level(click[...], sampleRate: rate)), -60, accuracy: 0.001) XCTAssertEqual( - SpeakerFilter.decide(turns: two, voices: [.mine, .unjudged]), .unfiltered(.onlyYou)) + try XCTUnwrap(SpeakerFilter.level((pause + loud + pause)[20..<40], sampleRate: rate)), -20, + accuracy: 0.001) } func testOnlyTheOtherVoiceIsCutWhateverTheDiarizerCalledIt() { let turns = [turn("a", 0, 3), turn("a", 3.5, 7), turn("a", 7.2, 9)] - XCTAssertEqual( - SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .mine]), .drop([3.5...7])) + XCTAssertEqual(decide(turns, [.mine, .other, .mine]), .drop([3.5...7])) } func testAShortTurnIsCutOnlyWhenItsSpeakerWasAlwaysSomeoneElse() { let turns = [ - turn("a", 0, 3), turn("b", 3, 6), turn("b", 6.2, 6.8), turn("a", 7, 7.5), turn("c", 8, 8.6), + turn("a", 0, 5), turn("b", 5, 8), turn("b", 8.2, 8.8), turn("a", 9, 9.5), turn("c", 10, 10.6), ] - XCTAssertEqual( - SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .unjudged, .unjudged, .unjudged]), - .drop([3...6, 6.2...6.8])) + let voices: [TurnVoice] = [.mine, .other, .unjudged, .unjudged, .unjudged] + XCTAssertEqual(decide(turns, voices), .drop([5...8, 8.2...8.8])) + XCTAssertEqual(decide(turns, voices, levels: [-20, -35, -22, -35, -35]), .drop([5...8])) - let merged = [turn("a", 0, 3), turn("a", 3, 6), turn("a", 6.2, 6.8)] - XCTAssertEqual( - SpeakerFilter.decide(turns: merged, voices: [.mine, .other, .unjudged]), .drop([3...6])) + let merged = [turn("a", 0, 4), turn("a", 4, 7), turn("a", 7.2, 7.8)] + XCTAssertEqual(decide(merged, [.mine, .other, .unjudged]), .drop([4...7])) } private func assertClose( @@ -127,24 +169,17 @@ final class SpeakerFilterTests: XCTestCase { } func testSpeechOverTheUsersOwnTurnIsNotCut() { + XCTAssertEqual(decide([turn("a", 0, 4), turn("b", 3, 6)], [.mine, .other]), .drop([4...6])) + XCTAssertEqual(decide([turn("a", 0, 3), turn("b", 2, 8)], [.other, .mine]), .drop([0...2])) XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3, 6)], voices: [.mine, .other]), - .drop([4...6])) - XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 6), turn("b", 2, 3)], voices: [.other, .mine]), - .drop([0...2, 3...6])) + decide([turn("a", 0, 4), turn("b", 1, 3)], [.mine, .other]), .unfiltered(.overlapping)) XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 1, 3)], voices: [.mine, .other]), - .unfiltered(.overlapping)) - XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3.9, 4.1)], voices: [.mine, .other]), - .unfiltered(.overlapping)) + decide([turn("a", 0, 4), turn("b", 3.9, 4.1)], [.mine, .other]), .unfiltered(.overlapping)) } func testTouchingTurnsBecomeOneCut() { - let turns = [turn("a", 0, 2), turn("b", 2, 4), turn("c", 4, 6), turn("b", 3.5, 4.5)] - XCTAssertEqual( - SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .other, .other]), .drop([2...6])) + let turns = [turn("a", 0, 6), turn("b", 6, 8), turn("c", 8, 10), turn("b", 7.5, 8.5)] + XCTAssertEqual(decide(turns, [.mine, .other, .other, .other]), .drop([6...10])) } func testCosineDistance() { diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift index a2a8427688dc1..0076d31360b77 100644 --- a/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift @@ -107,6 +107,7 @@ private struct DiarizerBox: @unchecked Sendable { private struct HeardTurn: Sendable { var turn: SpeakerTurn var embedding: [Float]? + var level: Float? } private struct EnrollmentClip: Sendable { @@ -425,7 +426,7 @@ actor DictationEngine { let turns = heard.map(\.turn) let voices = SpeakerFilter.voices( turns: turns, embeddings: heard.map(\.embedding), profile: profile.embedding) - switch SpeakerFilter.decide(turns: turns, voices: voices) { + switch SpeakerFilter.decide(turns: turns, voices: voices, levels: heard.map(\.level)) { case .unfiltered(let reason): return (samples, .skipped(reason)) case .drop(let ranges): @@ -470,7 +471,13 @@ actor DictationEngine { turn, windows: windows, distances: Array(distances.prefix(windows.count))) } } - var heard = turns.map { HeardTurn(turn: $0, embedding: nil) } + var heard = turns.map { turn in + let from = min(Int(turn.start * Self.sampleRate), samples.count) + let to = min(max(Int(turn.end * Self.sampleRate), from), samples.count) + return HeardTurn( + turn: turn, embedding: nil, + level: SpeakerFilter.level(samples[from..= SpeakerFilter.minimumJudgedSeconds } diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift index 07c9d086ff20e..4d6a026a55e14 100644 --- a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift @@ -20,6 +20,8 @@ enum FilterSkip: String, Sendable { case onlyYou case tooShort case overlapping + case similarLevel + case mostlyOther case noProfile case notInstalled case failed @@ -49,6 +51,13 @@ enum SpeakerFilter { static let scanSeconds = 3.0 static let scanWindowSeconds = 1.5 static let scanHopSeconds = 0.5 + /// A voice is cut only when it is this much quieter than the user's: further from the phone. + /// One as loud as theirs may be their own, missed by the profile. + static let quieterDecibels: Float = 6 + static let levelFrameSeconds = 0.02 + static let levelQuantile = 0.9 + /// The user has to be heard for this many times the length of the speech that is cut. + static let dominanceRatio = 1.25 static func normalized(_ vector: [Float]) -> [Float] { let length = vector.reduce(Float(0)) { $0 + $1 * $1 }.squareRoot() @@ -152,31 +161,67 @@ enum SpeakerFilter { return zip(edges, edges.dropFirst()).map { SpeakerTurn(speaker: turn.speaker, start: $0, end: $1) } } - /// What to cut from a recording. Only turns judged to be another voice go, and only where no - /// kept turn overlaps them. A turn too short to judge goes when its speaker's judged turns were - /// all another voice. With no turn judged to be the user, nothing is cut: a missed match on - /// their own voice must not lose the dictation. - static func decide(turns: [SpeakerTurn], voices: [TurnVoice]) -> SpeakerDecision { - precondition(turns.count == voices.count) + /// How loud a stretch gets, in decibels: the power of its loudest tenth of 20 ms frames. The + /// mean would fall with every pause in it and rise with the noise under it. Nil when there is + /// nothing to measure. + static func level(_ samples: ArraySlice, sampleRate: Double) -> Float? { + let frame = max(Int(levelFrameSeconds * sampleRate), 1) + var powers: [Float] = [] + var start = samples.startIndex + while start + frame <= samples.endIndex { + powers.append(samples[start..<(start + frame)].reduce(Float(0)) { $0 + $1 * $1 } / Float(frame)) + start += frame + } + guard !powers.isEmpty else { return nil } + powers.sort() + let power = powers[min(Int(Double(powers.count) * levelQuantile), powers.count - 1)] + return power > 0 ? 10 * log10(power) : nil + } + + /// What to cut from a recording. A voiceprint alone has cut the user's own words, so a turn + /// goes only when three things agree: it was judged to be another voice (or is too short to + /// judge and its speaker's judged turns all were), it is clearly quieter than the user, and the + /// user was heard for longer than everything cut. No kept turn may overlap it. With no turn + /// judged to be the user, nothing is cut. + static func decide(turns: [SpeakerTurn], voices: [TurnVoice], levels: [Float?]) -> SpeakerDecision { + precondition(turns.count == voices.count && turns.count == levels.count) if turns.isEmpty { return .unfiltered(.noSpeech) } if voices.allSatisfy({ $0 == .unjudged }) { return .unfiltered(.tooShort) } if !voices.contains(.mine) { return .unfiltered(.noMatch) } if !voices.contains(.other) { return .unfiltered(.onlyYou) } + let mine = turns.indices.filter { voices[$0] == .mine && turns[$0].end > turns[$0].start } + var power: Float = 0 + var measured: Float = 0 + for index in mine { + guard let level = levels[index] else { continue } + power += pow(10, level / 10) * Float(turns[index].duration) + measured += Float(turns[index].duration) + } + guard measured > 0, power > 0 else { return .unfiltered(.similarLevel) } + let userLevel = 10 * log10(power / measured) + var judgedBySpeaker: [String: Set] = [:] for (turn, voice) in zip(turns, voices) where voice != .unjudged { judgedBySpeaker[turn.speaker, default: []].insert(voice) } var dropped: [ClosedRange] = [] var kept: [ClosedRange] = [] - for (turn, voice) in zip(turns, voices) where turn.end > turn.start { - let isOther = voice == .other || (voice == .unjudged && judgedBySpeaker[turn.speaker] == [.other]) - if isOther { dropped.append(turn.start...turn.end) } else { kept.append(turn.start...turn.end) } + for (index, turn) in turns.enumerated() where turn.end > turn.start { + let isOther = + voices[index] == .other + || (voices[index] == .unjudged && judgedBySpeaker[turn.speaker] == [.other]) + let isQuieter = levels[index].map { userLevel - $0 > quieterDecibels } ?? false + if isOther && isQuieter { dropped.append(turn.start...turn.end) } else { kept.append(turn.start...turn.end) } } + if dropped.isEmpty { return .unfiltered(.similarLevel) } let ranges = subtract(merged(dropped), merged(kept)).filter { $0.upperBound - $0.lowerBound >= minimumDropSeconds } - return ranges.isEmpty ? .unfiltered(.overlapping) : .drop(ranges) + if ranges.isEmpty { return .unfiltered(.overlapping) } + let userSeconds = seconds(merged(mine.map { turns[$0].start...turns[$0].end })) + if userSeconds < seconds(ranges) * dominanceRatio { return .unfiltered(.mostlyOther) } + return .drop(ranges) } static func merged(_ ranges: [ClosedRange]) -> [ClosedRange] { diff --git a/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx b/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx index 743c9e049d7fa..b226e29b65035 100644 --- a/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx +++ b/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx @@ -230,9 +230,10 @@ function DictationVoice(props: { ) : null} The filter compares each stretch of speech with your voice profile and removes the ones that - are someone else. It adds a short wait after each recording. If it does not recognise you, - nothing is removed. Your voice profile stays on this device. The filter model is by - pyannote, WeSpeaker and BUT Speech@FIT (CC BY 4.0), converted by Fluid Inference. + are someone else and quieter than you, further from the phone. A voice as loud as yours + stays. It adds a short wait after each recording. If it does not recognise you, nothing is + removed. Your voice profile stays on this device. The filter model is by pyannote, WeSpeaker + and BUT Speech@FIT (CC BY 4.0), converted by Fluid Inference. ); diff --git a/apps/mobile/src/features/dictation/dictationNative.ts b/apps/mobile/src/features/dictation/dictationNative.ts index 5095628588623..211f5123a512b 100644 --- a/apps/mobile/src/features/dictation/dictationNative.ts +++ b/apps/mobile/src/features/dictation/dictationNative.ts @@ -6,6 +6,8 @@ export type DictationFilterSkip = | "onlyYou" | "tooShort" | "overlapping" + | "similarLevel" + | "mostlyOther" | "noProfile" | "notInstalled" | "failed"; diff --git a/apps/mobile/src/features/dictation/runSummary.ts b/apps/mobile/src/features/dictation/runSummary.ts index 08f9578cbd8ed..63b9444a622e0 100644 --- a/apps/mobile/src/features/dictation/runSummary.ts +++ b/apps/mobile/src/features/dictation/runSummary.ts @@ -17,6 +17,10 @@ function describeFilter(filter: DictationFilterReport): string { return "Only your voice was heard."; case "noMatch": return "Your voice was not recognised, so nothing was removed."; + case "similarLevel": + return "Another voice was as loud as yours, so nothing was removed."; + case "mostlyOther": + return "Most of the recording did not match your voice, so nothing was removed."; case "noProfile": return "No voice profile is recorded, so nothing was removed."; case "notInstalled": From ac8236026f0be876701989aa7b9bbf72bbee8709 Mon Sep 17 00:00:00 2001 From: pseudo <27822378+pseudoseed@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:07:05 -0700 Subject: [PATCH 5/7] fix(mobile): recording starts without waiting for Apple's speech model The microphone waited for Apple to prepare its model before it recorded, so words spoken in that gap were never captured. Preparation now runs while the user talks and is awaited when the recording is transcribed. A language Apple cannot take is reported then; a downloaded model that was still loading takes the recording instead. Co-Authored-By: Claude Code --- .../dictation/dictationTranscriber.test.ts | 8 ++- .../dictation/dictationTranscriber.ts | 26 +++++-- .../src/native/voiceTranscription.ios.test.ts | 69 ++++++++++++++++++- .../src/native/voiceTranscription.ios.ts | 40 +++++++++-- 4 files changed, 127 insertions(+), 16 deletions(-) diff --git a/apps/mobile/src/features/dictation/dictationTranscriber.test.ts b/apps/mobile/src/features/dictation/dictationTranscriber.test.ts index a7846e0b5e94d..4e0e5801f416f 100644 --- a/apps/mobile/src/features/dictation/dictationTranscriber.test.ts +++ b/apps/mobile/src/features/dictation/dictationTranscriber.test.ts @@ -44,7 +44,8 @@ const TRANSCRIPT: DictationTranscript = { function setup( input: { readonly settings?: Partial; - readonly system?: "present" | "absent" | "refuses"; + /** "refuses" fails to prepare; "fails" prepares and then fails on the recording. */ + readonly system?: "present" | "absent" | "refuses" | "fails"; readonly systemLocale?: string; readonly native?: "present" | "absent"; readonly locale?: string; @@ -62,6 +63,7 @@ function setup( write: async () => {}, }); const systemTranscribe = vi.fn(async () => { + if (input.system === "fails") throw new Error("That language is not supported."); clock.now += 250; return SPOKEN; }); @@ -221,7 +223,7 @@ describe("with a downloaded model selected", () => { expect(native.transcribe.mock.calls[0]).toContain("en-US"); }); - it.each(["absent", "refuses"] as const)( + it.each(["absent", "refuses", "fails"] as const)( "waits for a loading model when the system recognizer %s", async (system) => { const { dictate, native, lastRun } = setup({ settings, system }); @@ -256,7 +258,7 @@ describe("with a downloaded model selected", () => { expect(lastRun()).toMatchObject({ engine: "apple", fallback: "failed" }); }); - it.each(["absent", "refuses"] as const)( + it.each(["absent", "refuses", "fails"] as const)( "reports the model's own failure when the system recognizer %s", async (system) => { const { dictate, native, lastRun } = setup({ settings, system }); diff --git a/apps/mobile/src/features/dictation/dictationTranscriber.ts b/apps/mobile/src/features/dictation/dictationTranscriber.ts index ca2a157256e36..84be6a59ce939 100644 --- a/apps/mobile/src/features/dictation/dictationTranscriber.ts +++ b/apps/mobile/src/features/dictation/dictationTranscriber.ts @@ -67,7 +67,10 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic fallback: DictationFallback | null, ): PreparedVoiceTranscription { return { - locale: prepared.locale, + // Read late: the system recognizer settles its locale while the user talks. + get locale() { + return prepared.locale; + }, async transcribe(uri, options) { const started = deps.now(); const text = await prepared.transcribe(uri, options); @@ -115,7 +118,12 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic const system = await prepareSystem(options).catch(() => null); throwIfVoiceTranscriptionAborted(signal); if (system === null) throw failure; - return viaSystem(system, "failed").transcribe(uri, options); + try { + return await viaSystem(system, "failed").transcribe(uri, options); + } catch { + throwIfVoiceTranscriptionAborted(signal); + throw failure; + } } finally { signal.removeEventListener("abort", cancel); } @@ -186,10 +194,16 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic () => false, ); if (loaded) return model.transcribe(uri, transcribeOptions); - return viaSystem(system, loadFailed ? "failed" : "warming-up").transcribe( - uri, - transcribeOptions, - ); + try { + return await viaSystem(system, loadFailed ? "failed" : "warming-up").transcribe( + uri, + transcribeOptions, + ); + } catch { + throwIfVoiceTranscriptionAborted(transcribeOptions.signal); + // The system recognizer can refuse the language only now, after the recording. + return model.transcribe(uri, transcribeOptions); + } }, }; } diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index d83c371bc2a90..8730bea3e6bfa 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -1,7 +1,8 @@ import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller"; const mocks = vi.hoisted(() => ({ isAvailable: vi.fn<(locale: string) => boolean>(), @@ -43,8 +44,39 @@ function deferred() { return { promise, resolve }; } +/** Drives one on-device dictation through the controller the composer uses. */ +function createDictation() { + const recorder = { + uri: "file:///voice.m4a", + prepareToRecordAsync: async () => {}, + record: vi.fn(), + stop: vi.fn(async () => {}), + }; + const commits: string[] = []; + const controller = new VoiceInputController({ + recorder, + getTranscriber: () => ({ + prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options), + }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => {}, + deleteRecording: () => {}, + readDraft: () => ({ + ownerKey: "environment:thread", + text: "", + selection: { start: 0, end: 0 }, + revision: 1, + }), + commitDraft: (text) => commits.push(text), + onStateChange: () => {}, + }); + return { controller, recorder, commits }; +} + beforeEach(() => { vi.resetAllMocks(); + resetVoiceInputGlobalsForTests(); mocks.isAvailable.mockReturnValue(true); mocks.prepare.mockResolvedValue("sv-SE"); mocks.readAudio.mockResolvedValue(audio); @@ -56,6 +88,41 @@ afterEach(() => { }); describe("getLocalVoiceTranscriber", () => { + it("records while Apple prepares its speech model and transcribes once it is ready", async () => { + const nativePrepare = deferred(); + mocks.prepare.mockReturnValue(nativePrepare.promise); + const dictation = createDictation(); + + await dictation.controller.start(); + expect(dictation.recorder.record).toHaveBeenCalledOnce(); + expect(dictation.controller.currentState.phase).toBe("recording"); + + const stopping = dictation.controller.stop(); + nativePrepare.resolve("sv-SE"); + await stopping; + + expect(mocks.transcribe).toHaveBeenCalledExactlyOnceWith(audio, "sv-SE"); + expect(dictation.commits).toEqual(["Hej världen."]); + }); + + it.each([ + ["AppleTranscriptionUnsupportedLocale", "unsupported-locale"], + ["Network", "preparation-failed"], + ])( + "reports a %s failure to prepare when the recording is transcribed", + async (code, expected) => { + mocks.prepare.mockRejectedValue(Object.assign(new Error("failed"), { code })); + const options = { signal: new AbortController().signal }; + const error = await (await getLocalVoiceTranscriber())! + .prepare(options) + .then((prepared) => prepared.transcribe("file:///voice.m4a", options)) + .catch((cause: unknown) => cause); + + expect(error).toMatchObject({ code: expected }); + expect(mocks.transcribe).not.toHaveBeenCalled(); + }, + ); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = deferred(); const readResult = deferred(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 5fc6c9a0b8a2b..607b9ceb446eb 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -43,24 +43,52 @@ export async function getLocalVoiceTranscriber(): Promise { throwIfVoiceTranscriptionAborted(signal); - if (!AppleTranscription.isAvailable(locale)) { + if (!AppleTranscription.isAvailable(deviceLocale)) { throw new VoiceTranscriptionError({ code: "unavailable", message: "Voice transcription requires a supported device with iOS 26 or later.", }); } + let locale = deviceLocale; + // Not awaited: recording starts now and Apple's model loads while the user talks. + const model = prepareLocale(deviceLocale, signal).then((supported) => { + locale = supported; + }); + // Recorder setup can fail before the recording is transcribed. + void model.catch(() => {}); + + async function ready(signal: AbortSignal) { + throwIfVoiceTranscriptionAborted(signal); + const cancelled = Promise.withResolvers(); + const cancel = () => cancelled.resolve(); + signal.addEventListener("abort", cancel, { once: true }); + try { + await Promise.race([model, cancelled.promise]); + throwIfVoiceTranscriptionAborted(signal); + return locale; + } finally { + signal.removeEventListener("abort", cancel); + } + } + return { + get locale() { + return locale; + }, + transcribe: async (uri, options) => + transcribeVoiceRecording(uri, await ready(options.signal), options), + }; +} + +async function prepareLocale(locale: string, signal: AbortSignal): Promise { try { const supportedLocale = await AppleTranscription.prepare(locale); throwIfVoiceTranscriptionAborted(signal); - return { - locale: supportedLocale, - transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options), - }; + return supportedLocale; } catch (error) { throwIfVoiceTranscriptionAborted(signal); if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { From af08ab2301b0c5f2f37b0fb8292ea1426353d59a Mon Sep 17 00:00:00 2001 From: pseudo <27822378+pseudoseed@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:07:45 -0700 Subject: [PATCH 6/7] fix(mobile): keep the words spoken before a recording is interrupted A call, Siri, a recorder error or leaving the app threw the whole recording away. expo-audio also pauses the recorder for an audio interruption without an event and never resumes it, so the app kept showing a recording that captured nothing more. An interrupted recording is now transcribed and its words are added, with a notice that it stopped early. The metering loop notices a recorder that stopped capturing and finishes the same way. Co-Authored-By: Claude Code --- .../voice-input/voiceInputSession.test.ts | 28 +++++++++ .../features/voice-input/voiceInputSession.ts | 9 ++- docs/internals/voice-input.md | 16 +++-- docs/user/composer.md | 10 ++-- .../src/voice-input/controller.test.ts | 60 ++++++++++++++----- .../src/voice-input/controller.ts | 24 +++++--- 6 files changed, 116 insertions(+), 31 deletions(-) diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 9d30e376b54e2..832945a2407c5 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -399,6 +399,34 @@ describe("global voice input", () => { } }); + it("finishes with the captured audio when the recorder stops capturing on its own", async () => { + vi.useFakeTimers(); + try { + let sample: VoiceRecordingSample | null = null; + const { session, recorder } = createSession({ sampleRecording: () => sample }); + const commit = vi.fn(); + const interrupt = vi.spyOn(session.controller, "interruptRecording"); + + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + // Not capturing yet is not a recorder that stopped. + vi.advanceTimersByTime(80); + expect(interrupt).not.toHaveBeenCalled(); + + sample = { durationMillis: 3_000, metering: -20 }; + vi.advanceTimersByTime(80); + // iOS pauses the recorder for a call or Siri and never reports or resumes it. + sample = null; + vi.advanceTimersByTime(80); + expect(interrupt).toHaveBeenCalledOnce(); + await interrupt.mock.results[0]!.value; + + expect(recorder.stop).toHaveBeenCalled(); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + } finally { + vi.useRealTimers(); + } + }); + it("publishes which owners are focused", () => { const { session } = createSession(); const listener = vi.fn(); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 5bf9395daf59c..34491a27808ea 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -96,6 +96,7 @@ export class VoiceInputSession { private audioLevels = SILENT_LEVELS; private releaseKeepAwake: (() => void) | null = null; private meteringInterval: ReturnType | null = null; + private capturing = false; constructor(dependencies: VoiceInputSessionDependencies) { this.dependencies = dependencies; @@ -181,6 +182,7 @@ export class VoiceInputSession { private startRecordingActivity(): void { this.releaseKeepAwake = this.dependencies.keepAwake(); + this.capturing = false; this.sampleRecording(); this.meteringInterval = setInterval(() => this.sampleRecording(), VOICE_METERING_INTERVAL_MS); } @@ -195,7 +197,12 @@ export class VoiceInputSession { private sampleRecording(): void { if (this.controller.currentState.phase !== "recording") return; const sample = this.dependencies.sampleRecording(); - if (!sample) return; + if (!sample) { + // The system pauses the recorder for a call or Siri without an event, and never resumes it. + if (this.capturing) void this.controller.interruptRecording(); + return; + } + this.capturing = true; const level = normalizeVoiceInputDecibels(sample.metering); if (level !== 0 || this.audioLevels.some((sample) => sample !== 0)) { diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 6b39541d78046..24d7778e10645 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -28,16 +28,24 @@ the system one and runs a downloaded model through the `features/dictation`, `modules/pseudo-dictation` and `plugins/withDictationPods.cjs`. Upstream files only import from them, at the end of their import block, so upstream merges stay small. -- The controller awaits preparation before it records, so preparation never waits on a model. - The model loads while the user talks, and the system recognizer stands in for a model that is - missing, still compiling, or failed on the recording. +- The controller awaits preparation before it records, so preparation never waits on a model, + Apple's included. The model loads while the user talks, and the system recognizer stands in for + a model that is missing, still compiling, or failed on the recording. A language Apple cannot + take is therefore known only when the recording is transcribed. +- expo-audio pauses the recorder for an audio-session interruption without an event and never + resumes it. The metering loop notices that capture stopped, and the controller transcribes the + captured file. That file wins over the recorder's current one, because a media-services reset + hands the recorder a new, empty file. - FluidAudio is a git pod pinned to a commit and added by the config plugin to the app target only. The widget extension must not link it. - The engine reaches the network only inside an explicit download. - The voice filter never runs on the GPU, because iOS refuses GPU work in the background, where a dictation may finish. - The thresholds in [SpeakerFilter](../../apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift) - come from measurement and lean towards keeping a stranger's words over cutting the user's. + come from measurement and lean towards keeping a stranger's words over cutting the user's. A + voiceprint alone cut the user's own words in long dictations, so a turn is cut only when it is + also clearly quieter than the user. Loudness is a heuristic, not identity: a voice as loud as + the user's stays. `RealVoiceTests` measures them on recordings of real people; rerun it before changing one. - `ios/Core` builds as a Swift package on macOS so `swift test` covers it without a simulator. The Expo module file is outside that package and compiles only in an iOS build. diff --git a/docs/user/composer.md b/docs/user/composer.md index 4b844b3ef3b1b..e4e2c56d52685 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -201,8 +201,8 @@ transcribing. The first use may download a speech model and needs a network connection. Later transcription works offline for that language. On Android, transcription processes the full recording at roughly its original duration. Recordings can be up to five -minutes long. Canceling or an audio interruption discards the recording and -preserves your existing draft. Navigating to another screen keeps dictation active; +minutes long. Canceling discards the recording and preserves your existing draft. When a call, +Siri or another app interrupts a recording, the words spoken before it are added to the draft. Navigating to another screen keeps dictation active; use the floating controls to finish or cancel. While recording, the screen stays awake; it can sleep normally once recording stops. @@ -235,8 +235,10 @@ Apple's transcription, the microphone button works once a model is downloaded. is transcribed. Download the voice filter (about 21 MB), then record 3 short samples of your voice. It works with a downloaded model, not with Apple, and adds a short wait after each recording. -When the filter does not recognise your voice in a recording, it removes nothing. People talking -over you cannot be separated from you. Your voice profile stays on the device; record it again or +The filter removes a voice that is not yours and is quieter than you, further from the phone. A +voice as loud as yours stays, so that your own words are never the ones removed. When the filter +does not recognise your voice in a recording, it removes nothing. People talking over you cannot be +separated from you. Your voice profile stays on the device; record it again or delete it on the same page. ### Corrections diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 52069d1bd67b9..1f364d83577b1 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -501,32 +501,64 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState.error).toContain("no longer available"); }); - it("discards recorder errors and audio interruptions without transcribing", async () => { - const transcribe = vi.fn(async () => "ignored"); - const preparationEntered = deferred(); + it.each([ + [ + "a recorder error", + (controller: VoiceInputController) => + controller.handleRecorderStatus({ + isFinished: true, + hasError: true, + error: "Audio route changed", + url: null, + }), + ], + ["the background", (controller: VoiceInputController) => controller.appMovedToBackground()], + ])("adds the words captured before %s stopped the recording", async (_name, interrupt) => { + const transcribe = vi.fn(async () => "captured words"); const harness = createHarness({ - getTranscriber: () => ({ - prepare: async ({ signal }) => { - preparationEntered.resolve(signal); - return preparedTranscription(transcribe); - }, - }), + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), }); await harness.controller.start(); - const signal = await preparationEntered.promise; + // A media-services reset points the recorder at a new, empty file. harness.recorder.uri = "file:///reset-empty.m4a"; + await interrupt(harness.controller); + + expect(transcribe).toHaveBeenCalledWith("file:///voice.m4a", expect.anything()); + expect(harness.commits).toEqual([ + { text: "hello captured words", selection: { start: 20, end: 20 } }, + ]); + expect(harness.controller.currentState).toEqual({ + phase: "error", + error: "Voice recording stopped early. The words it captured were added.", + errorAction: "retry", + }); + expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); + }); + + it.each([ + ["nothing was said", async () => " "], + [ + "the audio cannot be read", + async (): Promise => { + throw new Error("unreadable"); + }, + ], + ])("reports the interruption when %s before it", async (_name, transcribe) => { + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released")); await harness.controller.handleRecorderStatus({ isFinished: true, hasError: true, error: "Audio route changed", - url: "file:///voice.m4a", + url: null, }); expect(harness.commits).toEqual([]); - expect(transcribe).not.toHaveBeenCalled(); - expect(signal.aborted).toBe(true); expect(harness.controller.currentState.error).toBe("Audio route changed"); - expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); }); it("cancels preparation when the app reaches the background", async () => { diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 5b057c61fab23..4e7eac828f4ab 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -187,6 +187,8 @@ function transcriptionErrorMessage(error: unknown): string { } const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; +const RECORDING_STOPPED_EARLY_MESSAGE = + "Voice recording stopped early. The words it captured were added."; export class VoiceInputController { private readonly dependencies: VoiceInputControllerDependencies; @@ -325,9 +327,7 @@ export class VoiceInputController { completedUri: string | null = null, ): Promise | void { if (this.state.phase !== "recording") return; - this.rememberRecordingUri(completedUri); - this.recordingUri = completedUri ?? this.recordingUri; - return this.discardRecording(message); + return this.finishRecording(completedUri !== null, completedUri, message); } appMovedToBackground(): Promise | void { @@ -371,9 +371,11 @@ export class VoiceInputController { } } + /** `interruption` is set when the recording ended without the user stopping it. */ private async finishRecording( alreadyStopped: boolean, completedUri: string | null, + interruption: string | null = null, ): Promise { if (this.finishing || this.state.phase !== "recording") return; this.finishing = true; @@ -381,9 +383,14 @@ export class VoiceInputController { this.setState({ phase: "transcribing", error: null, errorAction: null }); try { - if (!alreadyStopped) await this.dependencies.recorder.stop(); + if (!alreadyStopped) { + const stopping = this.dependencies.recorder.stop(); + // An interrupted recorder may already be gone; the file it wrote is still there. + await (interruption === null ? stopping : stopping.catch(() => {})); + } await this.releaseAudioSession(); - this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri; + // The file capture started in wins: a media-services reset gives the recorder a new, empty one. + this.recordingUri = completedUri ?? this.recordingUri ?? this.dependencies.recorder.uri; this.rememberRecordingUri(this.recordingUri); if (!this.isCurrent(operationToken)) return; if ( @@ -407,7 +414,7 @@ export class VoiceInputController { ); } catch (error) { if (this.isCurrent(operationToken)) { - this.setError(transcriptionErrorMessage(error), "retry"); + this.setError(interruption ?? transcriptionErrorMessage(error), "retry"); } return; } @@ -427,12 +434,13 @@ export class VoiceInputController { return; } if (result.kind === "empty") { - this.setError("No speech was detected.", "retry"); + this.setError(interruption ?? "No speech was detected.", "retry"); return; } this.dependencies.commitDraft(result.text, result.selection); - this.setState(IDLE_STATE); + if (interruption === null) this.setState(IDLE_STATE); + else this.setError(RECORDING_STOPPED_EARLY_MESSAGE, "retry"); } catch { if (this.isCurrent(operationToken)) { this.setError("Could not finish voice recording.", "retry"); From dca681ef164afb1d308697929ed60788b665ae8c Mon Sep 17 00:00:00 2001 From: pseudo <27822378+pseudoseed@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:21:25 -0700 Subject: [PATCH 7/7] fix(mobile): dictation recordings are kept so spoken words can be recovered Every recording was deleted the moment its dictation ended, so a failed, empty or wrong transcript could not be recovered, and a draft that changed during the dictation dropped the whole transcript. - The recorder's file is moved to a kept folder as soon as recording stops, before transcription. The last 5 stay, canceled ones included. - A transcription that throws offers "Transcribe again", which runs the kept file and adds it to the draft. - Settings > Microphone > Model lists the recent recordings. Each runs through the model in use without the voice filter, with a copy button. - A transcript for an edited draft goes at its end. It is dropped only when the draft is gone. - The 5-minute limit says so, and a notice after a commit offers no retry at an old caret. Co-Authored-By: Claude Code --- .../SettingsDictationModelsRouteScreen.tsx | 166 ++++++++++--- .../features/dictation/dictationRuntime.ts | 2 + .../dictation/dictationTranscriber.test.ts | 13 + .../dictation/dictationTranscriber.ts | 4 +- .../components/ScheduledTaskPromptField.tsx | 1 + .../src/features/threads/ThreadComposer.tsx | 1 + .../voice-input/ComposerDictationControl.tsx | 18 ++ .../voice-input/GlobalVoiceInputControl.tsx | 20 +- .../voice-input/VoiceInputProvider.tsx | 2 + .../features/voice-input/keptRecordings.ts | 55 +++++ .../voice-input/lazyVoiceRecorder.test.ts | 1 + .../voice-input/useVoiceInputController.ts | 2 + .../voice-input/voiceInputSession.test.ts | 120 ++++++--- .../features/voice-input/voiceInputSession.ts | 14 +- .../native/voiceTranscription.android.test.ts | 7 +- .../src/native/voiceTranscription.ios.test.ts | 1 + docs/internals/voice-input.md | 24 +- docs/user/composer.md | 14 +- .../src/voice-input/controller.test.ts | 184 +++++++++++--- .../src/voice-input/controller.ts | 233 +++++++++++++----- 20 files changed, 697 insertions(+), 185 deletions(-) create mode 100644 apps/mobile/src/features/voice-input/keptRecordings.ts diff --git a/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx b/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx index 7f00c9b406609..1f79d8b11dfcf 100644 --- a/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx +++ b/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx @@ -1,12 +1,22 @@ import { useAtomValue } from "@effect/atom-react"; -import { useEffect, useState, useSyncExternalStore } from "react"; +import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; +import * as Clipboard from "expo-clipboard"; +import { useFocusEffect } from "@react-navigation/native"; +import { File } from "expo-file-system"; +import { useCallback, useEffect, useState, useSyncExternalStore } from "react"; import { Alert, View } from "react-native"; import { AppText as Text } from "../../components/ui/AppText"; import { ListActionRow } from "../../components/ui/ListActionRow"; import { ListChoiceRow } from "../../components/ui/ListChoiceRow"; import { ListRowLayout } from "../../components/ui/ListRowLayout"; +import { getLocalVoiceTranscriber as getSystemVoiceTranscriber } from "../../native/voiceTranscription"; import { SettingsSection } from "../settings/components/SettingsSection"; +import { + KEPT_VOICE_RECORDINGS, + keptVoiceRecordings, + type KeptVoiceRecording, +} from "../voice-input/keptRecordings"; import { localVoiceTranscriberAtom } from "../voice-input/VoiceInputProvider"; import { useAppleDictationAvailable } from "./appleDictation"; import { nativeErrorMessage, type DictationDownloads } from "./dictationDownloads"; @@ -23,6 +33,7 @@ import { currentDictationSettings, deviceLocale, dictationDownloads, + makeDictationTranscriber, updateDictationSettings, useDictationState, } from "./dictationRuntime"; @@ -36,12 +47,25 @@ import { describeRun } from "./runSummary"; const TITLE = "Dictation model"; -type SampleResult = - | { readonly status: "idle" } +type TrialResult = | { readonly status: "running" } | { readonly status: "done"; readonly text: string } | { readonly status: "failed"; readonly message: string }; +const SAMPLE = "sample"; + +/** `of` is the sample or a kept recording's URI. One result is on screen at a time, so the run details under it are its own. */ +type Trial = { readonly of: string; readonly result: TrialResult }; + +function recordedAtLabel(recordedAt: number): string { + return new Date(recordedAt).toLocaleString([], { + weekday: "long", + hour: "numeric", + minute: "2-digit", + second: "2-digit", + }); +} + export function SettingsDictationModelsRouteScreen() { const downloads = dictationDownloads(); if (dictationNative === null || downloads === null) return ; @@ -58,7 +82,9 @@ function DictationModels(props: { const transcriber = useAtomValue(localVoiceTranscriberAtom); const appleMissing = useAppleDictationAvailable() === false; const [device] = useState(() => native.deviceInfo()); - const [sample, setSample] = useState({ status: "idle" }); + const [trial, setTrial] = useState(null); + const [copied, setCopied] = useState(false); + const [recordings, setRecordings] = useState(keptVoiceRecordings); const suggested = recommendedDictationModel({ physicalMemoryBytes: device.physicalMemoryBytes, locale: deviceLocale(), @@ -69,6 +95,8 @@ function DictationModels(props: { void downloads.refresh(); }, [downloads]); + useFocusEffect(useCallback(() => setRecordings(keptVoiceRecordings()), [])); + const describe = (model: LocalDictationModel) => model.id === suggested ? `${model.summary} Suggested for this device.` : model.summary; const installed = LOCAL_DICTATION_MODELS.filter( @@ -108,27 +136,87 @@ function DictationModels(props: { ], ); - const transcribeSample = async () => { - const uri = native.sampleClipUri(); - if (uri === null || transcriber === null || transcriber === "checking") { - setSample({ status: "failed", message: "Dictation is not available right now." }); - return; - } - setSample({ status: "running" }); + const transcribe = async ( + of: Trial["of"], + uri: string, + using: VoiceTranscriber, + failure: string, + ) => { + setCopied(false); + setTrial({ of, result: { status: "running" } }); const { signal } = new AbortController(); try { - // Loaded first so the sample runs on the chosen model, not on the stand-in for a cold one. + // Loaded first so this runs on the chosen model, not on the stand-in for a cold one. if (installed.some((model) => model.id === settings.modelId)) { await native.prepare(settings.modelId, DICTATION_PROCESSING); } - const prepared = await transcriber.prepare({ signal }); - setSample({ status: "done", text: await prepared.transcribe(uri, { signal }) }); + const prepared = await using.prepare({ signal }); + const text = await prepared.transcribe(uri, { signal }); + setTrial({ of, result: { status: "done", text } }); } catch (error) { - setSample({ - status: "failed", - message: nativeErrorMessage(error, "The sample could not be transcribed."), + setTrial({ of, result: { status: "failed", message: nativeErrorMessage(error, failure) } }); + } + }; + + const transcribeSample = () => { + const uri = native.sampleClipUri(); + if (uri === null || transcriber === null || transcriber === "checking") { + setTrial({ + of: SAMPLE, + result: { status: "failed", message: "Dictation is not available right now." }, }); + return; } + void transcribe(SAMPLE, uri, transcriber, "The sample could not be transcribed."); + }; + + const transcribeRecording = (recording: KeptVoiceRecording) => { + if (!new File(recording.uri).exists) { + setRecordings(keptVoiceRecordings()); + setTrial(null); + return; + } + void transcribe( + recording.uri, + recording.uri, + makeDictationTranscriber(getSystemVoiceTranscriber, { ignoreVoiceFilter: true }), + "The recording could not be transcribed.", + ); + }; + + const running = trial?.result.status === "running" ? trial.of : null; + const resultOf = (of: Trial["of"]) => { + const result = trial?.of === of ? trial.result : null; + if (result?.status === "failed") { + return ( + + {result.message} + + ); + } + if (result?.status !== "done") return null; + const heard = result.text.trim().length > 0; + return ( + <> + + + {heard ? result.text : "No speech was found in this recording."} + + {lastRun === null ? null : ( + {describeRun(lastRun)} + )} + + {heard && of !== SAMPLE ? ( + + void Clipboard.setStringAsync(result.text).then(() => setCopied(true))} + /> + + ) : null} + + ); }; return ( @@ -188,22 +276,36 @@ function DictationModels(props: { icon="play" label="Transcribe a sample" description="Runs a built-in recording through the model in use." - loading={sample.status === "running"} - onPress={() => void transcribeSample()} + loading={running === SAMPLE} + disabled={running !== null} + onPress={transcribeSample} /> - {sample.status === "done" ? ( - - {sample.text} - {lastRun === null ? null : ( - {describeRun(lastRun)} - )} - - ) : null} - {sample.status === "failed" ? ( - - {sample.message} - - ) : null} + {resultOf(SAMPLE)} + + + + {recordings.length === 0 ? ( + + ) : ( + recordings.map((recording, index) => ( + 0 ? "border-t border-border-subtle" : undefined} + > + transcribeRecording(recording)} + /> + {resultOf(recording.uri)} + + )) + )} {installed.length === 0 ? null : ( diff --git a/apps/mobile/src/features/dictation/dictationRuntime.ts b/apps/mobile/src/features/dictation/dictationRuntime.ts index ae68cc611cf5c..34d34a1102d31 100644 --- a/apps/mobile/src/features/dictation/dictationRuntime.ts +++ b/apps/mobile/src/features/dictation/dictationRuntime.ts @@ -49,6 +49,7 @@ export function deviceLocale(): string { export function makeDictationTranscriber( getSystemTranscriber: () => Promise, + options: { readonly ignoreVoiceFilter?: boolean } = {}, ): VoiceTranscriber { const { store, pending } = dictationRuntime(); return createDictationTranscriber({ @@ -59,6 +60,7 @@ export function makeDictationTranscriber( getLocale: deviceLocale, now: Date.now, newRequestId: () => `dictation-${Date.now()}-${++requests}`, + ignoreVoiceFilter: options.ignoreVoiceFilter, }); } diff --git a/apps/mobile/src/features/dictation/dictationTranscriber.test.ts b/apps/mobile/src/features/dictation/dictationTranscriber.test.ts index 4e0e5801f416f..c51bd71e07307 100644 --- a/apps/mobile/src/features/dictation/dictationTranscriber.test.ts +++ b/apps/mobile/src/features/dictation/dictationTranscriber.test.ts @@ -49,6 +49,7 @@ function setup( readonly systemLocale?: string; readonly native?: "present" | "absent"; readonly locale?: string; + readonly ignoreVoiceFilter?: boolean; } = {}, ) { const clock = { now: 1_000 }; @@ -89,6 +90,7 @@ function setup( now: () => clock.now, pending, newRequestId: () => "request-1", + ignoreVoiceFilter: input.ignoreVoiceFilter, }); const abort = new AbortController(); const dictate = async () => { @@ -164,6 +166,17 @@ describe("with a downloaded model selected", () => { ); expect(native.prepare).toHaveBeenCalledWith("parakeet-v3", "auto"); expect(getSystemTranscriber).not.toHaveBeenCalled(); + + const again = setup({ settings, ignoreVoiceFilter: true }); + await again.dictate(); + expect(again.native.transcribe).toHaveBeenCalledWith( + expect.any(String), + URI, + "parakeet-v3", + "auto", + false, + "en-US", + ); expect(lastRun()).toEqual({ at: 1_000, engine: "parakeet-v3", diff --git a/apps/mobile/src/features/dictation/dictationTranscriber.ts b/apps/mobile/src/features/dictation/dictationTranscriber.ts index 84be6a59ce939..dcbdb47a18d2b 100644 --- a/apps/mobile/src/features/dictation/dictationTranscriber.ts +++ b/apps/mobile/src/features/dictation/dictationTranscriber.ts @@ -28,6 +28,8 @@ export interface DictationTranscriberDeps { readonly now: () => number; readonly pending: PendingDictations; readonly newRequestId: () => string; + /** For a recording transcribed again: the filter may be what removed its words. */ + readonly ignoreVoiceFilter?: boolean; } const NO_MODEL_MESSAGE = "Download a dictation model in Settings > Microphone."; @@ -105,7 +107,7 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic uri, modelId, DICTATION_PROCESSING, - store.getSnapshot().settings.onlyMyVoice, + !deps.ignoreVoiceFilter && store.getSnapshot().settings.onlyMyVoice, locale, ); } catch (cause) { diff --git a/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx b/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx index 62cfea826b5ad..05a8a9fb5220b 100644 --- a/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx +++ b/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx @@ -58,6 +58,7 @@ export function ScheduledTaskPromptField(props: { {showsDictation ? ( diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 686076628ce51..7ae177e21e996 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -889,6 +889,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps > {isExpanded ? editingAttachments : null} diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index 615914589e80e..d9df88eab80e8 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -321,6 +321,8 @@ function VoiceActionButton(props: { export function ComposerDictationError(props: { readonly error: string | null; + /** Set when the recording behind the error is still there to transcribe again. */ + readonly onTranscribeAgain?: (() => void) | null; readonly onDismiss: () => void; }) { if (!props.error) return null; @@ -329,6 +331,22 @@ export function ComposerDictationError(props: { {props.error} + {props.onTranscribeAgain ? ( + + + + ) : null} ) : isError ? ( - voice.isAvailable ? ( + voice.isAvailable && errorAction !== null ? ( { - if (!openSettings) { + if (errorAction !== "settings") { void voice.session.retry(); return; } @@ -214,7 +220,11 @@ function EdgeDictationPill(props: { void Linking.openSettings(); }} > - + ) : null ) : ( diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index b6459da93fb18..fbe010e6e8539 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -20,6 +20,7 @@ import { voiceInputBlocksSubmission, type VoiceTranscriber, } from "@t3tools/client-runtime/voice-input"; +import { keepVoiceRecording } from "./keptRecordings"; import { createLazyVoiceRecorder } from "./lazyVoiceRecorder"; import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; import { VoiceInputSession } from "./voiceInputSession"; @@ -134,6 +135,7 @@ const voiceInputAtom = Atom.make((get) => { audioToken = null; }, deleteRecording: (uri) => new File(uri).delete(), + keepRecording: keepVoiceRecording, sampleRecording: () => { const status = recorder.getStatus(); return status?.isRecording diff --git a/apps/mobile/src/features/voice-input/keptRecordings.ts b/apps/mobile/src/features/voice-input/keptRecordings.ts new file mode 100644 index 0000000000000..2e27250d62c71 --- /dev/null +++ b/apps/mobile/src/features/voice-input/keptRecordings.ts @@ -0,0 +1,55 @@ +import { Directory, File, Paths } from "expo-file-system"; + +/** Enough that a new, retried or canceled recording does not push out the one being recovered. */ +export const KEPT_VOICE_RECORDINGS = 5; + +const FILE_NAME = /^dictation-(\d+)\.m4a$/; + +export interface KeptVoiceRecording { + readonly uri: string; + readonly recordedAt: number; +} + +function folder(): Directory { + return new Directory(Paths.cache, "dictation-recordings"); +} + +/** The dictation recordings still on the device, newest first. */ +export function keptVoiceRecordings(): ReadonlyArray { + try { + const kept = folder(); + if (!kept.exists) return []; + return kept + .list() + .flatMap((entry) => { + const recordedAt = entry instanceof File ? FILE_NAME.exec(entry.name)?.[1] : undefined; + return recordedAt === undefined ? [] : [{ uri: entry.uri, recordedAt: Number(recordedAt) }]; + }) + .sort((left, right) => right.recordedAt - left.recordedAt); + } catch { + return []; + } +} + +/** Returns where the recording is now: the kept place, or where it was if it could not be moved. */ +export function keepVoiceRecording(uri: string): string { + let keptUri: string; + try { + const kept = folder(); + if (!kept.exists) kept.create({ intermediates: true, idempotent: true }); + const destination = new File(kept, `dictation-${Date.now()}.m4a`); + // The synchronous move: the recording is transcribed from its new place straight after. + new File(uri).moveSync(destination); + keptUri = destination.uri; + } catch { + return uri; + } + for (const old of keptVoiceRecordings().slice(KEPT_VOICE_RECORDINGS)) { + try { + new File(old.uri).delete(); + } catch { + // It is removed after a later recording. + } + } + return keptUri; +} diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts index fbb6059099a8a..7852a0b65079d 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -112,6 +112,7 @@ describe("createLazyVoiceRecorder", () => { configureRecording: async () => undefined, releaseRecording: async () => undefined, deleteRecording: (uri) => deleted.push(uri), + keepRecording: (uri) => uri, readDraft: () => ({ ownerKey: "thread", text: "", diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index e26672b5b4809..cfc8d18261f0e 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -74,6 +74,8 @@ export function useVoiceInputController(input: { freezesEditor: isBusy, blocksSubmission: isBusy, start, + /** Null unless the error on screen kept its recording. */ + transcribeAgain: state.errorAction === "transcribe" ? () => void session.retry() : null, stop: global.stop, cancel: global.cancel, }; diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 832945a2407c5..7357d47552877 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -64,6 +64,7 @@ function createSession( configureRecording: async () => {}, releaseRecording: vi.fn(async () => {}), deleteRecording: vi.fn(), + keepRecording: (uri) => uri, sampleRecording: () => null, publishAudioLevels, keepAwake, @@ -241,19 +242,45 @@ describe("global voice input", () => { }, ); - it.each(["changed", "removed"] as const)( - "does not overwrite a %s starting draft", - async (change) => { - const { session } = createSession(); - let text: string | null = "first"; - const commit = vi.fn(); - await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); - text = change === "removed" ? null : "edited prompt"; - await session.controller.stop(); - expect(commit).not.toHaveBeenCalled(); - expect(session.controller.currentState.error).toContain("draft changed"); - }, - ); + it("adds the words after an edit made to the draft during the recording", async () => { + const { session } = createSession(); + let text = "first"; + const commit = vi.fn(); + await session.start(createTarget("first", () => text, commit, { start: 0, end: 0 })); + text = "edited prompt"; + await session.controller.stop(); + expect(commit).toHaveBeenCalledWith("edited prompt spoken text", { start: 25, end: 25 }); + }); + + it("does not write to a draft that was removed during the recording", async () => { + const { session } = createSession(); + let text: string | null = "first"; + const commit = vi.fn(); + await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); + text = null; + await session.controller.stop(); + expect(commit).not.toHaveBeenCalled(); + expect(session.controller.currentState.error).toContain("draft is gone"); + }); + + it("retries a failed transcription from the recording, without recording again", async () => { + const { session, recorder, prepare } = createSession(); + const transcribe = vi + .fn() + .mockRejectedValueOnce(new Error("model failed")) + .mockResolvedValue("spoken text"); + prepare.mockResolvedValue({ locale: "en-US", transcribe }); + const commit = vi.fn(); + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + await session.controller.stop(); + expect(session.getSnapshot().state.errorAction).toBe("transcribe"); + + await session.retry(); + + expect(recorder.record).toHaveBeenCalledOnce(); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(session.getSnapshot().state.phase).toBe("idle"); + }); it("finishes the original draft at the recording limit while it is off screen", async () => { const { session, recorder } = createSession(); @@ -266,10 +293,14 @@ describe("global voice input", () => { url: recorder.uri, }); expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); - expect(session.controller.currentState.phase).toBe("idle"); + expect(session.controller.currentState).toEqual({ + phase: "error", + error: "Voice recording reached the 5 minute limit. The words up to it were added.", + errorAction: null, + }); }); - it("rejects a transcript when its off-screen draft changes and returns to the original text", async () => { + it("adds the transcript at the end when its off-screen draft changed and changed back", async () => { const registry = AtomRegistry.make(); const draft = Atom.make("hello world"); const unsubscribe = vi.fn(); @@ -292,8 +323,7 @@ describe("global voice input", () => { registry.set(draft, "changed"); registry.set(draft, "hello world"); await session.controller.stop(); - expect(commit).not.toHaveBeenCalled(); - expect(session.controller.currentState.error).toContain("draft changed"); + expect(commit).toHaveBeenCalledWith("hello world spoken text", { start: 23, end: 23 }); expect(unsubscribe).toHaveBeenCalledTimes(1); registry.dispose(); }); @@ -399,33 +429,41 @@ describe("global voice input", () => { } }); - it("finishes with the captured audio when the recorder stops capturing on its own", async () => { - vi.useFakeTimers(); - try { - let sample: VoiceRecordingSample | null = null; - const { session, recorder } = createSession({ sampleRecording: () => sample }); - const commit = vi.fn(); - const interrupt = vi.spyOn(session.controller, "interruptRecording"); + it.each([ + ["interruptRecording", 3_000, "stopped early"], + // The recorder can stop at the limit before its finish event arrives. + ["recordingReachedLimit", 299_950, "5 minute limit"], + ] as const)( + "finishes with the captured audio through %s when the recorder stops on its own", + async (method, durationMillis, notice) => { + vi.useFakeTimers(); + try { + let sample: VoiceRecordingSample | null = null; + const { session, recorder } = createSession({ sampleRecording: () => sample }); + const commit = vi.fn(); + const interrupt = vi.spyOn(session.controller, method); - await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); - // Not capturing yet is not a recorder that stopped. - vi.advanceTimersByTime(80); - expect(interrupt).not.toHaveBeenCalled(); + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + // Not capturing yet is not a recorder that stopped. + vi.advanceTimersByTime(80); + expect(interrupt).not.toHaveBeenCalled(); - sample = { durationMillis: 3_000, metering: -20 }; - vi.advanceTimersByTime(80); - // iOS pauses the recorder for a call or Siri and never reports or resumes it. - sample = null; - vi.advanceTimersByTime(80); - expect(interrupt).toHaveBeenCalledOnce(); - await interrupt.mock.results[0]!.value; + sample = { durationMillis, metering: -20 }; + vi.advanceTimersByTime(80); + // iOS pauses the recorder for a call or Siri without reporting it. + sample = null; + vi.advanceTimersByTime(80); + expect(interrupt).toHaveBeenCalledOnce(); + await interrupt.mock.results[0]!.value; - expect(recorder.stop).toHaveBeenCalled(); - expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); - } finally { - vi.useRealTimers(); - } - }); + expect(recorder.stop).toHaveBeenCalled(); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(session.controller.currentState.error).toContain(notice); + } finally { + vi.useRealTimers(); + } + }, + ); it("publishes which owners are focused", () => { const { session } = createSession(); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 34491a27808ea..30317ceb13647 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -135,7 +135,11 @@ export class VoiceInputSession { if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel(); } + /** Does what the error on screen offers: the kept recording again, or a new recording. */ retry(): Promise { + if (this.controller.currentState.errorAction === "transcribe") { + return this.controller.retranscribe(); + } return this.target ? this.start(this.target) : Promise.resolve(); } @@ -198,8 +202,14 @@ export class VoiceInputSession { if (this.controller.currentState.phase !== "recording") return; const sample = this.dependencies.sampleRecording(); if (!sample) { - // The system pauses the recorder for a call or Siri without an event, and never resumes it. - if (this.capturing) void this.controller.interruptRecording(); + if (!this.capturing) return; + // The recorder stops at the limit just before it says so, and the system pauses it for a + // call or Siri without saying so at all. Either way the words so far are transcribed now. + if (this.snapshot.elapsedSeconds >= VOICE_RECORDING_LIMIT_SECONDS - 1) { + void this.controller.recordingReachedLimit(); + } else { + void this.controller.interruptRecording(); + } return; } this.capturing = true; diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts index 7a2e39e4304e8..e4a580b5d23d3 100644 --- a/apps/mobile/src/native/voiceTranscription.android.test.ts +++ b/apps/mobile/src/native/voiceTranscription.android.test.ts @@ -70,6 +70,7 @@ it.each([ configureRecording: async () => {}, releaseRecording: async () => {}, deleteRecording: () => {}, + keepRecording: (uri) => uri, readDraft: () => draft, commitDraft: commit, onStateChange: () => {}, @@ -77,7 +78,11 @@ it.each([ await controller.start(); if (phase === "transcription") await controller.stop(); - expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + // Once something was recorded, the error offers the kept recording instead of a new one. + expect(controller.currentState).toMatchObject({ + phase: "error", + errorAction: phase === "transcription" ? "transcribe" : "retry", + }); expect(commit).not.toHaveBeenCalled(); expect(draft.text).toBe("Preserved draft."); expect(controller.currentState.error).toMatch(expected); diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index 8730bea3e6bfa..4a2b64567bf6f 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -62,6 +62,7 @@ function createDictation() { configureRecording: async () => {}, releaseRecording: async () => {}, deleteRecording: () => {}, + keepRecording: (uri) => uri, readDraft: () => ({ ownerKey: "environment:thread", text: "", diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 24d7778e10645..3dc4c25c3286c 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -8,8 +8,17 @@ environment-backed transcription is not implemented. The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts) owns the operation while the client supplies capture and transcription. Preparation binds the transcriber and resolved locale for the whole recording. Draft ownership, -text, and revision are captured before recording and checked before insertion, so -a late transcript cannot overwrite a draft that was edited or replaced. +text, and revision are captured before recording and checked before insertion. A +transcript is dropped only when its draft is gone or belongs to another owner. In a +draft that was edited, the recorded caret means nothing, so the words go at its end. + +Recordings outlive their dictation. The controller hands the file to `keepRecording` +as soon as the recorder stops, before transcription, and cleanup never deletes it. +Whatever goes wrong afterwards (a thrown transcription, an empty result, a dropped +transcript, words the voice filter removed) is then recoverable: `retranscribe` runs +the latest file again, and mobile Settings runs any of the last few without the +voice filter. Mobile keeps several, not one, because the next recording is often the +user trying again, and it must not destroy the one they want back. Cancellation invalidates a result immediately, but resources stay owned until the underlying work settles. Apple's native transcription call cannot be interrupted @@ -32,10 +41,13 @@ files only import from them, at the end of their import block, so upstream merge Apple's included. The model loads while the user talks, and the system recognizer stands in for a model that is missing, still compiling, or failed on the recording. A language Apple cannot take is therefore known only when the recording is transcribed. -- expo-audio pauses the recorder for an audio-session interruption without an event and never - resumes it. The metering loop notices that capture stopped, and the controller transcribes the - captured file. That file wins over the recorder's current one, because a media-services reset - hands the recorder a new, empty file. +- expo-audio pauses the recorder for an audio-session interruption without an event, and resumes + it only if the system later says capture may continue. The metering loop notices that capture + stopped, and the controller transcribes the captured file at once instead of waiting. That file + wins over the recorder's current one, because a media-services reset hands the recorder a new, + empty file. +- At the five-minute limit the recorder stops just before it reports that it finished. A stop in + the last second is therefore treated as the limit, not as an interruption. - FluidAudio is a git pod pinned to a commit and added by the config plugin to the app target only. The widget extension must not link it. - The engine reaches the network only inside an explicit download. diff --git a/docs/user/composer.md b/docs/user/composer.md index e4e2c56d52685..9ed4b6e0dcbb8 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -201,11 +201,17 @@ transcribing. The first use may download a speech model and needs a network connection. Later transcription works offline for that language. On Android, transcription processes the full recording at roughly its original duration. Recordings can be up to five -minutes long. Canceling discards the recording and preserves your existing draft. When a call, -Siri or another app interrupts a recording, the words spoken before it are added to the draft. Navigating to another screen keeps dictation active; +minutes long; at the limit the recording stops and the words up to it are added. Canceling leaves +your draft as it was. When a call, Siri or another app interrupts a recording, the words spoken +before it are added to the draft. If the draft changed while you were dictating, the words are +added at its end. Navigating to another screen keeps dictation active; use the floating controls to finish or cancel. While recording, the screen stays awake; it can sleep normally once recording stops. +Your last 5 recordings stay on the device, canceled ones included. When transcription fails, the +error offers **Transcribe again**, which runs the same recording so you do not have to repeat +yourself. + Choose a preferred order in **Settings → Microphone**. Voice input requests the first connected microphone in the list. The list mixes kinds (built-in, other wired, other Bluetooth, and CarPlay on iOS) with wired and Bluetooth devices you have dictated @@ -229,6 +235,10 @@ Apple's transcription takes a recording made in that time where it is available, recording a model fails on. **Settings → Microphone** shows what the last dictation ran on. On a device without Apple's transcription, the microphone button works once a model is downloaded. +When a dictation came out empty, cut short or wrong, open **Settings → Microphone → Model** and +tap it under **Recent recordings**. It runs through the model in use, without the voice filter, +and the text is shown for you to copy. + ### Only my voice **Settings → Microphone → Only my voice** removes other people's speech from a recording before it diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 1f364d83577b1..5a3446d6b5913 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -48,6 +48,8 @@ function draft(overrides: Partial = {}): VoiceDraftSnapshot }; } +const KEPT_URI = "file:///last-dictation.m4a"; + function createHarness( overrides: Partial = {}, initialDraft = draft(), @@ -56,6 +58,7 @@ function createHarness( let currentDraft: VoiceDraftSnapshot | null = initialDraft; const commits: Array<{ text: string; selection: { start: number; end: number } }> = []; const deleted: string[] = []; + const kept: string[] = []; const dependencies: VoiceInputControllerDependencies = { recorder, getTranscriber: () => ({ prepare: async () => preparedTranscription() }), @@ -63,6 +66,10 @@ function createHarness( configureRecording: async () => undefined, releaseRecording: async () => undefined, deleteRecording: (uri) => deleted.push(uri), + keepRecording: (uri) => { + kept.push(uri); + return KEPT_URI; + }, readDraft: () => currentDraft, commitDraft: (text, selection) => commits.push({ text, selection }), onStateChange: vi.fn(), @@ -73,6 +80,7 @@ function createHarness( recorder, commits, deleted, + kept, setDraft: (next: VoiceDraftSnapshot | null) => { currentDraft = next; }, @@ -95,18 +103,23 @@ describe("resolveTranscriptCommit", () => { }); }); - it("does not replace text after the owner, text, or revision changes", () => { + it("does not add text to another owner's draft or to one that is gone", () => { const captured = draft(); expect( resolveTranscriptCommit(captured, draft({ ownerKey: "other" }), "text", "en-US"), - ).toEqual({ - kind: "stale", - }); - expect(resolveTranscriptCommit(captured, draft({ text: "newer" }), "text", "en-US")).toEqual({ - kind: "stale", - }); + ).toEqual({ kind: "stale" }); + expect(resolveTranscriptCommit(captured, null, "text", "en-US")).toEqual({ kind: "stale" }); + }); + + it("adds the words at the end of a draft that was edited during the recording", () => { + const captured = draft(); + expect( + resolveTranscriptCommit(captured, draft({ text: "newer draft" }), "text", "en-US"), + ).toEqual({ kind: "commit", text: "newer draft text", selection: { start: 16, end: 16 } }); expect(resolveTranscriptCommit(captured, draft({ revision: 2 }), "text", "en-US")).toEqual({ - kind: "stale", + kind: "commit", + text: "hello world text", + selection: { start: 16, end: 16 }, }); }); }); @@ -163,7 +176,7 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState).toMatchObject({ phase: "error", error: expect.any(String), - errorAction: failure === "permission" ? "settings" : "retry", + errorAction: failure === "permission" ? "settings" : "transcribe", }); harness.setDraft(draft({ ownerKey: "environment:other-thread" })); @@ -256,7 +269,84 @@ describe("VoiceInputController", () => { expect(harness.commits).toEqual([ { text: "hello new text", selection: { start: 14, end: 14 } }, ]); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(harness.deleted).toEqual([]); + }); + + it.each< + [ + string, + { + readonly transcribe?: PreparedVoiceTranscription["transcribe"]; + readonly act?: "lose the draft" | "fail to stop" | "cancel"; + readonly commits?: ReadonlyArray; + }, + ] + >([ + ["transcription fails", { transcribe: () => Promise.reject(new Error("model failed")) }], + ["nothing was heard", { transcribe: async () => " " }], + ["the draft is gone", { act: "lose the draft" }], + ["the recorder fails to stop", { act: "fail to stop", commits: ["hello new text"] }], + ["the user cancels the recording", { act: "cancel" }], + ])("keeps the recording when %s", async (_name, scenario) => { + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => preparedTranscription(scenario.transcribe), + }), + }); + await harness.controller.start(); + if (scenario.act === "fail to stop") { + harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released")); + } + if (scenario.act === "lose the draft") harness.setDraft(null); + if (scenario.act === "cancel") harness.controller.cancel(); + await harness.controller.stop(); + // The next recording starts only once this one has let go of the microphone. + const next = createHarness(); + await next.controller.start(); + await next.controller.interruptRecording(); + + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(harness.deleted).toEqual([]); + expect(harness.commits.map((commit) => commit.text)).toEqual(scenario.commits ?? []); + }); + + it.each([ + ["at the caret the recording started at", null, "hello second try"], + [ + "at the end of a draft edited since", + draft({ text: "hello world, and more", revision: 2 }), + "hello world, and more second try", + ], + ])("transcribes the kept recording again and adds it %s", async (_name, edited, expected) => { + const transcribe = vi + .fn() + .mockRejectedValueOnce(new Error("model failed")) + .mockResolvedValue("second try"); + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState).toEqual({ + phase: "error", + error: "Could not transcribe this recording.", + errorAction: "transcribe", + }); + + if (edited) harness.setDraft(edited); + await harness.controller.retranscribe(); + + expect(transcribe).toHaveBeenLastCalledWith(KEPT_URI, expect.anything()); + expect(harness.recorder.record).toHaveBeenCalledOnce(); + expect(harness.commits.map((commit) => commit.text)).toEqual([expected]); + expect(harness.controller.currentState.phase).toBe("idle"); + expect(harness.deleted).toEqual([]); + + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); }); it.each(["cancel", "dispose", "ownerChanged"] as const)( @@ -313,7 +403,8 @@ describe("VoiceInputController", () => { await nextStart; expect(harness.commits).toEqual([]); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(harness.deleted).toEqual([]); expect(harness.controller.currentState.phase).toBe("idle"); expect(next.controller.currentState.phase).toBe("recording"); @@ -362,7 +453,7 @@ describe("VoiceInputController", () => { await stopping; expect(harness.commits).toEqual([]); - expect(harness.controller.currentState.error).toContain("draft changed"); + expect(harness.controller.currentState.error).toContain("draft is gone"); }); it("queues the next recording until canceled preparation settles", async () => { @@ -445,7 +536,7 @@ describe("VoiceInputController", () => { }); await expect(harness.controller.interruptRecording()).resolves.toBeUndefined(); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); const next = createHarness(); await next.controller.start(); expect(next.controller.currentState.phase).toBe("recording"); @@ -504,6 +595,7 @@ describe("VoiceInputController", () => { it.each([ [ "a recorder error", + "Voice recording stopped early. The words it captured were added.", (controller: VoiceInputController) => controller.handleRecorderStatus({ isFinished: true, @@ -512,28 +604,48 @@ describe("VoiceInputController", () => { url: null, }), ], - ["the background", (controller: VoiceInputController) => controller.appMovedToBackground()], - ])("adds the words captured before %s stopped the recording", async (_name, interrupt) => { - const transcribe = vi.fn(async () => "captured words"); - const harness = createHarness({ - getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), - }); - await harness.controller.start(); - // A media-services reset points the recorder at a new, empty file. - harness.recorder.uri = "file:///reset-empty.m4a"; - await interrupt(harness.controller); - - expect(transcribe).toHaveBeenCalledWith("file:///voice.m4a", expect.anything()); - expect(harness.commits).toEqual([ - { text: "hello captured words", selection: { start: 20, end: 20 } }, - ]); - expect(harness.controller.currentState).toEqual({ - phase: "error", - error: "Voice recording stopped early. The words it captured were added.", - errorAction: "retry", - }); - expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); - }); + [ + "the background", + "Voice recording stopped early. The words it captured were added.", + (controller: VoiceInputController) => controller.appMovedToBackground(), + ], + [ + "the five-minute limit", + "Voice recording reached the 5 minute limit. The words up to it were added.", + (controller: VoiceInputController) => + controller.handleRecorderStatus({ + isFinished: true, + hasError: false, + error: null, + url: "file:///voice.m4a", + }), + ], + ])( + "adds the words captured before %s stopped the recording", + async (_name, notice, interrupt) => { + const transcribe = vi.fn(async () => "captured words"); + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + // A media-services reset points the recorder at a new, empty file. + harness.recorder.uri = "file:///reset-empty.m4a"; + await interrupt(harness.controller); + + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(transcribe).toHaveBeenCalledWith(KEPT_URI, expect.anything()); + expect(harness.commits).toEqual([ + { text: "hello captured words", selection: { start: 20, end: 20 } }, + ]); + // No retry is offered: a new recording would go in at the caret this one started at. + expect(harness.controller.currentState).toEqual({ + phase: "error", + error: notice, + errorAction: null, + }); + expect(harness.deleted).toEqual(["file:///reset-empty.m4a"]); + }, + ); it.each([ ["nothing was said", async () => " "], @@ -558,7 +670,7 @@ describe("VoiceInputController", () => { expect(harness.commits).toEqual([]); expect(harness.controller.currentState.error).toBe("Audio route changed"); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); }); it("cancels preparation when the app reaches the background", async () => { diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 4e7eac828f4ab..8d402a7b6c4a6 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -16,7 +16,8 @@ export type VoiceInputPhase = "idle" | "preparing" | "recording" | "transcribing export type VoiceInputState = { readonly phase: VoiceInputPhase; readonly error: string | null; - readonly errorAction: "retry" | "settings" | null; + /** "retry" records again; "transcribe" runs the recording that was kept through the model again. */ + readonly errorAction: "retry" | "transcribe" | "settings" | null; }; export function voiceInputBlocksSubmission(state: VoiceInputState): boolean { @@ -56,6 +57,8 @@ export type VoiceInputControllerDependencies = { readonly configureRecording: () => Promise; readonly releaseRecording: () => Promise; readonly deleteRecording: (uri: string) => void; + /** Moves a finished recording to where recordings are kept, and returns where it is now. */ + readonly keepRecording: (uri: string) => string; readonly readDraft: () => VoiceDraftSnapshot | null; readonly commitDraft: ( text: string, @@ -79,12 +82,7 @@ export function resolveTranscriptCommit( transcript: string, locale: string, ): TranscriptCommitResult { - if ( - !current || - current.ownerKey !== captured.ownerKey || - current.text !== captured.text || - current.revision !== captured.revision - ) { + if (!current || current.ownerKey !== captured.ownerKey) { return { kind: "stale" }; } @@ -93,13 +91,17 @@ export function resolveTranscriptCommit( return { kind: "empty" }; } - const isEmptySelection = captured.selection.start === captured.selection.end; + // The caret recorded at the start means nothing in an edited draft, so the words go at its end. + const edited = current.text !== captured.text || current.revision !== captured.revision; + const text = current.text; + const selection = edited ? { start: text.length, end: text.length } : captured.selection; + const isEmptySelection = selection.start === selection.end; const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); let insertion = replacement; if (isEmptySelection && usesEnglishSpacing) { - const left = captured.text[captured.selection.start - 1]; - const right = captured.text[captured.selection.start]; + const left = text[selection.start - 1]; + const right = text[selection.start]; const leftNeedsBoundary = left !== undefined && /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && @@ -111,12 +113,7 @@ export function resolveTranscriptCommit( insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; } - const result = replaceTextRange( - captured.text, - captured.selection.start, - captured.selection.end, - insertion, - ); + const result = replaceTextRange(text, selection.start, selection.end, insertion); return { kind: "commit", text: result.text, @@ -189,6 +186,15 @@ function transcriptionErrorMessage(error: unknown): string { const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; const RECORDING_STOPPED_EARLY_MESSAGE = "Voice recording stopped early. The words it captured were added."; +const RECORDING_LIMIT_MESSAGE = + "Voice recording reached the 5 minute limit. The words up to it were added."; + +type KeptRecording = { + readonly uri: string; + /** Where the recorder wrote it. The file is no longer there once it was moved. */ + readonly source: string; + readonly draft: VoiceDraftSnapshot | null; +}; export class VoiceInputController { private readonly dependencies: VoiceInputControllerDependencies; @@ -201,6 +207,8 @@ export class VoiceInputController { private recordingUri: string | null = null; private readonly ownedRecordingUris = new Set(); private recordingConfigured = false; + private recordingStarted = false; + private kept: KeptRecording | null = null; private finishing = false; constructor(dependencies: VoiceInputControllerDependencies) { @@ -281,6 +289,7 @@ export class VoiceInputController { } this.capturedDraft = capturedDraft; this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS }); + this.recordingStarted = true; this.setState({ phase: "recording", error: null, errorAction: null }); } catch { if (this.isCurrent(operationToken)) @@ -313,7 +322,7 @@ export class VoiceInputController { this.setState(IDLE_STATE); return; case "recording": - this.discardRecording(null); + void this.discardRecording(); return; case "transcribing": this.invalidateOperation(); @@ -322,12 +331,87 @@ export class VoiceInputController { } } + /** Runs the kept recording through the model again, after a transcription that failed. */ + async retranscribe(): Promise { + const kept = this.kept; + if (this.state.phase !== "error" || this.state.errorAction !== "transcribe" || !kept?.draft) { + return; + } + const capturedDraft = kept.draft; + let sessionToken: VoiceInputSession | null = null; + const operationToken = ++this.operationToken; + this.setState({ phase: "transcribing", error: null, errorAction: null }); + + try { + for ( + let previousSession = activeSession; + previousSession?.abandoned; + previousSession = activeSession + ) { + await previousSession.released; + if (!this.isCurrent(operationToken)) return; + } + sessionToken = acquireSession(); + if (!sessionToken) { + this.setError("Another voice recording is already active.", "transcribe"); + return; + } + this.sessionToken = sessionToken; + const abortController = new AbortController(); + this.transcriptionAbortController = abortController; + const { signal } = abortController; + + const transcriber = this.dependencies.getTranscriber(); + if (!transcriber) { + this.setError("Voice transcription is not available.", "transcribe"); + return; + } + + let transcript: string; + let locale: string; + try { + const transcription = await runTranscriptionOperation(() => + transcriber.prepare({ signal }), + ); + if (!this.isCurrent(operationToken)) return; + transcript = await runTranscriptionOperation(() => + transcription.transcribe(kept.uri, { signal }), + ); + locale = transcription.locale; + } catch (error) { + if (this.isCurrent(operationToken)) { + this.setError(transcriptionErrorMessage(error), "transcribe"); + } + return; + } + if (!this.isCurrent(operationToken)) return; + this.commitTranscript(capturedDraft, transcript, locale, null, null); + } catch { + if (this.isCurrent(operationToken)) { + this.setError("Could not finish voice recording.", "transcribe"); + } + } finally { + if (sessionToken && this.sessionToken === sessionToken) await this.releaseResources(); + } + } + interruptRecording( message = "Voice recording was interrupted.", completedUri: string | null = null, ): Promise | void { if (this.state.phase !== "recording") return; - return this.finishRecording(completedUri !== null, completedUri, message); + return this.finishRecording( + completedUri !== null, + completedUri, + RECORDING_STOPPED_EARLY_MESSAGE, + message, + ); + } + + /** The recorder ends a recording by itself, without an error, only at the limit. */ + recordingReachedLimit(completedUri: string | null = null): Promise | void { + if (this.state.phase !== "recording") return; + return this.finishRecording(completedUri !== null, completedUri, RECORDING_LIMIT_MESSAGE); } appMovedToBackground(): Promise | void { @@ -351,7 +435,7 @@ export class VoiceInputController { if (!status.url) { return this.interruptRecording(); } - return this.finishRecording(true, status.url); + return this.recordingReachedLimit(status.url); } } @@ -362,7 +446,7 @@ export class VoiceInputController { dispose(): void { if (this.state.phase === "recording") { - this.discardRecording(null); + void this.discardRecording(); return; } if (this.state.phase === "preparing" || this.state.phase === "transcribing") { @@ -371,30 +455,39 @@ export class VoiceInputController { } } - /** `interruption` is set when the recording ended without the user stopping it. */ + /** + * `notice` is set when the recording ended without the user stopping it, and is shown once its + * words are added. `failure` is shown instead when there were no words to add. + */ private async finishRecording( alreadyStopped: boolean, completedUri: string | null, - interruption: string | null = null, + notice: string | null = null, + failure: string | null = null, ): Promise { if (this.finishing || this.state.phase !== "recording") return; this.finishing = true; const operationToken = this.operationToken; + let keptRecording = false; this.setState({ phase: "transcribing", error: null, errorAction: null }); try { if (!alreadyStopped) { - const stopping = this.dependencies.recorder.stop(); - // An interrupted recorder may already be gone; the file it wrote is still there. - await (interruption === null ? stopping : stopping.catch(() => {})); + try { + await this.dependencies.recorder.stop(); + } catch { + // A recorder that fails to stop, or is already gone, has still written its file. + } } await this.releaseAudioSession(); // The file capture started in wins: a media-services reset gives the recorder a new, empty one. - this.recordingUri = completedUri ?? this.recordingUri ?? this.dependencies.recorder.uri; - this.rememberRecordingUri(this.recordingUri); + const recordingUri = this.keep( + completedUri ?? this.recordingUri ?? this.dependencies.recorder.uri, + ); + keptRecording = recordingUri !== null; if (!this.isCurrent(operationToken)) return; if ( - !this.recordingUri || + !recordingUri || !this.transcription || !this.transcriptionAbortController || !this.capturedDraft @@ -403,7 +496,6 @@ export class VoiceInputController { return; } - const recordingUri = this.recordingUri; const transcription = this.transcription; const signal = this.transcriptionAbortController.signal; const capturedDraft = this.capturedDraft; @@ -414,36 +506,15 @@ export class VoiceInputController { ); } catch (error) { if (this.isCurrent(operationToken)) { - this.setError(interruption ?? transcriptionErrorMessage(error), "retry"); + this.setError(failure ?? transcriptionErrorMessage(error), "transcribe"); } return; } if (!this.isCurrent(operationToken)) return; - - const result = resolveTranscriptCommit( - capturedDraft, - this.dependencies.readDraft(), - transcript, - transcription.locale, - ); - if (result.kind === "stale") { - this.setError( - "The draft changed while voice input was running. The transcript was not added.", - "retry", - ); - return; - } - if (result.kind === "empty") { - this.setError(interruption ?? "No speech was detected.", "retry"); - return; - } - - this.dependencies.commitDraft(result.text, result.selection); - if (interruption === null) this.setState(IDLE_STATE); - else this.setError(RECORDING_STOPPED_EARLY_MESSAGE, "retry"); + this.commitTranscript(capturedDraft, transcript, transcription.locale, notice, failure); } catch { if (this.isCurrent(operationToken)) { - this.setError("Could not finish voice recording.", "retry"); + this.setError("Could not finish voice recording.", keptRecording ? "transcribe" : "retry"); } } finally { this.finishing = false; @@ -451,16 +522,57 @@ export class VoiceInputController { } } - private async discardRecording(error: string | null): Promise { - this.invalidateOperation(); - this.setState( - error - ? { phase: "error", error, errorAction: "retry" } - : { phase: "idle", error: null, errorAction: null }, + private commitTranscript( + capturedDraft: VoiceDraftSnapshot, + transcript: string, + locale: string, + notice: string | null, + failure: string | null, + ): void { + const result = resolveTranscriptCommit( + capturedDraft, + this.dependencies.readDraft(), + transcript, + locale, ); + if (result.kind === "stale") { + this.setError( + "The draft is gone, so the words were not added. The recording is kept.", + "retry", + ); + return; + } + if (result.kind === "empty") { + this.setError(failure ?? "No speech was detected.", "retry"); + return; + } + + this.dependencies.commitDraft(result.text, result.selection); + // No retry with a notice: a new recording would go in at the caret this one started at. + if (notice === null) this.setState(IDLE_STATE); + else this.setError(notice, null); + } + + private async discardRecording(): Promise { + this.invalidateOperation(); + this.setState(IDLE_STATE); await this.releaseResources(); } + /** A recording outlives its dictation, whatever became of it, so its words can be recovered. */ + private keep(uri: string | null): string | null { + this.recordingStarted = false; + if (!uri) return null; + let keptUri = uri; + try { + keptUri = this.dependencies.keepRecording(uri); + } catch { + // It stays where the recorder wrote it. + } + this.kept = { uri: keptUri, source: uri, draft: this.capturedDraft }; + return keptUri; + } + private async releaseResources(): Promise { this.rememberRecordingUri(this.recordingUri); try { @@ -471,8 +583,11 @@ export class VoiceInputController { await this.dependencies.recorder.stop(); } catch {} } + // A recording that was canceled, or abandoned before it finished, is kept as well. + if (this.recordingStarted) this.keep(this.recordingUri); this.recordingUri = null; for (const uri of this.ownedRecordingUris) { + if (uri === this.kept?.uri || uri === this.kept?.source) continue; try { this.dependencies.deleteRecording(uri); } catch {}