diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift index 1b91c8e913e1e..d9dee71a8ad34 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift @@ -193,8 +193,12 @@ final class RealModelTests: XCTestCase { func testOnlyMyVoiceDropsTheOtherSpeaker() async throws { try await download(.fast) _ = try await enroll(voice: Self.me) - let mine = try DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine")) + // The user does most of the talking, and the other voice is further from the phone. + let mine = + try DictationEngine.decode(speak(Self.enrollment[1], voice: Self.me, name: "opening")) + + silence(0.6) + DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine")) let theirs = try DictationEngine.decode(speak(Self.theirs, voice: Self.other, name: "theirs")) + .map { $0 * 0.25 } let url = try write(mine + silence(0.6) + theirs, name: "both") let unfiltered = try await engine.transcribe( diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift index 3859bbf1e6469..490126211d6df 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift @@ -85,6 +85,14 @@ final class RealVoiceTests: XCTestCase { Array(samples.prefix(Int(seconds * DictationEngine.sampleRate))) } + /// The voice 12 dB under `own`: about four times further from the phone than the person holding + /// it. The corpus records each reader at a different level, so the gap is set, not assumed. + private static func distant(_ samples: [Float], from own: [Float]) -> [Float] { + func power(_ clip: [Float]) -> Float { clip.reduce(Float(0)) { $0 + $1 * $1 } / Float(clip.count) } + let gain = 0.25 * (power(own) / power(samples)).squareRoot() + return samples.map { $0 * gain } + } + private func write(_ samples: [Float]) throws -> URL { written += 1 let url = clips.appendingPathComponent("clip-\(written).wav") @@ -92,9 +100,16 @@ final class RealVoiceTests: XCTestCase { return url } - private func words(_ samples: [Float], onlyMyVoice: Bool) async throws -> (Set, FilterOutcome?) { + private func words( + _ samples: [Float], onlyMyVoice: Bool, model: DictationModel = .fast, asPhoneRecording: Bool = false + ) async throws -> (Set, FilterOutcome?) { + var url = try write(samples) + if asPhoneRecording { + url = url.deletingPathExtension().appendingPathExtension("m4a") + try writePhoneRecording(samples, to: url) + } let transcript = try await engine.transcribe( - requestId: UUID().uuidString, url: try write(samples), model: .fast, processing: .auto, + requestId: UUID().uuidString, url: url, model: model, processing: .auto, onlyMyVoice: onlyMyVoice, language: "en-US") let words = transcript.text.lowercased().split(whereSeparator: { !$0.isLetter }).map(String.init) return (Set(words), transcript.filter) @@ -139,6 +154,133 @@ final class RealVoiceTests: XCTestCase { XCTAssertLessThanOrEqual(Double(mixAccepted.count) / Double(mixes), 0.05) } + // MARK: Long recordings + + /// People dictate for longer than any other clip here, and the phone hands over AAC, not WAV. + func testAHalfMinuteOfSpeechFromThePhoneIsTranscribedInFull() async throws { + try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in }) + var lost: [String] = [] + for model in [DictationModel.fast, .balanced, .accurate] { + try await engine.download(model, processing: .auto, onProgress: { _ in }) + for speaker in speakers.prefix(4) { + try await engine.saveVoiceProfile(try speaker.enrollment.map(write)) + var recording: [Float] = [] + var expected: Set = [] + for utterance in speaker.held where seconds(recording) < 30 { + recording += utterance + expected.formUnion(try await words(utterance, onlyMyVoice: false, model: model).0) + } + XCTAssertGreaterThanOrEqual(seconds(recording), 30) + for onlyMyVoice in [false, true] { + let (heard, outcome) = try await words( + recording, onlyMyVoice: onlyMyVoice, model: model, asPhoneRecording: true) + let missing = expected.subtracting(heard).count + print( + "MEASURE long, \(model.rawValue) \(speaker.name) \(Int(seconds(recording))) s, filter " + + "\(onlyMyVoice): lost \(percent(missing, expected.count)), " + + "cut \(outcome?.droppedSeconds ?? 0) s") + if Double(missing) / Double(expected.count) > 0.05 { + lost.append("\(model.rawValue) \(speaker.name) filter \(onlyMyVoice): \(percent(missing, expected.count))") + } + } + } + } + XCTAssertEqual(lost, []) + } + + // MARK: Sound unlike enrollment + + /// White noise under the speech, `snr` decibels below it. The seed is fixed so runs agree. + private static func noisy(_ samples: [Float], snr: Float) -> [Float] { + let power = samples.reduce(Float(0)) { $0 + $1 * $1 } / Float(samples.count) + let peak = (3 * power).squareRoot() / pow(10, snr / 20) + var state: UInt64 = 0x9E37_79B9_7F4A_7C15 + return samples.map { sample in + state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407 + return sample + peak * (Float(state >> 40) / Float(1 << 24) * 2 - 1) + } + } + + /// Roughly a telephone headset: the top and the bottom of the voice are gone. + private static func narrowband(_ samples: [Float]) -> [Float] { + var output = samples + var low: Float = 0 + var lower: Float = 0 + var rumble: Float = 0 + for index in samples.indices { + low += 0.72 * (samples[index] - low) + lower += 0.72 * (low - lower) + rumble = 0.89 * rumble + 0.11 * lower + output[index] = index % 2 == 0 ? lower - rumble : output[index - 1] + } + return output + } + + /// A small room with hard walls. + private static func echoing(_ samples: [Float]) -> [Float] { + var output = samples + for (delay, gain) in [(464, Float(0.55)), (592, 0.5), (688, 0.45), (1104, 0.4)] { + var echo = [Float](repeating: 0, count: samples.count) + for index in samples.indices.dropFirst(delay) { + echo[index] = gain * (samples[index - delay] + echo[index - delay]) + output[index] += 0.35 * echo[index] + } + } + return output + } + + /// The profile is recorded once, somewhere quiet. People then dictate in a car, in a kitchen or + /// over a headset, where their voice sits further from it, and that must not cost them words. + func testTheFilterKeepsTheSpeakersWordsWhereTheSoundIsUnlikeEnrollment() async throws { + try await engine.download(.fast, processing: .auto, onProgress: { _ in }) + try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in }) + let conditions: [(name: String, change: ([Float]) -> [Float])] = [ + ("noise at 5 dB", { Self.noisy($0, snr: 5) }), + ("narrowband", Self.narrowband), + ("narrowband with noise at 5 dB", { Self.noisy(Self.narrowband($0), snr: 5) }), + ("echo with noise at 10 dB", { Self.noisy(Self.echoing($0), snr: 10) }), + ] + var tallies = conditions.map { _ in (own: 0, ownLost: 0, stranger: 0, strangerKept: 0) } + + for (index, speaker) in speakers.prefix(8).enumerated() { + try await engine.saveVoiceProfile(try speaker.enrollment.map(write)) + let stranger = speakers[(index + speakers.count / 2) % speakers.count] + var alone: [Float] = [] + for utterance in speaker.held where seconds(alone) < 30 { alone += utterance } + + for (position, condition) in conditions.enumerated() { + let recording = condition.change(alone) + let spoken = try await words(recording, onlyMyVoice: false).0 + let (kept, outcome) = try await words(recording, onlyMyVoice: true) + tallies[position].own += spoken.count + tallies[position].ownLost += spoken.subtracting(kept).count + print( + "MEASURE unlike enrollment, \(condition.name), \(speaker.name) alone for " + + "\(Int(seconds(alone))) s: lost \(percent(spoken.subtracting(kept).count, spoken.count)), " + + "cut \(outcome?.droppedSeconds ?? 0) s") + + // The other voice is across the room, under the same noise or through the same headset. + let own = Self.cut(speaker.held[0], 8) + let theirs = Self.distant(Self.cut(stranger.held[0], 4), from: own) + let unwanted = try await words(condition.change(own + theirs), onlyMyVoice: false).0 + .subtracting(try await words(condition.change(own), onlyMyVoice: false).0) + let heard = try await words(condition.change(own + theirs), onlyMyVoice: true).0 + tallies[position].stranger += unwanted.count + tallies[position].strangerKept += unwanted.intersection(heard).count + } + } + + for (condition, tally) in zip(conditions, tallies) { + print( + "MEASURE unlike enrollment, \(condition.name): speaker's words lost " + + "\(percent(tally.ownLost, tally.own)), stranger's words kept " + + "\(percent(tally.strangerKept, tally.stranger))") + XCTAssertLessThanOrEqual(Double(tally.ownLost) / Double(tally.own), 0.05, condition.name) + XCTAssertLessThanOrEqual( + Double(tally.strangerKept) / Double(tally.stranger), 0.15, condition.name) + } + } + // MARK: Voice filter func testTheFilterKeepsTheSpeakersWordsAndDropsTheStrangers() async throws { @@ -175,10 +317,11 @@ final class RealVoiceTests: XCTestCase { let (_, soloOutcome) = try await words(Self.cut(stranger.held[2], 6), onlyMyVoice: true) if soloOutcome?.applied == true { strangerSoloCut.append("\(stranger.name) as \(speaker.name)") } - let own = Self.cut(speaker.held[0], 5) + // A dictation is mostly its owner, and anyone else is further from the phone. + let own = Self.cut(speaker.held[0], 8) let ownSecond = Self.cut(speaker.held[1], 4) - let theirs = Self.cut(stranger.held[0], 5) - let theirsShort = Self.cut(stranger.held[1], 2) + let theirs = Self.distant(Self.cut(stranger.held[0], 4), from: own) + let theirsShort = Self.distant(Self.cut(stranger.held[1], 2), from: own) let cases: [(name: String, mine: [[Float]], theirs: [Float], samples: [Float])] = [ ("mine then theirs", [own], theirs, own + theirs), ("theirs then mine", [own], theirs, theirs + own), diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift index d7efa65802abe..4a00857032198 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift @@ -7,6 +7,14 @@ final class SpeakerFilterTests: XCTestCase { SpeakerTurn(speaker: speaker, start: start, end: end) } + /// Unless a test says otherwise the user speaks at -20 dB and everyone else 15 dB under them. + private func decide( + _ turns: [SpeakerTurn], _ voices: [TurnVoice], levels: [Float?]? = nil + ) -> SpeakerDecision { + SpeakerFilter.decide( + turns: turns, voices: voices, levels: levels ?? voices.map { $0 == .mine ? -20 : -35 }) + } + func testAShortTurnMaySitFurtherFromTheProfile() { XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.5), .mine) XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.7), .other) @@ -15,36 +23,96 @@ final class SpeakerFilterTests: XCTestCase { XCTAssertEqual(SpeakerFilter.judge(duration: 0.9, distance: 0.1), .unjudged) } + /// A unit vector at `distance` from the profile `[1, 0, 0]`, leaning towards the second or third axis. + private func voiceprint(_ distance: Float, stranger: Bool = false) -> [Float] { + let along = 1 - distance + let across = (1 - along * along).squareRoot() + return stranger ? [along, 0, across] : [along, across, 0] + } + + func testATurnTheProfileRejectsIsTheUsersWhenItSoundsLikeTheirOtherTurns() { + let profile: [Float] = [1, 0, 0] + // Noise or another microphone moves every turn away from the profile. The opening words pass + // as a short turn; the half minute after them is the same voice and must not be cut. + let turns = [turn("a", 0, 1.5), turn("a", 2, 30), turn("b", 30, 35), turn("b", 35, 35.5)] + let voices = SpeakerFilter.voices( + turns: turns, + embeddings: [voiceprint(0.7), voiceprint(0.65), voiceprint(0.65, stranger: true), nil], + profile: profile) + XCTAssertEqual(voices, [.mine, .mine, .other, .unjudged]) + XCTAssertEqual(decide(turns, voices), .drop([30...35.5])) + + XCTAssertEqual( + SpeakerFilter.voices( + turns: [turn("a", 0, 5), turn("b", 5, 10)], + embeddings: [voiceprint(0.9), voiceprint(0.9)], profile: profile), + [.other, .other]) + } + func testNothingIsCutUnlessTheUserAndSomeoneElseWereBothHeard() { let two = [turn("a", 0, 3), turn("b", 3, 6)] - XCTAssertEqual(SpeakerFilter.decide(turns: [], voices: []), .unfiltered(.noSpeech)) + XCTAssertEqual(decide([], []), .unfiltered(.noSpeech)) + XCTAssertEqual(decide(two, [.unjudged, .unjudged]), .unfiltered(.tooShort)) + XCTAssertEqual(decide(two, [.other, .other]), .unfiltered(.noMatch)) + XCTAssertEqual(decide(two, [.other, .unjudged]), .unfiltered(.noMatch)) + XCTAssertEqual(decide(two, [.mine, .mine]), .unfiltered(.onlyYou)) + XCTAssertEqual(decide(two, [.mine, .unjudged]), .unfiltered(.onlyYou)) + } + + func testNothingIsCutWhenTheUserWasHeardForLessThanTheSpeechToCut() { + // The profile took three words of a half-minute dictation for the user and the rest for + // someone else. That is a missed match on their own voice, not a stranger talking for 28 s. XCTAssertEqual( - SpeakerFilter.decide(turns: two, voices: [.unjudged, .unjudged]), .unfiltered(.tooShort)) - XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.other, .other]), .unfiltered(.noMatch)) + decide([turn("a", 0, 1.5), turn("a", 2, 30)], [.mine, .other]), .unfiltered(.mostlyOther)) XCTAssertEqual( - SpeakerFilter.decide(turns: two, voices: [.other, .unjudged]), .unfiltered(.noMatch)) - XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.mine, .mine]), .unfiltered(.onlyYou)) + decide([turn("a", 0, 5), turn("b", 5, 9.2)], [.mine, .other]), .unfiltered(.mostlyOther)) + XCTAssertEqual(decide([turn("a", 0, 5), turn("b", 5, 9)], [.mine, .other]), .drop([5...9])) + } + + func testAVoiceIsCutOnlyWhenItIsClearlyQuieterThanTheUsers() { + let turns = [turn("a", 0, 10), turn("a", 10, 14), turn("b", 14, 17)] + let voices: [TurnVoice] = [.mine, .other, .other] + // A stretch as loud as the user is as close to the phone as they are, so it may be them. + XCTAssertEqual(decide(turns, voices, levels: [-20, -22, -26]), .unfiltered(.similarLevel)) + XCTAssertEqual(decide(turns, voices, levels: [-20, -22, -27]), .drop([14...17])) + XCTAssertEqual(decide(turns, voices, levels: [-20, -27, -27]), .drop([10...17])) + XCTAssertEqual(decide(turns, voices, levels: [-20, nil, nil]), .unfiltered(.similarLevel)) + XCTAssertEqual(decide(turns, voices, levels: [nil, -40, -40]), .unfiltered(.similarLevel)) + } + + func testLevelIsHowLoudAStretchGetsNotItsMean() throws { + let rate = 1000.0 + let loud = [Float](repeating: 0.1, count: 20) + let pause = [Float](repeating: 0.001, count: 20) + XCTAssertNil(SpeakerFilter.level([], sampleRate: rate)) + XCTAssertNil(SpeakerFilter.level(loud[..<19], sampleRate: rate)) + XCTAssertNil(SpeakerFilter.level([Float](repeating: 0, count: 40)[...], sampleRate: rate)) + // Two frames in ten are speech: the pauses between them do not pull the level down. + let halting = (0..<10).flatMap { $0 % 5 == 0 ? loud : pause } + XCTAssertEqual(try XCTUnwrap(SpeakerFilter.level(halting[...], sampleRate: rate)), -20, accuracy: 0.001) + // One loud frame in twenty is a click, not the voice. + let click = (0..<20).flatMap { $0 == 7 ? loud : pause } + XCTAssertEqual(try XCTUnwrap(SpeakerFilter.level(click[...], sampleRate: rate)), -60, accuracy: 0.001) XCTAssertEqual( - SpeakerFilter.decide(turns: two, voices: [.mine, .unjudged]), .unfiltered(.onlyYou)) + try XCTUnwrap(SpeakerFilter.level((pause + loud + pause)[20..<40], sampleRate: rate)), -20, + accuracy: 0.001) } func testOnlyTheOtherVoiceIsCutWhateverTheDiarizerCalledIt() { let turns = [turn("a", 0, 3), turn("a", 3.5, 7), turn("a", 7.2, 9)] - XCTAssertEqual( - SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .mine]), .drop([3.5...7])) + XCTAssertEqual(decide(turns, [.mine, .other, .mine]), .drop([3.5...7])) } func testAShortTurnIsCutOnlyWhenItsSpeakerWasAlwaysSomeoneElse() { let turns = [ - turn("a", 0, 3), turn("b", 3, 6), turn("b", 6.2, 6.8), turn("a", 7, 7.5), turn("c", 8, 8.6), + turn("a", 0, 5), turn("b", 5, 8), turn("b", 8.2, 8.8), turn("a", 9, 9.5), turn("c", 10, 10.6), ] - XCTAssertEqual( - SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .unjudged, .unjudged, .unjudged]), - .drop([3...6, 6.2...6.8])) + let voices: [TurnVoice] = [.mine, .other, .unjudged, .unjudged, .unjudged] + XCTAssertEqual(decide(turns, voices), .drop([5...8, 8.2...8.8])) + XCTAssertEqual(decide(turns, voices, levels: [-20, -35, -22, -35, -35]), .drop([5...8])) - let merged = [turn("a", 0, 3), turn("a", 3, 6), turn("a", 6.2, 6.8)] - XCTAssertEqual( - SpeakerFilter.decide(turns: merged, voices: [.mine, .other, .unjudged]), .drop([3...6])) + let merged = [turn("a", 0, 4), turn("a", 4, 7), turn("a", 7.2, 7.8)] + XCTAssertEqual(decide(merged, [.mine, .other, .unjudged]), .drop([4...7])) } private func assertClose( @@ -101,24 +169,17 @@ final class SpeakerFilterTests: XCTestCase { } func testSpeechOverTheUsersOwnTurnIsNotCut() { + XCTAssertEqual(decide([turn("a", 0, 4), turn("b", 3, 6)], [.mine, .other]), .drop([4...6])) + XCTAssertEqual(decide([turn("a", 0, 3), turn("b", 2, 8)], [.other, .mine]), .drop([0...2])) XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3, 6)], voices: [.mine, .other]), - .drop([4...6])) - XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 6), turn("b", 2, 3)], voices: [.other, .mine]), - .drop([0...2, 3...6])) + decide([turn("a", 0, 4), turn("b", 1, 3)], [.mine, .other]), .unfiltered(.overlapping)) XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 1, 3)], voices: [.mine, .other]), - .unfiltered(.overlapping)) - XCTAssertEqual( - SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3.9, 4.1)], voices: [.mine, .other]), - .unfiltered(.overlapping)) + decide([turn("a", 0, 4), turn("b", 3.9, 4.1)], [.mine, .other]), .unfiltered(.overlapping)) } func testTouchingTurnsBecomeOneCut() { - let turns = [turn("a", 0, 2), turn("b", 2, 4), turn("c", 4, 6), turn("b", 3.5, 4.5)] - XCTAssertEqual( - SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .other, .other]), .drop([2...6])) + let turns = [turn("a", 0, 6), turn("b", 6, 8), turn("c", 8, 10), turn("b", 7.5, 8.5)] + XCTAssertEqual(decide(turns, [.mine, .other, .other, .other]), .drop([6...10])) } func testCosineDistance() { diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift index 9ced2e26e3b10..284d886b0ef08 100644 --- a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift @@ -22,3 +22,27 @@ func writeWav(_ samples: [Float], to url: URL) throws { ]) try file.write(from: buffer) } + +/// What the app's recorder writes: AAC at 44.1 kHz in stereo. +func writePhoneRecording(_ samples: [Float], to url: URL) throws { + let rate = 44_100.0 + let file = try AVAudioFile( + forWriting: url, + settings: [ + AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: rate, AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: 128_000, + ]) + let count = Int(Double(samples.count) * rate / DictationEngine.sampleRate) + let buffer = try XCTUnwrap( + AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(count))) + buffer.frameLength = AVAudioFrameCount(count) + for frame in 0.. TurnVoice in - guard let embedding = turn.embedding else { return .unjudged } - return SpeakerFilter.judge( - duration: turn.turn.duration, - distance: SpeakerFilter.cosineDistance(embedding, profile.embedding)) - } - switch SpeakerFilter.decide(turns: heard.map(\.turn), voices: voices) { + let turns = heard.map(\.turn) + let voices = SpeakerFilter.voices( + turns: turns, embeddings: heard.map(\.embedding), profile: profile.embedding) + switch SpeakerFilter.decide(turns: turns, voices: voices, levels: heard.map(\.level)) { case .unfiltered(let reason): return (samples, .skipped(reason)) case .drop(let ranges): @@ -473,7 +471,13 @@ actor DictationEngine { turn, windows: windows, distances: Array(distances.prefix(windows.count))) } } - var heard = turns.map { HeardTurn(turn: $0, embedding: nil) } + var heard = turns.map { turn in + let from = min(Int(turn.start * Self.sampleRate), samples.count) + let to = min(max(Int(turn.end * Self.sampleRate), from), samples.count) + return HeardTurn( + turn: turn, embedding: nil, + level: SpeakerFilter.level(samples[from..= SpeakerFilter.minimumJudgedSeconds } diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift index 79626d73289bc..4d6a026a55e14 100644 --- a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift @@ -20,6 +20,8 @@ enum FilterSkip: String, Sendable { case onlyYou case tooShort case overlapping + case similarLevel + case mostlyOther case noProfile case notInstalled case failed @@ -49,6 +51,13 @@ enum SpeakerFilter { static let scanSeconds = 3.0 static let scanWindowSeconds = 1.5 static let scanHopSeconds = 0.5 + /// A voice is cut only when it is this much quieter than the user's: further from the phone. + /// One as loud as theirs may be their own, missed by the profile. + static let quieterDecibels: Float = 6 + static let levelFrameSeconds = 0.02 + static let levelQuantile = 0.9 + /// The user has to be heard for this many times the length of the speech that is cut. + static let dominanceRatio = 1.25 static func normalized(_ vector: [Float]) -> [Float] { let length = vector.reduce(Float(0)) { $0 + $1 * $1 }.squareRoot() @@ -92,6 +101,30 @@ enum SpeakerFilter { return distance <= limit ? .mine : .other } + /// Whose voice each turn is. A turn too short for a voiceprint has nil in `embeddings`. + /// + /// Noise or another microphone moves the user's own turns away from the profile, which was + /// recorded somewhere else. So a turn the profile rejects is held against the turns it accepted + /// in this recording, and is the user's when it matches them. + static func voices(turns: [SpeakerTurn], embeddings: [[Float]?], profile: [Float]) -> [TurnVoice] { + precondition(turns.count == embeddings.count) + var voices = zip(turns, embeddings).map { turn, embedding -> TurnVoice in + guard let embedding else { return .unjudged } + return judge(duration: turn.duration, distance: cosineDistance(embedding, profile)) + } + let mine = voices.indices.filter { voices[$0] == .mine } + guard + let heard = meanEmbedding( + mine.compactMap { embeddings[$0] }, weights: mine.map { Float(turns[$0].duration) }) + else { return voices } + for index in voices.indices where voices[index] == .other { + if let embedding = embeddings[index], cosineDistance(embedding, heard) <= matchDistance { + voices[index] = .mine + } + } + return voices + } + /// The windows that scan a turn, evenly spaced, the last one ending where the turn does. static func scanWindows(_ turn: SpeakerTurn) -> [Range] { guard turn.duration >= scanSeconds else { return [] } @@ -128,31 +161,67 @@ enum SpeakerFilter { return zip(edges, edges.dropFirst()).map { SpeakerTurn(speaker: turn.speaker, start: $0, end: $1) } } - /// What to cut from a recording. Only turns judged to be another voice go, and only where no - /// kept turn overlaps them. A turn too short to judge goes when its speaker's judged turns were - /// all another voice. With no turn judged to be the user, nothing is cut: a missed match on - /// their own voice must not lose the dictation. - static func decide(turns: [SpeakerTurn], voices: [TurnVoice]) -> SpeakerDecision { - precondition(turns.count == voices.count) + /// How loud a stretch gets, in decibels: the power of its loudest tenth of 20 ms frames. The + /// mean would fall with every pause in it and rise with the noise under it. Nil when there is + /// nothing to measure. + static func level(_ samples: ArraySlice, sampleRate: Double) -> Float? { + let frame = max(Int(levelFrameSeconds * sampleRate), 1) + var powers: [Float] = [] + var start = samples.startIndex + while start + frame <= samples.endIndex { + powers.append(samples[start..<(start + frame)].reduce(Float(0)) { $0 + $1 * $1 } / Float(frame)) + start += frame + } + guard !powers.isEmpty else { return nil } + powers.sort() + let power = powers[min(Int(Double(powers.count) * levelQuantile), powers.count - 1)] + return power > 0 ? 10 * log10(power) : nil + } + + /// What to cut from a recording. A voiceprint alone has cut the user's own words, so a turn + /// goes only when three things agree: it was judged to be another voice (or is too short to + /// judge and its speaker's judged turns all were), it is clearly quieter than the user, and the + /// user was heard for longer than everything cut. No kept turn may overlap it. With no turn + /// judged to be the user, nothing is cut. + static func decide(turns: [SpeakerTurn], voices: [TurnVoice], levels: [Float?]) -> SpeakerDecision { + precondition(turns.count == voices.count && turns.count == levels.count) if turns.isEmpty { return .unfiltered(.noSpeech) } if voices.allSatisfy({ $0 == .unjudged }) { return .unfiltered(.tooShort) } if !voices.contains(.mine) { return .unfiltered(.noMatch) } if !voices.contains(.other) { return .unfiltered(.onlyYou) } + let mine = turns.indices.filter { voices[$0] == .mine && turns[$0].end > turns[$0].start } + var power: Float = 0 + var measured: Float = 0 + for index in mine { + guard let level = levels[index] else { continue } + power += pow(10, level / 10) * Float(turns[index].duration) + measured += Float(turns[index].duration) + } + guard measured > 0, power > 0 else { return .unfiltered(.similarLevel) } + let userLevel = 10 * log10(power / measured) + var judgedBySpeaker: [String: Set] = [:] for (turn, voice) in zip(turns, voices) where voice != .unjudged { judgedBySpeaker[turn.speaker, default: []].insert(voice) } var dropped: [ClosedRange] = [] var kept: [ClosedRange] = [] - for (turn, voice) in zip(turns, voices) where turn.end > turn.start { - let isOther = voice == .other || (voice == .unjudged && judgedBySpeaker[turn.speaker] == [.other]) - if isOther { dropped.append(turn.start...turn.end) } else { kept.append(turn.start...turn.end) } + for (index, turn) in turns.enumerated() where turn.end > turn.start { + let isOther = + voices[index] == .other + || (voices[index] == .unjudged && judgedBySpeaker[turn.speaker] == [.other]) + let isQuieter = levels[index].map { userLevel - $0 > quieterDecibels } ?? false + if isOther && isQuieter { dropped.append(turn.start...turn.end) } else { kept.append(turn.start...turn.end) } } + if dropped.isEmpty { return .unfiltered(.similarLevel) } let ranges = subtract(merged(dropped), merged(kept)).filter { $0.upperBound - $0.lowerBound >= minimumDropSeconds } - return ranges.isEmpty ? .unfiltered(.overlapping) : .drop(ranges) + if ranges.isEmpty { return .unfiltered(.overlapping) } + let userSeconds = seconds(merged(mine.map { turns[$0].start...turns[$0].end })) + if userSeconds < seconds(ranges) * dominanceRatio { return .unfiltered(.mostlyOther) } + return .drop(ranges) } static func merged(_ ranges: [ClosedRange]) -> [ClosedRange] { diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift index 48eb6fdb2d9ec..c5c490a8a8253 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift @@ -445,6 +445,7 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro private var isReadOnly = false private var isApplyingControlledValue = false private var nativeEventCount = 0 + private var hasControlledDocument = false private var resetGeneration = 0 private var lastContentSize = CGSize.zero private var iconImages: [String: UIImage] = [:] @@ -666,12 +667,15 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro } if isReset { resetGeneration = document.resetGeneration - nativeEventCount = document.mostRecentEventCount endComposition() } - if document.isNativeEcho && !isReset && textView.serializedText() != document.value { + // iOS destroys this view under display: none and mounts a fresh one while JS keeps its + // revision and draft. Continue from them, or JS drops every event the new view sends. + nativeEventCount = document.mostRecentEventCount + if document.isNativeEcho && !isReset && hasControlledDocument && textView.serializedText() != document.value { return } + hasControlledDocument = true if tokensJson != document.tokensJson { tokensJson = document.tokensJson tokens = decode([ComposerTokenPayload].self, from: document.tokensJson) ?? [] diff --git a/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift b/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift index 2266f1a451cc3..d898f18ba68f5 100644 --- a/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift +++ b/apps/mobile/modules/t3-composer-editor/tests/ios/main.swift @@ -91,6 +91,20 @@ do { check(view.text == "abc", "A stale controlled value overwrote newer native typing") } +// iOS mounts a new view after display: none (dictation) while JS keeps its revision and draft. +do { + let view = T3ComposerEditorView() + view.frame = CGRect(x: 0, y: 0, width: 320, height: 200) + view.setControlledDocumentJson(document("dictated", revision: 40, isNativeEcho: true)) + check(view.text == "dictated", "A remounted editor did not take the draft JS holds: \(view.text)") + view.editor.insertText("!") + let typed = view.onComposerChange.payloads.last + check( + (typed?["eventCount"] as? Int ?? 0) > 40, + "Typing in a remounted editor must be newer than the parent's revision, or JS drops it" + ) +} + // A send clear during an IME composition must not come back when the composition commits. do { let view = makeEditor("hello ") diff --git a/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx b/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx index 7f00c9b406609..1f79d8b11dfcf 100644 --- a/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx +++ b/apps/mobile/src/features/dictation/SettingsDictationModelsRouteScreen.tsx @@ -1,12 +1,22 @@ import { useAtomValue } from "@effect/atom-react"; -import { useEffect, useState, useSyncExternalStore } from "react"; +import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; +import * as Clipboard from "expo-clipboard"; +import { useFocusEffect } from "@react-navigation/native"; +import { File } from "expo-file-system"; +import { useCallback, useEffect, useState, useSyncExternalStore } from "react"; import { Alert, View } from "react-native"; import { AppText as Text } from "../../components/ui/AppText"; import { ListActionRow } from "../../components/ui/ListActionRow"; import { ListChoiceRow } from "../../components/ui/ListChoiceRow"; import { ListRowLayout } from "../../components/ui/ListRowLayout"; +import { getLocalVoiceTranscriber as getSystemVoiceTranscriber } from "../../native/voiceTranscription"; import { SettingsSection } from "../settings/components/SettingsSection"; +import { + KEPT_VOICE_RECORDINGS, + keptVoiceRecordings, + type KeptVoiceRecording, +} from "../voice-input/keptRecordings"; import { localVoiceTranscriberAtom } from "../voice-input/VoiceInputProvider"; import { useAppleDictationAvailable } from "./appleDictation"; import { nativeErrorMessage, type DictationDownloads } from "./dictationDownloads"; @@ -23,6 +33,7 @@ import { currentDictationSettings, deviceLocale, dictationDownloads, + makeDictationTranscriber, updateDictationSettings, useDictationState, } from "./dictationRuntime"; @@ -36,12 +47,25 @@ import { describeRun } from "./runSummary"; const TITLE = "Dictation model"; -type SampleResult = - | { readonly status: "idle" } +type TrialResult = | { readonly status: "running" } | { readonly status: "done"; readonly text: string } | { readonly status: "failed"; readonly message: string }; +const SAMPLE = "sample"; + +/** `of` is the sample or a kept recording's URI. One result is on screen at a time, so the run details under it are its own. */ +type Trial = { readonly of: string; readonly result: TrialResult }; + +function recordedAtLabel(recordedAt: number): string { + return new Date(recordedAt).toLocaleString([], { + weekday: "long", + hour: "numeric", + minute: "2-digit", + second: "2-digit", + }); +} + export function SettingsDictationModelsRouteScreen() { const downloads = dictationDownloads(); if (dictationNative === null || downloads === null) return ; @@ -58,7 +82,9 @@ function DictationModels(props: { const transcriber = useAtomValue(localVoiceTranscriberAtom); const appleMissing = useAppleDictationAvailable() === false; const [device] = useState(() => native.deviceInfo()); - const [sample, setSample] = useState({ status: "idle" }); + const [trial, setTrial] = useState(null); + const [copied, setCopied] = useState(false); + const [recordings, setRecordings] = useState(keptVoiceRecordings); const suggested = recommendedDictationModel({ physicalMemoryBytes: device.physicalMemoryBytes, locale: deviceLocale(), @@ -69,6 +95,8 @@ function DictationModels(props: { void downloads.refresh(); }, [downloads]); + useFocusEffect(useCallback(() => setRecordings(keptVoiceRecordings()), [])); + const describe = (model: LocalDictationModel) => model.id === suggested ? `${model.summary} Suggested for this device.` : model.summary; const installed = LOCAL_DICTATION_MODELS.filter( @@ -108,27 +136,87 @@ function DictationModels(props: { ], ); - const transcribeSample = async () => { - const uri = native.sampleClipUri(); - if (uri === null || transcriber === null || transcriber === "checking") { - setSample({ status: "failed", message: "Dictation is not available right now." }); - return; - } - setSample({ status: "running" }); + const transcribe = async ( + of: Trial["of"], + uri: string, + using: VoiceTranscriber, + failure: string, + ) => { + setCopied(false); + setTrial({ of, result: { status: "running" } }); const { signal } = new AbortController(); try { - // Loaded first so the sample runs on the chosen model, not on the stand-in for a cold one. + // Loaded first so this runs on the chosen model, not on the stand-in for a cold one. if (installed.some((model) => model.id === settings.modelId)) { await native.prepare(settings.modelId, DICTATION_PROCESSING); } - const prepared = await transcriber.prepare({ signal }); - setSample({ status: "done", text: await prepared.transcribe(uri, { signal }) }); + const prepared = await using.prepare({ signal }); + const text = await prepared.transcribe(uri, { signal }); + setTrial({ of, result: { status: "done", text } }); } catch (error) { - setSample({ - status: "failed", - message: nativeErrorMessage(error, "The sample could not be transcribed."), + setTrial({ of, result: { status: "failed", message: nativeErrorMessage(error, failure) } }); + } + }; + + const transcribeSample = () => { + const uri = native.sampleClipUri(); + if (uri === null || transcriber === null || transcriber === "checking") { + setTrial({ + of: SAMPLE, + result: { status: "failed", message: "Dictation is not available right now." }, }); + return; } + void transcribe(SAMPLE, uri, transcriber, "The sample could not be transcribed."); + }; + + const transcribeRecording = (recording: KeptVoiceRecording) => { + if (!new File(recording.uri).exists) { + setRecordings(keptVoiceRecordings()); + setTrial(null); + return; + } + void transcribe( + recording.uri, + recording.uri, + makeDictationTranscriber(getSystemVoiceTranscriber, { ignoreVoiceFilter: true }), + "The recording could not be transcribed.", + ); + }; + + const running = trial?.result.status === "running" ? trial.of : null; + const resultOf = (of: Trial["of"]) => { + const result = trial?.of === of ? trial.result : null; + if (result?.status === "failed") { + return ( + + {result.message} + + ); + } + if (result?.status !== "done") return null; + const heard = result.text.trim().length > 0; + return ( + <> + + + {heard ? result.text : "No speech was found in this recording."} + + {lastRun === null ? null : ( + {describeRun(lastRun)} + )} + + {heard && of !== SAMPLE ? ( + + void Clipboard.setStringAsync(result.text).then(() => setCopied(true))} + /> + + ) : null} + + ); }; return ( @@ -188,22 +276,36 @@ function DictationModels(props: { icon="play" label="Transcribe a sample" description="Runs a built-in recording through the model in use." - loading={sample.status === "running"} - onPress={() => void transcribeSample()} + loading={running === SAMPLE} + disabled={running !== null} + onPress={transcribeSample} /> - {sample.status === "done" ? ( - - {sample.text} - {lastRun === null ? null : ( - {describeRun(lastRun)} - )} - - ) : null} - {sample.status === "failed" ? ( - - {sample.message} - - ) : null} + {resultOf(SAMPLE)} + + + + {recordings.length === 0 ? ( + + ) : ( + recordings.map((recording, index) => ( + 0 ? "border-t border-border-subtle" : undefined} + > + transcribeRecording(recording)} + /> + {resultOf(recording.uri)} + + )) + )} {installed.length === 0 ? null : ( diff --git a/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx b/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx index 743c9e049d7fa..b226e29b65035 100644 --- a/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx +++ b/apps/mobile/src/features/dictation/SettingsDictationVoiceRouteScreen.tsx @@ -230,9 +230,10 @@ function DictationVoice(props: { ) : null} The filter compares each stretch of speech with your voice profile and removes the ones that - are someone else. It adds a short wait after each recording. If it does not recognise you, - nothing is removed. Your voice profile stays on this device. The filter model is by - pyannote, WeSpeaker and BUT Speech@FIT (CC BY 4.0), converted by Fluid Inference. + are someone else and quieter than you, further from the phone. A voice as loud as yours + stays. It adds a short wait after each recording. If it does not recognise you, nothing is + removed. Your voice profile stays on this device. The filter model is by pyannote, WeSpeaker + and BUT Speech@FIT (CC BY 4.0), converted by Fluid Inference. ); diff --git a/apps/mobile/src/features/dictation/dictationNative.ts b/apps/mobile/src/features/dictation/dictationNative.ts index 5095628588623..211f5123a512b 100644 --- a/apps/mobile/src/features/dictation/dictationNative.ts +++ b/apps/mobile/src/features/dictation/dictationNative.ts @@ -6,6 +6,8 @@ export type DictationFilterSkip = | "onlyYou" | "tooShort" | "overlapping" + | "similarLevel" + | "mostlyOther" | "noProfile" | "notInstalled" | "failed"; diff --git a/apps/mobile/src/features/dictation/dictationRuntime.ts b/apps/mobile/src/features/dictation/dictationRuntime.ts index ae68cc611cf5c..34d34a1102d31 100644 --- a/apps/mobile/src/features/dictation/dictationRuntime.ts +++ b/apps/mobile/src/features/dictation/dictationRuntime.ts @@ -49,6 +49,7 @@ export function deviceLocale(): string { export function makeDictationTranscriber( getSystemTranscriber: () => Promise, + options: { readonly ignoreVoiceFilter?: boolean } = {}, ): VoiceTranscriber { const { store, pending } = dictationRuntime(); return createDictationTranscriber({ @@ -59,6 +60,7 @@ export function makeDictationTranscriber( getLocale: deviceLocale, now: Date.now, newRequestId: () => `dictation-${Date.now()}-${++requests}`, + ignoreVoiceFilter: options.ignoreVoiceFilter, }); } diff --git a/apps/mobile/src/features/dictation/dictationTranscriber.test.ts b/apps/mobile/src/features/dictation/dictationTranscriber.test.ts index a7846e0b5e94d..c51bd71e07307 100644 --- a/apps/mobile/src/features/dictation/dictationTranscriber.test.ts +++ b/apps/mobile/src/features/dictation/dictationTranscriber.test.ts @@ -44,10 +44,12 @@ const TRANSCRIPT: DictationTranscript = { function setup( input: { readonly settings?: Partial; - readonly system?: "present" | "absent" | "refuses"; + /** "refuses" fails to prepare; "fails" prepares and then fails on the recording. */ + readonly system?: "present" | "absent" | "refuses" | "fails"; readonly systemLocale?: string; readonly native?: "present" | "absent"; readonly locale?: string; + readonly ignoreVoiceFilter?: boolean; } = {}, ) { const clock = { now: 1_000 }; @@ -62,6 +64,7 @@ function setup( write: async () => {}, }); const systemTranscribe = vi.fn(async () => { + if (input.system === "fails") throw new Error("That language is not supported."); clock.now += 250; return SPOKEN; }); @@ -87,6 +90,7 @@ function setup( now: () => clock.now, pending, newRequestId: () => "request-1", + ignoreVoiceFilter: input.ignoreVoiceFilter, }); const abort = new AbortController(); const dictate = async () => { @@ -162,6 +166,17 @@ describe("with a downloaded model selected", () => { ); expect(native.prepare).toHaveBeenCalledWith("parakeet-v3", "auto"); expect(getSystemTranscriber).not.toHaveBeenCalled(); + + const again = setup({ settings, ignoreVoiceFilter: true }); + await again.dictate(); + expect(again.native.transcribe).toHaveBeenCalledWith( + expect.any(String), + URI, + "parakeet-v3", + "auto", + false, + "en-US", + ); expect(lastRun()).toEqual({ at: 1_000, engine: "parakeet-v3", @@ -221,7 +236,7 @@ describe("with a downloaded model selected", () => { expect(native.transcribe.mock.calls[0]).toContain("en-US"); }); - it.each(["absent", "refuses"] as const)( + it.each(["absent", "refuses", "fails"] as const)( "waits for a loading model when the system recognizer %s", async (system) => { const { dictate, native, lastRun } = setup({ settings, system }); @@ -256,7 +271,7 @@ describe("with a downloaded model selected", () => { expect(lastRun()).toMatchObject({ engine: "apple", fallback: "failed" }); }); - it.each(["absent", "refuses"] as const)( + it.each(["absent", "refuses", "fails"] as const)( "reports the model's own failure when the system recognizer %s", async (system) => { const { dictate, native, lastRun } = setup({ settings, system }); diff --git a/apps/mobile/src/features/dictation/dictationTranscriber.ts b/apps/mobile/src/features/dictation/dictationTranscriber.ts index ca2a157256e36..dcbdb47a18d2b 100644 --- a/apps/mobile/src/features/dictation/dictationTranscriber.ts +++ b/apps/mobile/src/features/dictation/dictationTranscriber.ts @@ -28,6 +28,8 @@ export interface DictationTranscriberDeps { readonly now: () => number; readonly pending: PendingDictations; readonly newRequestId: () => string; + /** For a recording transcribed again: the filter may be what removed its words. */ + readonly ignoreVoiceFilter?: boolean; } const NO_MODEL_MESSAGE = "Download a dictation model in Settings > Microphone."; @@ -67,7 +69,10 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic fallback: DictationFallback | null, ): PreparedVoiceTranscription { return { - locale: prepared.locale, + // Read late: the system recognizer settles its locale while the user talks. + get locale() { + return prepared.locale; + }, async transcribe(uri, options) { const started = deps.now(); const text = await prepared.transcribe(uri, options); @@ -102,7 +107,7 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic uri, modelId, DICTATION_PROCESSING, - store.getSnapshot().settings.onlyMyVoice, + !deps.ignoreVoiceFilter && store.getSnapshot().settings.onlyMyVoice, locale, ); } catch (cause) { @@ -115,7 +120,12 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic const system = await prepareSystem(options).catch(() => null); throwIfVoiceTranscriptionAborted(signal); if (system === null) throw failure; - return viaSystem(system, "failed").transcribe(uri, options); + try { + return await viaSystem(system, "failed").transcribe(uri, options); + } catch { + throwIfVoiceTranscriptionAborted(signal); + throw failure; + } } finally { signal.removeEventListener("abort", cancel); } @@ -186,10 +196,16 @@ export function createDictationTranscriber(deps: DictationTranscriberDeps): Voic () => false, ); if (loaded) return model.transcribe(uri, transcribeOptions); - return viaSystem(system, loadFailed ? "failed" : "warming-up").transcribe( - uri, - transcribeOptions, - ); + try { + return await viaSystem(system, loadFailed ? "failed" : "warming-up").transcribe( + uri, + transcribeOptions, + ); + } catch { + throwIfVoiceTranscriptionAborted(transcribeOptions.signal); + // The system recognizer can refuse the language only now, after the recording. + return model.transcribe(uri, transcribeOptions); + } }, }; } diff --git a/apps/mobile/src/features/dictation/runSummary.ts b/apps/mobile/src/features/dictation/runSummary.ts index 08f9578cbd8ed..63b9444a622e0 100644 --- a/apps/mobile/src/features/dictation/runSummary.ts +++ b/apps/mobile/src/features/dictation/runSummary.ts @@ -17,6 +17,10 @@ function describeFilter(filter: DictationFilterReport): string { return "Only your voice was heard."; case "noMatch": return "Your voice was not recognised, so nothing was removed."; + case "similarLevel": + return "Another voice was as loud as yours, so nothing was removed."; + case "mostlyOther": + return "Most of the recording did not match your voice, so nothing was removed."; case "noProfile": return "No voice profile is recorded, so nothing was removed."; case "notInstalled": diff --git a/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx b/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx index 62cfea826b5ad..05a8a9fb5220b 100644 --- a/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx +++ b/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx @@ -58,6 +58,7 @@ export function ScheduledTaskPromptField(props: { {showsDictation ? ( diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 686076628ce51..7ae177e21e996 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -889,6 +889,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps > {isExpanded ? editingAttachments : null} diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index 615914589e80e..d9df88eab80e8 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -321,6 +321,8 @@ function VoiceActionButton(props: { export function ComposerDictationError(props: { readonly error: string | null; + /** Set when the recording behind the error is still there to transcribe again. */ + readonly onTranscribeAgain?: (() => void) | null; readonly onDismiss: () => void; }) { if (!props.error) return null; @@ -329,6 +331,22 @@ export function ComposerDictationError(props: { {props.error} + {props.onTranscribeAgain ? ( + + + + ) : null} ) : isError ? ( - voice.isAvailable ? ( + voice.isAvailable && errorAction !== null ? ( { - if (!openSettings) { + if (errorAction !== "settings") { void voice.session.retry(); return; } @@ -214,7 +220,11 @@ function EdgeDictationPill(props: { void Linking.openSettings(); }} > - + ) : null ) : ( diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index b6459da93fb18..fbe010e6e8539 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -20,6 +20,7 @@ import { voiceInputBlocksSubmission, type VoiceTranscriber, } from "@t3tools/client-runtime/voice-input"; +import { keepVoiceRecording } from "./keptRecordings"; import { createLazyVoiceRecorder } from "./lazyVoiceRecorder"; import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; import { VoiceInputSession } from "./voiceInputSession"; @@ -134,6 +135,7 @@ const voiceInputAtom = Atom.make((get) => { audioToken = null; }, deleteRecording: (uri) => new File(uri).delete(), + keepRecording: keepVoiceRecording, sampleRecording: () => { const status = recorder.getStatus(); return status?.isRecording diff --git a/apps/mobile/src/features/voice-input/keptRecordings.ts b/apps/mobile/src/features/voice-input/keptRecordings.ts new file mode 100644 index 0000000000000..2e27250d62c71 --- /dev/null +++ b/apps/mobile/src/features/voice-input/keptRecordings.ts @@ -0,0 +1,55 @@ +import { Directory, File, Paths } from "expo-file-system"; + +/** Enough that a new, retried or canceled recording does not push out the one being recovered. */ +export const KEPT_VOICE_RECORDINGS = 5; + +const FILE_NAME = /^dictation-(\d+)\.m4a$/; + +export interface KeptVoiceRecording { + readonly uri: string; + readonly recordedAt: number; +} + +function folder(): Directory { + return new Directory(Paths.cache, "dictation-recordings"); +} + +/** The dictation recordings still on the device, newest first. */ +export function keptVoiceRecordings(): ReadonlyArray { + try { + const kept = folder(); + if (!kept.exists) return []; + return kept + .list() + .flatMap((entry) => { + const recordedAt = entry instanceof File ? FILE_NAME.exec(entry.name)?.[1] : undefined; + return recordedAt === undefined ? [] : [{ uri: entry.uri, recordedAt: Number(recordedAt) }]; + }) + .sort((left, right) => right.recordedAt - left.recordedAt); + } catch { + return []; + } +} + +/** Returns where the recording is now: the kept place, or where it was if it could not be moved. */ +export function keepVoiceRecording(uri: string): string { + let keptUri: string; + try { + const kept = folder(); + if (!kept.exists) kept.create({ intermediates: true, idempotent: true }); + const destination = new File(kept, `dictation-${Date.now()}.m4a`); + // The synchronous move: the recording is transcribed from its new place straight after. + new File(uri).moveSync(destination); + keptUri = destination.uri; + } catch { + return uri; + } + for (const old of keptVoiceRecordings().slice(KEPT_VOICE_RECORDINGS)) { + try { + new File(old.uri).delete(); + } catch { + // It is removed after a later recording. + } + } + return keptUri; +} diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts index fbb6059099a8a..7852a0b65079d 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -112,6 +112,7 @@ describe("createLazyVoiceRecorder", () => { configureRecording: async () => undefined, releaseRecording: async () => undefined, deleteRecording: (uri) => deleted.push(uri), + keepRecording: (uri) => uri, readDraft: () => ({ ownerKey: "thread", text: "", diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index e26672b5b4809..cfc8d18261f0e 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -74,6 +74,8 @@ export function useVoiceInputController(input: { freezesEditor: isBusy, blocksSubmission: isBusy, start, + /** Null unless the error on screen kept its recording. */ + transcribeAgain: state.errorAction === "transcribe" ? () => void session.retry() : null, stop: global.stop, cancel: global.cancel, }; diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 9d30e376b54e2..7357d47552877 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -64,6 +64,7 @@ function createSession( configureRecording: async () => {}, releaseRecording: vi.fn(async () => {}), deleteRecording: vi.fn(), + keepRecording: (uri) => uri, sampleRecording: () => null, publishAudioLevels, keepAwake, @@ -241,19 +242,45 @@ describe("global voice input", () => { }, ); - it.each(["changed", "removed"] as const)( - "does not overwrite a %s starting draft", - async (change) => { - const { session } = createSession(); - let text: string | null = "first"; - const commit = vi.fn(); - await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); - text = change === "removed" ? null : "edited prompt"; - await session.controller.stop(); - expect(commit).not.toHaveBeenCalled(); - expect(session.controller.currentState.error).toContain("draft changed"); - }, - ); + it("adds the words after an edit made to the draft during the recording", async () => { + const { session } = createSession(); + let text = "first"; + const commit = vi.fn(); + await session.start(createTarget("first", () => text, commit, { start: 0, end: 0 })); + text = "edited prompt"; + await session.controller.stop(); + expect(commit).toHaveBeenCalledWith("edited prompt spoken text", { start: 25, end: 25 }); + }); + + it("does not write to a draft that was removed during the recording", async () => { + const { session } = createSession(); + let text: string | null = "first"; + const commit = vi.fn(); + await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); + text = null; + await session.controller.stop(); + expect(commit).not.toHaveBeenCalled(); + expect(session.controller.currentState.error).toContain("draft is gone"); + }); + + it("retries a failed transcription from the recording, without recording again", async () => { + const { session, recorder, prepare } = createSession(); + const transcribe = vi + .fn() + .mockRejectedValueOnce(new Error("model failed")) + .mockResolvedValue("spoken text"); + prepare.mockResolvedValue({ locale: "en-US", transcribe }); + const commit = vi.fn(); + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + await session.controller.stop(); + expect(session.getSnapshot().state.errorAction).toBe("transcribe"); + + await session.retry(); + + expect(recorder.record).toHaveBeenCalledOnce(); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(session.getSnapshot().state.phase).toBe("idle"); + }); it("finishes the original draft at the recording limit while it is off screen", async () => { const { session, recorder } = createSession(); @@ -266,10 +293,14 @@ describe("global voice input", () => { url: recorder.uri, }); expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); - expect(session.controller.currentState.phase).toBe("idle"); + expect(session.controller.currentState).toEqual({ + phase: "error", + error: "Voice recording reached the 5 minute limit. The words up to it were added.", + errorAction: null, + }); }); - it("rejects a transcript when its off-screen draft changes and returns to the original text", async () => { + it("adds the transcript at the end when its off-screen draft changed and changed back", async () => { const registry = AtomRegistry.make(); const draft = Atom.make("hello world"); const unsubscribe = vi.fn(); @@ -292,8 +323,7 @@ describe("global voice input", () => { registry.set(draft, "changed"); registry.set(draft, "hello world"); await session.controller.stop(); - expect(commit).not.toHaveBeenCalled(); - expect(session.controller.currentState.error).toContain("draft changed"); + expect(commit).toHaveBeenCalledWith("hello world spoken text", { start: 23, end: 23 }); expect(unsubscribe).toHaveBeenCalledTimes(1); registry.dispose(); }); @@ -399,6 +429,42 @@ describe("global voice input", () => { } }); + it.each([ + ["interruptRecording", 3_000, "stopped early"], + // The recorder can stop at the limit before its finish event arrives. + ["recordingReachedLimit", 299_950, "5 minute limit"], + ] as const)( + "finishes with the captured audio through %s when the recorder stops on its own", + async (method, durationMillis, notice) => { + vi.useFakeTimers(); + try { + let sample: VoiceRecordingSample | null = null; + const { session, recorder } = createSession({ sampleRecording: () => sample }); + const commit = vi.fn(); + const interrupt = vi.spyOn(session.controller, method); + + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + // Not capturing yet is not a recorder that stopped. + vi.advanceTimersByTime(80); + expect(interrupt).not.toHaveBeenCalled(); + + sample = { durationMillis, metering: -20 }; + vi.advanceTimersByTime(80); + // iOS pauses the recorder for a call or Siri without reporting it. + sample = null; + vi.advanceTimersByTime(80); + expect(interrupt).toHaveBeenCalledOnce(); + await interrupt.mock.results[0]!.value; + + expect(recorder.stop).toHaveBeenCalled(); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(session.controller.currentState.error).toContain(notice); + } finally { + vi.useRealTimers(); + } + }, + ); + it("publishes which owners are focused", () => { const { session } = createSession(); const listener = vi.fn(); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 5bf9395daf59c..30317ceb13647 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -96,6 +96,7 @@ export class VoiceInputSession { private audioLevels = SILENT_LEVELS; private releaseKeepAwake: (() => void) | null = null; private meteringInterval: ReturnType | null = null; + private capturing = false; constructor(dependencies: VoiceInputSessionDependencies) { this.dependencies = dependencies; @@ -134,7 +135,11 @@ export class VoiceInputSession { if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel(); } + /** Does what the error on screen offers: the kept recording again, or a new recording. */ retry(): Promise { + if (this.controller.currentState.errorAction === "transcribe") { + return this.controller.retranscribe(); + } return this.target ? this.start(this.target) : Promise.resolve(); } @@ -181,6 +186,7 @@ export class VoiceInputSession { private startRecordingActivity(): void { this.releaseKeepAwake = this.dependencies.keepAwake(); + this.capturing = false; this.sampleRecording(); this.meteringInterval = setInterval(() => this.sampleRecording(), VOICE_METERING_INTERVAL_MS); } @@ -195,7 +201,18 @@ export class VoiceInputSession { private sampleRecording(): void { if (this.controller.currentState.phase !== "recording") return; const sample = this.dependencies.sampleRecording(); - if (!sample) return; + if (!sample) { + if (!this.capturing) return; + // The recorder stops at the limit just before it says so, and the system pauses it for a + // call or Siri without saying so at all. Either way the words so far are transcribed now. + if (this.snapshot.elapsedSeconds >= VOICE_RECORDING_LIMIT_SECONDS - 1) { + void this.controller.recordingReachedLimit(); + } else { + void this.controller.interruptRecording(); + } + return; + } + this.capturing = true; const level = normalizeVoiceInputDecibels(sample.metering); if (level !== 0 || this.audioLevels.some((sample) => sample !== 0)) { diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts index 7a2e39e4304e8..e4a580b5d23d3 100644 --- a/apps/mobile/src/native/voiceTranscription.android.test.ts +++ b/apps/mobile/src/native/voiceTranscription.android.test.ts @@ -70,6 +70,7 @@ it.each([ configureRecording: async () => {}, releaseRecording: async () => {}, deleteRecording: () => {}, + keepRecording: (uri) => uri, readDraft: () => draft, commitDraft: commit, onStateChange: () => {}, @@ -77,7 +78,11 @@ it.each([ await controller.start(); if (phase === "transcription") await controller.stop(); - expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + // Once something was recorded, the error offers the kept recording instead of a new one. + expect(controller.currentState).toMatchObject({ + phase: "error", + errorAction: phase === "transcription" ? "transcribe" : "retry", + }); expect(commit).not.toHaveBeenCalled(); expect(draft.text).toBe("Preserved draft."); expect(controller.currentState.error).toMatch(expected); diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index d83c371bc2a90..4a2b64567bf6f 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -1,7 +1,8 @@ import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; +import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller"; const mocks = vi.hoisted(() => ({ isAvailable: vi.fn<(locale: string) => boolean>(), @@ -43,8 +44,40 @@ function deferred() { return { promise, resolve }; } +/** Drives one on-device dictation through the controller the composer uses. */ +function createDictation() { + const recorder = { + uri: "file:///voice.m4a", + prepareToRecordAsync: async () => {}, + record: vi.fn(), + stop: vi.fn(async () => {}), + }; + const commits: string[] = []; + const controller = new VoiceInputController({ + recorder, + getTranscriber: () => ({ + prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options), + }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => {}, + deleteRecording: () => {}, + keepRecording: (uri) => uri, + readDraft: () => ({ + ownerKey: "environment:thread", + text: "", + selection: { start: 0, end: 0 }, + revision: 1, + }), + commitDraft: (text) => commits.push(text), + onStateChange: () => {}, + }); + return { controller, recorder, commits }; +} + beforeEach(() => { vi.resetAllMocks(); + resetVoiceInputGlobalsForTests(); mocks.isAvailable.mockReturnValue(true); mocks.prepare.mockResolvedValue("sv-SE"); mocks.readAudio.mockResolvedValue(audio); @@ -56,6 +89,41 @@ afterEach(() => { }); describe("getLocalVoiceTranscriber", () => { + it("records while Apple prepares its speech model and transcribes once it is ready", async () => { + const nativePrepare = deferred(); + mocks.prepare.mockReturnValue(nativePrepare.promise); + const dictation = createDictation(); + + await dictation.controller.start(); + expect(dictation.recorder.record).toHaveBeenCalledOnce(); + expect(dictation.controller.currentState.phase).toBe("recording"); + + const stopping = dictation.controller.stop(); + nativePrepare.resolve("sv-SE"); + await stopping; + + expect(mocks.transcribe).toHaveBeenCalledExactlyOnceWith(audio, "sv-SE"); + expect(dictation.commits).toEqual(["Hej världen."]); + }); + + it.each([ + ["AppleTranscriptionUnsupportedLocale", "unsupported-locale"], + ["Network", "preparation-failed"], + ])( + "reports a %s failure to prepare when the recording is transcribed", + async (code, expected) => { + mocks.prepare.mockRejectedValue(Object.assign(new Error("failed"), { code })); + const options = { signal: new AbortController().signal }; + const error = await (await getLocalVoiceTranscriber())! + .prepare(options) + .then((prepared) => prepared.transcribe("file:///voice.m4a", options)) + .catch((cause: unknown) => cause); + + expect(error).toMatchObject({ code: expected }); + expect(mocks.transcribe).not.toHaveBeenCalled(); + }, + ); + it("does not start native transcription after cancellation during a file read", async () => { const enteredRead = deferred(); const readResult = deferred(); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 5fc6c9a0b8a2b..607b9ceb446eb 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -43,24 +43,52 @@ export async function getLocalVoiceTranscriber(): Promise { throwIfVoiceTranscriptionAborted(signal); - if (!AppleTranscription.isAvailable(locale)) { + if (!AppleTranscription.isAvailable(deviceLocale)) { throw new VoiceTranscriptionError({ code: "unavailable", message: "Voice transcription requires a supported device with iOS 26 or later.", }); } + let locale = deviceLocale; + // Not awaited: recording starts now and Apple's model loads while the user talks. + const model = prepareLocale(deviceLocale, signal).then((supported) => { + locale = supported; + }); + // Recorder setup can fail before the recording is transcribed. + void model.catch(() => {}); + + async function ready(signal: AbortSignal) { + throwIfVoiceTranscriptionAborted(signal); + const cancelled = Promise.withResolvers(); + const cancel = () => cancelled.resolve(); + signal.addEventListener("abort", cancel, { once: true }); + try { + await Promise.race([model, cancelled.promise]); + throwIfVoiceTranscriptionAborted(signal); + return locale; + } finally { + signal.removeEventListener("abort", cancel); + } + } + return { + get locale() { + return locale; + }, + transcribe: async (uri, options) => + transcribeVoiceRecording(uri, await ready(options.signal), options), + }; +} + +async function prepareLocale(locale: string, signal: AbortSignal): Promise { try { const supportedLocale = await AppleTranscription.prepare(locale); throwIfVoiceTranscriptionAborted(signal); - return { - locale: supportedLocale, - transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options), - }; + return supportedLocale; } catch (error) { throwIfVoiceTranscriptionAborted(signal); if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 6b39541d78046..3dc4c25c3286c 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -8,8 +8,17 @@ environment-backed transcription is not implemented. The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts) owns the operation while the client supplies capture and transcription. Preparation binds the transcriber and resolved locale for the whole recording. Draft ownership, -text, and revision are captured before recording and checked before insertion, so -a late transcript cannot overwrite a draft that was edited or replaced. +text, and revision are captured before recording and checked before insertion. A +transcript is dropped only when its draft is gone or belongs to another owner. In a +draft that was edited, the recorded caret means nothing, so the words go at its end. + +Recordings outlive their dictation. The controller hands the file to `keepRecording` +as soon as the recorder stops, before transcription, and cleanup never deletes it. +Whatever goes wrong afterwards (a thrown transcription, an empty result, a dropped +transcript, words the voice filter removed) is then recoverable: `retranscribe` runs +the latest file again, and mobile Settings runs any of the last few without the +voice filter. Mobile keeps several, not one, because the next recording is often the +user trying again, and it must not destroy the one they want back. Cancellation invalidates a result immediately, but resources stay owned until the underlying work settles. Apple's native transcription call cannot be interrupted @@ -28,16 +37,27 @@ the system one and runs a downloaded model through the `features/dictation`, `modules/pseudo-dictation` and `plugins/withDictationPods.cjs`. Upstream files only import from them, at the end of their import block, so upstream merges stay small. -- The controller awaits preparation before it records, so preparation never waits on a model. - The model loads while the user talks, and the system recognizer stands in for a model that is - missing, still compiling, or failed on the recording. +- The controller awaits preparation before it records, so preparation never waits on a model, + Apple's included. The model loads while the user talks, and the system recognizer stands in for + a model that is missing, still compiling, or failed on the recording. A language Apple cannot + take is therefore known only when the recording is transcribed. +- expo-audio pauses the recorder for an audio-session interruption without an event, and resumes + it only if the system later says capture may continue. The metering loop notices that capture + stopped, and the controller transcribes the captured file at once instead of waiting. That file + wins over the recorder's current one, because a media-services reset hands the recorder a new, + empty file. +- At the five-minute limit the recorder stops just before it reports that it finished. A stop in + the last second is therefore treated as the limit, not as an interruption. - FluidAudio is a git pod pinned to a commit and added by the config plugin to the app target only. The widget extension must not link it. - The engine reaches the network only inside an explicit download. - The voice filter never runs on the GPU, because iOS refuses GPU work in the background, where a dictation may finish. - The thresholds in [SpeakerFilter](../../apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift) - come from measurement and lean towards keeping a stranger's words over cutting the user's. + come from measurement and lean towards keeping a stranger's words over cutting the user's. A + voiceprint alone cut the user's own words in long dictations, so a turn is cut only when it is + also clearly quieter than the user. Loudness is a heuristic, not identity: a voice as loud as + the user's stays. `RealVoiceTests` measures them on recordings of real people; rerun it before changing one. - `ios/Core` builds as a Swift package on macOS so `swift test` covers it without a simulator. The Expo module file is outside that package and compiles only in an iOS build. diff --git a/docs/user/composer.md b/docs/user/composer.md index 4b844b3ef3b1b..9ed4b6e0dcbb8 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -201,11 +201,17 @@ transcribing. The first use may download a speech model and needs a network connection. Later transcription works offline for that language. On Android, transcription processes the full recording at roughly its original duration. Recordings can be up to five -minutes long. Canceling or an audio interruption discards the recording and -preserves your existing draft. Navigating to another screen keeps dictation active; +minutes long; at the limit the recording stops and the words up to it are added. Canceling leaves +your draft as it was. When a call, Siri or another app interrupts a recording, the words spoken +before it are added to the draft. If the draft changed while you were dictating, the words are +added at its end. Navigating to another screen keeps dictation active; use the floating controls to finish or cancel. While recording, the screen stays awake; it can sleep normally once recording stops. +Your last 5 recordings stay on the device, canceled ones included. When transcription fails, the +error offers **Transcribe again**, which runs the same recording so you do not have to repeat +yourself. + Choose a preferred order in **Settings → Microphone**. Voice input requests the first connected microphone in the list. The list mixes kinds (built-in, other wired, other Bluetooth, and CarPlay on iOS) with wired and Bluetooth devices you have dictated @@ -229,14 +235,20 @@ Apple's transcription takes a recording made in that time where it is available, recording a model fails on. **Settings → Microphone** shows what the last dictation ran on. On a device without Apple's transcription, the microphone button works once a model is downloaded. +When a dictation came out empty, cut short or wrong, open **Settings → Microphone → Model** and +tap it under **Recent recordings**. It runs through the model in use, without the voice filter, +and the text is shown for you to copy. + ### Only my voice **Settings → Microphone → Only my voice** removes other people's speech from a recording before it is transcribed. Download the voice filter (about 21 MB), then record 3 short samples of your voice. It works with a downloaded model, not with Apple, and adds a short wait after each recording. -When the filter does not recognise your voice in a recording, it removes nothing. People talking -over you cannot be separated from you. Your voice profile stays on the device; record it again or +The filter removes a voice that is not yours and is quieter than you, further from the phone. A +voice as loud as yours stays, so that your own words are never the ones removed. When the filter +does not recognise your voice in a recording, it removes nothing. People talking over you cannot be +separated from you. Your voice profile stays on the device; record it again or delete it on the same page. ### Corrections diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 52069d1bd67b9..5a3446d6b5913 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -48,6 +48,8 @@ function draft(overrides: Partial = {}): VoiceDraftSnapshot }; } +const KEPT_URI = "file:///last-dictation.m4a"; + function createHarness( overrides: Partial = {}, initialDraft = draft(), @@ -56,6 +58,7 @@ function createHarness( let currentDraft: VoiceDraftSnapshot | null = initialDraft; const commits: Array<{ text: string; selection: { start: number; end: number } }> = []; const deleted: string[] = []; + const kept: string[] = []; const dependencies: VoiceInputControllerDependencies = { recorder, getTranscriber: () => ({ prepare: async () => preparedTranscription() }), @@ -63,6 +66,10 @@ function createHarness( configureRecording: async () => undefined, releaseRecording: async () => undefined, deleteRecording: (uri) => deleted.push(uri), + keepRecording: (uri) => { + kept.push(uri); + return KEPT_URI; + }, readDraft: () => currentDraft, commitDraft: (text, selection) => commits.push({ text, selection }), onStateChange: vi.fn(), @@ -73,6 +80,7 @@ function createHarness( recorder, commits, deleted, + kept, setDraft: (next: VoiceDraftSnapshot | null) => { currentDraft = next; }, @@ -95,18 +103,23 @@ describe("resolveTranscriptCommit", () => { }); }); - it("does not replace text after the owner, text, or revision changes", () => { + it("does not add text to another owner's draft or to one that is gone", () => { const captured = draft(); expect( resolveTranscriptCommit(captured, draft({ ownerKey: "other" }), "text", "en-US"), - ).toEqual({ - kind: "stale", - }); - expect(resolveTranscriptCommit(captured, draft({ text: "newer" }), "text", "en-US")).toEqual({ - kind: "stale", - }); + ).toEqual({ kind: "stale" }); + expect(resolveTranscriptCommit(captured, null, "text", "en-US")).toEqual({ kind: "stale" }); + }); + + it("adds the words at the end of a draft that was edited during the recording", () => { + const captured = draft(); + expect( + resolveTranscriptCommit(captured, draft({ text: "newer draft" }), "text", "en-US"), + ).toEqual({ kind: "commit", text: "newer draft text", selection: { start: 16, end: 16 } }); expect(resolveTranscriptCommit(captured, draft({ revision: 2 }), "text", "en-US")).toEqual({ - kind: "stale", + kind: "commit", + text: "hello world text", + selection: { start: 16, end: 16 }, }); }); }); @@ -163,7 +176,7 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState).toMatchObject({ phase: "error", error: expect.any(String), - errorAction: failure === "permission" ? "settings" : "retry", + errorAction: failure === "permission" ? "settings" : "transcribe", }); harness.setDraft(draft({ ownerKey: "environment:other-thread" })); @@ -256,7 +269,84 @@ describe("VoiceInputController", () => { expect(harness.commits).toEqual([ { text: "hello new text", selection: { start: 14, end: 14 } }, ]); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(harness.deleted).toEqual([]); + }); + + it.each< + [ + string, + { + readonly transcribe?: PreparedVoiceTranscription["transcribe"]; + readonly act?: "lose the draft" | "fail to stop" | "cancel"; + readonly commits?: ReadonlyArray; + }, + ] + >([ + ["transcription fails", { transcribe: () => Promise.reject(new Error("model failed")) }], + ["nothing was heard", { transcribe: async () => " " }], + ["the draft is gone", { act: "lose the draft" }], + ["the recorder fails to stop", { act: "fail to stop", commits: ["hello new text"] }], + ["the user cancels the recording", { act: "cancel" }], + ])("keeps the recording when %s", async (_name, scenario) => { + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => preparedTranscription(scenario.transcribe), + }), + }); + await harness.controller.start(); + if (scenario.act === "fail to stop") { + harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released")); + } + if (scenario.act === "lose the draft") harness.setDraft(null); + if (scenario.act === "cancel") harness.controller.cancel(); + await harness.controller.stop(); + // The next recording starts only once this one has let go of the microphone. + const next = createHarness(); + await next.controller.start(); + await next.controller.interruptRecording(); + + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(harness.deleted).toEqual([]); + expect(harness.commits.map((commit) => commit.text)).toEqual(scenario.commits ?? []); + }); + + it.each([ + ["at the caret the recording started at", null, "hello second try"], + [ + "at the end of a draft edited since", + draft({ text: "hello world, and more", revision: 2 }), + "hello world, and more second try", + ], + ])("transcribes the kept recording again and adds it %s", async (_name, edited, expected) => { + const transcribe = vi + .fn() + .mockRejectedValueOnce(new Error("model failed")) + .mockResolvedValue("second try"); + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState).toEqual({ + phase: "error", + error: "Could not transcribe this recording.", + errorAction: "transcribe", + }); + + if (edited) harness.setDraft(edited); + await harness.controller.retranscribe(); + + expect(transcribe).toHaveBeenLastCalledWith(KEPT_URI, expect.anything()); + expect(harness.recorder.record).toHaveBeenCalledOnce(); + expect(harness.commits.map((commit) => commit.text)).toEqual([expected]); + expect(harness.controller.currentState.phase).toBe("idle"); + expect(harness.deleted).toEqual([]); + + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); }); it.each(["cancel", "dispose", "ownerChanged"] as const)( @@ -313,7 +403,8 @@ describe("VoiceInputController", () => { await nextStart; expect(harness.commits).toEqual([]); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(harness.deleted).toEqual([]); expect(harness.controller.currentState.phase).toBe("idle"); expect(next.controller.currentState.phase).toBe("recording"); @@ -362,7 +453,7 @@ describe("VoiceInputController", () => { await stopping; expect(harness.commits).toEqual([]); - expect(harness.controller.currentState.error).toContain("draft changed"); + expect(harness.controller.currentState.error).toContain("draft is gone"); }); it("queues the next recording until canceled preparation settles", async () => { @@ -445,7 +536,7 @@ describe("VoiceInputController", () => { }); await expect(harness.controller.interruptRecording()).resolves.toBeUndefined(); - expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); const next = createHarness(); await next.controller.start(); expect(next.controller.currentState.phase).toBe("recording"); @@ -501,32 +592,85 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState.error).toContain("no longer available"); }); - it("discards recorder errors and audio interruptions without transcribing", async () => { - const transcribe = vi.fn(async () => "ignored"); - const preparationEntered = deferred(); + it.each([ + [ + "a recorder error", + "Voice recording stopped early. The words it captured were added.", + (controller: VoiceInputController) => + controller.handleRecorderStatus({ + isFinished: true, + hasError: true, + error: "Audio route changed", + url: null, + }), + ], + [ + "the background", + "Voice recording stopped early. The words it captured were added.", + (controller: VoiceInputController) => controller.appMovedToBackground(), + ], + [ + "the five-minute limit", + "Voice recording reached the 5 minute limit. The words up to it were added.", + (controller: VoiceInputController) => + controller.handleRecorderStatus({ + isFinished: true, + hasError: false, + error: null, + url: "file:///voice.m4a", + }), + ], + ])( + "adds the words captured before %s stopped the recording", + async (_name, notice, interrupt) => { + const transcribe = vi.fn(async () => "captured words"); + const harness = createHarness({ + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), + }); + await harness.controller.start(); + // A media-services reset points the recorder at a new, empty file. + harness.recorder.uri = "file:///reset-empty.m4a"; + await interrupt(harness.controller); + + expect(harness.kept).toEqual(["file:///voice.m4a"]); + expect(transcribe).toHaveBeenCalledWith(KEPT_URI, expect.anything()); + expect(harness.commits).toEqual([ + { text: "hello captured words", selection: { start: 20, end: 20 } }, + ]); + // No retry is offered: a new recording would go in at the caret this one started at. + expect(harness.controller.currentState).toEqual({ + phase: "error", + error: notice, + errorAction: null, + }); + expect(harness.deleted).toEqual(["file:///reset-empty.m4a"]); + }, + ); + + it.each([ + ["nothing was said", async () => " "], + [ + "the audio cannot be read", + async (): Promise => { + throw new Error("unreadable"); + }, + ], + ])("reports the interruption when %s before it", async (_name, transcribe) => { const harness = createHarness({ - getTranscriber: () => ({ - prepare: async ({ signal }) => { - preparationEntered.resolve(signal); - return preparedTranscription(transcribe); - }, - }), + getTranscriber: () => ({ prepare: async () => preparedTranscription(transcribe) }), }); await harness.controller.start(); - const signal = await preparationEntered.promise; - harness.recorder.uri = "file:///reset-empty.m4a"; + harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released")); await harness.controller.handleRecorderStatus({ isFinished: true, hasError: true, error: "Audio route changed", - url: "file:///voice.m4a", + url: null, }); expect(harness.commits).toEqual([]); - expect(transcribe).not.toHaveBeenCalled(); - expect(signal.aborted).toBe(true); expect(harness.controller.currentState.error).toBe("Audio route changed"); - expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); + expect(harness.kept).toEqual(["file:///voice.m4a"]); }); it("cancels preparation when the app reaches the background", async () => { diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 5b057c61fab23..8d402a7b6c4a6 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -16,7 +16,8 @@ export type VoiceInputPhase = "idle" | "preparing" | "recording" | "transcribing export type VoiceInputState = { readonly phase: VoiceInputPhase; readonly error: string | null; - readonly errorAction: "retry" | "settings" | null; + /** "retry" records again; "transcribe" runs the recording that was kept through the model again. */ + readonly errorAction: "retry" | "transcribe" | "settings" | null; }; export function voiceInputBlocksSubmission(state: VoiceInputState): boolean { @@ -56,6 +57,8 @@ export type VoiceInputControllerDependencies = { readonly configureRecording: () => Promise; readonly releaseRecording: () => Promise; readonly deleteRecording: (uri: string) => void; + /** Moves a finished recording to where recordings are kept, and returns where it is now. */ + readonly keepRecording: (uri: string) => string; readonly readDraft: () => VoiceDraftSnapshot | null; readonly commitDraft: ( text: string, @@ -79,12 +82,7 @@ export function resolveTranscriptCommit( transcript: string, locale: string, ): TranscriptCommitResult { - if ( - !current || - current.ownerKey !== captured.ownerKey || - current.text !== captured.text || - current.revision !== captured.revision - ) { + if (!current || current.ownerKey !== captured.ownerKey) { return { kind: "stale" }; } @@ -93,13 +91,17 @@ export function resolveTranscriptCommit( return { kind: "empty" }; } - const isEmptySelection = captured.selection.start === captured.selection.end; + // The caret recorded at the start means nothing in an edited draft, so the words go at its end. + const edited = current.text !== captured.text || current.revision !== captured.revision; + const text = current.text; + const selection = edited ? { start: text.length, end: text.length } : captured.selection; + const isEmptySelection = selection.start === selection.end; const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); let insertion = replacement; if (isEmptySelection && usesEnglishSpacing) { - const left = captured.text[captured.selection.start - 1]; - const right = captured.text[captured.selection.start]; + const left = text[selection.start - 1]; + const right = text[selection.start]; const leftNeedsBoundary = left !== undefined && /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && @@ -111,12 +113,7 @@ export function resolveTranscriptCommit( insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; } - const result = replaceTextRange( - captured.text, - captured.selection.start, - captured.selection.end, - insertion, - ); + const result = replaceTextRange(text, selection.start, selection.end, insertion); return { kind: "commit", text: result.text, @@ -187,6 +184,17 @@ function transcriptionErrorMessage(error: unknown): string { } const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; +const RECORDING_STOPPED_EARLY_MESSAGE = + "Voice recording stopped early. The words it captured were added."; +const RECORDING_LIMIT_MESSAGE = + "Voice recording reached the 5 minute limit. The words up to it were added."; + +type KeptRecording = { + readonly uri: string; + /** Where the recorder wrote it. The file is no longer there once it was moved. */ + readonly source: string; + readonly draft: VoiceDraftSnapshot | null; +}; export class VoiceInputController { private readonly dependencies: VoiceInputControllerDependencies; @@ -199,6 +207,8 @@ export class VoiceInputController { private recordingUri: string | null = null; private readonly ownedRecordingUris = new Set(); private recordingConfigured = false; + private recordingStarted = false; + private kept: KeptRecording | null = null; private finishing = false; constructor(dependencies: VoiceInputControllerDependencies) { @@ -279,6 +289,7 @@ export class VoiceInputController { } this.capturedDraft = capturedDraft; this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS }); + this.recordingStarted = true; this.setState({ phase: "recording", error: null, errorAction: null }); } catch { if (this.isCurrent(operationToken)) @@ -311,7 +322,7 @@ export class VoiceInputController { this.setState(IDLE_STATE); return; case "recording": - this.discardRecording(null); + void this.discardRecording(); return; case "transcribing": this.invalidateOperation(); @@ -320,14 +331,87 @@ export class VoiceInputController { } } + /** Runs the kept recording through the model again, after a transcription that failed. */ + async retranscribe(): Promise { + const kept = this.kept; + if (this.state.phase !== "error" || this.state.errorAction !== "transcribe" || !kept?.draft) { + return; + } + const capturedDraft = kept.draft; + let sessionToken: VoiceInputSession | null = null; + const operationToken = ++this.operationToken; + this.setState({ phase: "transcribing", error: null, errorAction: null }); + + try { + for ( + let previousSession = activeSession; + previousSession?.abandoned; + previousSession = activeSession + ) { + await previousSession.released; + if (!this.isCurrent(operationToken)) return; + } + sessionToken = acquireSession(); + if (!sessionToken) { + this.setError("Another voice recording is already active.", "transcribe"); + return; + } + this.sessionToken = sessionToken; + const abortController = new AbortController(); + this.transcriptionAbortController = abortController; + const { signal } = abortController; + + const transcriber = this.dependencies.getTranscriber(); + if (!transcriber) { + this.setError("Voice transcription is not available.", "transcribe"); + return; + } + + let transcript: string; + let locale: string; + try { + const transcription = await runTranscriptionOperation(() => + transcriber.prepare({ signal }), + ); + if (!this.isCurrent(operationToken)) return; + transcript = await runTranscriptionOperation(() => + transcription.transcribe(kept.uri, { signal }), + ); + locale = transcription.locale; + } catch (error) { + if (this.isCurrent(operationToken)) { + this.setError(transcriptionErrorMessage(error), "transcribe"); + } + return; + } + if (!this.isCurrent(operationToken)) return; + this.commitTranscript(capturedDraft, transcript, locale, null, null); + } catch { + if (this.isCurrent(operationToken)) { + this.setError("Could not finish voice recording.", "transcribe"); + } + } finally { + if (sessionToken && this.sessionToken === sessionToken) await this.releaseResources(); + } + } + interruptRecording( message = "Voice recording was interrupted.", completedUri: string | null = null, ): Promise | void { if (this.state.phase !== "recording") return; - this.rememberRecordingUri(completedUri); - this.recordingUri = completedUri ?? this.recordingUri; - return this.discardRecording(message); + return this.finishRecording( + completedUri !== null, + completedUri, + RECORDING_STOPPED_EARLY_MESSAGE, + message, + ); + } + + /** The recorder ends a recording by itself, without an error, only at the limit. */ + recordingReachedLimit(completedUri: string | null = null): Promise | void { + if (this.state.phase !== "recording") return; + return this.finishRecording(completedUri !== null, completedUri, RECORDING_LIMIT_MESSAGE); } appMovedToBackground(): Promise | void { @@ -351,7 +435,7 @@ export class VoiceInputController { if (!status.url) { return this.interruptRecording(); } - return this.finishRecording(true, status.url); + return this.recordingReachedLimit(status.url); } } @@ -362,7 +446,7 @@ export class VoiceInputController { dispose(): void { if (this.state.phase === "recording") { - this.discardRecording(null); + void this.discardRecording(); return; } if (this.state.phase === "preparing" || this.state.phase === "transcribing") { @@ -371,23 +455,39 @@ export class VoiceInputController { } } + /** + * `notice` is set when the recording ended without the user stopping it, and is shown once its + * words are added. `failure` is shown instead when there were no words to add. + */ private async finishRecording( alreadyStopped: boolean, completedUri: string | null, + notice: string | null = null, + failure: string | null = null, ): Promise { if (this.finishing || this.state.phase !== "recording") return; this.finishing = true; const operationToken = this.operationToken; + let keptRecording = false; this.setState({ phase: "transcribing", error: null, errorAction: null }); try { - if (!alreadyStopped) await this.dependencies.recorder.stop(); + if (!alreadyStopped) { + try { + await this.dependencies.recorder.stop(); + } catch { + // A recorder that fails to stop, or is already gone, has still written its file. + } + } await this.releaseAudioSession(); - this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri; - this.rememberRecordingUri(this.recordingUri); + // The file capture started in wins: a media-services reset gives the recorder a new, empty one. + const recordingUri = this.keep( + completedUri ?? this.recordingUri ?? this.dependencies.recorder.uri, + ); + keptRecording = recordingUri !== null; if (!this.isCurrent(operationToken)) return; if ( - !this.recordingUri || + !recordingUri || !this.transcription || !this.transcriptionAbortController || !this.capturedDraft @@ -396,7 +496,6 @@ export class VoiceInputController { return; } - const recordingUri = this.recordingUri; const transcription = this.transcription; const signal = this.transcriptionAbortController.signal; const capturedDraft = this.capturedDraft; @@ -407,35 +506,15 @@ export class VoiceInputController { ); } catch (error) { if (this.isCurrent(operationToken)) { - this.setError(transcriptionErrorMessage(error), "retry"); + this.setError(failure ?? transcriptionErrorMessage(error), "transcribe"); } return; } if (!this.isCurrent(operationToken)) return; - - const result = resolveTranscriptCommit( - capturedDraft, - this.dependencies.readDraft(), - transcript, - transcription.locale, - ); - if (result.kind === "stale") { - this.setError( - "The draft changed while voice input was running. The transcript was not added.", - "retry", - ); - return; - } - if (result.kind === "empty") { - this.setError("No speech was detected.", "retry"); - return; - } - - this.dependencies.commitDraft(result.text, result.selection); - this.setState(IDLE_STATE); + this.commitTranscript(capturedDraft, transcript, transcription.locale, notice, failure); } catch { if (this.isCurrent(operationToken)) { - this.setError("Could not finish voice recording.", "retry"); + this.setError("Could not finish voice recording.", keptRecording ? "transcribe" : "retry"); } } finally { this.finishing = false; @@ -443,16 +522,57 @@ export class VoiceInputController { } } - private async discardRecording(error: string | null): Promise { - this.invalidateOperation(); - this.setState( - error - ? { phase: "error", error, errorAction: "retry" } - : { phase: "idle", error: null, errorAction: null }, + private commitTranscript( + capturedDraft: VoiceDraftSnapshot, + transcript: string, + locale: string, + notice: string | null, + failure: string | null, + ): void { + const result = resolveTranscriptCommit( + capturedDraft, + this.dependencies.readDraft(), + transcript, + locale, ); + if (result.kind === "stale") { + this.setError( + "The draft is gone, so the words were not added. The recording is kept.", + "retry", + ); + return; + } + if (result.kind === "empty") { + this.setError(failure ?? "No speech was detected.", "retry"); + return; + } + + this.dependencies.commitDraft(result.text, result.selection); + // No retry with a notice: a new recording would go in at the caret this one started at. + if (notice === null) this.setState(IDLE_STATE); + else this.setError(notice, null); + } + + private async discardRecording(): Promise { + this.invalidateOperation(); + this.setState(IDLE_STATE); await this.releaseResources(); } + /** A recording outlives its dictation, whatever became of it, so its words can be recovered. */ + private keep(uri: string | null): string | null { + this.recordingStarted = false; + if (!uri) return null; + let keptUri = uri; + try { + keptUri = this.dependencies.keepRecording(uri); + } catch { + // It stays where the recorder wrote it. + } + this.kept = { uri: keptUri, source: uri, draft: this.capturedDraft }; + return keptUri; + } + private async releaseResources(): Promise { this.rememberRecordingUri(this.recordingUri); try { @@ -463,8 +583,11 @@ export class VoiceInputController { await this.dependencies.recorder.stop(); } catch {} } + // A recording that was canceled, or abandoned before it finished, is kept as well. + if (this.recordingStarted) this.keep(this.recordingUri); this.recordingUri = null; for (const uri of this.ownedRecordingUris) { + if (uri === this.kept?.uri || uri === this.kept?.source) continue; try { this.dependencies.deleteRecording(uri); } catch {}