fix(mobile): dictated speech is no longer lost on iOS

apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift

@@ -193,8 +193,12 @@ final class RealModelTests: XCTestCase {
   func testOnlyMyVoiceDropsTheOtherSpeaker() async throws {
     try await download(.fast)
     _ = try await enroll(voice: Self.me)
-    let mine = try DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine"))
+    // The user does most of the talking, and the other voice is further from the phone.
+    let mine =
+      try DictationEngine.decode(speak(Self.enrollment[1], voice: Self.me, name: "opening"))
+      + silence(0.6) + DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine"))
     let theirs = try DictationEngine.decode(speak(Self.theirs, voice: Self.other, name: "theirs"))
+      .map { $0 * 0.25 }
     let url = try write(mine + silence(0.6) + theirs, name: "both")
 
     let unfiltered = try await engine.transcribe(

apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift

@@ -85,16 +85,31 @@ final class RealVoiceTests: XCTestCase {
     Array(samples.prefix(Int(seconds * DictationEngine.sampleRate)))
   }
 
+  /// The voice 12 dB under `own`: about four times further from the phone than the person holding
+  /// it. The corpus records each reader at a different level, so the gap is set, not assumed.
+  private static func distant(_ samples: [Float], from own: [Float]) -> [Float] {
+    func power(_ clip: [Float]) -> Float { clip.reduce(Float(0)) { $0 + $1 * $1 } / Float(clip.count) }
+    let gain = 0.25 * (power(own) / power(samples)).squareRoot()
+    return samples.map { $0 * gain }
+  }
+
   private func write(_ samples: [Float]) throws -> URL {
     written += 1
     let url = clips.appendingPathComponent("clip-\(written).wav")
     try writeWav(samples, to: url)
     return url
   }
 
-  private func words(_ samples: [Float], onlyMyVoice: Bool) async throws -> (Set<String>, FilterOutcome?) {
+  private func words(
+    _ samples: [Float], onlyMyVoice: Bool, model: DictationModel = .fast, asPhoneRecording: Bool = false
+  ) async throws -> (Set<String>, FilterOutcome?) {
+    var url = try write(samples)
+    if asPhoneRecording {
+      url = url.deletingPathExtension().appendingPathExtension("m4a")
+      try writePhoneRecording(samples, to: url)
+    }
     let transcript = try await engine.transcribe(
-      requestId: UUID().uuidString, url: try write(samples), model: .fast, processing: .auto,
+      requestId: UUID().uuidString, url: url, model: model, processing: .auto,
       onlyMyVoice: onlyMyVoice, language: "en-US")
     let words = transcript.text.lowercased().split(whereSeparator: { !$0.isLetter }).map(String.init)
     return (Set(words), transcript.filter)
@@ -139,6 +154,133 @@ final class RealVoiceTests: XCTestCase {
     XCTAssertLessThanOrEqual(Double(mixAccepted.count) / Double(mixes), 0.05)
   }
 
+  // MARK: Long recordings
+
+  /// People dictate for longer than any other clip here, and the phone hands over AAC, not WAV.
+  func testAHalfMinuteOfSpeechFromThePhoneIsTranscribedInFull() async throws {
+    try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in })
+    var lost: [String] = []
+    for model in [DictationModel.fast, .balanced, .accurate] {
+      try await engine.download(model, processing: .auto, onProgress: { _ in })
+      for speaker in speakers.prefix(4) {
+        try await engine.saveVoiceProfile(try speaker.enrollment.map(write))
+        var recording: [Float] = []
+        var expected: Set<String> = []
+        for utterance in speaker.held where seconds(recording) < 30 {
+          recording += utterance
+          expected.formUnion(try await words(utterance, onlyMyVoice: false, model: model).0)
+        }
+        XCTAssertGreaterThanOrEqual(seconds(recording), 30)
+        for onlyMyVoice in [false, true] {
+          let (heard, outcome) = try await words(
+            recording, onlyMyVoice: onlyMyVoice, model: model, asPhoneRecording: true)
+          let missing = expected.subtracting(heard).count
+          print(
+            "MEASURE long, \(model.rawValue) \(speaker.name) \(Int(seconds(recording))) s, filter "
+              + "\(onlyMyVoice): lost \(percent(missing, expected.count)), "
+              + "cut \(outcome?.droppedSeconds ?? 0) s")
+          if Double(missing) / Double(expected.count) > 0.05 {
+            lost.append("\(model.rawValue) \(speaker.name) filter \(onlyMyVoice): \(percent(missing, expected.count))")
+          }
+        }
+      }
+    }
+    XCTAssertEqual(lost, [])
+  }
+
+  // MARK: Sound unlike enrollment
+
+  /// White noise under the speech, `snr` decibels below it. The seed is fixed so runs agree.
+  private static func noisy(_ samples: [Float], snr: Float) -> [Float] {
+    let power = samples.reduce(Float(0)) { $0 + $1 * $1 } / Float(samples.count)
+    let peak = (3 * power).squareRoot() / pow(10, snr / 20)
+    var state: UInt64 = 0x9E37_79B9_7F4A_7C15
+    return samples.map { sample in
+      state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407
+      return sample + peak * (Float(state >> 40) / Float(1 << 24) * 2 - 1)
+    }
+  }
+
+  /// Roughly a telephone headset: the top and the bottom of the voice are gone.
+  private static func narrowband(_ samples: [Float]) -> [Float] {
+    var output = samples
+    var low: Float = 0
+    var lower: Float = 0
+    var rumble: Float = 0
+    for index in samples.indices {
+      low += 0.72 * (samples[index] - low)
+      lower += 0.72 * (low - lower)
+      rumble = 0.89 * rumble + 0.11 * lower
+      output[index] = index % 2 == 0 ? lower - rumble : output[index - 1]
+    }
+    return output
+  }
+
+  /// A small room with hard walls.
+  private static func echoing(_ samples: [Float]) -> [Float] {
+    var output = samples
+    for (delay, gain) in [(464, Float(0.55)), (592, 0.5), (688, 0.45), (1104, 0.4)] {
+      var echo = [Float](repeating: 0, count: samples.count)
+      for index in samples.indices.dropFirst(delay) {
+        echo[index] = gain * (samples[index - delay] + echo[index - delay])
+        output[index] += 0.35 * echo[index]
+      }
+    }
+    return output
+  }
+
+  /// The profile is recorded once, somewhere quiet. People then dictate in a car, in a kitchen or
+  /// over a headset, where their voice sits further from it, and that must not cost them words.
+  func testTheFilterKeepsTheSpeakersWordsWhereTheSoundIsUnlikeEnrollment() async throws {
+    try await engine.download(.fast, processing: .auto, onProgress: { _ in })
+    try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in })
+    let conditions: [(name: String, change: ([Float]) -> [Float])] = [
+      ("noise at 5 dB", { Self.noisy($0, snr: 5) }),
+      ("narrowband", Self.narrowband),
+      ("narrowband with noise at 5 dB", { Self.noisy(Self.narrowband($0), snr: 5) }),
+      ("echo with noise at 10 dB", { Self.noisy(Self.echoing($0), snr: 10) }),
+    ]
+    var tallies = conditions.map { _ in (own: 0, ownLost: 0, stranger: 0, strangerKept: 0) }
+
+    for (index, speaker) in speakers.prefix(8).enumerated() {
+      try await engine.saveVoiceProfile(try speaker.enrollment.map(write))
+      let stranger = speakers[(index + speakers.count / 2) % speakers.count]
+      var alone: [Float] = []
+      for utterance in speaker.held where seconds(alone) < 30 { alone += utterance }
+
+      for (position, condition) in conditions.enumerated() {
+        let recording = condition.change(alone)
+        let spoken = try await words(recording, onlyMyVoice: false).0
+        let (kept, outcome) = try await words(recording, onlyMyVoice: true)
+        tallies[position].own += spoken.count
+        tallies[position].ownLost += spoken.subtracting(kept).count
+        print(
+          "MEASURE unlike enrollment, \(condition.name), \(speaker.name) alone for "
+            + "\(Int(seconds(alone))) s: lost \(percent(spoken.subtracting(kept).count, spoken.count)), "
+            + "cut \(outcome?.droppedSeconds ?? 0) s")
+
+        // The other voice is across the room, under the same noise or through the same headset.
+        let own = Self.cut(speaker.held[0], 8)
+        let theirs = Self.distant(Self.cut(stranger.held[0], 4), from: own)
+        let unwanted = try await words(condition.change(own + theirs), onlyMyVoice: false).0
+          .subtracting(try await words(condition.change(own), onlyMyVoice: false).0)
+        let heard = try await words(condition.change(own + theirs), onlyMyVoice: true).0
+        tallies[position].stranger += unwanted.count
+        tallies[position].strangerKept += unwanted.intersection(heard).count
+      }
+    }
+
+    for (condition, tally) in zip(conditions, tallies) {
+      print(
+        "MEASURE unlike enrollment, \(condition.name): speaker's words lost "
+          + "\(percent(tally.ownLost, tally.own)), stranger's words kept "
+          + "\(percent(tally.strangerKept, tally.stranger))")
+      XCTAssertLessThanOrEqual(Double(tally.ownLost) / Double(tally.own), 0.05, condition.name)
+      XCTAssertLessThanOrEqual(
+        Double(tally.strangerKept) / Double(tally.stranger), 0.15, condition.name)
+    }
+  }
+
   // MARK: Voice filter
 
   func testTheFilterKeepsTheSpeakersWordsAndDropsTheStrangers() async throws {
@@ -175,10 +317,11 @@ final class RealVoiceTests: XCTestCase {
         let (_, soloOutcome) = try await words(Self.cut(stranger.held[2], 6), onlyMyVoice: true)
         if soloOutcome?.applied == true { strangerSoloCut.append("\(stranger.name) as \(speaker.name)") }
 
-        let own = Self.cut(speaker.held[0], 5)
+        // A dictation is mostly its owner, and anyone else is further from the phone.
+        let own = Self.cut(speaker.held[0], 8)
         let ownSecond = Self.cut(speaker.held[1], 4)
-        let theirs = Self.cut(stranger.held[0], 5)
-        let theirsShort = Self.cut(stranger.held[1], 2)
+        let theirs = Self.distant(Self.cut(stranger.held[0], 4), from: own)
+        let theirsShort = Self.distant(Self.cut(stranger.held[1], 2), from: own)
         let cases: [(name: String, mine: [[Float]], theirs: [Float], samples: [Float])] = [
           ("mine then theirs", [own], theirs, own + theirs),
           ("theirs then mine", [own], theirs, theirs + own),

apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift

@@ -7,6 +7,14 @@ final class SpeakerFilterTests: XCTestCase {
     SpeakerTurn(speaker: speaker, start: start, end: end)
   }
 
+  /// Unless a test says otherwise the user speaks at -20 dB and everyone else 15 dB under them.
+  private func decide(
+    _ turns: [SpeakerTurn], _ voices: [TurnVoice], levels: [Float?]? = nil
+  ) -> SpeakerDecision {
+    SpeakerFilter.decide(
+      turns: turns, voices: voices, levels: levels ?? voices.map { $0 == .mine ? -20 : -35 })
+  }
+
   func testAShortTurnMaySitFurtherFromTheProfile() {
     XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.5), .mine)
     XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.7), .other)
@@ -15,36 +23,96 @@ final class SpeakerFilterTests: XCTestCase {
     XCTAssertEqual(SpeakerFilter.judge(duration: 0.9, distance: 0.1), .unjudged)
   }
 
+  /// A unit vector at `distance` from the profile `[1, 0, 0]`, leaning towards the second or third axis.
+  private func voiceprint(_ distance: Float, stranger: Bool = false) -> [Float] {
+    let along = 1 - distance
+    let across = (1 - along * along).squareRoot()
+    return stranger ? [along, 0, across] : [along, across, 0]
+  }
+
+  func testATurnTheProfileRejectsIsTheUsersWhenItSoundsLikeTheirOtherTurns() {
+    let profile: [Float] = [1, 0, 0]
+    // Noise or another microphone moves every turn away from the profile. The opening words pass
+    // as a short turn; the half minute after them is the same voice and must not be cut.
+    let turns = [turn("a", 0, 1.5), turn("a", 2, 30), turn("b", 30, 35), turn("b", 35, 35.5)]
+    let voices = SpeakerFilter.voices(
+      turns: turns,
+      embeddings: [voiceprint(0.7), voiceprint(0.65), voiceprint(0.65, stranger: true), nil],
+      profile: profile)
+    XCTAssertEqual(voices, [.mine, .mine, .other, .unjudged])
+    XCTAssertEqual(decide(turns, voices), .drop([30...35.5]))
+
+    XCTAssertEqual(
+      SpeakerFilter.voices(
+        turns: [turn("a", 0, 5), turn("b", 5, 10)],
+        embeddings: [voiceprint(0.9), voiceprint(0.9)], profile: profile),
+      [.other, .other])
+  }
+
   func testNothingIsCutUnlessTheUserAndSomeoneElseWereBothHeard() {
     let two = [turn("a", 0, 3), turn("b", 3, 6)]
-    XCTAssertEqual(SpeakerFilter.decide(turns: [], voices: []), .unfiltered(.noSpeech))
+    XCTAssertEqual(decide([], []), .unfiltered(.noSpeech))
+    XCTAssertEqual(decide(two, [.unjudged, .unjudged]), .unfiltered(.tooShort))
+    XCTAssertEqual(decide(two, [.other, .other]), .unfiltered(.noMatch))
+    XCTAssertEqual(decide(two, [.other, .unjudged]), .unfiltered(.noMatch))
+    XCTAssertEqual(decide(two, [.mine, .mine]), .unfiltered(.onlyYou))
+    XCTAssertEqual(decide(two, [.mine, .unjudged]), .unfiltered(.onlyYou))
+  }
+
+  func testNothingIsCutWhenTheUserWasHeardForLessThanTheSpeechToCut() {
+    // The profile took three words of a half-minute dictation for the user and the rest for
+    // someone else. That is a missed match on their own voice, not a stranger talking for 28 s.
     XCTAssertEqual(
-      SpeakerFilter.decide(turns: two, voices: [.unjudged, .unjudged]), .unfiltered(.tooShort))
-    XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.other, .other]), .unfiltered(.noMatch))
+      decide([turn("a", 0, 1.5), turn("a", 2, 30)], [.mine, .other]), .unfiltered(.mostlyOther))
     XCTAssertEqual(
-      SpeakerFilter.decide(turns: two, voices: [.other, .unjudged]), .unfiltered(.noMatch))
-    XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.mine, .mine]), .unfiltered(.onlyYou))
+      decide([turn("a", 0, 5), turn("b", 5, 9.2)], [.mine, .other]), .unfiltered(.mostlyOther))
+    XCTAssertEqual(decide([turn("a", 0, 5), turn("b", 5, 9)], [.mine, .other]), .drop([5...9]))
+  }
+
+  func testAVoiceIsCutOnlyWhenItIsClearlyQuieterThanTheUsers() {
+    let turns = [turn("a", 0, 10), turn("a", 10, 14), turn("b", 14, 17)]
+    let voices: [TurnVoice] = [.mine, .other, .other]
+    // A stretch as loud as the user is as close to the phone as they are, so it may be them.
+    XCTAssertEqual(decide(turns, voices, levels: [-20, -22, -26]), .unfiltered(.similarLevel))
+    XCTAssertEqual(decide(turns, voices, levels: [-20, -22, -27]), .drop([14...17]))
+    XCTAssertEqual(decide(turns, voices, levels: [-20, -27, -27]), .drop([10...17]))
+    XCTAssertEqual(decide(turns, voices, levels: [-20, nil, nil]), .unfiltered(.similarLevel))
+    XCTAssertEqual(decide(turns, voices, levels: [nil, -40, -40]), .unfiltered(.similarLevel))
+  }
+
+  func testLevelIsHowLoudAStretchGetsNotItsMean() throws {
+    let rate = 1000.0
+    let loud = [Float](repeating: 0.1, count: 20)
+    let pause = [Float](repeating: 0.001, count: 20)
+    XCTAssertNil(SpeakerFilter.level([], sampleRate: rate))
+    XCTAssertNil(SpeakerFilter.level(loud[..<19], sampleRate: rate))
+    XCTAssertNil(SpeakerFilter.level([Float](repeating: 0, count: 40)[...], sampleRate: rate))
+    // Two frames in ten are speech: the pauses between them do not pull the level down.
+    let halting = (0..<10).flatMap { $0 % 5 == 0 ? loud : pause }
+    XCTAssertEqual(try XCTUnwrap(SpeakerFilter.level(halting[...], sampleRate: rate)), -20, accuracy: 0.001)
+    // One loud frame in twenty is a click, not the voice.
+    let click = (0..<20).flatMap { $0 == 7 ? loud : pause }
+    XCTAssertEqual(try XCTUnwrap(SpeakerFilter.level(click[...], sampleRate: rate)), -60, accuracy: 0.001)
     XCTAssertEqual(
-      SpeakerFilter.decide(turns: two, voices: [.mine, .unjudged]), .unfiltered(.onlyYou))
+      try XCTUnwrap(SpeakerFilter.level((pause + loud + pause)[20..<40], sampleRate: rate)), -20,
+      accuracy: 0.001)
   }
 
   func testOnlyTheOtherVoiceIsCutWhateverTheDiarizerCalledIt() {
     let turns = [turn("a", 0, 3), turn("a", 3.5, 7), turn("a", 7.2, 9)]
-    XCTAssertEqual(
-      SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .mine]), .drop([3.5...7]))
+    XCTAssertEqual(decide(turns, [.mine, .other, .mine]), .drop([3.5...7]))
   }
 
   func testAShortTurnIsCutOnlyWhenItsSpeakerWasAlwaysSomeoneElse() {
     let turns = [
-      turn("a", 0, 3), turn("b", 3, 6), turn("b", 6.2, 6.8), turn("a", 7, 7.5), turn("c", 8, 8.6),
+      turn("a", 0, 5), turn("b", 5, 8), turn("b", 8.2, 8.8), turn("a", 9, 9.5), turn("c", 10, 10.6),
     ]
-    XCTAssertEqual(
-      SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .unjudged, .unjudged, .unjudged]),
-      .drop([3...6, 6.2...6.8]))
+    let voices: [TurnVoice] = [.mine, .other, .unjudged, .unjudged, .unjudged]
+    XCTAssertEqual(decide(turns, voices), .drop([5...8, 8.2...8.8]))
+    XCTAssertEqual(decide(turns, voices, levels: [-20, -35, -22, -35, -35]), .drop([5...8]))
 
-    let merged = [turn("a", 0, 3), turn("a", 3, 6), turn("a", 6.2, 6.8)]
-    XCTAssertEqual(
-      SpeakerFilter.decide(turns: merged, voices: [.mine, .other, .unjudged]), .drop([3...6]))
+    let merged = [turn("a", 0, 4), turn("a", 4, 7), turn("a", 7.2, 7.8)]
+    XCTAssertEqual(decide(merged, [.mine, .other, .unjudged]), .drop([4...7]))
   }
 
   private func assertClose(
@@ -101,24 +169,17 @@ final class SpeakerFilterTests: XCTestCase {
   }
 
   func testSpeechOverTheUsersOwnTurnIsNotCut() {
+    XCTAssertEqual(decide([turn("a", 0, 4), turn("b", 3, 6)], [.mine, .other]), .drop([4...6]))
+    XCTAssertEqual(decide([turn("a", 0, 3), turn("b", 2, 8)], [.other, .mine]), .drop([0...2]))
     XCTAssertEqual(
-      SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3, 6)], voices: [.mine, .other]),
-      .drop([4...6]))
-    XCTAssertEqual(
-      SpeakerFilter.decide(turns: [turn("a", 0, 6), turn("b", 2, 3)], voices: [.other, .mine]),
-      .drop([0...2, 3...6]))
+      decide([turn("a", 0, 4), turn("b", 1, 3)], [.mine, .other]), .unfiltered(.overlapping))
     XCTAssertEqual(
-      SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 1, 3)], voices: [.mine, .other]),
-      .unfiltered(.overlapping))
-    XCTAssertEqual(
-      SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3.9, 4.1)], voices: [.mine, .other]),
-      .unfiltered(.overlapping))
+      decide([turn("a", 0, 4), turn("b", 3.9, 4.1)], [.mine, .other]), .unfiltered(.overlapping))
   }
 
   func testTouchingTurnsBecomeOneCut() {
-    let turns = [turn("a", 0, 2), turn("b", 2, 4), turn("c", 4, 6), turn("b", 3.5, 4.5)]
-    XCTAssertEqual(
-      SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .other, .other]), .drop([2...6]))
+    let turns = [turn("a", 0, 6), turn("b", 6, 8), turn("c", 8, 10), turn("b", 7.5, 8.5)]
+    XCTAssertEqual(decide(turns, [.mine, .other, .other, .other]), .drop([6...10]))
   }
 
   func testCosineDistance() {

apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift

@@ -22,3 +22,27 @@ func writeWav(_ samples: [Float], to url: URL) throws {
     ])
   try file.write(from: buffer)
 }
+
+/// What the app's recorder writes: AAC at 44.1 kHz in stereo.
+func writePhoneRecording(_ samples: [Float], to url: URL) throws {
+  let rate = 44_100.0
+  let file = try AVAudioFile(
+    forWriting: url,
+    settings: [
+      AVFormatIDKey: kAudioFormatMPEG4AAC, AVSampleRateKey: rate, AVNumberOfChannelsKey: 2,
+      AVEncoderBitRateKey: 128_000,
+    ])
+  let count = Int(Double(samples.count) * rate / DictationEngine.sampleRate)
+  let buffer = try XCTUnwrap(
+    AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(count)))
+  buffer.frameLength = AVAudioFrameCount(count)
+  for frame in 0..<count {
+    let position = Double(frame) * DictationEngine.sampleRate / rate
+    let index = min(Int(position), samples.count - 1)
+    let next = min(index + 1, samples.count - 1)
+    let value = samples[index] + (samples[next] - samples[index]) * Float(position - Double(index))
+    buffer.floatChannelData![0][frame] = value
+    buffer.floatChannelData![1][frame] = value
+  }
+  try file.write(from: buffer)
+}