diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index bae349e57d1f4..05f15c179679a 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -490,6 +490,7 @@ const config: ExpoConfig = { }, ], "./plugins/withIosCocoaPodsUuidCache.cjs", + "./plugins/withDictationPods.cjs", // Compile out the unused pedometer to avoid an iOS motion purpose string. ["expo-sensors", { motionPermission: false }], // Mods run in reverse order; expo-widgets must create and wipe the target before assets are added. diff --git a/apps/mobile/modules/pseudo-dictation/.gitignore b/apps/mobile/modules/pseudo-dictation/.gitignore new file mode 100644 index 0000000000000..2d9f16e2d2761 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/.gitignore @@ -0,0 +1,2 @@ +.build/ +.swiftpm/ diff --git a/apps/mobile/modules/pseudo-dictation/Package.resolved b/apps/mobile/modules/pseudo-dictation/Package.resolved new file mode 100644 index 0000000000000..ce5718c2a8280 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Package.resolved @@ -0,0 +1,14 @@ +{ + "originHash" : "7e42f7e16e5e2a1f9c0c8684754ec366d7be77a442743064faa607cb53a64078", + "pins" : [ + { + "identity" : "fluidaudio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/FluidInference/FluidAudio.git", + "state" : { + "revision" : "c8b41f98eae668794dd0083bbe028fe7e6585051" + } + } + ], + "version" : 3 +} diff --git a/apps/mobile/modules/pseudo-dictation/Package.swift b/apps/mobile/modules/pseudo-dictation/Package.swift new file mode 100644 index 0000000000000..16333ed2c7dec --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Package.swift @@ -0,0 +1,29 @@ +// swift-tools-version: 6.0 +import PackageDescription + +// The app builds this module through CocoaPods. This manifest exists so the core can be compiled +// and tested on a Mac without an iOS build. Keep the FluidAudio revision equal to the one in +// `apps/mobile/plugins/withDictationPods.cjs`. +let package = Package( + name: "PseudoDictation", + platforms: [.macOS(.v14), .iOS(.v18)], + dependencies: [ + .package( + url: "https://github.com/FluidInference/FluidAudio.git", + revision: "c8b41f98eae668794dd0083bbe028fe7e6585051") + ], + targets: [ + .target( + name: "PseudoDictationCore", + dependencies: [.product(name: "FluidAudio", package: "FluidAudio")], + path: "ios/Core", + swiftSettings: [.swiftLanguageMode(.v5)] + ), + .testTarget( + name: "PseudoDictationCoreTests", + dependencies: ["PseudoDictationCore"], + path: "Tests/PseudoDictationCoreTests", + swiftSettings: [.swiftLanguageMode(.v5)] + ), + ] +) diff --git a/apps/mobile/modules/pseudo-dictation/THIRD_PARTY_NOTICES.md b/apps/mobile/modules/pseudo-dictation/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000000000..d5670f785ee47 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/THIRD_PARTY_NOTICES.md @@ -0,0 +1,32 @@ +# Third-party notices + +On-device dictation in PseudoCode is built on the work below. + +## Shipped in the app + +**FluidAudio** by Fluid Inference, Apache License 2.0. +https://github.com/FluidInference/FluidAudio (commit `c8b41f98eae668794dd0083bbe028fe7e6585051`, v0.18.3). +Its licence text and the notices for the code it bundles are in that repository's `LICENSE` and +`ThirdPartyLicenses/`. + +**`ios/Resources/dictation-sample.wav`** is FluidAudio's test fixture +`Tests/FluidAudioTests/ASR/Parakeet/SlidingWindow/Fixtures/03-diff-explanation-16.9s.wav`, unchanged, +under the same Apache License 2.0. + +## Downloaded on the device when the user asks for them + +These models are not part of the app. They are fetched from Hugging Face. All are Core ML +conversions by Fluid Inference, modified from the originals named here. + +| Model in the app | Repository | Original | Licence | +| ---------------- | --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | --------- | +| Fast | `FluidInference/parakeet-tdt-ctc-110m-coreml` | NVIDIA `parakeet-tdt_ctc-110m` | CC BY 4.0 | +| Balanced | `FluidInference/parakeet-tdt-0.6b-v3-coreml` | NVIDIA `parakeet-tdt-0.6b-v3` | CC BY 4.0 | +| Most accurate | `FluidInference/parakeet-ultra-coreml` | moondream `parakeet-ultra`, trained from NVIDIA Parakeet v3 | CC BY 4.0 | +| Voice filter | `FluidInference/speaker-diarization-coreml` (`Segmentation`, `FBank`, `Embedding`, `PldaRho`) | `pyannote/speaker-diarization-community-1` by pyannote, with WeSpeaker embeddings and PLDA parameters from BUT Speech@FIT | CC BY 4.0 | + +CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ + +The voice filter repository holds files under more than one licence. Its `NOTICE.md` says which +licence covers which file; the 4 files above are in its CC BY 4.0 set: +https://huggingface.co/FluidInference/speaker-diarization-coreml/blob/main/NOTICE.md diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/DictationEngineTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/DictationEngineTests.swift new file mode 100644 index 0000000000000..1e0ab6e27ac69 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/DictationEngineTests.swift @@ -0,0 +1,174 @@ +import FluidAudio +import XCTest + +@testable import PseudoDictationCore + +private final class Recorder: @unchecked Sendable { + private let lock = NSLock() + private var stored: [DownloadUpdate] = [] + + var updates: [DownloadUpdate] { + lock.lock() + defer { lock.unlock() } + return stored + } + + func add(_ update: DownloadUpdate) { + lock.lock() + stored.append(update) + lock.unlock() + } +} + +private func environment(freeDisk: UInt64? = nil, availableMemory: UInt64 = 0) -> DictationEnvironment { + DictationEnvironment( + appBuild: "1.0(1)", osBuild: "test", isSimulator: true, + availableMemory: { availableMemory }, freeDisk: { _ in freeDisk }) +} + +final class ProgressRelayTests: XCTestCase { + func testLibraryProgressMapsDownloadToTheWholeBar() { + XCTAssertEqual( + DownloadUpdate(DownloadProgress(fractionCompleted: 0, phase: .listing)), + DownloadUpdate(phase: .downloading, fraction: 0)) + XCTAssertEqual( + DownloadUpdate( + DownloadProgress( + fractionCompleted: 0.25, phase: .downloading(completedFiles: 1, totalFiles: 2))), + DownloadUpdate(phase: .downloading, fraction: 0.5)) + XCTAssertEqual( + DownloadUpdate( + DownloadProgress( + fractionCompleted: 0.5, phase: .downloading(completedFiles: 0, totalFiles: 0))), + DownloadUpdate(phase: .downloading, fraction: 1)) + XCTAssertEqual( + DownloadUpdate( + DownloadProgress(fractionCompleted: 0.75, phase: .compiling(modelName: "Encoder"))), + DownloadUpdate(phase: .preparing, fraction: 1)) + } + + func testProgressNeverMovesBackwardsAndSkipsSmallSteps() { + let recorder = Recorder() + let relay = ProgressRelay(emit: { recorder.add($0) }) + for fraction in [0, 0.004, 0.2, 0.1, 0.204, 0.21, 0.996, 1] { + relay.report(DownloadUpdate(phase: .downloading, fraction: fraction)) + } + XCTAssertEqual(recorder.updates.map(\.fraction), [0, 0.2, 0.21, 0.996]) + } + + func testACachedLaterComponentDoesNotReopenTheDownloadPhase() { + let recorder = Recorder() + let relay = ProgressRelay(emit: { recorder.add($0) }) + relay.report(DownloadUpdate(phase: .downloading, fraction: 0.9)) + relay.report(DownloadUpdate(phase: .preparing, fraction: 1)) + relay.report(DownloadUpdate(phase: .downloading, fraction: 1)) + relay.report(DownloadUpdate(phase: .preparing, fraction: 1)) + XCTAssertEqual(recorder.updates.map(\.phase), [.downloading, .preparing]) + } +} + +final class DictationEngineTests: XCTestCase { + private var storage: DictationStorage! + + override func setUpWithError() throws { + storage = try makeTemporaryStorage() + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: storage.root) + } + + private func assertThrows( + _ expected: DictationError, _ body: () async throws -> T, file: StaticString = #filePath, + line: UInt = #line + ) async { + do { + _ = try await body() + XCTFail("Expected \(expected)", file: file, line: line) + } catch { + XCTAssertEqual(error as? DictationError, expected, file: file, line: line) + } + } + + func testDownloadIsRefusedBeforeAnyNetworkUseWhenStorageIsShort() async { + let engine = DictationEngine(storage: storage, environment: environment(freeDisk: 10 << 20)) + await assertThrows( + .notEnoughDisk(requiredBytes: DictationModel.fast.downloadDiskBytes, availableBytes: 10 << 20) + ) { + try await engine.download(.fast, processing: .auto, onProgress: { _ in }) + } + XCTAssertFalse(FileManager.default.fileExists(atPath: storage.modelDirectory(.fast).path)) + } + + func testPrepareAndTranscribeRefuseAModelThatIsNotDownloaded() async { + let engine = DictationEngine(storage: storage, environment: environment()) + await assertThrows(.notInstalled(.fast)) { + try await engine.prepare(.fast, processing: .auto) + } + await assertThrows(.notInstalled(.balanced)) { + try await engine.transcribe( + requestId: "1", url: URL(fileURLWithPath: "/dev/null"), model: .balanced, + processing: .auto, onlyMyVoice: false, language: "en-US") + } + } + + func testACancelThatArrivesFirstStopsItsTranscriptionAndNoOther() async { + let engine = DictationEngine(storage: storage, environment: environment()) + let transcribe = { (requestId: String) in + try await engine.transcribe( + requestId: requestId, url: URL(fileURLWithPath: "/dev/null"), model: .fast, + processing: .auto, onlyMyVoice: false, language: "en-US") + } + await engine.cancelTranscription(requestId: "early") + await assertThrows(.notInstalled(.fast)) { try await transcribe("other") } + await assertThrows(.cancelled) { try await transcribe("early") } + await assertThrows(.notInstalled(.fast)) { try await transcribe("early") } + } + + func testTheVoiceFilterIsNotASpeechModel() async { + let engine = DictationEngine(storage: storage, environment: environment()) + await assertThrows(.unknownModel("voice-filter")) { + try await engine.prepare(.voiceFilter, processing: .auto) + } + } + + func testStatusOfAModelThatIsNotDownloaded() async { + let engine = DictationEngine(storage: storage, environment: environment()) + let status = await engine.status(.fast, processing: .auto) + XCTAssertEqual( + status, ModelStatus(installed: false, warm: false, downloading: false, sizeOnDiskBytes: 0)) + } + + func testAWarmMarkerDoesNotOutliveTheModelFiles() async throws { + try storage.markWarm( + .fast, key: WarmKey(appBuild: "1.0(1)", osBuild: "test", processing: "0")) + let engine = DictationEngine(storage: storage, environment: environment()) + let status = await engine.status(.fast, processing: .auto) + XCTAssertFalse(status.warm) + } + + func testSavingAProfileNeedsEveryClip() async { + let engine = DictationEngine(storage: storage, environment: environment()) + await assertThrows(.enrollmentRejected(.tooFewClips)) { + try await engine.saveVoiceProfile([URL(fileURLWithPath: "/dev/null")]) + } + let status = await engine.voiceProfileStatus() + XCTAssertFalse(status.enrolled) + } + + func testLanguageHintComesFromTheLocalePrefix() { + XCTAssertEqual(DictationEngine.languageHint("en-US"), .english) + XCTAssertEqual(DictationEngine.languageHint("de_DE"), .german) + XCTAssertEqual(DictationEngine.languageHint("FR"), .french) + XCTAssertNil(DictationEngine.languageHint("ja-JP")) + XCTAssertNil(DictationEngine.languageHint(nil)) + XCTAssertNil(DictationEngine.languageHint("")) + } + + func testForeignErrorsAreDescribedAsCodedDictationErrors() { + XCTAssertEqual(DictationEngine.describe(CancellationError()), .cancelled) + XCTAssertEqual(DictationEngine.describe(DictationError.noVoiceProfile), .noVoiceProfile) + XCTAssertEqual(DictationEngine.describe(DictationError.notInstalled(.fast)).code, "not-installed") + XCTAssertEqual(DictationEngine.describe(URLError(.timedOut)).code, "failed") + } +} diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/DictationStorageTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/DictationStorageTests.swift new file mode 100644 index 0000000000000..0b1b0aa7d30cd --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/DictationStorageTests.swift @@ -0,0 +1,131 @@ +import FluidAudio +import XCTest + +@testable import PseudoDictationCore + +func makeTemporaryStorage() throws -> DictationStorage { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("pseudo-dictation-tests", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let storage = DictationStorage(root: root) + try storage.prepare() + return storage +} + +/// Writes the files a finished voice filter download leaves behind, without real model weights. +func writeVoiceFilterFiles(_ storage: DictationStorage) throws { + let manager = FileManager.default + let directory = storage.modelDirectory(.voiceFilter) + for name in ModelNames.OfflineDiarizer.requiredModels { + let url = directory.appendingPathComponent(name) + if name.hasSuffix(".mlmodelc") { + try manager.createDirectory(at: url, withIntermediateDirectories: true) + try Data([1]).write(to: url.appendingPathComponent("coremldata.bin")) + } else { + try manager.createDirectory(at: directory, withIntermediateDirectories: true) + try Data([1]).write(to: url) + } + } +} + +final class DictationStorageTests: XCTestCase { + private var storage: DictationStorage! + + override func setUpWithError() throws { + storage = try makeTemporaryStorage() + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: storage.root) + } + + func testModelsStayOutOfBackups() throws { + let values = try storage.root.resourceValues(forKeys: [.isExcludedFromBackupKey]) + XCTAssertEqual(values.isExcludedFromBackup, true) + } + + func testFilesWithoutTheMarkerAreNotInstalled() throws { + try writeVoiceFilterFiles(storage) + XCTAssertFalse(storage.isInstalled(.voiceFilter)) + try storage.markInstalled(.voiceFilter) + XCTAssertTrue(storage.isInstalled(.voiceFilter)) + } + + func testTheMarkerWithoutFilesIsNotInstalled() throws { + try storage.markInstalled(.voiceFilter) + try storage.markInstalled(.fast) + XCTAssertFalse(storage.isInstalled(.voiceFilter)) + XCTAssertFalse(storage.isInstalled(.fast)) + } + + func testAnInterruptedFileIsNotInstalled() throws { + try writeVoiceFilterFiles(storage) + try storage.markInstalled(.voiceFilter) + let partial = storage.modelDirectory(.voiceFilter) + .appendingPathComponent("Embedding.mlmodelc/weights/weight.bin.partial") + try FileManager.default.createDirectory( + at: partial.deletingLastPathComponent(), withIntermediateDirectories: true) + try Data([1]).write(to: partial) + XCTAssertFalse(storage.isInstalled(.voiceFilter)) + } + + func testAModelMissingItsCompiledDataIsNotInstalled() throws { + try writeVoiceFilterFiles(storage) + try storage.markInstalled(.voiceFilter) + try FileManager.default.removeItem( + at: storage.modelDirectory(.voiceFilter).appendingPathComponent("FBank.mlmodelc/coremldata.bin")) + XCTAssertFalse(storage.isInstalled(.voiceFilter)) + } + + func testDeleteRemovesFilesAndBothMarkers() throws { + let key = WarmKey(appBuild: "1(1)", osBuild: "os", processing: "2") + try writeVoiceFilterFiles(storage) + try storage.markInstalled(.voiceFilter) + try storage.markWarm(.voiceFilter, key: key) + XCTAssertGreaterThan(storage.sizeOnDisk(.voiceFilter), 0) + + try storage.delete(.voiceFilter) + + XCTAssertFalse(FileManager.default.fileExists(atPath: storage.modelDirectory(.voiceFilter).path)) + XCTAssertEqual(storage.sizeOnDisk(.voiceFilter), 0) + XCTAssertFalse(storage.isWarm(.voiceFilter, key: key)) + try writeVoiceFilterFiles(storage) + XCTAssertFalse(storage.isInstalled(.voiceFilter)) + } + + func testWarmOnlyForTheBuildOsAndProcessingItWasLoadedUnder() throws { + let key = WarmKey(appBuild: "1.0(7)", osBuild: "Version 18.0", processing: "2") + XCTAssertFalse(storage.isWarm(.fast, key: key)) + try storage.markWarm(.fast, key: key) + XCTAssertTrue(storage.isWarm(.fast, key: key)) + XCTAssertFalse(storage.isWarm(.balanced, key: key)) + XCTAssertFalse(storage.isWarm(.fast, key: WarmKey(appBuild: "1.0(8)", osBuild: "Version 18.0", processing: "2"))) + XCTAssertFalse(storage.isWarm(.fast, key: WarmKey(appBuild: "1.0(7)", osBuild: "Version 18.1", processing: "2"))) + XCTAssertFalse(storage.isWarm(.fast, key: WarmKey(appBuild: "1.0(7)", osBuild: "Version 18.0", processing: "0"))) + } + + func testVoiceProfileRoundTrip() throws { + let profile = VoiceProfile( + extractor: VoiceProfile.currentExtractor, embedding: [0.6, 0.8], clipCount: 3, + createdAt: Date(timeIntervalSince1970: 1_700_000_000)) + XCTAssertNil(storage.loadVoiceProfile()) + try storage.saveVoiceProfile(profile) + XCTAssertEqual(storage.loadVoiceProfile(), profile) + try storage.deleteVoiceProfile() + XCTAssertNil(storage.loadVoiceProfile()) + XCTAssertNoThrow(try storage.deleteVoiceProfile()) + } + + func testAProfileThisBuildCannotCompareReadsAsAbsent() throws { + let url = storage.root.appendingPathComponent("voice-profile.json") + try storage.saveVoiceProfile( + VoiceProfile(extractor: "another-extractor", embedding: [1, 0], clipCount: 3, createdAt: Date())) + XCTAssertNil(storage.loadVoiceProfile()) + try storage.saveVoiceProfile( + VoiceProfile( + extractor: VoiceProfile.currentExtractor, embedding: [], clipCount: 3, createdAt: Date())) + XCTAssertNil(storage.loadVoiceProfile()) + try Data("{ not json".utf8).write(to: url) + XCTAssertNil(storage.loadVoiceProfile()) + } +} diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift new file mode 100644 index 0000000000000..1b91c8e913e1e --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealModelTests.swift @@ -0,0 +1,296 @@ +import AVFoundation +import FluidAudio +import XCTest + +@testable import PseudoDictationCore + +/// Runs the engine against the real models with voices from the Mac's speech synthesiser. +/// Skipped unless `PSEUDO_DICTATION_MODELS` names a directory to download into (about 2 GB): +/// +/// PSEUDO_DICTATION_MODELS=/path/to/models swift test --filter RealModelTests +final class RealModelTests: XCTestCase { + private static let me = "Samantha" + private static let other = "Daniel" + private static let enrollment = [ + "The quick brown fox jumps over the lazy dog while the sun sets behind the quiet hills.", + "Please review the pull request and tell me whether the tests cover the new settings screen.", + "I would like to schedule the release for Thursday afternoon after the team meeting ends.", + ] + private static let mine = "Open the settings screen and rename the project to umbrella." + private static let theirs = "Yesterday we climbed the volcano and watched the lava glow at night." + + private var storage: DictationStorage! + private var engine: DictationEngine! + private var clips: URL! + + override func setUpWithError() throws { + guard let root = ProcessInfo.processInfo.environment["PSEUDO_DICTATION_MODELS"], !root.isEmpty + else { + throw XCTSkip("Set PSEUDO_DICTATION_MODELS to a directory to run against the real models.") + } + storage = DictationStorage(root: URL(fileURLWithPath: root, isDirectory: true)) + try storage.prepare() + // A voice profile left by another test must not leak into this one. + try storage.deleteVoiceProfile() + engine = DictationEngine( + storage: storage, + environment: DictationEnvironment( + appBuild: "harness", osBuild: ProcessInfo.processInfo.operatingSystemVersionString, + isSimulator: false, availableMemory: { 0 }, freeDisk: { _ in nil })) + clips = FileManager.default.temporaryDirectory + .appendingPathComponent("pseudo-dictation-clips-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: clips, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + if let clips { try? FileManager.default.removeItem(at: clips) } + } + + // MARK: Fixtures + + private func speak(_ text: String, voice: String, name: String) throws -> URL { + let url = clips.appendingPathComponent("\(name).wav") + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/say") + process.arguments = [ + "-v", voice, "-o", url.path, "--file-format=WAVE", "--data-format=LEI16@16000", text, + ] + try process.run() + process.waitUntilExit() + XCTAssertEqual(process.terminationStatus, 0, "say failed for \(voice)") + return url + } + + private func write(_ samples: [Float], name: String) throws -> URL { + let url = clips.appendingPathComponent("\(name).wav") + try writeWav(samples, to: url) + return url + } + + private func silence(_ seconds: Double) -> [Float] { + [Float](repeating: 0, count: Int(seconds * DictationEngine.sampleRate)) + } + + private func download(_ model: DictationModel) async throws { + let started = Date() + try await engine.download(model, processing: .auto, onProgress: { _ in }) + let megabytes = storage.sizeOnDisk(model) / 1_000_000 + print("MEASURE download \(model.rawValue): \(megabytes) MB on disk, \(Int(Date().timeIntervalSince(started))) s") + } + + private func enroll(voice: String) async throws -> [URL] { + try await download(.voiceFilter) + let urls = try Self.enrollment.enumerated().map { index, text in + try speak(text, voice: voice, name: "enroll-\(voice)-\(index)") + } + try await engine.saveVoiceProfile(urls) + return urls + } + + private func words(_ text: String) -> Set { + Set(text.lowercased().split(whereSeparator: { !$0.isLetter }).map(String.init)) + } + + // MARK: Speech models + + func testEveryModelDownloadsLoadsAndTranscribes() async throws { + let url = try speak(Self.mine, voice: Self.me, name: "mine") + for model in [DictationModel.fast, .balanced, .accurate] { + try await download(model) + let status = await engine.status(model, processing: .auto) + XCTAssertTrue(status.installed, model.rawValue) + XCTAssertTrue(status.warm, model.rawValue) + XCTAssertGreaterThan(status.sizeOnDiskBytes, 50_000_000, model.rawValue) + XCTAssertLessThanOrEqual(status.sizeOnDiskBytes, model.downloadDiskBytes, model.rawValue) + + let transcript = try await engine.transcribe( + requestId: model.rawValue, url: url, model: model, processing: .auto, onlyMyVoice: false, + language: "en-US") + print( + "MEASURE transcribe \(model.rawValue): \(transcript.audioSeconds) s audio in " + + "\(transcript.processingSeconds) s: \(transcript.text)") + let heard = words(transcript.text) + for expected in ["settings", "rename", "project", "umbrella"] { + XCTAssertTrue(heard.contains(expected), "\(model.rawValue) missed \(expected): \(transcript.text)") + } + XCTAssertNil(transcript.filter) + } + } + + func testAModelLoadsWithTheNetworkOff() async throws { + try await download(.fast) + await engine.releaseIfIdle() + XCTAssertTrue(ModelHub.offlineMode) + let url = try speak(Self.mine, voice: Self.me, name: "mine") + let transcript = try await engine.transcribe( + requestId: "offline", url: url, model: .fast, processing: .cpu, onlyMyVoice: false, + language: nil) + XCTAssertTrue(words(transcript.text).contains("umbrella"), transcript.text) + } + + func testSilenceIsNotSentToTheModel() async throws { + try await download(.fast) + let url = try write(silence(2), name: "silence") + let transcript = try await engine.transcribe( + requestId: "silence", url: url, model: .fast, processing: .auto, onlyMyVoice: true, + language: "en-US") + XCTAssertEqual(transcript.text, "") + XCTAssertEqual(transcript.filter, .skipped(.noSpeech)) + } + + func testACancelledTranscriptionReportsCancelled() async throws { + try await download(.fast) + let url = try speak(Self.enrollment.joined(separator: " "), voice: Self.me, name: "long") + let engine = try XCTUnwrap(self.engine) + let running = Task { + try await engine.transcribe( + requestId: "cancel", url: url, model: .fast, processing: .auto, onlyMyVoice: false, + language: "en-US") + } + running.cancel() + do { + _ = try await running.value + XCTFail("Expected the transcription to be cancelled") + } catch { + XCTAssertEqual(error as? DictationError, .cancelled) + } + let after = try await engine.transcribe( + requestId: "after", url: url, model: .fast, processing: .auto, onlyMyVoice: false, + language: "en-US") + XCTAssertTrue(words(after.text).contains("fox"), after.text) + } + + func testACancelledDownloadLeavesNothingBehind() async throws { + let storage = DictationStorage( + root: clips.appendingPathComponent("cancelled-download", isDirectory: true)) + let engine = DictationEngine( + storage: storage, + environment: DictationEnvironment( + appBuild: "harness", osBuild: ProcessInfo.processInfo.operatingSystemVersionString, + isSimulator: false, availableMemory: { 0 }, freeDisk: { _ in nil })) + let (progress, continuation) = AsyncStream.makeStream() + let running = Task { + defer { continuation.finish() } + try await engine.download(.fast, processing: .auto, onProgress: { continuation.yield($0) }) + } + for await update in progress where update.fraction > 0 { break } + XCTAssertTrue(FileManager.default.fileExists(atPath: storage.modelDirectory(.fast).path)) + await engine.cancelDownload(.fast) + do { + try await running.value + XCTFail("Expected the download to be cancelled") + } catch { + XCTAssertEqual(error as? DictationError, .cancelled) + } + XCTAssertFalse(FileManager.default.fileExists(atPath: storage.modelDirectory(.fast).path)) + let status = await engine.status(.fast, processing: .auto) + XCTAssertEqual( + status, ModelStatus(installed: false, warm: false, downloading: false, sizeOnDiskBytes: 0)) + } + + // MARK: Voice filter + + func testOnlyMyVoiceDropsTheOtherSpeaker() async throws { + try await download(.fast) + _ = try await enroll(voice: Self.me) + let mine = try DictationEngine.decode(speak(Self.mine, voice: Self.me, name: "mine")) + let theirs = try DictationEngine.decode(speak(Self.theirs, voice: Self.other, name: "theirs")) + let url = try write(mine + silence(0.6) + theirs, name: "both") + + let unfiltered = try await engine.transcribe( + requestId: "unfiltered", url: url, model: .fast, processing: .auto, onlyMyVoice: false, + language: "en-US") + XCTAssertTrue(words(unfiltered.text).contains("umbrella"), unfiltered.text) + XCTAssertTrue(words(unfiltered.text).contains("volcano"), unfiltered.text) + + let filtered = try await engine.transcribe( + requestId: "filtered", url: url, model: .fast, processing: .auto, onlyMyVoice: true, + language: "en-US") + print("MEASURE filter: \(String(describing: filtered.filter)) -> \(filtered.text)") + XCTAssertTrue(words(filtered.text).contains("umbrella"), filtered.text) + XCTAssertFalse(words(filtered.text).contains("volcano"), filtered.text) + XCTAssertFalse(words(filtered.text).contains("lava"), filtered.text) + let outcome = try XCTUnwrap(filtered.filter) + XCTAssertTrue(outcome.applied) + XCTAssertEqual(outcome.keptSeconds, Double(mine.count) / DictationEngine.sampleRate, accuracy: 1) + XCTAssertEqual( + outcome.droppedSeconds, Double(theirs.count) / DictationEngine.sampleRate, accuracy: 1) + } + + func testOnlyMyVoiceKeepsEverythingWhenIAmTheOnlySpeaker() async throws { + try await download(.fast) + _ = try await enroll(voice: Self.me) + let url = try speak(Self.mine, voice: Self.me, name: "mine") + let transcript = try await engine.transcribe( + requestId: "solo", url: url, model: .fast, processing: .auto, onlyMyVoice: true, + language: "en-US") + XCTAssertEqual(transcript.filter, .skipped(.onlyYou)) + XCTAssertTrue(words(transcript.text).contains("umbrella"), transcript.text) + } + + func testOnlyMyVoiceKeepsEverythingWhenNoSpeakerMatches() async throws { + try await download(.fast) + _ = try await enroll(voice: Self.me) + let url = try speak(Self.theirs, voice: Self.other, name: "theirs") + let transcript = try await engine.transcribe( + requestId: "stranger", url: url, model: .fast, processing: .auto, onlyMyVoice: true, + language: "en-US") + XCTAssertEqual(transcript.filter, .skipped(.noMatch)) + XCTAssertTrue(words(transcript.text).contains("volcano"), transcript.text) + } + + func testOnlyMyVoiceWithoutAProfileStillTranscribes() async throws { + try await download(.fast) + let url = try speak(Self.mine, voice: Self.me, name: "mine") + let transcript = try await engine.transcribe( + requestId: "no-profile", url: url, model: .fast, processing: .auto, onlyMyVoice: true, + language: "en-US") + XCTAssertEqual(transcript.filter, .skipped(.noProfile)) + XCTAssertTrue(words(transcript.text).contains("umbrella"), transcript.text) + } + + // MARK: Enrollment + + func testEnrollmentAcceptsOneVoiceAndSavesAProfile() async throws { + let urls = try await enroll(voice: Self.me) + let check = try await engine.checkEnrollmentClip(urls[0]) + XCTAssertNil(check.problem) + XCTAssertGreaterThan(check.speechSeconds, DictationEngine.minimumEnrollmentSpeechSeconds) + let status = await engine.voiceProfileStatus() + XCTAssertTrue(status.enrolled) + XCTAssertEqual(status.clipCount, 3) + XCTAssertEqual(storage.loadVoiceProfile()?.embedding.count, 256) + } + + func testEnrollmentRejectsClipsItCannotUse() async throws { + try await download(.voiceFilter) + let quiet = try await engine.checkEnrollmentClip(write(silence(4), name: "quiet")) + XCTAssertEqual(quiet.problem, .tooQuiet) + + let short = try await engine.checkEnrollmentClip(speak("Hello there.", voice: Self.me, name: "short")) + XCTAssertEqual(short.problem, .tooShort) + + let mine = try DictationEngine.decode(speak(Self.enrollment[0], voice: Self.me, name: "a")) + let theirs = try DictationEngine.decode(speak(Self.enrollment[1], voice: Self.other, name: "b")) + let shared = try await engine.checkEnrollmentClip( + write(mine + silence(0.5) + theirs, name: "shared")) + XCTAssertEqual(shared.problem, .multipleVoices) + } + + func testEnrollmentRejectsClipsFromDifferentPeople() async throws { + try await download(.voiceFilter) + let urls = [ + try speak(Self.enrollment[0], voice: Self.me, name: "c0"), + try speak(Self.enrollment[1], voice: Self.me, name: "c1"), + try speak(Self.enrollment[2], voice: Self.other, name: "c2"), + ] + do { + try await engine.saveVoiceProfile(urls) + XCTFail("Expected the clips to disagree") + } catch { + XCTAssertEqual(error as? DictationError, .enrollmentRejected(.clipsDisagree)) + } + let status = await engine.voiceProfileStatus() + XCTAssertFalse(status.enrolled) + } +} diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift new file mode 100644 index 0000000000000..3859bbf1e6469 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/RealVoiceTests.swift @@ -0,0 +1,235 @@ +import XCTest + +@testable import PseudoDictationCore + +/// Measures enrollment and the voice filter on recordings of real people, through the engine. +/// The synthetic voices in `RealModelTests` are far easier to tell apart than people are, so the +/// thresholds in `SpeakerFilter` are judged here. +/// +/// Skipped unless `PSEUDO_DICTATION_MODELS` names the model directory and `PSEUDO_DICTATION_VOICES` +/// names a LibriSpeech-layout directory (`//*.flac`), such as Mini LibriSpeech +/// `dev-clean-2` from https://www.openslr.org/31/: +/// +/// PSEUDO_DICTATION_MODELS=/path/to/models PSEUDO_DICTATION_VOICES=/path/to/dev-clean-2 \ +/// swift test --filter RealVoiceTests +final class RealVoiceTests: XCTestCase { + private struct Speaker { + var name: String + var enrollment: [[Float]] + var held: [[Float]] + } + + private var storage: DictationStorage! + private var engine: DictationEngine! + private var clips: URL! + private var speakers: [Speaker] = [] + private var written = 0 + + override func setUpWithError() throws { + let environment = ProcessInfo.processInfo.environment + guard let models = environment["PSEUDO_DICTATION_MODELS"], !models.isEmpty, + let voices = environment["PSEUDO_DICTATION_VOICES"], !voices.isEmpty + else { + throw XCTSkip("Set PSEUDO_DICTATION_MODELS and PSEUDO_DICTATION_VOICES to run on real voices.") + } + storage = DictationStorage(root: URL(fileURLWithPath: models, isDirectory: true)) + try storage.prepare() + try storage.deleteVoiceProfile() + engine = DictationEngine( + storage: storage, + environment: DictationEnvironment( + appBuild: "harness", osBuild: ProcessInfo.processInfo.operatingSystemVersionString, + isSimulator: false, availableMemory: { 0 }, freeDisk: { _ in nil })) + clips = FileManager.default.temporaryDirectory + .appendingPathComponent("pseudo-dictation-voices-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: clips, withIntermediateDirectories: true) + speakers = try Self.loadSpeakers(URL(fileURLWithPath: voices, isDirectory: true)) + XCTAssertGreaterThanOrEqual(speakers.count, 10, "Too few speakers with enough long utterances") + } + + override func tearDownWithError() throws { + if let clips { try? FileManager.default.removeItem(at: clips) } + try storage?.deleteVoiceProfile() + } + + // MARK: Fixtures + + /// Three utterances of at least 7 s to enroll with and at least three more to test on, taken + /// from across the speaker's chapters so the sessions differ. + private static func loadSpeakers(_ root: URL) throws -> [Speaker] { + let files = FileManager.default + var speakers: [Speaker] = [] + for name in try files.contentsOfDirectory(atPath: root.path).sorted() where Int(name) != nil { + var utterances: [URL] = [] + let directory = root.appendingPathComponent(name) + for chapter in try files.contentsOfDirectory(atPath: directory.path).sorted() { + let chapterDirectory = directory.appendingPathComponent(chapter) + utterances += try files.contentsOfDirectory(atPath: chapterDirectory.path).sorted() + .filter { $0.hasSuffix(".flac") }.map { chapterDirectory.appendingPathComponent($0) } + } + var long: [[Float]] = [] + for index in stride(from: 0, to: utterances.count, by: max(1, utterances.count / 14)) { + let samples = try DictationEngine.decode(utterances[index]) + if samples.count >= Int(7 * DictationEngine.sampleRate) { long.append(samples) } + if long.count == 9 { break } + } + guard long.count >= 6 else { continue } + speakers.append( + Speaker( + name: name, enrollment: long.prefix(3).map { cut($0, 8) }, held: Array(long.dropFirst(3)))) + } + return speakers + } + + private static func cut(_ samples: [Float], _ seconds: Double) -> [Float] { + Array(samples.prefix(Int(seconds * DictationEngine.sampleRate))) + } + + private func write(_ samples: [Float]) throws -> URL { + written += 1 + let url = clips.appendingPathComponent("clip-\(written).wav") + try writeWav(samples, to: url) + return url + } + + private func words(_ samples: [Float], onlyMyVoice: Bool) async throws -> (Set, FilterOutcome?) { + let transcript = try await engine.transcribe( + requestId: UUID().uuidString, url: try write(samples), model: .fast, processing: .auto, + onlyMyVoice: onlyMyVoice, language: "en-US") + let words = transcript.text.lowercased().split(whereSeparator: { !$0.isLetter }).map(String.init) + return (Set(words), transcript.filter) + } + + private func seconds(_ samples: [Float]) -> Double { + Double(samples.count) / DictationEngine.sampleRate + } + + private func percent(_ part: Int, _ whole: Int) -> String { + String(format: "%d of %d (%.1f%%)", part, whole, whole == 0 ? 0 : 100 * Double(part) / Double(whole)) + } + + // MARK: Enrollment + + func testEnrollmentAcceptsOnePersonAndRejectsAMixOfTwo() async throws { + try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in }) + var ownRejected: [String] = [] + var mixAccepted: [String] = [] + var mixes = 0 + for (index, speaker) in speakers.enumerated() { + do { + try await engine.saveVoiceProfile(try speaker.enrollment.map(write)) + } catch { + ownRejected.append("\(speaker.name): \(error)") + } + for offset in [1, speakers.count / 2] { + let stranger = speakers[(index + offset) % speakers.count] + mixes += 1 + do { + try await engine.saveVoiceProfile( + try [speaker.enrollment[0], speaker.enrollment[1], stranger.enrollment[2]].map(write)) + mixAccepted.append("\(speaker.name) with \(stranger.name)") + } catch { + XCTAssertEqual(error as? DictationError, .enrollmentRejected(.clipsDisagree)) + } + } + } + print("MEASURE enrollment, one person rejected: \(percent(ownRejected.count, speakers.count)) \(ownRejected)") + print("MEASURE enrollment, two people accepted: \(percent(mixAccepted.count, mixes)) \(mixAccepted)") + XCTAssertEqual(ownRejected, []) + XCTAssertLessThanOrEqual(Double(mixAccepted.count) / Double(mixes), 0.05) + } + + // MARK: Voice filter + + func testTheFilterKeepsTheSpeakersWordsAndDropsTheStrangers() async throws { + try await engine.download(.fast, processing: .auto, onProgress: { _ in }) + try await engine.download(.voiceFilter, processing: .auto, onProgress: { _ in }) + + var recordings = 0 + var unfilteredRecordings: [String] = [] + var ownWords = 0 + var ownWordsLost = 0 + var strangerWords = 0 + var strangerWordsKept = 0 + var cutError: [Double] = [] + var layouts: [String: (ownWords: Int, ownLost: Int, strangerWords: Int, strangerKept: Int)] = [:] + var soloRecordings = 0 + var soloCut: [String] = [] + var strangerSolo = 0 + var strangerSoloCut: [String] = [] + + for (index, speaker) in speakers.enumerated() { + try await engine.saveVoiceProfile(try speaker.enrollment.map(write)) + + for utterance in speaker.held.prefix(3) { + soloRecordings += 1 + let (_, outcome) = try await words(Self.cut(utterance, 6), onlyMyVoice: true) + if outcome?.applied == true { + soloCut.append("\(speaker.name) lost \(outcome?.droppedSeconds ?? 0) s") + } + } + + for offset in [1, speakers.count / 2] { + let stranger = speakers[(index + offset) % speakers.count] + strangerSolo += 1 + let (_, soloOutcome) = try await words(Self.cut(stranger.held[2], 6), onlyMyVoice: true) + if soloOutcome?.applied == true { strangerSoloCut.append("\(stranger.name) as \(speaker.name)") } + + let own = Self.cut(speaker.held[0], 5) + let ownSecond = Self.cut(speaker.held[1], 4) + let theirs = Self.cut(stranger.held[0], 5) + let theirsShort = Self.cut(stranger.held[1], 2) + let cases: [(name: String, mine: [[Float]], theirs: [Float], samples: [Float])] = [ + ("mine then theirs", [own], theirs, own + theirs), + ("theirs then mine", [own], theirs, theirs + own), + ("theirs between mine", [own, ownSecond], theirsShort, own + theirsShort + ownSecond), + ] + for recording in cases { + recordings += 1 + var expected: Set = [] + for piece in recording.mine { + expected.formUnion(try await words(piece, onlyMyVoice: false).0) + } + let unwanted = try await words(recording.theirs, onlyMyVoice: false).0.subtracting(expected) + let (heard, outcome) = try await words(recording.samples, onlyMyVoice: true) + ownWords += expected.count + ownWordsLost += expected.subtracting(heard).count + strangerWords += unwanted.count + strangerWordsKept += unwanted.intersection(heard).count + var layout = layouts[recording.name] ?? (0, 0, 0, 0) + layout.ownWords += expected.count + layout.ownLost += expected.subtracting(heard).count + layout.strangerWords += unwanted.count + layout.strangerKept += unwanted.intersection(heard).count + layouts[recording.name] = layout + if let outcome, outcome.applied { + cutError.append(abs(outcome.droppedSeconds - seconds(recording.theirs))) + } else { + unfilteredRecordings.append( + "\(speaker.name)/\(stranger.name) \(recording.name): \(String(describing: outcome?.skipped))") + } + } + } + } + + let sortedError = cutError.sorted() + print("MEASURE filter, speakers: \(speakers.count), mixed recordings: \(recordings)") + print("MEASURE filter, mixed recordings left whole: \(percent(unfilteredRecordings.count, recordings)) \(unfilteredRecordings)") + print("MEASURE filter, speaker's words lost: \(percent(ownWordsLost, ownWords))") + print("MEASURE filter, stranger's words kept: \(percent(strangerWordsKept, strangerWords))") + for (name, layout) in layouts.sorted(by: { $0.key < $1.key }) { + print( + "MEASURE filter, \(name): speaker's words lost \(percent(layout.ownLost, layout.ownWords)), " + + "stranger's words kept \(percent(layout.strangerKept, layout.strangerWords))") + } + print( + "MEASURE filter, cut length error: median \(sortedError[sortedError.count / 2]) s, " + + "worst \(sortedError.last ?? -1) s") + print("MEASURE filter, speaker alone and cut: \(percent(soloCut.count, soloRecordings)) \(soloCut)") + print("MEASURE filter, stranger alone and cut: \(percent(strangerSoloCut.count, strangerSolo)) \(strangerSoloCut)") + + XCTAssertLessThanOrEqual(Double(ownWordsLost) / Double(ownWords), 0.05) + XCTAssertLessThanOrEqual(Double(strangerWordsKept) / Double(strangerWords), 0.15) + XCTAssertLessThanOrEqual(Double(soloCut.count) / Double(soloRecordings), 0.02) + } +} diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift new file mode 100644 index 0000000000000..d7efa65802abe --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/SpeakerFilterTests.swift @@ -0,0 +1,182 @@ +import XCTest + +@testable import PseudoDictationCore + +final class SpeakerFilterTests: XCTestCase { + private func turn(_ speaker: String, _ start: Double, _ end: Double) -> SpeakerTurn { + SpeakerTurn(speaker: speaker, start: start, end: end) + } + + func testAShortTurnMaySitFurtherFromTheProfile() { + XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.5), .mine) + XCTAssertEqual(SpeakerFilter.judge(duration: 3, distance: 0.7), .other) + XCTAssertEqual(SpeakerFilter.judge(duration: 1.5, distance: 0.7), .mine) + XCTAssertEqual(SpeakerFilter.judge(duration: 1.5, distance: 0.8), .other) + XCTAssertEqual(SpeakerFilter.judge(duration: 0.9, distance: 0.1), .unjudged) + } + + func testNothingIsCutUnlessTheUserAndSomeoneElseWereBothHeard() { + let two = [turn("a", 0, 3), turn("b", 3, 6)] + XCTAssertEqual(SpeakerFilter.decide(turns: [], voices: []), .unfiltered(.noSpeech)) + XCTAssertEqual( + SpeakerFilter.decide(turns: two, voices: [.unjudged, .unjudged]), .unfiltered(.tooShort)) + XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.other, .other]), .unfiltered(.noMatch)) + XCTAssertEqual( + SpeakerFilter.decide(turns: two, voices: [.other, .unjudged]), .unfiltered(.noMatch)) + XCTAssertEqual(SpeakerFilter.decide(turns: two, voices: [.mine, .mine]), .unfiltered(.onlyYou)) + XCTAssertEqual( + SpeakerFilter.decide(turns: two, voices: [.mine, .unjudged]), .unfiltered(.onlyYou)) + } + + func testOnlyTheOtherVoiceIsCutWhateverTheDiarizerCalledIt() { + let turns = [turn("a", 0, 3), turn("a", 3.5, 7), turn("a", 7.2, 9)] + XCTAssertEqual( + SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .mine]), .drop([3.5...7])) + } + + func testAShortTurnIsCutOnlyWhenItsSpeakerWasAlwaysSomeoneElse() { + let turns = [ + turn("a", 0, 3), turn("b", 3, 6), turn("b", 6.2, 6.8), turn("a", 7, 7.5), turn("c", 8, 8.6), + ] + XCTAssertEqual( + SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .unjudged, .unjudged, .unjudged]), + .drop([3...6, 6.2...6.8])) + + let merged = [turn("a", 0, 3), turn("a", 3, 6), turn("a", 6.2, 6.8)] + XCTAssertEqual( + SpeakerFilter.decide(turns: merged, voices: [.mine, .other, .unjudged]), .drop([3...6])) + } + + private func assertClose( + _ actual: [Double], _ expected: [Double], file: StaticString = #filePath, line: UInt = #line + ) { + XCTAssertEqual(actual.count, expected.count, "\(actual)", file: file, line: line) + for (a, b) in zip(actual, expected) { + XCTAssertEqual(a, b, accuracy: 0.001, "\(actual)", file: file, line: line) + } + } + + func testALongTurnIsScannedFromItsStartToItsEnd() { + XCTAssertEqual(SpeakerFilter.scanWindows(turn("a", 0, 2.9)), []) + let even = SpeakerFilter.scanWindows(turn("a", 1, 4)) + assertClose(even.map(\.lowerBound), [1, 1.5, 2, 2.5]) + assertClose(even.map(\.upperBound), [2.5, 3, 3.5, 4]) + let uneven = SpeakerFilter.scanWindows(turn("a", 0, 3.2)) + assertClose(uneven.map(\.lowerBound), [0, 0.5, 1, 1.7]) + assertClose(uneven.map(\.upperBound), [1.5, 2, 2.5, 3.2]) + } + + func testATurnIsSplitWhereItsVoiceChanges() { + let long = turn("a", 0, 6) + let windows = SpeakerFilter.scanWindows(long) + XCTAssertEqual(windows.count, 10) + XCTAssertEqual( + SpeakerFilter.split(long, windows: windows, distances: [Float](repeating: 0.2, count: 10)), [long]) + XCTAssertEqual(SpeakerFilter.split(turn("a", 0, 2), windows: [], distances: []), [turn("a", 0, 2)]) + + // Windows centred at 0.75, 1.25 ... 5.25. The voice changes between 1.75 and 2.25, and back + // between 3.75 and 4.25, nearer the window that sits closer to the limit of 0.6. + let pieces = SpeakerFilter.split( + long, windows: windows, distances: [0.2, 0.2, 0.4, 0.8, 0.9, 0.9, 0.7, 0.3, 0.2, 0.2]) + XCTAssertEqual(pieces.map(\.speaker), ["a", "a", "a"]) + assertClose(pieces.map(\.start), [0, 2, 3.875]) + assertClose(pieces.map(\.end), [2, 3.875, 6]) + } + + func testABlipTooShortToJudgeDoesNotSplitATurn() { + let long = turn("a", 0, 6) + let windows = SpeakerFilter.scanWindows(long) + XCTAssertEqual( + SpeakerFilter.split( + long, windows: windows, distances: [0.2, 0.2, 0.2, 0.7, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]), + [long]) + // The blip inside the other voice is ignored too, so that stretch stays one piece. + let pieces = SpeakerFilter.split( + long, windows: windows, distances: [0.2, 0.2, 1.0, 1.0, 0.5, 1.0, 1.0, 1.0, 0.2, 0.2]) + assertClose(pieces.map(\.start), [0, 1.5, 4.5]) + // Of three changes close together, the closest two are the blip. + let late = SpeakerFilter.split( + long, windows: windows, distances: [0.2, 0.2, 1.0, 0.5, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2]) + assertClose(late.map(\.start), [0, 1.5, 4.5]) + } + + func testSpeechOverTheUsersOwnTurnIsNotCut() { + XCTAssertEqual( + SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3, 6)], voices: [.mine, .other]), + .drop([4...6])) + XCTAssertEqual( + SpeakerFilter.decide(turns: [turn("a", 0, 6), turn("b", 2, 3)], voices: [.other, .mine]), + .drop([0...2, 3...6])) + XCTAssertEqual( + SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 1, 3)], voices: [.mine, .other]), + .unfiltered(.overlapping)) + XCTAssertEqual( + SpeakerFilter.decide(turns: [turn("a", 0, 4), turn("b", 3.9, 4.1)], voices: [.mine, .other]), + .unfiltered(.overlapping)) + } + + func testTouchingTurnsBecomeOneCut() { + let turns = [turn("a", 0, 2), turn("b", 2, 4), turn("c", 4, 6), turn("b", 3.5, 4.5)] + XCTAssertEqual( + SpeakerFilter.decide(turns: turns, voices: [.mine, .other, .other, .other]), .drop([2...6])) + } + + func testCosineDistance() { + XCTAssertEqual(SpeakerFilter.cosineDistance([1, 0], [2, 0]), 0, accuracy: 1e-6) + XCTAssertEqual(SpeakerFilter.cosineDistance([1, 0], [0, 3]), 1, accuracy: 1e-6) + XCTAssertEqual(SpeakerFilter.cosineDistance([1, 0], [-1, 0]), 2, accuracy: 1e-6) + XCTAssertEqual(SpeakerFilter.cosineDistance([1, 0], [1, 0, 0]), .infinity) + XCTAssertEqual(SpeakerFilter.cosineDistance([0, 0], [1, 0]), .infinity) + } + + func testMeanEmbeddingWeighsClipsEquallyWhateverTheirLength() throws { + let mean = try XCTUnwrap(SpeakerFilter.meanEmbedding([[10, 0], [0, 0.1]])) + XCTAssertEqual(mean[0], mean[1], accuracy: 1e-6) + XCTAssertEqual(mean[0] * mean[0] + mean[1] * mean[1], 1, accuracy: 1e-6) + } + + func testMeanEmbeddingRejectsVectorsItCannotAverage() { + XCTAssertNil(SpeakerFilter.meanEmbedding([])) + XCTAssertNil(SpeakerFilter.meanEmbedding([[1, 0], [1, 0, 0]])) + XCTAssertNil(SpeakerFilter.meanEmbedding([[1, 0], [-1, 0]])) + } + + func testMeanEmbeddingFollowsItsWeights() throws { + let mean = try XCTUnwrap(SpeakerFilter.meanEmbedding([[5, 0], [0, 0.2]], weights: [3, 1])) + XCTAssertEqual(mean[0] / mean[1], 3, accuracy: 1e-5) + XCTAssertNil(SpeakerFilter.meanEmbedding([[1, 0], [0, 1]], weights: [1])) + } + + func testRemoveCutsTheRangesAndLeavesASilenceAtEachCut() { + let samples = (0..<100).map(Float.init) + let output = SpeakerFilter.remove(samples: samples, ranges: [2...4, 6...7], sampleRate: 10) + let silence = [Float](repeating: 0, count: Int(SpeakerFilter.joinSilenceSeconds * 10)) + XCTAssertEqual( + output, + Array(samples[0..<20]) + silence + Array(samples[40..<60]) + silence + Array(samples[70..<100])) + } + + func testRemoveAddsNoSilenceAtTheEndsOfTheRecording() { + let samples = (0..<100).map(Float.init) + let output = SpeakerFilter.remove(samples: samples, ranges: [0...1, 9...12], sampleRate: 10) + XCTAssertEqual(output, Array(samples[10..<90])) + } + + func testAudioGate() { + let rate = 16_000.0 + let silence = [Float](repeating: 0, count: 16_000) + let hiss = (0..<16_000).map { _ in Float.random(in: -0.0005...0.0005) } + let tone = (0..<16_000).map { Float(sin(Double($0) * 2 * .pi * 220 / rate)) * 0.1 } + XCTAssertFalse(AudioGate.hasSpeech(samples: silence, sampleRate: rate)) + XCTAssertFalse(AudioGate.hasSpeech(samples: hiss, sampleRate: rate)) + XCTAssertTrue(AudioGate.hasSpeech(samples: tone, sampleRate: rate)) + XCTAssertTrue(AudioGate.hasSpeech(samples: silence + Array(tone.prefix(1600)), sampleRate: rate)) + XCTAssertFalse(AudioGate.hasSpeech(samples: Array(tone.prefix(3200)), sampleRate: rate)) + } + + func testMemoryGateTreatsNoReadingAsEnough() { + XCTAssertTrue(hasEnoughMemory(available: 0, required: 100)) + XCTAssertTrue(hasEnoughMemory(available: 100, required: 100)) + XCTAssertFalse(hasEnoughMemory(available: 99, required: 100)) + } +} diff --git a/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift new file mode 100644 index 0000000000000..9ced2e26e3b10 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/Tests/PseudoDictationCoreTests/TestAudio.swift @@ -0,0 +1,24 @@ +import AVFoundation +import XCTest + +@testable import PseudoDictationCore + +func writeWav(_ samples: [Float], to url: URL) throws { + let format = try XCTUnwrap( + AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: DictationEngine.sampleRate, channels: 1, + interleaved: false)) + let buffer = try XCTUnwrap( + AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count))) + buffer.frameLength = AVAudioFrameCount(samples.count) + samples.withUnsafeBufferPointer { source in + buffer.floatChannelData![0].update(from: source.baseAddress!, count: samples.count) + } + let file = try AVAudioFile( + forWriting: url, + settings: [ + AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: DictationEngine.sampleRate, + AVNumberOfChannelsKey: 1, AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, + ]) + try file.write(from: buffer) +} diff --git a/apps/mobile/modules/pseudo-dictation/expo-module.config.json b/apps/mobile/modules/pseudo-dictation/expo-module.config.json new file mode 100644 index 0000000000000..2a5dc44d9621f --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/expo-module.config.json @@ -0,0 +1,6 @@ +{ + "platforms": ["apple"], + "apple": { + "modules": ["PseudoDictationModule"] + } +} diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift new file mode 100644 index 0000000000000..02a4c475bd921 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationEngine.swift @@ -0,0 +1,647 @@ +import CoreML +import FluidAudio +import Foundation + +struct ModelStatus: Equatable, Sendable { + var installed: Bool + var warm: Bool + var downloading: Bool + var sizeOnDiskBytes: UInt64 +} + +struct DownloadUpdate: Equatable, Sendable { + enum Phase: String, Sendable { + case downloading + case preparing + } + + var phase: Phase + var fraction: Double + + /// FluidAudio spends the first half of its fraction on the download and the rest on compiling. + init(_ progress: DownloadProgress) { + switch progress.phase { + case .listing: + self.init(phase: .downloading, fraction: 0) + case .downloading: + self.init(phase: .downloading, fraction: min(max(progress.fractionCompleted * 2, 0), 1)) + case .compiling: + self.init(phase: .preparing, fraction: 1) + } + } + + init(phase: Phase, fraction: Double) { + self.phase = phase + self.fraction = fraction + } +} + +/// Passes on a download's progress in whole percent steps and never backwards, so the bridge is not +/// flooded and a resumed file cannot make the bar jump back. +final class ProgressRelay: @unchecked Sendable { + private let lock = NSLock() + private var last: DownloadUpdate? + private let emit: @Sendable (DownloadUpdate) -> Void + + init(emit: @escaping @Sendable (DownloadUpdate) -> Void) { + self.emit = emit + } + + func report(_ update: DownloadUpdate) { + lock.lock() + var next = update + if let last { + if last.phase == .preparing && update.phase == .downloading { + lock.unlock() + return + } + if last.phase == update.phase { + next.fraction = max(last.fraction, update.fraction) + // Whole percents, so a subtraction's rounding error cannot swallow a step or the final 100. + if (next.fraction * 100).rounded() <= (last.fraction * 100).rounded() { + lock.unlock() + return + } + } + } + last = next + lock.unlock() + emit(next) + } +} + +struct FilterOutcome: Equatable, Sendable { + var applied: Bool + var keptSeconds: Double + var droppedSeconds: Double + var skipped: FilterSkip? + + static func skipped(_ reason: FilterSkip) -> FilterOutcome { + FilterOutcome(applied: false, keptSeconds: 0, droppedSeconds: 0, skipped: reason) + } +} + +struct Transcript: Sendable { + var text: String + var audioSeconds: Double + var processingSeconds: Double + var filter: FilterOutcome? +} + +struct ClipCheck: Equatable, Sendable { + var speechSeconds: Double + var problem: EnrollmentProblem? +} + +struct VoiceProfileStatus: Equatable, Sendable { + var enrolled: Bool + var clipCount: Int + var createdAt: Date? +} + +/// The diarizer is a plain class. It is only ever used from the engine's queue, one job at a time. +private struct DiarizerBox: @unchecked Sendable { + let manager: OfflineDiarizerManager +} + +private struct HeardTurn: Sendable { + var turn: SpeakerTurn + var embedding: [Float]? +} + +private struct EnrollmentClip: Sendable { + var speechSeconds: Double + var embedding: [Float] + var problem: EnrollmentProblem? +} + +actor DictationEngine { + static let sampleRate = 16_000.0 + static let idleReleaseSeconds = 300.0 + static let minimumEnrollmentSpeechSeconds = 3.0 + static let dominantSpeakerShare = 0.8 + static let enrollmentClipCount = 3 + static let rememberedCancels = 8 + + private struct LoadKey: Equatable { + var model: DictationModel + var processing: DictationProcessing + } + + private let storage: DictationStorage + private let environment: DictationEnvironment + private var asr: (key: LoadKey, task: Task)? + private var diarizer: Task? + private var downloads: [DictationModel: Task] = [:] + private var requests: [String: Task] = [:] + /// Cancels that arrived before their transcription did: each bridge call runs in its own task. + private var cancelledEarly: [String] = [] + private var sizes: [DictationModel: UInt64] = [:] + private var queueTail: Task? + private var queued = 0 + private var idleRelease: Task? + + /// Dictation never reaches for the network. Only an explicit download lifts this. Set once, so + /// a second engine cannot cut off a download the first one is running. + private static let offlineByDefault: Void = { ModelHub.offlineMode = true }() + + init(storage: DictationStorage, environment: DictationEnvironment = .current) { + self.storage = storage + self.environment = environment + _ = Self.offlineByDefault + } + + // MARK: Models + + func status(_ model: DictationModel, processing: DictationProcessing) -> ModelStatus { + let installed = storage.isInstalled(model) + // Walking the model's files is too slow for a call made on every tap of the microphone. + if installed, sizes[model] == nil { sizes[model] = storage.sizeOnDisk(model) } + return ModelStatus( + installed: installed, + warm: installed && storage.isWarm(model, key: warmKey(processing)), + downloading: downloads[model] != nil, + sizeOnDiskBytes: installed ? sizes[model] ?? 0 : 0 + ) + } + + func download( + _ model: DictationModel, + processing: DictationProcessing, + onProgress: @escaping @Sendable (DownloadUpdate) -> Void + ) async throws { + if let running = downloads[model] { + do { + return try await running.value + } catch { + throw running.isCancelled ? DictationError.cancelled : Self.describe(error) + } + } + try storage.prepare() + let free = storage.isInstalled(model) ? nil : environment.freeDisk(storage.root) + if let free, free < model.downloadDiskBytes { + throw DictationError.notEnoughDisk(requiredBytes: model.downloadDiskBytes, availableBytes: free) + } + + let relay = ProgressRelay(emit: onProgress) + let storage = storage + let isSimulator = environment.isSimulator + let units = processing.computeUnits(isSimulator: isSimulator) + ModelHub.offlineMode = false + let task = Task.detached(priority: .userInitiated) { + let handler: ProgressHandler = { relay.report(DownloadUpdate($0)) } + if let version = model.asrVersion { + _ = try await AsrModels.download( + to: storage.modelDirectory(model), version: version, encoderComputeUnits: units, + progressHandler: handler) + } else { + _ = try await OfflineDiarizerModels.load( + from: storage.modelsRoot, + configuration: Self.diarizerConfiguration(isSimulator: isSimulator), + progressHandler: handler) + } + try Task.checkCancellation() + try storage.markInstalled(model) + } + downloads[model] = task + let result = await task.result + downloads[model] = nil + sizes[model] = nil + if downloads.isEmpty { ModelHub.offlineMode = true } + + do { + try result.get() + } catch { + // A cancelled download gives its space back. A failed one keeps its files so a retry resumes. + if task.isCancelled, !storage.isInstalled(model) { try? storage.delete(model) } + throw task.isCancelled ? DictationError.cancelled : Self.describe(error) + } + if model.asrVersion != nil { + relay.report(DownloadUpdate(phase: .preparing, fraction: 1)) + // The model is installed either way. A load that fails here is tried again by a dictation. + try? await prepare(model, processing: processing) + } + } + + func cancelDownload(_ model: DictationModel) { + downloads[model]?.cancel() + } + + func delete(_ model: DictationModel) async throws { + downloads[model]?.cancel() + _ = await downloads[model]?.result + let work = enqueue { try await self.deleteNow(model) } + try await finish(work) + } + + private func deleteNow(_ model: DictationModel) async throws { + if model == .voiceFilter { + diarizer = nil + } else if asr?.key.model == model { + await dropAsr() + } + sizes[model] = nil + try storage.delete(model) + } + + /// Loads a model and records that this build, OS and processing choice have compiled it. + func prepare(_ model: DictationModel, processing: DictationProcessing) async throws { + let key = LoadKey(model: model, processing: processing) + let work = enqueue { + _ = try await self.value(of: try await self.loadTask(key), key: key) + try await self.storage.markWarm(model, key: self.warmKey(processing)) + } + try await finish(work) + } + + /// Frees the loaded models unless one is in use. Called when idle and on a memory warning. + func releaseIfIdle() async { + guard queued == 0 else { return } + idleRelease?.cancel() + idleRelease = nil + diarizer = nil + await dropAsr() + } + + // MARK: Transcription + + func transcribe( + requestId: String, + url: URL, + model: DictationModel, + processing: DictationProcessing, + onlyMyVoice: Bool, + language: String? + ) async throws -> Transcript { + if let early = cancelledEarly.firstIndex(of: requestId) { + cancelledEarly.remove(at: early) + throw DictationError.cancelled + } + let work = enqueue { + try await self.runTranscription( + url: url, model: model, processing: processing, onlyMyVoice: onlyMyVoice, language: language) + } + requests[requestId] = work + defer { requests[requestId] = nil } + return try await withTaskCancellationHandler { + try await finish(work) + } onCancel: { + work.cancel() + } + } + + func cancelTranscription(requestId: String) { + if let request = requests[requestId] { return request.cancel() } + cancelledEarly = Array((cancelledEarly + [requestId]).suffix(Self.rememberedCancels)) + } + + private func runTranscription( + url: URL, + model: DictationModel, + processing: DictationProcessing, + onlyMyVoice: Bool, + language: String? + ) async throws -> Transcript { + let started = Date() + let key = LoadKey(model: model, processing: processing) + // The load starts before decoding so the two overlap. + let loading = try await loadTask(key) + let samples = try await Self.decodeOffActor(url) + let audioSeconds = Double(samples.count) / Self.sampleRate + guard AudioGate.hasSpeech(samples: samples, sampleRate: Self.sampleRate) else { + return Transcript( + text: "", audioSeconds: audioSeconds, + processingSeconds: Date().timeIntervalSince(started), + filter: onlyMyVoice ? .skipped(.noSpeech) : nil) + } + + var input = samples + var filter: FilterOutcome? + if onlyMyVoice { + let filtered = await filterToEnrolledVoice(samples) + input = filtered.samples + filter = filtered.outcome + } + try Task.checkCancellation() + + let manager = try await value(of: loading, key: key) + var decoderState = TdtDecoderState.make(decoderLayers: await manager.decoderLayerCount) + let result = try await manager.transcribe( + input, decoderState: &decoderState, language: Self.languageHint(language)) + try Task.checkCancellation() + try? storage.markWarm(model, key: warmKey(processing)) + return Transcript( + text: result.text.trimmingCharacters(in: .whitespacesAndNewlines), + audioSeconds: audioSeconds, + processingSeconds: Date().timeIntervalSince(started), + filter: filter) + } + + // MARK: Voice profile + + func voiceProfileStatus() -> VoiceProfileStatus { + let profile = storage.loadVoiceProfile() + return VoiceProfileStatus( + enrolled: profile != nil, clipCount: profile?.clipCount ?? 0, createdAt: profile?.createdAt) + } + + func checkEnrollmentClip(_ url: URL) async throws -> ClipCheck { + let work = enqueue { try await self.enrollmentClip(url) } + let clip = try await finish(work) + return ClipCheck(speechSeconds: clip.speechSeconds, problem: clip.problem) + } + + func saveVoiceProfile(_ urls: [URL]) async throws { + guard urls.count >= Self.enrollmentClipCount else { + throw DictationError.enrollmentRejected(.tooFewClips) + } + let work = enqueue { () async throws -> [[Float]] in + var embeddings: [[Float]] = [] + for url in urls { + let clip = try await self.enrollmentClip(url) + if let problem = clip.problem { throw DictationError.enrollmentRejected(problem) } + embeddings.append(clip.embedding) + } + return embeddings + } + let embeddings = try await finish(work) + // Pair by pair: against their mean, one stranger's clip pulls the mean far enough to pass. + let agree = embeddings.indices.allSatisfy { first in + embeddings.indices.allSatisfy { second in + SpeakerFilter.cosineDistance(embeddings[first], embeddings[second]) + <= SpeakerFilter.agreementDistance + } + } + guard agree, let mean = SpeakerFilter.meanEmbedding(embeddings) else { + throw DictationError.enrollmentRejected(.clipsDisagree) + } + try storage.saveVoiceProfile( + VoiceProfile( + extractor: VoiceProfile.currentExtractor, embedding: mean, clipCount: embeddings.count, + createdAt: Date())) + } + + func deleteVoiceProfile() throws { + try storage.deleteVoiceProfile() + } + + private func enrollmentClip(_ url: URL) async throws -> EnrollmentClip { + let samples = try await Self.decodeOffActor(url) + guard AudioGate.hasSpeech(samples: samples, sampleRate: Self.sampleRate) else { + return EnrollmentClip(speechSeconds: 0, embedding: [], problem: .tooQuiet) + } + let heard = try await heardTurns(samples).compactMap { turn in + turn.embedding.map { (seconds: turn.turn.duration, embedding: $0) } + } + guard let longest = heard.max(by: { $0.seconds < $1.seconds }) else { + return EnrollmentClip(speechSeconds: 0, embedding: [], problem: .tooShort) + } + // The longest turn is taken to be the speaker; the rest of the clip has to sound like it. + let same = heard.filter { + let distance = SpeakerFilter.cosineDistance($0.embedding, longest.embedding) + return SpeakerFilter.judge(duration: $0.seconds, distance: distance) == .mine + } + let speechSeconds = same.reduce(0) { $0 + $1.seconds } + let heardSeconds = heard.reduce(0) { $0 + $1.seconds } + if speechSeconds / heardSeconds < Self.dominantSpeakerShare { + return EnrollmentClip(speechSeconds: speechSeconds, embedding: [], problem: .multipleVoices) + } + guard speechSeconds >= Self.minimumEnrollmentSpeechSeconds, + let embedding = SpeakerFilter.meanEmbedding( + same.map(\.embedding), weights: same.map { Float($0.seconds) }) + else { + return EnrollmentClip(speechSeconds: speechSeconds, embedding: [], problem: .tooShort) + } + return EnrollmentClip(speechSeconds: speechSeconds, embedding: embedding, problem: nil) + } + + /// Any failure here leaves the recording whole: the filter may be skipped, never the dictation. + private func filterToEnrolledVoice(_ samples: [Float]) async -> (samples: [Float], outcome: FilterOutcome) { + guard let profile = storage.loadVoiceProfile() else { return (samples, .skipped(.noProfile)) } + guard storage.isInstalled(.voiceFilter) else { return (samples, .skipped(.notInstalled)) } + guard let heard = try? await heardTurns(samples, splitAgainst: profile.embedding) else { + return (samples, .skipped(.failed)) + } + let voices = heard.map { turn -> TurnVoice in + guard let embedding = turn.embedding else { return .unjudged } + return SpeakerFilter.judge( + duration: turn.turn.duration, + distance: SpeakerFilter.cosineDistance(embedding, profile.embedding)) + } + switch SpeakerFilter.decide(turns: heard.map(\.turn), voices: voices) { + case .unfiltered(let reason): + return (samples, .skipped(reason)) + case .drop(let ranges): + let droppedSeconds = SpeakerFilter.seconds(ranges) + return ( + SpeakerFilter.remove(samples: samples, ranges: ranges, sampleRate: Self.sampleRate), + FilterOutcome( + applied: true, keptSeconds: Double(samples.count) / Self.sampleRate - droppedSeconds, + droppedSeconds: droppedSeconds, skipped: nil) + ) + } + } + + /// The recording's speaker turns, with a voiceprint for each one long enough to judge. + /// + /// The diarizer finds the turns well, but in a recording of a few seconds it labels every turn + /// as one speaker. So its labels are not trusted: each turn gets its own voiceprint. It also + /// joins a short interjection to the turn beside it, so with a voice profile to compare against, + /// long turns are split where the voice changes. + private func heardTurns(_ samples: [Float], splitAgainst profile: [Float]? = nil) async throws -> [HeardTurn] { + let manager = try await diarizerManager() + let segments: [TimedSpeakerSegment] + do { + segments = try await manager.process(audio: samples).segments + } catch OfflineDiarizationError.noSpeechDetected { + return [] + } + let totalSeconds = Double(samples.count) / Self.sampleRate + let source = ArrayAudioSampleSource(samples: samples) + var turns = segments.map { + SpeakerTurn( + speaker: $0.speakerId, start: max(Double($0.startTimeSeconds), 0), + end: min(Double($0.endTimeSeconds), totalSeconds)) + } + let scans = turns.map(SpeakerFilter.scanWindows) + if let profile, scans.contains(where: { !$0.isEmpty }) { + var distances = try await manager.embedSpans(audioSource: source, spans: scans.flatMap { $0 }) + .map { SpeakerFilter.cosineDistance($0, profile) }[...] + turns = zip(turns, scans).flatMap { turn, windows in + defer { distances = distances.dropFirst(windows.count) } + return SpeakerFilter.split( + turn, windows: windows, distances: Array(distances.prefix(windows.count))) + } + } + var heard = turns.map { HeardTurn(turn: $0, embedding: nil) } + let judged = heard.indices.filter { + heard[$0].turn.duration >= SpeakerFilter.minimumJudgedSeconds + } + guard !judged.isEmpty else { return heard } + let embeddings = try await manager.embedSpans( + audioSource: source, spans: judged.map { heard[$0].turn.start..(_ work: @escaping @Sendable () async throws -> T) -> Task { + let previous = queueTail + queued += 1 + idleRelease?.cancel() + let task = Task { + await previous?.value + try Task.checkCancellation() + return try await work() + } + queueTail = Task { _ = await task.result } + return task + } + + @discardableResult + private func finish(_ task: Task) async throws -> T { + let result = await task.result + queued -= 1 + if queued == 0 { scheduleIdleRelease() } + do { + return try result.get() + } catch { + throw task.isCancelled ? DictationError.cancelled : Self.describe(error) + } + } + + private func scheduleIdleRelease() { + idleRelease?.cancel() + idleRelease = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(Self.idleReleaseSeconds * 1_000_000_000)) + guard !Task.isCancelled else { return } + await self?.releaseIfIdle() + } + } + + // MARK: Loading + + private func warmKey(_ processing: DictationProcessing) -> WarmKey { + WarmKey( + appBuild: environment.appBuild, osBuild: environment.osBuild, + processing: String(processing.computeUnits(isSimulator: environment.isSimulator).rawValue)) + } + + private func value(of task: Task, key: LoadKey) async throws -> AsrManager { + do { + return try await task.value + } catch { + if asr?.key == key { asr = nil } + throw error + } + } + + /// Only called from queued work, so the suspension below cannot interleave two loads. + private func loadTask(_ key: LoadKey) async throws -> Task { + if let asr, asr.key == key { return asr.task } + guard let version = key.model.asrVersion else { + throw DictationError.unknownModel(key.model.rawValue) + } + guard storage.isInstalled(key.model) else { throw DictationError.notInstalled(key.model) } + // The old model goes first: two at once would double the memory the gate below allows for. + await dropAsr() + let available = environment.availableMemory() + guard hasEnoughMemory(available: available, required: key.model.loadMemoryBytes) else { + throw DictationError.notEnoughMemory( + requiredBytes: key.model.loadMemoryBytes, availableBytes: available) + } + let directory = storage.modelDirectory(key.model) + let units = key.processing.computeUnits(isSimulator: environment.isSimulator) + let task = Task.detached(priority: .userInitiated) { () throws -> AsrManager in + let configuration = AsrModels.defaultConfiguration() + configuration.computeUnits = units + let models = try AsrModels.loadLocal( + from: directory, version: version, configuration: configuration, + encoderComputeUnits: units) + let manager = AsrManager(config: .default) + try await manager.loadModels(models) + return manager + } + asr = (key, task) + return task + } + + private func dropAsr() async { + guard let current = asr else { return } + asr = nil + await (try? current.task.value)?.cleanup() + } + + private func diarizerManager() async throws -> OfflineDiarizerManager { + if let diarizer { return try await diarizer.value.manager } + guard storage.isInstalled(.voiceFilter) else { throw DictationError.notInstalled(.voiceFilter) } + let available = environment.availableMemory() + guard hasEnoughMemory(available: available, required: DictationModel.voiceFilter.loadMemoryBytes) + else { + throw DictationError.notEnoughMemory( + requiredBytes: DictationModel.voiceFilter.loadMemoryBytes, availableBytes: available) + } + let root = storage.modelsRoot + let isSimulator = environment.isSimulator + let task = Task.detached(priority: .userInitiated) { () throws -> DiarizerBox in + let models = try await OfflineDiarizerModels.load( + from: root, configuration: Self.diarizerConfiguration(isSimulator: isSimulator)) + var config = OfflineDiarizerConfig.default + // Dictations are short, and smaller batches keep the peak memory down on a phone. + config.segmentation.batchSize = min(config.segmentation.batchSize, 8) + config.embedding.batchSize = min(config.embedding.batchSize, 8) + let manager = OfflineDiarizerManager(config: config) + manager.initialize(models: models) + return DiarizerBox(manager: manager) + } + diarizer = task + do { + return try await task.value.manager + } catch { + diarizer = nil + throw error + } + } + + /// Never the GPU: iOS refuses GPU work in the background, where a dictation may finish. + private static func diarizerConfiguration(isSimulator: Bool) -> MLModelConfiguration { + let configuration = MLModelConfiguration() + configuration.computeUnits = isSimulator ? .cpuOnly : .cpuAndNeuralEngine + return configuration + } + + // MARK: Helpers + + static func decode(_ url: URL) throws -> [Float] { + do { + return try AudioConverter().resampleAudioFile(url) + } catch { + throw DictationError.audioUnreadable(error.localizedDescription) + } + } + + /// Decoding and resampling a recording takes long enough to hold up a cancel on the actor. + private static func decodeOffActor(_ url: URL) async throws -> [Float] { + try await Task.detached(priority: .userInitiated) { try decode(url) }.value + } + + /// The hint only steers the multilingual models away from the wrong script; an unknown language + /// means no hint. + static func languageHint(_ locale: String?) -> Language? { + guard let code = locale?.split(whereSeparator: { $0 == "-" || $0 == "_" }).first else { + return nil + } + return Language(rawValue: code.lowercased()) + } + + static func describe(_ error: Error) -> DictationError { + if let error = error as? DictationError { return error } + if error is CancellationError { return .cancelled } + return .failed(error.localizedDescription) + } +} diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/DictationStorage.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationStorage.swift new file mode 100644 index 0000000000000..cb0ce47ab089d --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationStorage.swift @@ -0,0 +1,142 @@ +import FluidAudio +import Foundation + +/// What a compiled model was last loaded under. Core ML recompiles for the Neural Engine when any of +/// these change, and that first load can take minutes, so it must not happen on a dictation tap. +struct WarmKey: Codable, Equatable { + var appBuild: String + var osBuild: String + var processing: String +} + +struct VoiceProfile: Codable, Equatable { + static let currentExtractor = "fluidaudio-offline-span-256" + + var extractor: String + var embedding: [Float] + var clipCount: Int + var createdAt: Date +} + +struct DictationStorage: Sendable { + let root: URL + + static var applicationSupport: DictationStorage { + let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + return DictationStorage(root: base.appendingPathComponent("PseudoDictation", isDirectory: true)) + } + + var modelsRoot: URL { root.appendingPathComponent("Models", isDirectory: true) } + private var stateRoot: URL { root.appendingPathComponent("State", isDirectory: true) } + private var profileURL: URL { root.appendingPathComponent("voice-profile.json") } + + func modelDirectory(_ model: DictationModel) -> URL { + modelsRoot.appendingPathComponent(model.repo.folderName, isDirectory: true) + } + + private func installMarker(_ model: DictationModel) -> URL { + stateRoot.appendingPathComponent("\(model.rawValue).installed") + } + + private func warmMarker(_ model: DictationModel) -> URL { + stateRoot.appendingPathComponent("\(model.rawValue).warm.json") + } + + /// Models are large and can be downloaded again, so they stay out of device backups. + func prepare() throws { + let manager = FileManager.default + try manager.createDirectory(at: modelsRoot, withIntermediateDirectories: true) + try manager.createDirectory(at: stateRoot, withIntermediateDirectories: true) + var values = URLResourceValues() + values.isExcludedFromBackup = true + var target = root + try target.setResourceValues(values) + } + + func markInstalled(_ model: DictationModel) throws { + try prepare() + try Data().write(to: installMarker(model), options: .atomic) + } + + /// The marker says a download finished; the file check catches a folder damaged or cleared since. + func isInstalled(_ model: DictationModel) -> Bool { + guard FileManager.default.fileExists(atPath: installMarker(model).path) else { return false } + let directory = modelDirectory(model) + if let version = model.asrVersion { + return AsrModels.modelsExist(at: directory, version: version) + } + return ModelNames.OfflineDiarizer.requiredModels.allSatisfy { name in + Self.isLoadReady(directory.appendingPathComponent(name)) + } + } + + private static func isLoadReady(_ url: URL) -> Bool { + let manager = FileManager.default + var isDirectory: ObjCBool = false + guard manager.fileExists(atPath: url.path, isDirectory: &isDirectory) else { return false } + guard isDirectory.boolValue else { return true } + guard manager.fileExists(atPath: url.appendingPathComponent("coremldata.bin").path) else { + return false + } + let contents = manager.enumerator(at: url, includingPropertiesForKeys: nil) + while let item = contents?.nextObject() as? URL { + if item.pathExtension == "partial" { return false } + } + return true + } + + func delete(_ model: DictationModel) throws { + let manager = FileManager.default + for url in [installMarker(model), warmMarker(model), modelDirectory(model)] + where manager.fileExists(atPath: url.path) { + try manager.removeItem(at: url) + } + } + + func sizeOnDisk(_ model: DictationModel) -> UInt64 { + let keys: [URLResourceKey] = [.totalFileAllocatedSizeKey, .isRegularFileKey] + let contents = FileManager.default.enumerator( + at: modelDirectory(model), includingPropertiesForKeys: keys) + var total: UInt64 = 0 + while let item = contents?.nextObject() as? URL { + let values = try? item.resourceValues(forKeys: Set(keys)) + if values?.isRegularFile == true { + total += UInt64(values?.totalFileAllocatedSize ?? 0) + } + } + return total + } + + func isWarm(_ model: DictationModel, key: WarmKey) -> Bool { + guard let data = try? Data(contentsOf: warmMarker(model)) else { return false } + return (try? JSONDecoder().decode(WarmKey.self, from: data)) == key + } + + func markWarm(_ model: DictationModel, key: WarmKey) throws { + try prepare() + try JSONEncoder().encode(key).write(to: warmMarker(model), options: .atomic) + } + + /// A profile from another extractor holds vectors this build cannot compare, so it reads as absent. + func loadVoiceProfile() -> VoiceProfile? { + guard + let data = try? Data(contentsOf: profileURL), + let profile = try? JSONDecoder().decode(VoiceProfile.self, from: data), + profile.extractor == VoiceProfile.currentExtractor, + !profile.embedding.isEmpty + else { return nil } + return profile + } + + func saveVoiceProfile(_ profile: VoiceProfile) throws { + try prepare() + try JSONEncoder().encode(profile).write( + to: profileURL, options: [.atomic, .completeFileProtectionUntilFirstUserAuthentication]) + } + + func deleteVoiceProfile() throws { + if FileManager.default.fileExists(atPath: profileURL.path) { + try FileManager.default.removeItem(at: profileURL) + } + } +} diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/DictationTypes.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationTypes.swift new file mode 100644 index 0000000000000..b8b7a093ae31e --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/DictationTypes.swift @@ -0,0 +1,162 @@ +import CoreML +import FluidAudio +import Foundation + +enum DictationModel: String, CaseIterable, Sendable { + case fast = "parakeet-110m" + case balanced = "parakeet-v3" + case accurate = "parakeet-ultra" + case voiceFilter = "voice-filter" + + var asrVersion: AsrModelVersion? { + switch self { + case .fast: return .tdtCtc110m + case .balanced: return .v3 + case .accurate: return .ultra + case .voiceFilter: return nil + } + } + + var repo: Repo { + switch self { + case .fast: return .parakeetTdtCtc110m + case .balanced: return .parakeetV3 + case .accurate: return .parakeetUltra + case .voiceFilter: return .diarizer + } + } + + /// Free memory a load needs before it starts. Lower bounds, not measured peaks: a load attempted + /// with less than this is expected to end in the system killing the app. + var loadMemoryBytes: UInt64 { + switch self { + case .fast: return 150 << 20 + case .balanced: return 350 << 20 + case .accurate: return 450 << 20 + case .voiceFilter: return 120 << 20 + } + } + + /// Disk needed to start a download, with room for the compiled copies Core ML keeps. + var downloadDiskBytes: UInt64 { + switch self { + case .fast: return 600 << 20 + case .balanced: return 1200 << 20 + case .accurate: return 1500 << 20 + case .voiceFilter: return 200 << 20 + } + } +} + +enum DictationProcessing: String, Sendable { + case auto + case neuralEngine + case gpu + case cpu + + func computeUnits(isSimulator: Bool) -> MLComputeUnits { + if isSimulator { return .cpuOnly } + switch self { + case .auto, .neuralEngine: return .cpuAndNeuralEngine + case .gpu: return .cpuAndGPU + case .cpu: return .cpuOnly + } + } +} + +enum DictationError: Error, Equatable { + case unknownModel(String) + case notInstalled(DictationModel) + case notEnoughMemory(requiredBytes: UInt64, availableBytes: UInt64) + case notEnoughDisk(requiredBytes: UInt64, availableBytes: UInt64) + case noVoiceProfile + case enrollmentRejected(EnrollmentProblem) + case audioUnreadable(String) + case cancelled + case failed(String) + + var code: String { + switch self { + case .unknownModel: return "unknown-model" + case .notInstalled: return "not-installed" + case .notEnoughMemory: return "not-enough-memory" + case .notEnoughDisk: return "not-enough-disk" + case .noVoiceProfile: return "no-voice-profile" + case .enrollmentRejected: return "enrollment-rejected" + case .audioUnreadable: return "audio-unreadable" + case .cancelled: return "cancelled" + case .failed: return "failed" + } + } + + var message: String { + switch self { + case .unknownModel(let id): return "Unknown dictation model \(id)." + case .notInstalled: return "This dictation model is not downloaded." + case .notEnoughMemory: return "Not enough free memory to load this dictation model." + case .notEnoughDisk(let required, _): + return "Not enough free storage. This download needs about \(required >> 20) MB free." + case .noVoiceProfile: return "No voice profile is saved." + case .enrollmentRejected(let problem): return problem.message + case .audioUnreadable(let detail): return "The recording could not be read. \(detail)" + case .cancelled: return "Cancelled." + case .failed(let detail): return detail + } + } +} + +enum EnrollmentProblem: String, Sendable { + case tooQuiet + case tooShort + case multipleVoices + case clipsDisagree + case tooFewClips + + var message: String { + switch self { + case .tooQuiet: return "No speech was heard. Move closer to the microphone and try again." + case .tooShort: return "Not enough speech. Read the whole prompt." + case .multipleVoices: return "More than one voice was heard. Record somewhere quieter." + case .clipsDisagree: return "The recordings do not sound like the same voice. Record them again." + case .tooFewClips: return "Record every sample before saving." + } + } +} + +/// `available` of 0 means the system gave no reading, which is what the simulator and macOS return. +func hasEnoughMemory(available: UInt64, required: UInt64) -> Bool { + available == 0 || available >= required +} + +struct DictationEnvironment: Sendable { + var appBuild: String + var osBuild: String + var isSimulator: Bool + var availableMemory: @Sendable () -> UInt64 + var freeDisk: @Sendable (URL) -> UInt64? + + static var current: DictationEnvironment { + let info = Bundle.main.infoDictionary + #if targetEnvironment(simulator) + let isSimulator = true + #else + let isSimulator = false + #endif + return DictationEnvironment( + appBuild: "\(info?["CFBundleShortVersionString"] ?? "0")(\(info?["CFBundleVersion"] ?? "0"))", + osBuild: ProcessInfo.processInfo.operatingSystemVersionString, + isSimulator: isSimulator, + availableMemory: { + #if os(iOS) && !targetEnvironment(simulator) + return UInt64(os_proc_available_memory()) + #else + return 0 + #endif + }, + freeDisk: { url in + let values = try? url.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + return values?.volumeAvailableCapacityForImportantUsage.map { UInt64(max($0, 0)) } + } + ) + } +} diff --git a/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift new file mode 100644 index 0000000000000..79626d73289bc --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/ios/Core/SpeakerFilter.swift @@ -0,0 +1,229 @@ +import Foundation + +struct SpeakerTurn: Equatable, Sendable { + var speaker: String + var start: Double + var end: Double + + var duration: Double { end - start } +} + +enum TurnVoice: Equatable, Sendable { + case mine + case other + case unjudged +} + +enum FilterSkip: String, Sendable { + case noSpeech + case noMatch + case onlyYou + case tooShort + case overlapping + case noProfile + case notInstalled + case failed +} + +enum SpeakerDecision: Equatable { + case drop([ClosedRange]) + case unfiltered(FilterSkip) +} + +enum SpeakerFilter { + // Set from 19 Mini LibriSpeech speakers; RealVoiceTests measures the result. They lean towards + // keeping a stranger's turn, because cutting the user's own words is the worse error. + /// Cosine distance to the voice profile at or under which a turn is the enrolled voice. + static let matchDistance: Float = 0.6 + /// A turn under `shortTurnSeconds` gives a noisier voiceprint, so it has to be further off. + static let shortTurnMatchDistance: Float = 0.75 + static let shortTurnSeconds = 2.0 + /// Under this a turn says too little about its voice to be judged at all. + static let minimumJudgedSeconds = 1.0 + /// Enrollment clips further apart than this are not one voice. + static let agreementDistance: Float = 0.45 + static let minimumDropSeconds = 0.2 + static let joinSilenceSeconds = 0.15 + /// The diarizer joins a short interjection to the turn beside it, so a turn this long is + /// scanned for a change of voice inside it. + static let scanSeconds = 3.0 + static let scanWindowSeconds = 1.5 + static let scanHopSeconds = 0.5 + + static func normalized(_ vector: [Float]) -> [Float] { + let length = vector.reduce(Float(0)) { $0 + $1 * $1 }.squareRoot() + guard length > 0, length.isFinite else { return vector } + return vector.map { $0 / length } + } + + /// The unit-length mean of the vectors' directions. Nil when they differ in size or cancel out. + static func meanEmbedding(_ vectors: [[Float]], weights: [Float]? = nil) -> [Float]? { + guard let size = vectors.first?.count, size > 0, vectors.allSatisfy({ $0.count == size }), + weights == nil || weights?.count == vectors.count + else { + return nil + } + var sum = [Float](repeating: 0, count: size) + for (position, vector) in vectors.enumerated() { + let weight = weights?[position] ?? 1 + for (index, value) in normalized(vector).enumerated() { sum[index] += value * weight } + } + let mean = normalized(sum) + return mean.contains(where: { $0 != 0 }) ? mean : nil + } + + static func cosineDistance(_ a: [Float], _ b: [Float]) -> Float { + guard a.count == b.count, !a.isEmpty else { return .infinity } + var dot: Float = 0 + var lengthA: Float = 0 + var lengthB: Float = 0 + for index in a.indices { + dot += a[index] * b[index] + lengthA += a[index] * a[index] + lengthB += b[index] * b[index] + } + guard lengthA > 0, lengthB > 0 else { return .infinity } + return 1 - dot / (lengthA.squareRoot() * lengthB.squareRoot()) + } + + static func judge(duration: Double, distance: Float) -> TurnVoice { + if duration < minimumJudgedSeconds { return .unjudged } + let limit = duration < shortTurnSeconds ? shortTurnMatchDistance : matchDistance + return distance <= limit ? .mine : .other + } + + /// The windows that scan a turn, evenly spaced, the last one ending where the turn does. + static func scanWindows(_ turn: SpeakerTurn) -> [Range] { + guard turn.duration >= scanSeconds else { return [] } + let lastStart = turn.end - scanWindowSeconds + var starts: [Double] = [] + var start = turn.start + while start < lastStart - scanHopSeconds / 2 { + starts.append(start) + start += scanHopSeconds + } + starts.append(lastStart) + return starts.map { $0..<($0 + scanWindowSeconds) } + } + + /// The turn cut where its voice changes: where the windows' distances to the voice profile + /// cross `matchDistance`, placed between the two window centres on either side. A stretch + /// between two changes that is too short to judge is taken for noise and left in place. + static func split(_ turn: SpeakerTurn, windows: [Range], distances: [Float]) -> [SpeakerTurn] { + precondition(windows.count == distances.count) + var changes: [Double] = [] + for index in windows.indices.dropLast() { + let (near, far) = (distances[index], distances[index + 1]) + guard (near <= matchDistance) != (far <= matchDistance) else { continue } + let from = (windows[index].lowerBound + windows[index].upperBound) / 2 + let to = (windows[index + 1].lowerBound + windows[index + 1].upperBound) / 2 + changes.append(from + Double((matchDistance - near) / (far - near)) * (to - from)) + } + while let closest = changes.indices.dropLast().min(by: { + changes[$0 + 1] - changes[$0] < changes[$1 + 1] - changes[$1] + }), changes[closest + 1] - changes[closest] < minimumJudgedSeconds { + changes.removeSubrange(closest...closest + 1) + } + let edges = [turn.start] + changes.filter { $0 > turn.start && $0 < turn.end } + [turn.end] + return zip(edges, edges.dropFirst()).map { SpeakerTurn(speaker: turn.speaker, start: $0, end: $1) } + } + + /// What to cut from a recording. Only turns judged to be another voice go, and only where no + /// kept turn overlaps them. A turn too short to judge goes when its speaker's judged turns were + /// all another voice. With no turn judged to be the user, nothing is cut: a missed match on + /// their own voice must not lose the dictation. + static func decide(turns: [SpeakerTurn], voices: [TurnVoice]) -> SpeakerDecision { + precondition(turns.count == voices.count) + if turns.isEmpty { return .unfiltered(.noSpeech) } + if voices.allSatisfy({ $0 == .unjudged }) { return .unfiltered(.tooShort) } + if !voices.contains(.mine) { return .unfiltered(.noMatch) } + if !voices.contains(.other) { return .unfiltered(.onlyYou) } + + var judgedBySpeaker: [String: Set] = [:] + for (turn, voice) in zip(turns, voices) where voice != .unjudged { + judgedBySpeaker[turn.speaker, default: []].insert(voice) + } + var dropped: [ClosedRange] = [] + var kept: [ClosedRange] = [] + for (turn, voice) in zip(turns, voices) where turn.end > turn.start { + let isOther = voice == .other || (voice == .unjudged && judgedBySpeaker[turn.speaker] == [.other]) + if isOther { dropped.append(turn.start...turn.end) } else { kept.append(turn.start...turn.end) } + } + let ranges = subtract(merged(dropped), merged(kept)).filter { + $0.upperBound - $0.lowerBound >= minimumDropSeconds + } + return ranges.isEmpty ? .unfiltered(.overlapping) : .drop(ranges) + } + + static func merged(_ ranges: [ClosedRange]) -> [ClosedRange] { + var output: [ClosedRange] = [] + for range in ranges.sorted(by: { $0.lowerBound < $1.lowerBound }) { + if let last = output.last, range.lowerBound <= last.upperBound { + output[output.count - 1] = last.lowerBound...max(last.upperBound, range.upperBound) + } else { + output.append(range) + } + } + return output + } + + /// `ranges` with every part inside a hole removed. Both lists are sorted and merged. + static func subtract(_ ranges: [ClosedRange], _ holes: [ClosedRange]) -> [ClosedRange] { + var output: [ClosedRange] = [] + for range in ranges { + var start = range.lowerBound + for hole in holes where hole.upperBound > start && hole.lowerBound < range.upperBound { + if hole.lowerBound > start { output.append(start...hole.lowerBound) } + start = max(start, hole.upperBound) + } + if start < range.upperBound { output.append(start...range.upperBound) } + } + return output + } + + /// The recording without `ranges`, with a short silence at each cut so words do not run together. + static func remove(samples: [Float], ranges: [ClosedRange], sampleRate: Double) -> [Float] { + let silence = [Float](repeating: 0, count: Int(joinSilenceSeconds * sampleRate)) + var output: [Float] = [] + var cursor = 0 + func keep(upTo end: Int) { + guard end > cursor else { return } + if !output.isEmpty { output.append(contentsOf: silence) } + output.append(contentsOf: samples[cursor..]) -> Double { + ranges.reduce(0) { $0 + ($1.upperBound - $1.lowerBound) } + } +} + +enum AudioGate { + static let minimumSeconds = 0.3 + static let floorDecibels: Float = -55 + private static let frameSeconds = 0.02 + + /// False for recordings too short or too quiet to hold speech. Speech models write plausible + /// words for silence, so those recordings never reach one. + static func hasSpeech(samples: [Float], sampleRate: Double) -> Bool { + guard Double(samples.count) >= minimumSeconds * sampleRate else { return false } + let frame = max(Int(frameSeconds * sampleRate), 1) + var loudest: Float = 0 + var index = 0 + while index + frame <= samples.count { + var energy: Float = 0 + for sample in samples[index..<(index + frame)] { energy += sample * sample } + loudest = max(loudest, energy / Float(frame)) + index += frame + } + guard loudest > 0 else { return false } + return 10 * log10(loudest) >= floorDecibels + } +} diff --git a/apps/mobile/modules/pseudo-dictation/ios/PseudoDictation.podspec b/apps/mobile/modules/pseudo-dictation/ios/PseudoDictation.podspec new file mode 100644 index 0000000000000..107e501521b90 --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/ios/PseudoDictation.podspec @@ -0,0 +1,25 @@ +Pod::Spec.new do |s| + s.name = 'PseudoDictation' + s.version = '1.0.0' + s.summary = 'On-device dictation models for PseudoCode.' + s.description = 'Local speech recognition, voice filtering and model storage built on FluidAudio.' + s.author = 'pseudoseed' + s.homepage = 'https://github.com/pseudoseed/t3code_dev' + s.platforms = { + :ios => '18.0', + } + s.source = { :path => '.' } + s.static_framework = true + s.swift_version = '5.9' + + s.dependency 'ExpoModulesCore' + s.dependency 'FluidAudio' + s.frameworks = 'AVFoundation', 'CoreML' + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + } + s.source_files = '**/*.{h,m,mm,swift}' + s.resource_bundles = { + 'PseudoDictationResources' => ['Resources/*.wav'], + } +end diff --git a/apps/mobile/modules/pseudo-dictation/ios/PseudoDictationModule.swift b/apps/mobile/modules/pseudo-dictation/ios/PseudoDictationModule.swift new file mode 100644 index 0000000000000..8f41fe4a6febb --- /dev/null +++ b/apps/mobile/modules/pseudo-dictation/ios/PseudoDictationModule.swift @@ -0,0 +1,180 @@ +import ExpoModulesCore +import Foundation +import UIKit + +/// The JavaScript face of `DictationEngine`. It decodes arguments, calls one engine method, and +/// turns `DictationError` into a coded exception. Behaviour belongs in `Core`. +public final class PseudoDictationModule: Module { + private let engine = DictationEngine(storage: .applicationSupport) + private var observers: [NSObjectProtocol] = [] + + public func definition() -> ModuleDefinition { + let engine = self.engine + let reportProgress: @Sendable (String, DownloadUpdate) -> Void = { [weak self] id, update in + self?.sendEvent( + "onDownloadProgress", + ["modelId": id, "phase": update.phase.rawValue, "fraction": update.fraction]) + } + + Name("PseudoDictation") + + Events("onDownloadProgress") + + OnCreate { + // A loaded model in a suspended app is the first thing iOS reclaims the whole app for. + self.observers = [ + UIApplication.didReceiveMemoryWarningNotification, + UIApplication.didEnterBackgroundNotification, + ].map { name in + NotificationCenter.default.addObserver(forName: name, object: nil, queue: nil) { _ in + Task { await engine.releaseIfIdle() } + } + } + } + + OnDestroy { + for observer in self.observers { NotificationCenter.default.removeObserver(observer) } + self.observers = [] + } + + Function("deviceInfo") { () -> [String: Any] in + let environment = DictationEnvironment.current + return [ + "physicalMemoryBytes": Double(ProcessInfo.processInfo.physicalMemory), + "isSimulator": environment.isSimulator, + ] + } + + Function("sampleClipUri") { () -> String? in + Self.sampleClip?.absoluteString + } + + AsyncFunction("modelStatus") { (id: String, processing: String) -> [String: Any] in + let status = await engine.status(try Self.model(id), processing: Self.processing(processing)) + return [ + "installed": status.installed, + "warm": status.warm, + "downloading": status.downloading, + "sizeOnDiskBytes": Double(status.sizeOnDiskBytes), + ] + } + + AsyncFunction("downloadModel") { (id: String, processing: String) in + try await Self.coded { + try await engine.download( + try Self.model(id), processing: Self.processing(processing), + onProgress: { reportProgress(id, $0) }) + } + } + + AsyncFunction("cancelDownload") { (id: String) in + await engine.cancelDownload(try Self.model(id)) + } + + AsyncFunction("deleteModel") { (id: String) in + try await Self.coded { try await engine.delete(try Self.model(id)) } + } + + AsyncFunction("prepare") { (id: String, processing: String) in + try await Self.coded { + try await engine.prepare(try Self.model(id), processing: Self.processing(processing)) + } + } + + AsyncFunction("release") { () in + await engine.releaseIfIdle() + } + + AsyncFunction("transcribe") { (requestId: String, uri: String, id: String, processing: String, onlyMyVoice: Bool, language: String?) -> [String: Any?] in + let transcript = try await Self.coded { + try await engine.transcribe( + requestId: requestId, url: try Self.fileURL(uri), model: try Self.model(id), + processing: Self.processing(processing), onlyMyVoice: onlyMyVoice, language: language) + } + return [ + "text": transcript.text, + "audioSeconds": transcript.audioSeconds, + "processingSeconds": transcript.processingSeconds, + "filter": transcript.filter.map { filter -> [String: Any?] in + [ + "applied": filter.applied, + "keptSeconds": filter.keptSeconds, + "droppedSeconds": filter.droppedSeconds, + "skipped": filter.skipped?.rawValue, + ] + }, + ] + } + + AsyncFunction("cancelTranscription") { (requestId: String) in + await engine.cancelTranscription(requestId: requestId) + } + + AsyncFunction("voiceProfileStatus") { () -> [String: Any?] in + let status = await engine.voiceProfileStatus() + return [ + "enrolled": status.enrolled, + "clipCount": status.clipCount, + "createdAt": status.createdAt.map { $0.timeIntervalSince1970 * 1000 }, + ] + } + + AsyncFunction("checkEnrollmentClip") { (uri: String) -> [String: Any?] in + let check = try await Self.coded { try await engine.checkEnrollmentClip(try Self.fileURL(uri)) } + return [ + "speechSeconds": check.speechSeconds, + "problem": check.problem?.rawValue, + "message": check.problem?.message, + ] + } + + AsyncFunction("saveVoiceProfile") { (uris: [String]) in + try await Self.coded { try await engine.saveVoiceProfile(try uris.map(Self.fileURL)) } + } + + AsyncFunction("deleteVoiceProfile") { () in + try await Self.coded { try await engine.deleteVoiceProfile() } + } + } + + private static var sampleClip: URL? { + for bundle in [Bundle(for: PseudoDictationModule.self), Bundle.main] { + let resources = bundle.url(forResource: "PseudoDictationResources", withExtension: "bundle") + let url = resources.flatMap(Bundle.init(url:))? + .url(forResource: "dictation-sample", withExtension: "wav") + if let url { return url } + } + return nil + } + + private static func model(_ id: String) throws -> DictationModel { + guard let model = DictationModel(rawValue: id) else { throw exception(.unknownModel(id)) } + return model + } + + private static func processing(_ value: String) -> DictationProcessing { + DictationProcessing(rawValue: value) ?? .auto + } + + private static func fileURL(_ uri: String) throws -> URL { + if uri.hasPrefix("/") { return URL(fileURLWithPath: uri) } + guard let url = URL(string: uri), url.isFileURL else { + throw exception(.audioUnreadable("Not a local file.")) + } + return url + } + + private static func exception(_ error: DictationError) -> Exception { + Exception(name: "DictationError", description: error.message, code: error.code) + } + + private static func coded(_ work: () async throws -> T) async throws -> T { + do { + return try await work() + } catch let error as Exception { + throw error + } catch { + throw exception(DictationEngine.describe(error)) + } + } +} diff --git a/apps/mobile/modules/pseudo-dictation/ios/Resources/dictation-sample.wav b/apps/mobile/modules/pseudo-dictation/ios/Resources/dictation-sample.wav new file mode 100644 index 0000000000000..e21adbac24b77 Binary files /dev/null and b/apps/mobile/modules/pseudo-dictation/ios/Resources/dictation-sample.wav differ diff --git a/apps/mobile/plugins/withDictationPods.cjs b/apps/mobile/plugins/withDictationPods.cjs new file mode 100644 index 0000000000000..56be9b2dc0056 --- /dev/null +++ b/apps/mobile/plugins/withDictationPods.cjs @@ -0,0 +1,40 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +const { withDangerousMod } = require("expo/config-plugins"); + +// CocoaPods trunk stops at FluidAudio 0.7.8, so the pod comes from git, pinned to the v0.18.3 commit. +// It is declared here and not in `extraPods` because those are added to every target that links +// Expo modules, and the widget extension must not carry a speech model runtime. +const FLUID_AUDIO_COMMIT = "c8b41f98eae668794dd0083bbe028fe7e6585051"; +const MARKER = "# pseudocode: on-device dictation runtime, app target only"; +const APP_TARGET_MODULES = " use_expo_modules!\n"; +const POD = ` ${MARKER} + pod 'FluidAudio', :git => 'https://github.com/FluidInference/FluidAudio.git', :commit => '${FLUID_AUDIO_COMMIT}' +`; + +function addDictationPods(podfile) { + if (podfile.includes(MARKER)) return podfile; + const first = podfile.indexOf(APP_TARGET_MODULES); + if (first === -1 || podfile.indexOf(APP_TARGET_MODULES, first + 1) !== -1) { + throw new Error( + "Unable to add the dictation pods: expected exactly one `use_expo_modules!` line in the Podfile.", + ); + } + const insertAt = first + APP_TARGET_MODULES.length; + return podfile.slice(0, insertAt) + POD + podfile.slice(insertAt); +} + +function withDictationPods(config) { + return withDangerousMod(config, [ + "ios", + (nextConfig) => { + const podfilePath = path.join(nextConfig.modRequest.platformProjectRoot, "Podfile"); + fs.writeFileSync(podfilePath, addDictationPods(fs.readFileSync(podfilePath, "utf8")), "utf8"); + return nextConfig; + }, + ]); +} + +withDictationPods.addDictationPods = addDictationPods; +module.exports = withDictationPods; diff --git a/apps/mobile/plugins/withDictationPods.test.ts b/apps/mobile/plugins/withDictationPods.test.ts new file mode 100644 index 0000000000000..c748e3325b462 --- /dev/null +++ b/apps/mobile/plugins/withDictationPods.test.ts @@ -0,0 +1,46 @@ +import { expect, it } from "vite-plus/test"; + +import plugin from "./withDictationPods.cjs"; + +const { addDictationPods } = plugin; + +const PODFILE = `platform :ios, '18.0' + +target 'PseudoCode' do + use_expo_modules! + + use_react_native!(:path => config[:reactNativePath]) +end + +target "ExpoWidgetsTarget" do + use_expo_modules_widgets! +end +`; + +it("adds FluidAudio to the app target and leaves the widget target alone", () => { + const result: string = addDictationPods(PODFILE); + const lines = result.split("\n"); + const pod = lines.findIndex((line) => line.includes("pod 'FluidAudio'")); + + expect(lines[pod]).toMatch( + /^ {2}pod 'FluidAudio', :git => 'https:\/\/github\.com\/FluidInference\/FluidAudio\.git', :commit => '[0-9a-f]{40}'$/, + ); + expect(pod).toBeGreaterThan(lines.indexOf("target 'PseudoCode' do")); + expect(pod).toBeLessThan(lines.indexOf("end")); + expect(result.replace(/^.*(pseudocode: on-device|pod 'FluidAudio').*\n/gm, "")).toBe(PODFILE); +}); + +it("changes nothing on a Podfile it already changed", () => { + const once = addDictationPods(PODFILE); + expect(addDictationPods(once)).toBe(once); +}); + +it.each([ + ["no app target line", PODFILE.replace(" use_expo_modules!\n", "")], + [ + "two app target lines", + PODFILE.replace(" use_expo_modules!\n", " use_expo_modules!\n use_expo_modules!\n"), + ], +])("fails the prebuild on a Podfile with %s", (_name, podfile) => { + expect(() => addDictationPods(podfile)).toThrow("expected exactly one `use_expo_modules!` line"); +}); diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index de0d2585b82bc..4f4040fbaba75 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -145,6 +145,7 @@ import { deriveLayout } from "./lib/layout"; import { useNativeLayoutMetrics } from "./native/native-layout-metrics"; import { HORIZONTAL_FORM_SHEET_OPTIONS, formSheetDetents } from "./native/sheet-surface"; import { workspaceDetailRouter } from "./features/workspace/workspace-detail-selection"; +import { dictationSettingsScreens } from "./features/dictation/dictationSettingsScreens"; // Hiding Android's native header after mount can leave detail panes measured short. // These screens render their own Material toolbar, so hide it from the first layout. @@ -940,6 +941,7 @@ const RootStackConfig = createWorkspaceStackNavigator({ }, }), ...SETTINGS_SCREENS, + ...dictationSettingsScreens(SETTINGS_PAGE_OPTIONS), SettingsLegal: createNativeStackScreen({ screen: SettingsLegalRouteScreen, linking: "settings/legal", diff --git a/apps/mobile/src/features/dictation/DictationSettingsPage.tsx b/apps/mobile/src/features/dictation/DictationSettingsPage.tsx new file mode 100644 index 0000000000000..d47705a3582b6 --- /dev/null +++ b/apps/mobile/src/features/dictation/DictationSettingsPage.tsx @@ -0,0 +1,98 @@ +import type { ReactNode } from "react"; +import { View } from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { AppText as Text } from "../../components/ui/AppText"; +import { Button } from "../../components/ui/Button"; +import { ListActionRow } from "../../components/ui/ListActionRow"; +import { ListRowLayout } from "../../components/ui/ListRowLayout"; +import { Progress } from "../../components/ui/Progress"; +import { ScreenScrollView as ScrollView } from "../../components/ui/ScreenScrollView"; +import { SettingsScreen } from "../settings/components/SettingsScreen"; +import { SettingsSection } from "../settings/components/SettingsSection"; +import type { ModelDownloadState } from "./dictationDownloads"; + +export function DictationSettingsPage(props: { + readonly title: string; + readonly children: ReactNode; +}) { + const insets = useSafeAreaInsets(); + return ( + + + {props.children} + + + ); +} + +/** Shown where the native dictation module is not in the build, such as on Android. */ +export function DictationUnavailable(props: { readonly title: string }) { + return ( + + + + + + ); +} + +export function Footnote(props: { readonly children: ReactNode }) { + return {props.children}; +} + +/** A model that is not on the device: a download button, or the download's progress. */ +export function DownloadRow(props: { + readonly name: string; + readonly description: string; + readonly state: ModelDownloadState; + readonly onDownload: () => void; + readonly onCancel: () => void; +}) { + const { progress, error } = props.state; + return ( + + {progress === null ? ( + + ) : ( + <> + + ) : undefined + } + /> + + + + + )} + {error === null ? null : ( + {error} + )} + + ); +} diff --git a/apps/mobile/src/features/dictation/DictationSettingsSection.tsx b/apps/mobile/src/features/dictation/DictationSettingsSection.tsx new file mode 100644 index 0000000000000..e33902b95c327 --- /dev/null +++ b/apps/mobile/src/features/dictation/DictationSettingsSection.tsx @@ -0,0 +1,70 @@ +import { useNavigation } from "@react-navigation/native"; +import { View } from "react-native"; + +import { ListRow } from "../../components/ui/ListRow"; +import { ListToggleRow } from "../../components/ui/ListToggleRow"; +import { SettingsSection } from "../settings/components/SettingsSection"; +import { useAppleDictationAvailable } from "./appleDictation"; +import { dictationModelName } from "./dictationModels"; +import { dictationNative } from "./dictationNative"; +import { updateDictationSettings, useDictationState } from "./dictationRuntime"; +import { Footnote } from "./DictationSettingsPage"; +import { describeRun } from "./runSummary"; + +/** The fork's dictation settings, shown on the Microphone page. */ +export function DictationSettingsSection() { + return dictationNative === null ? null : ; +} + +function DictationSettings() { + const navigation = useNavigation(); + const { settings, learning, lastRun } = useDictationState(); + const appleMissing = useAppleDictationAvailable() === false; + const voiceDetail = !settings.onlyMyVoice + ? "Off" + : settings.modelId === "apple" + ? "Needs a model" + : "On"; + const suggestions = learning.candidates.filter((candidate) => candidate.suggested).length; + return ( + <> + + navigation.navigate("SettingsDictationModels")} + /> + + navigation.navigate("SettingsDictationVoice")} + /> + + + 0 ? `${suggestions} to review` : `${learning.rules.length}`} + onPress={() => navigation.navigate("SettingsDictationCorrections")} + /> + + + updateDictationSettings({ removeFillers })} + /> + + + {lastRun === null ? null : Last dictation: {describeRun(lastRun)}} + + ); +} diff --git a/apps/mobile/src/features/dictation/SettingsDictationCorrectionsRouteScreen.tsx b/apps/mobile/src/features/dictation/SettingsDictationCorrectionsRouteScreen.tsx new file mode 100644 index 0000000000000..bb8fa913fb5a4 --- /dev/null +++ b/apps/mobile/src/features/dictation/SettingsDictationCorrectionsRouteScreen.tsx @@ -0,0 +1,157 @@ +import { useState } from "react"; +import { Alert, View } from "react-native"; + +import { AppText as Text, AppTextInput } from "../../components/ui/AppText"; +import { Button } from "../../components/ui/Button"; +import { ListRow } from "../../components/ui/ListRow"; +import { ListToggleRow } from "../../components/ui/ListToggleRow"; +import { SettingsSection } from "../settings/components/SettingsSection"; +import { forgetPair, saveRule, type CorrectionPair } from "./correctionLearning"; +import { dictationNative } from "./dictationNative"; +import { + updateDictationLearning, + updateDictationSettings, + useDictationState, +} from "./dictationRuntime"; +import { DictationSettingsPage, DictationUnavailable, Footnote } from "./DictationSettingsPage"; +import { ruleKey } from "./replacementRules"; + +const FIELD_CLASS_NAME = + "min-h-0 flex-1 rounded-none border-0 bg-transparent p-0 text-base leading-[23px]"; + +const TITLE = "Corrections"; + +export function SettingsDictationCorrectionsRouteScreen() { + return dictationNative === null ? ( + + ) : ( + + ); +} + +function DictationCorrections() { + const { settings, learning } = useDictationState(); + const [heard, setHeard] = useState(""); + const [written, setWritten] = useState(""); + const suggestions = learning.candidates.filter((candidate) => candidate.suggested); + const canAdd = ruleKey(heard).length > 0 && written.trim().length > 0; + + const add = () => { + updateDictationLearning((state) => saveRule(state, { heard, written }, "manual", Date.now())); + setHeard(""); + setWritten(""); + }; + + const review = (pair: CorrectionPair) => + Alert.alert(`Replace "${pair.heard}" with "${pair.written}"?`, undefined, [ + { + text: "Always replace", + onPress: () => + updateDictationLearning((state) => saveRule(state, pair, "learned", Date.now())), + }, + { + text: "Never", + style: "destructive", + onPress: () => updateDictationLearning((state) => forgetPair(state, pair)), + }, + { text: "Cancel", style: "cancel" }, + ]); + + const confirmDelete = (pair: CorrectionPair) => + Alert.alert(`Stop replacing "${pair.heard}"?`, undefined, [ + { text: "Cancel", style: "cancel" }, + { + text: "Delete", + style: "destructive", + onPress: () => updateDictationLearning((state) => forgetPair(state, pair)), + }, + ]); + + return ( + + + updateDictationSettings({ learnCorrections })} + /> + + + {suggestions.length === 0 ? null : ( + + {suggestions.map((pair, index) => ( + 0 ? "border-t border-border-subtle" : undefined} + > + review(pair)} + /> + + ))} + + )} + + + + Heard + + + + Written + { + if (canAdd) add(); + }} + /> + + +