From 6834f3b5cb4ccb980d1c9480b4bcd8f8ba591a0d Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Thu, 8 Oct 2026 23:53:20 -0400 Subject: [PATCH 01/10] EmbeddingGemma 2 audio: index hours of audio in seconds, search it with text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EmbeddingGemma2Audio runs the model's audio encoder (Gemma 4 USM conformer, 12 layers, log-mel computed in-graph) from the new EmbeddingGemma2Audio.mlpackage on FluidInference/embeddinggemma-2-coreml (revision c0fefd0a, pinned + checksummed; the text bundle keeps its own revision and compile stamp, so nothing recompiles). Each 10 s window of 16 kHz audio becomes 250 tokens, which the text model embeds as <|audio> tokens through embed_256 — audio lands in the text space and is searchable with a text query. - Audio encoder on the GPU: the chunked local attention's 5-D blocks fall back to the CPU on the ANE (46 ms per window vs 13.6 ms on the GPU). Windows run four in flight across all files, so the GPU (audio) and the Neural Engine (text) overlap: 1 hour in 4.6-5.7 s (626-777x real time), 4 h 36 min across 2,301 files in 34.7 s (478x). - fp16 drifts on a few quiet tokens; on a 1-hour index the top hit matched fp32 for 15/15 queries (top-5 overlap 95%). Swift end to end vs sentence-transformers fp32: cos 0.9995 on a LibriSpeech clip. - Any AVFoundation format decoded to 16 kHz mono; window/frame arithmetic matches the HF feature extractor (999 frames / 250 tokens per 10 s, 347 / 87 for 3.48 s), covered by unit tests. AudioSearchCheck: index files or folders, timing, --query, --dump. AudioSearchDemo: index the public datasets on this Mac (Earnings-22 hour, FLEURS en/fr, ESC-50) or --audio= folders, live speed tiles, search as you type (one sgemv over all windows), play the matching 10 s; demo.sh adds macmon + the live log. Speech queries work well; sound events are weak in the model itself (ESC-50 zero-shot 24% over 50 classes, same as PyTorch fp32). --- Package.swift | 2 + Sources/AudioSearchCheck/main.swift | 74 +++++ .../AudioSearchDemo/AudioSearchDemoApp.swift | 24 ++ .../AudioSearchDemo/AudioSearchModel.swift | 313 ++++++++++++++++++ Sources/AudioSearchDemo/ContentView.swift | 211 ++++++++++++ Sources/AudioSearchDemo/DemoLog.swift | 29 ++ Sources/AudioSearchDemo/README.md | 37 +++ Sources/AudioSearchDemo/demo.sh | 34 ++ .../EmbeddingGemma2Audio.swift | 183 ++++++++++ .../EmbeddingGemma2Manager.swift | 46 ++- .../EmbeddingGemma2ModelStore.swift | 54 ++- .../FluidUseTests/EmbeddingGemma2Tests.swift | 21 ++ 12 files changed, 1014 insertions(+), 14 deletions(-) create mode 100644 Sources/AudioSearchCheck/main.swift create mode 100644 Sources/AudioSearchDemo/AudioSearchDemoApp.swift create mode 100644 Sources/AudioSearchDemo/AudioSearchModel.swift create mode 100644 Sources/AudioSearchDemo/ContentView.swift create mode 100644 Sources/AudioSearchDemo/DemoLog.swift create mode 100644 Sources/AudioSearchDemo/README.md create mode 100755 Sources/AudioSearchDemo/demo.sh create mode 100644 Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift diff --git a/Package.swift b/Package.swift index 4f21087..424ceb9 100644 --- a/Package.swift +++ b/Package.swift @@ -53,6 +53,8 @@ let package = Package( .target(name: "BookmarkSort", dependencies: ["FluidUse"]), .executableTarget(name: "BookmarkSortCheck", dependencies: ["BookmarkSort", "FluidUse"]), .executableTarget(name: "BookmarkSortDemo", dependencies: ["BookmarkSort"]), + .executableTarget(name: "AudioSearchCheck", dependencies: ["FluidUse"]), + .executableTarget(name: "AudioSearchDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh"]), .executableTarget( name: "TopicSortDemo", dependencies: ["BookmarkSort", "FluidUse"], exclude: ["README.md", "demo.sh"], resources: [.copy("Resources")]), diff --git a/Sources/AudioSearchCheck/main.swift b/Sources/AudioSearchCheck/main.swift new file mode 100644 index 0000000..b303ae6 --- /dev/null +++ b/Sources/AudioSearchCheck/main.swift @@ -0,0 +1,74 @@ +import Foundation +import FluidUse + +/// Indexes audio files with EmbeddingGemma 2 (10 s windows) and searches them with text. +/// +/// swift run -c release AudioSearchCheck file.wav [more files…] [--query="…"]… [--dump=vectors.jsonl] +@main +struct AudioSearchCheck { + static func main() async throws { + let arguments = CommandLine.arguments.dropFirst() + // Folders expand to the audio files inside them, sorted. + let audioExtensions: Set = ["wav", "flac", "mp3", "m4a", "aiff", "caf"] + let files = arguments.filter { !$0.hasPrefix("--") }.flatMap { path -> [URL] in + let url = URL(fileURLWithPath: path) + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory), isDirectory.boolValue else { + return [url] + } + let names = (try? FileManager.default.contentsOfDirectory(atPath: path)) ?? [] + return names.filter { audioExtensions.contains(($0 as NSString).pathExtension.lowercased()) }.sorted() + .map { url.appendingPathComponent($0) } + } + let queries = arguments.filter { $0.hasPrefix("--query=") }.map { String($0.dropFirst(8)) } + let dump = arguments.first { $0.hasPrefix("--dump=") }.map { String($0.dropFirst(7)) } + guard !files.isEmpty else { + print("usage: AudioSearchCheck file.wav [more files…] [--query=\"…\"]… [--dump=vectors.jsonl]") + exit(2) + } + var start = DispatchTime.now().uptimeNanoseconds + let text = try await EmbeddingGemma2Manager.loadDefault() + let audio = try await EmbeddingGemma2Audio.load(text: text) + _ = try await audio.embed(window: [Float](repeating: 0, count: 16_000)) + print(String(format: "models loaded in %.1f s", seconds(since: start))) + start = DispatchTime.now().uptimeNanoseconds + let recordings = try files.map { try EmbeddingGemma2Audio.samples(contentsOf: $0) } + let decodeSeconds = seconds(since: start) + start = DispatchTime.now().uptimeNanoseconds + let embedded = try await audio.embed(recordings: recordings) + let embedSeconds = seconds(since: start) + let audioSeconds = recordings.reduce(0.0) { $0 + Double($1.count) } / Double(EmbeddingGemma2Audio.sampleRate) + let index: [(file: String, window: EmbeddingGemma2Audio.Window)] = zip(files, embedded).flatMap { + file, windows in + windows.map { (file: file.lastPathComponent, window: $0) } + } + print( + String( + format: "%.1f min of audio, %d windows: decode %.1f s, embed %.1f s = %.0fx real time (embed only)", + audioSeconds / 60, index.count, decodeSeconds, embedSeconds, audioSeconds / max(embedSeconds, 1e-6))) + if let dump { + let lines = index.map { item -> String in + let object: [String: Any] = ["file": item.file, "start": item.window.start, "v": item.window.embedding] + return String(decoding: try! JSONSerialization.data(withJSONObject: object), as: UTF8.self) + } + try lines.joined(separator: "\n").write(toFile: dump, atomically: true, encoding: .utf8) + } + for query in queries { + let vector = try await text.embed(query, prompt: .searchQuery) + let ranked = index.map { item in + (item, zip(vector, item.window.embedding).reduce(Float(0)) { $0 + $1.0 * $1.1 }) + } + .sorted { $0.1 > $1.1 }.prefix(3) + print("\(query):") + for (item, score) in ranked { + print(String(format: " %.3f %@ @ %@", score, item.file, clock(item.window.start))) + } + } + } + + static func seconds(since start: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 } + + static func clock(_ seconds: TimeInterval) -> String { + String(format: "%d:%02d", Int(seconds) / 60, Int(seconds) % 60) + } +} diff --git a/Sources/AudioSearchDemo/AudioSearchDemoApp.swift b/Sources/AudioSearchDemo/AudioSearchDemoApp.swift new file mode 100644 index 0000000..6e904d7 --- /dev/null +++ b/Sources/AudioSearchDemo/AudioSearchDemoApp.swift @@ -0,0 +1,24 @@ +import AppKit +import SwiftUI + +@main +struct AudioSearchDemoApp: App { + @StateObject private var model = AudioSearchModel() + + init() { + setvbuf(stdout, nil, _IOLBF, 0) + // Bare SwiftPM executables start as background processes; make this one a regular windowed app. + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("Search audio — on-device") { + ContentView() + .environmentObject(model) + .frame(minWidth: 900, minHeight: 640) + .task { await model.prepare() } + } + .defaultSize(width: 1100, height: 900) + } +} diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift new file mode 100644 index 0000000..b2e8278 --- /dev/null +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -0,0 +1,313 @@ +import Accelerate +import AVFoundation +import FluidUse +import Foundation +import SwiftUI +import os + +/// Indexes audio collections with EmbeddingGemma 2 (10 s windows: audio model on the GPU, text model on the Neural +/// Engine) and searches every window with a text query. +@MainActor +final class AudioSearchModel: ObservableObject { + enum Phase: Equatable { + case loading(String) + case ready + case indexing + case done + case failed(String) + } + + struct Collection: Identifiable { + let id: Int + let name: String + let color: Color + let files: [URL] + let suggestions: [String] + } + + struct Entry { + let collection: Int + let file: URL + let start: TimeInterval + let duration: TimeInterval + let embedding: [Float] + } + + struct Result: Identifiable { + let entry: Entry + let score: Float + var id: String { "\(entry.file.path)#\(entry.start)" } + } + + @Published private(set) var phase: Phase = .loading("Starting…") + @Published private(set) var collections: [Collection] = [] + @Published private(set) var windowsDone = 0 + @Published private(set) var windowsTotal = 0 + @Published private(set) var audioSeconds: Double = 0 + @Published private(set) var indexSeconds: Double = 0 + @Published private(set) var results: [Result] = [] + @Published private(set) var queryMilliseconds: Double = 0 + @Published private(set) var playing: String? + @Published var query = "" + + private var entries: [Entry] = [] + /// Every window's embedding, row after row, so a query is one matrix-vector product. + private var matrix: [Float] = [] + private var text: EmbeddingGemma2Manager? + private var audio: EmbeddingGemma2Audio? + private var player: AVAudioPlayer? + private var searchTask: Task? + private var runTask: Task? + + var realTimeFactor: Double { indexSeconds > 0 ? audioSeconds / indexSeconds : 0 } + var allSuggestions: [String] { collections.flatMap(\.suggestions) } + + func prepare() async { + do { + collections = Self.defaultCollections() + guard !collections.isEmpty else { + throw EmbeddingGemma2Error.invalidAsset("No audio found: pass --audio=") + } + DemoLog.event( + "Search audio · \(collections.map { "\($0.name) (\($0.files.count))" }.joined(separator: " · "))") + let loadStart = DispatchTime.now().uptimeNanoseconds + let ticker = Task { @MainActor [weak self] in + var seconds = 0 + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(1)) + seconds += 1 + guard let self, case .loading = self.phase else { continue } + self.phase = .loading( + "Loading EmbeddingGemma 2… \(seconds) s" + + (seconds >= 8 ? " (first launch downloads and compiles the models once)" : "")) + } + } + defer { ticker.cancel() } + let text = try await EmbeddingGemma2Manager.loadDefault() + let audio = try await EmbeddingGemma2Audio.load(text: text) + _ = try await audio.embed(window: [Float](repeating: 0, count: 16_000)) + _ = try await text.embed("warm up", prompt: .searchQuery) + self.text = text + self.audio = audio + DemoLog.model( + String( + format: "models loaded in %.1f s · audio encoder on the GPU, text model on the Neural Engine", + Double(DispatchTime.now().uptimeNanoseconds - loadStart) / 1e9)) + phase = .ready + if CommandLine.arguments.contains("--autostart") { start() } + } catch { + phase = .failed(error.localizedDescription) + DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) + } + } + + func start() { + guard phase == .ready else { return } + runTask = Task { await index() } + } + + func reset() { + runTask?.cancel() + runTask = nil + stop() + entries = [] + matrix = [] + results = [] + windowsDone = 0 + windowsTotal = 0 + audioSeconds = 0 + indexSeconds = 0 + if audio != nil { phase = .ready } + DemoLog.event("↺ reset") + } + + private func index() async { + guard let audio else { return } + phase = .indexing + DemoLog.event("▶ indexing \(collections.reduce(0) { $0 + $1.files.count }) files") + do { + let files = collections.flatMap { collection in collection.files.map { (collection.id, $0) } } + let decodeStart = DispatchTime.now().uptimeNanoseconds + let recordings = try await Task.detached(priority: .userInitiated) { + try files.map { try EmbeddingGemma2Audio.samples(contentsOf: $0.1) } + }.value + DemoLog.line( + String( + format: "decoded %d files to 16 kHz in %.1f s", files.count, + Double(DispatchTime.now().uptimeNanoseconds - decodeStart) / 1e9), color: 245) + let total = recordings.reduce(0.0) { $0 + Double($1.count) } / Double(EmbeddingGemma2Audio.sampleRate) + let start = DispatchTime.now().uptimeNanoseconds + let progress = IndexProgress() + // The embedding tasks report into `progress`; the UI samples it ten times a second. + let watcher = Task { @MainActor [weak self] in + while !Task.isCancelled { + try? await Task.sleep(for: .milliseconds(100)) + self?.refresh(progress: progress, total: total, start: start) + } + } + defer { watcher.cancel() } + let windows = try await audio.embed(recordings: recordings) { done, count in + progress.update(done: done, total: count) + } + try Task.checkCancellation() + entries = zip(files, windows).flatMap { file, windows in + windows.map { + Entry( + collection: file.0, file: file.1, start: $0.start, duration: $0.duration, + embedding: $0.embedding) + } + } + matrix = entries.flatMap(\.embedding) + refresh(progress: progress, total: total, start: start) + phase = .done + DemoLog.event( + String( + format: "indexed %@ of audio (%d windows) in %.1f s = %.0fx real time", Self.clock(total), + entries.count, indexSeconds, realTimeFactor)) + if let preset = CommandLine.arguments.first(where: { $0.hasPrefix("--query=") }) { + query = String(preset.dropFirst(8)) + } + if !query.isEmpty { search() } + } catch is CancellationError { + } catch { + phase = .failed(error.localizedDescription) + DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) + } + } + + private func refresh(progress: IndexProgress, total: Double, start: UInt64) { + let (done, count) = progress.snapshot + guard count > 0 else { return } + windowsDone = done + windowsTotal = count + audioSeconds = total * Double(done) / Double(count) + indexSeconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 + } + + /// Ranks every indexed window against the query (debounced while typing). + func search() { + searchTask?.cancel() + let query = query.trimmingCharacters(in: .whitespacesAndNewlines) + guard let text, !query.isEmpty, !entries.isEmpty else { + results = [] + return + } + searchTask = Task { + try? await Task.sleep(for: .milliseconds(120)) + guard !Task.isCancelled else { return } + let begin = DispatchTime.now().uptimeNanoseconds + guard let vector = try? await text.embed(query, prompt: .searchQuery), !Task.isCancelled else { return } + var scores = [Float](repeating: 0, count: entries.count) + cblas_sgemv( + CblasRowMajor, CblasNoTrans, Int32(entries.count), Int32(vector.count), 1, matrix, Int32(vector.count), + vector, 1, 0, &scores, 1) + let top = scores.indices.sorted { scores[$0] > scores[$1] }.prefix(25) + results = top.map { Result(entry: entries[$0], score: scores[$0]) } + queryMilliseconds = Double(DispatchTime.now().uptimeNanoseconds - begin) / 1e6 + DemoLog.line( + String(format: "🔎 “%@” → %d windows ranked in %.0f ms", query, entries.count, queryMilliseconds), + color: 81) + for result in results.prefix(3) { + DemoLog.line( + String( + format: " %.3f %@ %@ @ %@", result.score, collections[result.entry.collection].name, + result.entry.file.lastPathComponent, Self.clock(result.entry.start))) + } + } + } + + func play(_ result: Result) { + if playing == result.id { + stop() + return + } + stop() + guard let player = try? AVAudioPlayer(contentsOf: result.entry.file) else { return } + player.currentTime = result.entry.start + player.play() + self.player = player + playing = result.id + let id = result.id + Task { [weak self] in + try? await Task.sleep(for: .seconds(result.entry.duration)) + if self?.playing == id { self?.stop() } + } + } + + func stop() { + player?.stop() + player = nil + playing = nil + } + + static func clock(_ seconds: TimeInterval) -> String { + let total = Int(seconds.rounded()) + return total >= 3600 + ? String(format: "%d:%02d:%02d", total / 3600, total / 60 % 60, total % 60) + : String(format: "%d:%02d", total / 60, total % 60) + } + + /// `--audio=` (repeatable), else the public datasets already on this Mac. + static func defaultCollections() -> [Collection] { + let palette: [Color] = [.blue, .orange, .green, .pink, .purple, .teal] + let given = CommandLine.arguments.filter { $0.hasPrefix("--audio=") }.map { String($0.dropFirst(8)) } + if !given.isEmpty { + return given.enumerated().compactMap { index, path in + let files = audioFiles(at: URL(fileURLWithPath: path)) + guard !files.isEmpty else { return nil } + return Collection( + id: index, name: URL(fileURLWithPath: path).lastPathComponent, + color: palette[index % palette.count], files: files, suggestions: []) + } + } + let fluidAudio = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("FluidAudio") + let fluidUse = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent("FluidUse/Datasets") + let candidates: [(String, URL, [String])] = [ + ( + "Earnings calls", fluidAudio.appendingPathComponent("earnings22-1h"), + ["forward-looking statements disclaimer", "the operator opens the line for questions"] + ), + ( + "FLEURS English", fluidAudio.appendingPathComponent("FLEURS/en_us"), + ["the history of an ancient empire", "animals in the wild"] + ), + ("FLEURS French", fluidAudio.appendingPathComponent("FLEURS/fr_fr"), ["a story about a famous painter"]), + ( + "Sounds (ESC-50)", fluidUse.appendingPathComponent("esc50/wav"), + ["a baby crying", "someone laughing", "a dog barking", "church bells"] + ), + ] + var collections: [Collection] = [] + for candidate in candidates { + let files = audioFiles(at: candidate.1) + guard !files.isEmpty else { continue } + let index = collections.count + collections.append( + Collection( + id: index, name: candidate.0, color: palette[index % palette.count], files: files, + suggestions: candidate.2)) + } + return collections + } + + static func audioFiles(at url: URL) -> [URL] { + let extensions: Set = ["wav", "flac", "mp3", "m4a", "aiff", "caf"] + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { return [] } + guard isDirectory.boolValue else { return [url] } + let names = (try? FileManager.default.contentsOfDirectory(atPath: url.path)) ?? [] + return names.filter { extensions.contains(($0 as NSString).pathExtension.lowercased()) }.sorted() + .map { url.appendingPathComponent($0) } + } +} + +/// Window counts from the embedding tasks, read by the UI on a timer. +final class IndexProgress: Sendable { + private let counts = OSAllocatedUnfairLock(initialState: (done: 0, total: 0)) + + func update(done: Int, total: Int) { counts.withLock { $0 = (done, total) } } + + var snapshot: (Int, Int) { counts.withLock { ($0.done, $0.total) } } +} diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift new file mode 100644 index 0000000..2970f11 --- /dev/null +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -0,0 +1,211 @@ +import SwiftUI + +struct ContentView: View { + @EnvironmentObject private var model: AudioSearchModel + + var body: some View { + VStack(spacing: 0) { + Header() + Divider() + SearchBar() + Divider() + Results() + } + .background(Color(nsColor: .windowBackgroundColor)) + } +} + +struct Header: View { + @EnvironmentObject private var model: AudioSearchModel + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .center, spacing: 10) { + Text("Search audio").font(.title2.bold()) + Spacer() + Text(status).font(.callout).foregroundStyle(.secondary) + Button { + model.start() + } label: { + Label("Index", systemImage: "play.fill") + } + .keyboardShortcut(.return, modifiers: [.command]) + .disabled(model.phase != .ready) + Button { + model.reset() + } label: { + Label("Reset", systemImage: "arrow.counterclockwise") + } + .keyboardShortcut("r", modifiers: [.command]) + .disabled(model.phase != .done && model.phase != .indexing) + } + GeometryReader { proxy in + Capsule().fill(Color.primary.opacity(0.08)) + .overlay(alignment: .leading) { + Capsule().fill(Color.accentColor) + .frame( + width: proxy.size.width * Double(model.windowsDone) + / Double(max(model.windowsTotal, 1))) + } + } + .frame(height: 6) + HStack(spacing: 10) { + Tile(value: AudioSearchModel.clock(model.audioSeconds), caption: "audio indexed") + Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") + Tile( + value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), + caption: "faster than real time") + Tile(value: "\(model.windowsDone)", caption: "10 s windows") + } + HStack(spacing: 8) { + ForEach(model.collections) { collection in + HStack(spacing: 5) { + Circle().fill(collection.color).frame(width: 8, height: 8) + Text("\(collection.name) · \(collection.files.count)") + } + .font(.caption) + .padding(.horizontal, 8).padding(.vertical, 3) + .background(Capsule().fill(collection.color.opacity(0.12))) + } + Spacer() + Text("EmbeddingGemma 2 · audio on the GPU, text on the Neural Engine · nothing leaves this Mac") + .font(.caption).foregroundStyle(.secondary) + } + } + .padding(16) + } + + private var status: String { + switch model.phase { + case .loading(let message): message + case .ready: "Ready · press Index (⌘↩)" + case .indexing: "Indexing \(model.windowsDone)/\(model.windowsTotal) windows…" + case .done: "Indexed · search below" + case .failed(let message): message + } + } +} + +struct SearchBar: View { + @EnvironmentObject private var model: AudioSearchModel + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Image(systemName: "magnifyingglass").foregroundStyle(.secondary) + TextField("Describe what you're looking for: a phrase, a topic, a sound…", text: $model.query) + .textFieldStyle(.plain).font(.title3) + .onSubmit { model.search() } + .onChange(of: model.query) { model.search() } + if model.queryMilliseconds > 0, !model.results.isEmpty { + Text(String(format: "%.0f ms", model.queryMilliseconds)).font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + } + } + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 6) { + ForEach(model.allSuggestions, id: \.self) { suggestion in + Button(suggestion) { + model.query = suggestion + model.search() + } + .buttonStyle(.bordered).controlSize(.small) + } + } + } + } + .padding(.horizontal, 16).padding(.vertical, 12) + } +} + +struct Results: View { + @EnvironmentObject private var model: AudioSearchModel + + var body: some View { + ScrollView { + LazyVStack(spacing: 0) { + ForEach(Array(model.results.enumerated()), id: \.element.id) { rank, result in + ResultRow(rank: rank + 1, result: result) + Divider() + } + } + } + .overlay { + if model.results.isEmpty { + Text(model.phase == .done ? "Type a query or pick a suggestion" : "Index the audio, then search it") + .foregroundStyle(.secondary) + } + } + } +} + +struct ResultRow: View { + @EnvironmentObject private var model: AudioSearchModel + let rank: Int + let result: AudioSearchModel.Result + + var body: some View { + let collection = model.collections[result.entry.collection] + HStack(spacing: 12) { + Text("\(rank)").font(.callout.monospacedDigit()).foregroundStyle(.secondary).frame(width: 24) + Button { + model.play(result) + } label: { + Image(systemName: model.playing == result.id ? "stop.circle.fill" : "play.circle.fill") + .font(.system(size: 26)).foregroundStyle(collection.color) + } + .buttonStyle(.plain) + VStack(alignment: .leading, spacing: 3) { + HStack(spacing: 6) { + Text(collection.name).font(.caption.bold()).foregroundStyle(collection.color) + Text(result.entry.file.lastPathComponent).font(.callout).lineLimit(1) + } + Text( + "\(AudioSearchModel.clock(result.entry.start)) – " + + AudioSearchModel.clock(result.entry.start + result.entry.duration) + ) + .font(.caption.monospacedDigit()).foregroundStyle(.secondary) + } + Spacer() + ScoreBar(score: result.score, color: collection.color) + } + .padding(.horizontal, 16).padding(.vertical, 8) + .background(model.playing == result.id ? collection.color.opacity(0.08) : .clear) + } +} + +struct ScoreBar: View { + let score: Float + let color: Color + + var body: some View { + HStack(spacing: 6) { + GeometryReader { proxy in + Capsule().fill(Color.primary.opacity(0.08)) + .overlay(alignment: .leading) { + // Cosine scores here sit around 0.5–0.8; stretch that range across the bar. + Capsule().fill(color) + .frame(width: proxy.size.width * CGFloat(min(max((score - 0.5) / 0.35, 0.03), 1))) + } + } + .frame(width: 120, height: 6) + Text(String(format: "%.3f", score)).font(.caption.monospacedDigit()).foregroundStyle(.secondary) + } + } +} + +struct Tile: View { + let value: String + let caption: String + + var body: some View { + VStack(alignment: .leading, spacing: 2) { + Text(value).font(.system(size: 24, weight: .semibold, design: .rounded)).monospacedDigit() + .contentTransition(.numericText()) + Text(caption).font(.caption).foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(10) + .background(RoundedRectangle(cornerRadius: 10).fill(Color.primary.opacity(0.05))) + } +} diff --git a/Sources/AudioSearchDemo/DemoLog.swift b/Sources/AudioSearchDemo/DemoLog.swift new file mode 100644 index 0000000..7e5ddcd --- /dev/null +++ b/Sources/AudioSearchDemo/DemoLog.swift @@ -0,0 +1,29 @@ +import Foundation + +/// Colour-coded console lines for the side terminal (`demo.sh` tails stdout): model load, indexing, searches. +enum DemoLog { + /// 256-colour codes matching the collection colours. + private static let topicColors = [33, 208, 34, 205, 135, 37, 196, 61, 130, 121] + + static func line(_ text: String, color: Int? = nil, bold: Bool = false) { + let time = timeFormatter.string(from: Date()) + var styled = text + if let color { styled = "\u{1B}[38;5;\(color)m\(styled)\u{1B}[0m" } + if bold { styled = "\u{1B}[1m\(styled)\u{1B}[0m" } + print("\u{1B}[2m\(time)\u{1B}[0m \(styled)") + } + + static func colored(_ text: String, slot: Int) -> String { + "\u{1B}[38;5;\(topicColors[slot % topicColors.count])m\(text)\u{1B}[0m" + } + + static func event(_ text: String) { line(text, color: 220, bold: true) } + + static func model(_ text: String) { line("[model] " + text, color: 81) } + + private static let timeFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.dateFormat = "HH:mm:ss.SSS" + return formatter + }() +} diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md new file mode 100644 index 0000000..d652824 --- /dev/null +++ b/Sources/AudioSearchDemo/README.md @@ -0,0 +1,37 @@ +# Search audio + +Indexes hours of audio in seconds, then finds moments in it from a text query: a phrase someone said, a topic, a +sound. Everything runs on the Mac with EmbeddingGemma 2: its audio encoder turns each 10 s window into tokens on the +GPU, and its text model (Neural Engine) embeds those tokens into the same space as text queries. + +```bash +swift run -c release AudioSearchDemo # the public datasets already on this Mac (see below) +swift run -c release AudioSearchDemo --audio=~/Podcasts --audio=meeting.m4a +Sources/AudioSearchDemo/demo.sh # same app, plus one terminal: macmon above the live log +``` + +The first run downloads the models (about 1.1 GB, checksum-verified) from +[FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles +them once (a few minutes). Needs macOS 15+. + +- **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so + the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. +- Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. +- `--query="…"` runs a search right after indexing; `--autostart` indexes without waiting. + +Default audio, when present: the 1-hour Earnings-22 sample and FLEURS English/French used by FluidAudio's +benchmarks (`~/Library/Application Support/FluidAudio`), and ESC-50 sound clips +(`~/Library/Application Support/FluidUse/Datasets/esc50/wav`). Nothing is bundled. + +Numbers (M5 Pro, macOS 27): + +| | | +|---|---| +| 1 hour earnings call, 360 windows | 4.6–5.7 s, 626–777× real time | +| 4 h 36 min (2,301 files incl. 2,000 five-second clips) | 34.7 s, 478× real time | +| Swift vs sentence-transformers (fp32) | cosine 0.9995 on a LibriSpeech clip | + +Quality: speech search works well when the query names something said ("forward-looking statements" finds the +safe-harbor sentence; "operator opens the line" finds the call-in prompt). Sound events are weaker: zero-shot ESC-50 +accuracy is 24% over 50 classes (PyTorch fp32 gets the same, so that is the model, not the conversion); distinctive +sounds such as crying or laughing work, similar ones (rain vs waves vs helicopter) get mixed up. diff --git a/Sources/AudioSearchDemo/demo.sh b/Sources/AudioSearchDemo/demo.sh new file mode 100755 index 0000000..f88857e --- /dev/null +++ b/Sources/AudioSearchDemo/demo.sh @@ -0,0 +1,34 @@ +#!/bin/zsh +# Search audio demo: the app, plus one terminal (Ghostty if installed) with macmon (GPU / ANE / power; asitop as a +# fallback) on top and the live Neural Engine log below. +# Sources/AudioSearchDemo/demo.sh [app options, e.g. --autostart --auto-split --rate=200] +set -e +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +LOG=${TMPDIR:-/tmp}/audio-search-demo.log +# macmon (brew install macmon) needs no sudo and works on M5; asitop 0.0.24 crashes there (KeyError E0-Cluster_active). +if command -v macmon >/dev/null; then + MONITOR="$(command -v macmon)" +else + MONITOR="sudo sh -c 'pkill -x powermetrics; exec $(command -v asitop || echo "$HOME/.local/bin/asitop")'" +fi + +swift build -c release --product AudioSearchDemo --package-path "$ROOT" +: > "$LOG" +# One tmux session, reused across launches: the monitor on top, the log below. +if ! tmux has-session -t audio-search 2>/dev/null; then + # The pane stays open if the monitor exits, so the reason is visible. + tmux new-session -d -s audio-search -x 160 -y 70 \ + "$MONITOR; echo; echo \"monitor exited (\$?) — press Enter to retry\"; read; $MONITOR" + tmux set-option -t audio-search remain-on-exit on + tmux split-window -v -t audio-search "tail -n 300 -F '$LOG'" + if [[ -d /Applications/Ghostty.app ]]; then + open -na Ghostty --args -e "$(command -v tmux)" attach -t audio-search + else + osascript -e 'tell application "Terminal" to do script "tmux attach -t audio-search"' \ + -e 'tell application "Terminal" to activate' >/dev/null + fi +fi +pkill -f "release/AudioSearchDemo" 2>/dev/null || true +cd "$ROOT" +"$ROOT/.build/release/AudioSearchDemo" "$@" > "$LOG" 2> "$LOG.stderr" & +echo "demo pid $! · log $LOG · press Index (⌘↩) in the app" diff --git a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift new file mode 100644 index 0000000..6adeeff --- /dev/null +++ b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift @@ -0,0 +1,183 @@ +@preconcurrency import AVFoundation +@preconcurrency import CoreML +import Foundation + +/// EmbeddingGemma 2 audio embeddings: 10 s windows of 16 kHz mono audio go through `EmbeddingGemma2Audio` (log-mel + +/// USM conformer, on the GPU, where it runs 3x faster than on the Neural Engine) and then the text model (Neural +/// Engine), so audio lands in the same space as text and can be searched with a text query. +public final class EmbeddingGemma2Audio: Sendable { + public static let sampleRate = 16_000 + public static let windowSamples = 160_000 + static let leftPad = 160 + static let frames = 1_000 + public static let maxInFlight = 4 + + /// One embedded window of a recording. + public struct Window: Sendable { + public let start: TimeInterval + public let duration: TimeInterval + public let embedding: [Float] + } + + public let text: EmbeddingGemma2Manager + private let model: MLModel + + init(text: EmbeddingGemma2Manager, model: MLModel) { + self.text = text + self.model = model + } + + /// Downloads (once, checksum-verified) and loads the audio model next to an already loaded text model. + public static func load( + text: EmbeddingGemma2Manager, computeUnits: MLComputeUnits = .cpuAndGPU, + progress: EmbeddingGemma2ModelStore.Progress? = nil + ) async throws -> EmbeddingGemma2Audio { + let directory: URL + if let path = ProcessInfo.processInfo.environment["EMBEDDINGGEMMA2_MODEL_DIR"], !path.isEmpty { + directory = URL(fileURLWithPath: path) + } else { + directory = try await EmbeddingGemma2ModelStore.ensureAudio(progress: progress) + } + let compiled = directory.appendingPathComponent("EmbeddingGemma2Audio.mlmodelc") + let package = directory.appendingPathComponent("EmbeddingGemma2Audio.mlpackage") + let url: URL + if FileManager.default.fileExists(atPath: compiled.path) { + url = compiled + } else if FileManager.default.fileExists(atPath: package.path) { + url = try await MLModel.compileModel(at: package) + } else { + throw EmbeddingGemma2Error.invalidAsset("Missing EmbeddingGemma2Audio in \(directory.path)") + } + let configuration = MLModelConfiguration() + configuration.computeUnits = computeUnits + return EmbeddingGemma2Audio( + text: text, model: try await MLModel.load(contentsOf: url, configuration: configuration)) + } + + /// Decodes any file AVFoundation reads to 16 kHz mono Float samples. + public static func samples(contentsOf url: URL) throws -> [Float] { + let file = try AVAudioFile(forReading: url) + guard + let target = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: Double(sampleRate), channels: 1, interleaved: false), + let converter = AVAudioConverter(from: file.processingFormat, to: target) + else { throw EmbeddingGemma2Error.invalidAsset("Cannot convert \(url.lastPathComponent) to 16 kHz mono") } + let chunk: AVAudioFrameCount = 65_536 + var result: [Float] = [] + result.reserveCapacity(Int(Double(file.length) * Double(sampleRate) / file.processingFormat.sampleRate) + 1) + while true { + let capacity = + AVAudioFrameCount(Double(chunk) * Double(sampleRate) / file.processingFormat.sampleRate) + 1024 + guard let output = AVAudioPCMBuffer(pcmFormat: target, frameCapacity: capacity) else { break } + var error: NSError? + // Once the file is exhausted every read returns no frames, which ends the stream; no state to share. + let status = converter.convert(to: output, error: &error) { _, outStatus in + guard let input = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: chunk), + (try? file.read(into: input, frameCount: chunk)) != nil, input.frameLength > 0 + else { + outStatus.pointee = .endOfStream + return nil + } + outStatus.pointee = .haveData + return input + } + if let error { throw error } + if let channel = output.floatChannelData?[0], output.frameLength > 0 { + result.append(contentsOf: UnsafeBufferPointer(start: channel, count: Int(output.frameLength))) + } + if status == .endOfStream || status == .error { break } + } + return result + } + + /// Embeddings of consecutive 10 s windows of `samples` (16 kHz mono), in order. `progress` reports windows done. + public func embed( + samples: [Float], progress: (@Sendable (_ done: Int, _ total: Int) -> Void)? = nil + ) async throws -> [Window] { + try await embed(recordings: [samples], progress: progress).first ?? [] + } + + /// Windows of several recordings, embedded together so short files still keep the GPU and the Neural Engine + /// busy (one file at a time leaves a single window in flight). Result: one window list per recording, in order. + public func embed( + recordings: [[Float]], progress: (@Sendable (_ done: Int, _ total: Int) -> Void)? = nil + ) async throws -> [[Window]] { + var jobs: [(recording: Int, start: Int)] = [] + for (recording, samples) in recordings.enumerated() { + var starts = Array(stride(from: 0, to: samples.count, by: Self.windowSamples)) + // A tail under 0.5 s carries almost nothing and may be too short to produce a token. + if starts.count > 1, let last = starts.last, samples.count - last < Self.sampleRate / 2 { + starts.removeLast() + } + jobs += starts.map { (recording, $0) } + } + return try await withThrowingTaskGroup(of: (Int, Window).self) { group in + var windows = [Window?](repeating: nil, count: jobs.count) + var next = 0 + var done = 0 + func addJob() { + guard next < jobs.count else { return } + let index = next + next += 1 + let job = jobs[index] + let samples = recordings[job.recording] + let slice = Array(samples[job.start.. Int { + let real = leftPad + min(samples, windowSamples) + return real < 321 ? 0 : min(frames, (real - 321) / 160 + 1) + } + + /// Audio tokens for `validFrames` frames: two stride-2 convolutions, so token k is valid when frame 4k is. + static func tokenCount(validFrames: Int) -> Int { (validFrames + 3) / 4 } + + /// Embedding of up to 10 s of 16 kHz mono audio. + public func embed(window: [Float]) async throws -> [Float] { + let count = min(window.count, Self.windowSamples) + let waveform = try MLMultiArray( + shape: [1, NSNumber(value: Self.windowSamples + Self.leftPad)], dataType: .float32) + let mask = try MLMultiArray(shape: [1, NSNumber(value: Self.frames)], dataType: .float16) + let wave = waveform.dataPointer.assumingMemoryBound(to: Float.self) + wave.update(repeating: 0, count: Self.windowSamples + Self.leftPad) + window.withUnsafeBufferPointer { (wave + Self.leftPad).update(from: $0.baseAddress!, count: count) } + let maskPointer = mask.dataPointer.assumingMemoryBound(to: Float16.self) + let validFrames = Self.validFrames(samples: count) + for frame in 0.. 0 else { throw EmbeddingGemma2Error.predictionFailed("window too short to embed") } + return try await text.embed(audioTokens: tokens, count: tokenCount) + } +} diff --git a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Manager.swift b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Manager.swift index 4d6b3b2..c245b1f 100644 --- a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Manager.swift +++ b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Manager.swift @@ -206,15 +206,55 @@ public final class EmbeddingGemma2Manager: Sendable { } private func embed(ids: [Int32]) async throws -> [Float] { + let padded = ids + [Int32](repeating: tokenizer.padId, count: max(Self.lengths.last!, ids.count) - ids.count) let length = Self.lengths.first { $0 >= ids.count } ?? Self.lengths.last! + return try await run(length: length, count: ids.count) { destination in + writeEmbeddings(Array(padded.prefix(length)), to: destination) + } + } + + /// Embedding of one audio window: ` <|audio> tokens ` through the text model, where `tokens` + /// are the first `count` rows of an `EmbeddingGemma2Audio` output ([1, 250, 512], fp16 or fp32). + public func embed(audioTokens tokens: MLMultiArray, count: Int) async throws -> [Float] { + let total = count + 4 + guard let length = Self.lengths.first(where: { $0 >= total }) else { + throw EmbeddingGemma2Error.predictionFailed("\(count) audio tokens do not fit") + } + let tokenStride = tokens.strides[tokens.strides.count - 2].intValue + return try await run(length: length, count: total) { destination in + writeEmbeddings([bosId, Self.audioStartId], to: destination) + let target = destination + 2 * Self.hiddenSize + for row in 0.. + static let audioEndId: Int32 = 258_883 // + private var bosId: Int32 { tokenizer.bosId } + private var eosId: Int32 { tokenizer.eosId } + + /// One `embed_` call: `fill` writes `length` rows of fp16 inputs; the first `count` are attended. + private func run( + length: Int, count: Int, fill: (UnsafeMutablePointer) throws -> Void + ) async throws -> [Float] { guard let model = models[length] else { throw EmbeddingGemma2Error.predictionFailed("no model for \(length)") } let embeds = try MLMultiArray( shape: [1, NSNumber(value: length), NSNumber(value: Self.hiddenSize)], dataType: .float16) let mask = try MLMultiArray(shape: [1, NSNumber(value: length)], dataType: .float16) let maskPointer = mask.dataPointer.assumingMemoryBound(to: Float16.self) - let padded = ids + [Int32](repeating: tokenizer.padId, count: length - ids.count) - writeEmbeddings(padded, to: embeds.dataPointer.assumingMemoryBound(to: Float16.self)) - for position in 0...mlmodelc` beside it). Text and audio +/// are separate bundles, each pinned to the revision that added it, so adding audio never recompiles the text model. public enum EmbeddingGemma2ModelStore { public typealias Progress = @Sendable (_ file: String, _ bytes: Int64) -> Void @@ -16,6 +17,20 @@ public enum EmbeddingGemma2ModelStore { let sha256: String } + static let audioRevision = "c0fefd0ab4c1140ab35fb582ddbe51b4f93205e1" + + private static let audioAssets = [ + Asset( + path: "EmbeddingGemma2Audio.mlpackage/Manifest.json", + sha256: "65c8e9811516e7916a3d742261c7d5a02d4e43c113b437330ce86f98b3da3514"), + Asset( + path: "EmbeddingGemma2Audio.mlpackage/Data/com.apple.CoreML/model.mlmodel", + sha256: "d40896e5c88d62b02d15cc975f27e254c28a5774a1702e09bf5483071623548d"), + Asset( + path: "EmbeddingGemma2Audio.mlpackage/Data/com.apple.CoreML/weights/weight.bin", + sha256: "41d7b552caabe376e377dcbe1c8c5f9879853bc4f384a161bb1f886f19e84b56"), + ] + private static let assets = [ Asset(path: "config.json", sha256: "4ff891a28a5d55562b55ff3aa4ef362968d378629f9453f337cb70a9e1b45f61"), Asset(path: "tokenizer.json", sha256: "4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4"), @@ -31,15 +46,33 @@ public enum EmbeddingGemma2ModelStore { sha256: "75b97ab036ca87eae60560c4034a63d604201498dc4b6f6971dd0b1e4558d61f"), ] - /// Ensures every file exists and matches its checksum, and that the compiled model exists; returns the directory. + /// Ensures the text model, token table and tokenizer exist, match their checksums, and are compiled; returns the + /// directory. public static func ensure(cacheDirectory: URL? = nil, progress: Progress? = nil) async throws -> URL { + try await ensure( + assets: assets, revision: revision, package: "EmbeddingGemma2Text", verifiedName: ".verified-\(revision)", + cacheDirectory: cacheDirectory, progress: progress) + } + + /// Ensures the audio model (`EmbeddingGemma2Audio`) exists, matches its checksums, and is compiled; returns the + /// directory. The text bundle is needed too (`ensure`): audio tokens are embedded by the text model. + public static func ensureAudio(cacheDirectory: URL? = nil, progress: Progress? = nil) async throws -> URL { + try await ensure( + assets: audioAssets, revision: audioRevision, package: "EmbeddingGemma2Audio", + verifiedName: ".verified-audio-\(audioRevision)", cacheDirectory: cacheDirectory, progress: progress) + } + + private static func ensure( + assets: [Asset], revision: String, package: String, verifiedName: String, cacheDirectory: URL?, + progress: Progress? + ) async throws -> URL { let root = cacheDirectory ?? LayaModelStore.defaultCacheDirectory() let directory = root.appendingPathComponent(directoryName) let manager = FileManager.default try manager.createDirectory(at: directory, withIntermediateDirectories: true) - // Hashing ~550 MB takes a moment; skip it once this revision has been verified and nothing is missing. - let verified = directory.appendingPathComponent(".verified-\(revision)") - let compiled = directory.appendingPathComponent("EmbeddingGemma2Text.mlmodelc") + // Hashing hundreds of MB takes a moment; skip it once this revision has been verified and nothing is missing. + let verified = directory.appendingPathComponent(verifiedName) + let compiled = directory.appendingPathComponent("\(package).mlmodelc") let stamp = compiled.appendingPathComponent("fluiduse-revision") let allPresent = assets.allSatisfy { manager.fileExists(atPath: directory.appendingPathComponent($0.path).path) @@ -78,14 +111,13 @@ public enum EmbeddingGemma2ModelStore { // Recompile unless the compiled model carries this revision's stamp (a revision bump or an interrupted // compile leaves an old or missing stamp). Staged rename, so a reader never sees a half-written bundle. if (try? String(contentsOf: stamp, encoding: .utf8)) != revision { - progress?("EmbeddingGemma2Text.mlmodelc", 0) - let temporary = try await MLModel.compileModel( - at: directory.appendingPathComponent("EmbeddingGemma2Text.mlpackage")) + progress?("\(package).mlmodelc", 0) + let temporary = try await MLModel.compileModel(at: directory.appendingPathComponent("\(package).mlpackage")) try revision.write( to: temporary.appendingPathComponent("fluiduse-revision"), atomically: true, encoding: .utf8) - let staged = directory.appendingPathComponent("EmbeddingGemma2Text.\(UUID().uuidString).mlmodelc") + let staged = directory.appendingPathComponent("\(package).\(UUID().uuidString).mlmodelc") try manager.moveItem(at: temporary, to: staged) - let retired = directory.appendingPathComponent("EmbeddingGemma2Text.\(UUID().uuidString).old") + let retired = directory.appendingPathComponent("\(package).\(UUID().uuidString).old") if manager.fileExists(atPath: compiled.path) { try manager.moveItem(at: compiled, to: retired) } try manager.moveItem(at: staged, to: compiled) try? manager.removeItem(at: retired) diff --git a/Tests/FluidUseTests/EmbeddingGemma2Tests.swift b/Tests/FluidUseTests/EmbeddingGemma2Tests.swift index bd90dd0..b0746d4 100644 --- a/Tests/FluidUseTests/EmbeddingGemma2Tests.swift +++ b/Tests/FluidUseTests/EmbeddingGemma2Tests.swift @@ -88,3 +88,24 @@ final class EmbeddingGemma2Tests: XCTestCase { } } } + +/// Window arithmetic for the audio path, against the Hugging Face feature extractor's counts. +final class EmbeddingGemma2AudioWindowTests: XCTestCase { + func testFullWindowMatchesFeatureExtractor() { + // 10 s at 16 kHz: the extractor yields 999 frames and the encoder 250 tokens. + let frames = EmbeddingGemma2Audio.validFrames(samples: 160_000) + XCTAssertEqual(frames, 999) + XCTAssertEqual(EmbeddingGemma2Audio.tokenCount(validFrames: frames), 250) + } + + func testShortClipMatchesFeatureExtractor() { + // 55,680 samples (3.48 s): 347 frames, 87 audio tokens. + let frames = EmbeddingGemma2Audio.validFrames(samples: 55_680) + XCTAssertEqual(frames, 347) + XCTAssertEqual(EmbeddingGemma2Audio.tokenCount(validFrames: frames), 87) + } + + func testTooShortHasNoFrames() { + XCTAssertEqual(EmbeddingGemma2Audio.validFrames(samples: 100), 0) + } +} From 4c63fe2eefe26907cd1a1cfd62a625a122dd8a8f Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Thu, 8 Oct 2026 23:57:55 -0400 Subject: [PATCH 02/10] =?UTF-8?q?AudioSearchDemo:=20hands-free=20by=20defa?= =?UTF-8?q?ult=20=E2=80=94=20index=20on=20launch,=20type=20each=20query,?= =?UTF-8?q?=20play=20its=20top=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop/Auto (Space) toggles it; typing a query or pressing play ends it; --manual starts without it, --clip= sets seconds per result. Search now ranks through one shared path (rank) for typed and hands-free queries. --- .../AudioSearchDemo/AudioSearchModel.swift | 136 ++++++++++++++---- Sources/AudioSearchDemo/ContentView.swift | 9 +- Sources/AudioSearchDemo/README.md | 3 + Sources/AudioSearchDemo/demo.sh | 2 +- 4 files changed, 118 insertions(+), 32 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index b2e8278..ea20b0a 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -49,6 +49,9 @@ final class AudioSearchModel: ObservableObject { @Published private(set) var queryMilliseconds: Double = 0 @Published private(set) var playing: String? @Published var query = "" + /// Hands-free mode (default; `--manual` turns it off): index on launch, then type each suggested query and play + /// its top three windows, round and round. Typing or pressing play yourself switches it off. + @Published private(set) var autoPlay = !CommandLine.arguments.contains("--manual") private var entries: [Entry] = [] /// Every window's embedding, row after row, so a query is one matrix-vector product. @@ -58,6 +61,11 @@ final class AudioSearchModel: ObservableObject { private var player: AVAudioPlayer? private var searchTask: Task? private var runTask: Task? + private var autoTask: Task? + private var autoTyping = false + /// Seconds of each result to play in hands-free mode (`--clip=`). + private let clipSeconds = max( + 1, CommandLine.arguments.first { $0.hasPrefix("--clip=") }.flatMap { Double($0.dropFirst(7)) } ?? 6) var realTimeFactor: Double { indexSeconds > 0 ? audioSeconds / indexSeconds : 0 } var allSuggestions: [String] { collections.flatMap(\.suggestions) } @@ -94,7 +102,7 @@ final class AudioSearchModel: ObservableObject { format: "models loaded in %.1f s · audio encoder on the GPU, text model on the Neural Engine", Double(DispatchTime.now().uptimeNanoseconds - loadStart) / 1e9)) phase = .ready - if CommandLine.arguments.contains("--autostart") { start() } + if autoPlay || CommandLine.arguments.contains("--autostart") { start() } } catch { phase = .failed(error.localizedDescription) DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) @@ -107,6 +115,8 @@ final class AudioSearchModel: ObservableObject { } func reset() { + autoTask?.cancel() + autoTask = nil runTask?.cancel() runTask = nil stop() @@ -167,7 +177,11 @@ final class AudioSearchModel: ObservableObject { if let preset = CommandLine.arguments.first(where: { $0.hasPrefix("--query=") }) { query = String(preset.dropFirst(8)) } - if !query.isEmpty { search() } + if autoPlay { + startAutoPlay() + } else if !query.isEmpty { + search() + } } catch is CancellationError { } catch { phase = .failed(error.localizedDescription) @@ -184,54 +198,116 @@ final class AudioSearchModel: ObservableObject { indexSeconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 } - /// Ranks every indexed window against the query (debounced while typing). + /// Ranks every indexed window against the query (debounced while typing). Typing by hand ends hands-free mode. func search() { + if autoTyping { return } + stopAutoPlay() searchTask?.cancel() let query = query.trimmingCharacters(in: .whitespacesAndNewlines) - guard let text, !query.isEmpty, !entries.isEmpty else { + guard !query.isEmpty, !entries.isEmpty else { results = [] return } searchTask = Task { try? await Task.sleep(for: .milliseconds(120)) - guard !Task.isCancelled else { return } - let begin = DispatchTime.now().uptimeNanoseconds - guard let vector = try? await text.embed(query, prompt: .searchQuery), !Task.isCancelled else { return } - var scores = [Float](repeating: 0, count: entries.count) - cblas_sgemv( - CblasRowMajor, CblasNoTrans, Int32(entries.count), Int32(vector.count), 1, matrix, Int32(vector.count), - vector, 1, 0, &scores, 1) - let top = scores.indices.sorted { scores[$0] > scores[$1] }.prefix(25) - results = top.map { Result(entry: entries[$0], score: scores[$0]) } - queryMilliseconds = Double(DispatchTime.now().uptimeNanoseconds - begin) / 1e6 + guard !Task.isCancelled, let ranked = await rank(query), !Task.isCancelled else { return } + results = ranked + } + } + + /// Top 25 windows for `query`: one text embedding, then one matrix-vector product over every window. + private func rank(_ query: String) async -> [Result]? { + guard let text else { return nil } + let begin = DispatchTime.now().uptimeNanoseconds + guard let vector = try? await text.embed(query, prompt: .searchQuery) else { return nil } + var scores = [Float](repeating: 0, count: entries.count) + cblas_sgemv( + CblasRowMajor, CblasNoTrans, Int32(entries.count), Int32(vector.count), 1, matrix, Int32(vector.count), + vector, 1, 0, &scores, 1) + let ranked = scores.indices.sorted { scores[$0] > scores[$1] }.prefix(25).map { + Result(entry: entries[$0], score: scores[$0]) + } + queryMilliseconds = Double(DispatchTime.now().uptimeNanoseconds - begin) / 1e6 + DemoLog.line( + String(format: "🔎 “%@” → %d windows ranked in %.0f ms", query, entries.count, queryMilliseconds), color: 81) + for result in ranked.prefix(3) { DemoLog.line( - String(format: "🔎 “%@” → %d windows ranked in %.0f ms", query, entries.count, queryMilliseconds), - color: 81) - for result in results.prefix(3) { - DemoLog.line( - String( - format: " %.3f %@ %@ @ %@", result.score, collections[result.entry.collection].name, - result.entry.file.lastPathComponent, Self.clock(result.entry.start))) - } + String( + format: " %.3f %@ %@ @ %@", result.score, collections[result.entry.collection].name, + result.entry.file.lastPathComponent, Self.clock(result.entry.start))) } + return ranked } - func play(_ result: Result) { - if playing == result.id { - stop() - return + func toggleAutoPlay() { + if autoPlay { stopAutoPlay() } else if phase == .done { startAutoPlay() } + } + + private func startAutoPlay() { + autoTask?.cancel() + autoPlay = true + DemoLog.event("▶ hands-free: each query, then its top 3 (\(Int(clipSeconds)) s each)") + autoTask = Task { [weak self] in + while let self, !Task.isCancelled { + let queries = self.allSuggestions + guard !queries.isEmpty else { return } + for query in queries { + guard !Task.isCancelled else { return } + // Type it like a person would, then search. + self.autoTyping = true + self.query = "" + for character in query { + self.query.append(character) + try? await Task.sleep(for: .milliseconds(35)) + } + self.autoTyping = false + guard !Task.isCancelled, let ranked = await self.rank(query) else { return } + self.results = ranked + for (rank, result) in ranked.prefix(3).enumerated() { + guard !Task.isCancelled else { return } + DemoLog.line( + String( + format: " ▶ #%d %@ %@ @ %@", rank + 1, + self.collections[result.entry.collection].name, + result.entry.file.lastPathComponent, Self.clock(result.entry.start)), color: 120) + await self.playClip(result, seconds: min(self.clipSeconds, result.entry.duration)) + } + try? await Task.sleep(for: .seconds(1.2)) + } + } } + } + + private func stopAutoPlay() { + guard autoPlay || autoTask != nil else { return } + autoTask?.cancel() + autoTask = nil + autoTyping = false + autoPlay = false + stop() + DemoLog.event("❚❚ hands-free off") + } + + /// Plays `seconds` of a result's window and returns when it ends (or is stopped). + private func playClip(_ result: Result, seconds: TimeInterval) async { stop() guard let player = try? AVAudioPlayer(contentsOf: result.entry.file) else { return } player.currentTime = result.entry.start player.play() self.player = player playing = result.id - let id = result.id - Task { [weak self] in - try? await Task.sleep(for: .seconds(result.entry.duration)) - if self?.playing == id { self?.stop() } + try? await Task.sleep(for: .seconds(seconds)) + if playing == result.id { stop() } + } + + /// Play button: plays (or stops) one window; ends hands-free mode. + func play(_ result: Result) { + stopAutoPlay() + if playing == result.id { + stop() + return } + Task { await playClip(result, seconds: result.entry.duration) } } func stop() { diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index 2970f11..04912d1 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -31,6 +31,13 @@ struct Header: View { } .keyboardShortcut(.return, modifiers: [.command]) .disabled(model.phase != .ready) + Button { + model.toggleAutoPlay() + } label: { + Label(model.autoPlay ? "Stop" : "Auto", systemImage: model.autoPlay ? "pause.fill" : "sparkles") + } + .keyboardShortcut(.space, modifiers: []) + .disabled(model.phase != .done) Button { model.reset() } label: { @@ -80,7 +87,7 @@ struct Header: View { case .loading(let message): message case .ready: "Ready · press Index (⌘↩)" case .indexing: "Indexing \(model.windowsDone)/\(model.windowsTotal) windows…" - case .done: "Indexed · search below" + case .done: model.autoPlay ? "Hands-free · top 3 for each query" : "Indexed · search below" case .failed(let message): message } } diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index d652824..590773c 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -14,6 +14,9 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from [FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles them once (a few minutes). Needs macOS 15+. +- **Hands-free by default:** it indexes on launch, then types each suggested query, plays the top three windows + (6 s each, `--clip=`), and moves on, round and round. **Space** (Stop/Auto) or typing yourself ends it; + `--manual` starts without it. - **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. - Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. diff --git a/Sources/AudioSearchDemo/demo.sh b/Sources/AudioSearchDemo/demo.sh index f88857e..1566e88 100755 --- a/Sources/AudioSearchDemo/demo.sh +++ b/Sources/AudioSearchDemo/demo.sh @@ -31,4 +31,4 @@ fi pkill -f "release/AudioSearchDemo" 2>/dev/null || true cd "$ROOT" "$ROOT/.build/release/AudioSearchDemo" "$@" > "$LOG" 2> "$LOG.stderr" & -echo "demo pid $! · log $LOG · press Index (⌘↩) in the app" +echo "demo pid $! · log $LOG · indexes and plays on its own (Space stops hands-free mode)" From b6e1d9d1fe12abd6cde4f33d70b13a4430414b67 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 9 Oct 2026 00:23:55 -0400 Subject: [PATCH 03/10] AudioSearchDemo: alternate 30 s of speed (re-read everything) with 30 s of search Hands-free now alternates two segments (--segment=, default 30 s): - Speed: read and embed every file again as fast as it can, with live speed tiles. The first pass runs to completion and is searchable as it grows (windows stream in through EmbeddingGemma2Audio's new onWindow callback); later passes build a new index beside the current one and are cut at the segment end, so searches never see a partial index. - Search: type each suggested query and play its top three (4 s each, --clip=), continuing through the suggestion list across segments. Fixes found on the way: the field's change callback echoing a query the show typed (or cleared) switched hands-free off; the speed countdown returned before the indexing task had started, so a timed pass ran to the end. EmbeddingGemma2Audio checks for cancellation after each window. --- .../AudioSearchDemo/AudioSearchModel.swift | 212 ++++++++++++++---- Sources/AudioSearchDemo/ContentView.swift | 29 ++- Sources/AudioSearchDemo/README.md | 7 +- .../EmbeddingGemma2Audio.swift | 6 +- 4 files changed, 200 insertions(+), 54 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index ea20b0a..34ee555 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -25,7 +25,7 @@ final class AudioSearchModel: ObservableObject { let suggestions: [String] } - struct Entry { + struct Entry: Sendable { let collection: Int let file: URL let start: TimeInterval @@ -52,6 +52,15 @@ final class AudioSearchModel: ObservableObject { /// Hands-free mode (default; `--manual` turns it off): index on launch, then type each suggested query and play /// its top three windows, round and round. Typing or pressing play yourself switches it off. @Published private(set) var autoPlay = !CommandLine.arguments.contains("--manual") + /// Indexing passes so far. The first grows the index live; later ones rebuild it in the background while the + /// current index keeps answering searches, to keep showing the indexing speed. + @Published private(set) var pass = 0 + @Published private(set) var isIndexing = false + @Published private(set) var indexedWindows = 0 + /// Hands-free show: alternating `--segment=` seconds (default 30) of speed (reading everything again) and search. + enum Segment { case speed, search } + @Published private(set) var segment: Segment? + @Published private(set) var segmentRemaining = 0 private var entries: [Entry] = [] /// Every window's embedding, row after row, so a query is one matrix-vector product. @@ -63,9 +72,13 @@ final class AudioSearchModel: ObservableObject { private var runTask: Task? private var autoTask: Task? private var autoTyping = false + /// The last query text hands-free mode put in the field; the field's change callback echoing it is not typing. + private var autoQuery = "" /// Seconds of each result to play in hands-free mode (`--clip=`). private let clipSeconds = max( - 1, CommandLine.arguments.first { $0.hasPrefix("--clip=") }.flatMap { Double($0.dropFirst(7)) } ?? 6) + 1, CommandLine.arguments.first { $0.hasPrefix("--clip=") }.flatMap { Double($0.dropFirst(7)) } ?? 4) + private let segmentSeconds = max( + 5, CommandLine.arguments.first { $0.hasPrefix("--segment=") }.flatMap { Double($0.dropFirst(10)) } ?? 30) var realTimeFactor: Double { indexSeconds > 0 ? audioSeconds / indexSeconds : 0 } var allSuggestions: [String] { collections.flatMap(\.suggestions) } @@ -102,7 +115,11 @@ final class AudioSearchModel: ObservableObject { format: "models loaded in %.1f s · audio encoder on the GPU, text model on the Neural Engine", Double(DispatchTime.now().uptimeNanoseconds - loadStart) / 1e9)) phase = .ready - if autoPlay || CommandLine.arguments.contains("--autostart") { start() } + if autoPlay { + startAutoPlay() + } else if CommandLine.arguments.contains("--autostart") { + start() + } } catch { phase = .failed(error.localizedDescription) DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) @@ -110,7 +127,7 @@ final class AudioSearchModel: ObservableObject { } func start() { - guard phase == .ready else { return } + guard phase == .ready || phase == .done, !isIndexing else { return } runTask = Task { await index() } } @@ -122,6 +139,9 @@ final class AudioSearchModel: ObservableObject { stop() entries = [] matrix = [] + indexedWindows = 0 + pass = 0 + isIndexing = false results = [] windowsDone = 0 windowsTotal = 0 @@ -131,12 +151,24 @@ final class AudioSearchModel: ObservableObject { DemoLog.event("↺ reset") } + /// One indexing pass over every collection. Pass 1 adds windows to the live index as they finish (searchable at + /// once); later passes build a new index beside the current one and swap it in at the end. private func index() async { guard let audio else { return } + pass += 1 + isIndexing = true phase = .indexing - DemoLog.event("▶ indexing \(collections.reduce(0) { $0 + $1.files.count }) files") + let live = pass == 1 + // Round-robin across collections, so a growing index covers every collection early. + var queues = collections.map { collection in collection.files.map { (collection.id, $0) } } + var ordered: [(Int, URL)] = [] + while queues.contains(where: { !$0.isEmpty }) { + for index in queues.indices where !queues[index].isEmpty { ordered.append(queues[index].removeFirst()) } + } + let files = ordered + DemoLog.event( + "▶ pass \(pass): reading \(files.count) files" + (live ? "" : " (searches keep using pass \(pass - 1))")) do { - let files = collections.flatMap { collection in collection.files.map { (collection.id, $0) } } let decodeStart = DispatchTime.now().uptimeNanoseconds let recordings = try await Task.detached(priority: .userInitiated) { try files.map { try EmbeddingGemma2Audio.samples(contentsOf: $0.1) } @@ -148,47 +180,69 @@ final class AudioSearchModel: ObservableObject { let total = recordings.reduce(0.0) { $0 + Double($1.count) } / Double(EmbeddingGemma2Audio.sampleRate) let start = DispatchTime.now().uptimeNanoseconds let progress = IndexProgress() - // The embedding tasks report into `progress`; the UI samples it ten times a second. + let inbox = WindowInbox() + var staged: [Entry] = [] + // The embedding tasks report into `progress` and `inbox`; the UI drains them ten times a second. let watcher = Task { @MainActor [weak self] in while !Task.isCancelled { try? await Task.sleep(for: .milliseconds(100)) - self?.refresh(progress: progress, total: total, start: start) + guard let self else { return } + self.refresh(progress: progress, total: total, start: start) + let fresh = inbox.drain() + if live { self.add(fresh) } else { staged += fresh } } } - defer { watcher.cancel() } - let windows = try await audio.embed(recordings: recordings) { done, count in - progress.update(done: done, total: count) - } + _ = try await audio.embed( + recordings: recordings, progress: { done, count in progress.update(done: done, total: count) }, + onWindow: { recording, window in + inbox.append( + Entry( + collection: files[recording].0, file: files[recording].1, start: window.start, + duration: window.duration, embedding: window.embedding)) + }) + watcher.cancel() try Task.checkCancellation() - entries = zip(files, windows).flatMap { file, windows in - windows.map { - Entry( - collection: file.0, file: file.1, start: $0.start, duration: $0.duration, - embedding: $0.embedding) - } - } - matrix = entries.flatMap(\.embedding) refresh(progress: progress, total: total, start: start) + if live { + add(inbox.drain()) + } else { + staged += inbox.drain() + entries = staged + matrix = staged.flatMap(\.embedding) + indexedWindows = staged.count + } + isIndexing = false phase = .done DemoLog.event( String( - format: "indexed %@ of audio (%d windows) in %.1f s = %.0fx real time", Self.clock(total), - entries.count, indexSeconds, realTimeFactor)) - if let preset = CommandLine.arguments.first(where: { $0.hasPrefix("--query=") }) { + format: "pass %d: indexed %@ of audio (%d windows) in %.1f s = %.0fx real time", pass, + Self.clock(total), indexedWindows, indexSeconds, realTimeFactor)) + if let preset = CommandLine.arguments.first(where: { $0.hasPrefix("--query=") }), pass == 1 { query = String(preset.dropFirst(8)) } - if autoPlay { - startAutoPlay() - } else if !query.isEmpty { - search() - } + if !autoPlay, !query.isEmpty, pass == 1 { search() } } catch is CancellationError { + // Cut short (a timed speed run, or Reset): the previous index stays in place. + isIndexing = false + if phase == .indexing { phase = indexedWindows > 0 ? .done : .ready } + DemoLog.event( + String( + format: "pass %d: read %@ of audio in %.1f s = %.0fx real time (stopped there)", pass, + Self.clock(audioSeconds), indexSeconds, realTimeFactor)) } catch { + isIndexing = false phase = .failed(error.localizedDescription) DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) } } + private func add(_ fresh: [Entry]) { + guard !fresh.isEmpty else { return } + entries += fresh + matrix += fresh.flatMap(\.embedding) + indexedWindows = entries.count + } + private func refresh(progress: IndexProgress, total: Double, start: UInt64) { let (done, count) = progress.snapshot guard count > 0 else { return } @@ -200,7 +254,7 @@ final class AudioSearchModel: ObservableObject { /// Ranks every indexed window against the query (debounced while typing). Typing by hand ends hands-free mode. func search() { - if autoTyping { return } + if autoTyping || query == autoQuery { return } stopAutoPlay() searchTask?.cancel() let query = query.trimmingCharacters(in: .whitespacesAndNewlines) @@ -240,50 +294,116 @@ final class AudioSearchModel: ObservableObject { } func toggleAutoPlay() { - if autoPlay { stopAutoPlay() } else if phase == .done { startAutoPlay() } + if autoPlay { stopAutoPlay() } else if !entries.isEmpty { startAutoPlay() } } + /// The hands-free show: a speed segment (read every file again as fast as possible; the very first one runs + /// until the index is complete), then a search segment (type a query, play its top three, next query), each + /// `segmentSeconds` long, round and round. private func startAutoPlay() { autoTask?.cancel() autoPlay = true - DemoLog.event("▶ hands-free: each query, then its top 3 (\(Int(clipSeconds)) s each)") + DemoLog.event( + "▶ hands-free: \(Int(segmentSeconds)) s reading as fast as it can, then \(Int(segmentSeconds)) s of searches" + ) autoTask = Task { [weak self] in + var nextQuery = 0 while let self, !Task.isCancelled { + // ⚡ Speed + self.segment = .speed + self.stop() + self.results = [] + self.autoQuery = "" + self.query = "" + let firstPass = self.indexedWindows == 0 + // Mark it now: the countdown below polls `isIndexing` before the task gets to run. + self.isIndexing = true + let run = Task { await self.index() } + self.runTask = run + if firstPass { + let ticker = self.countUp() + await run.value + ticker.cancel() + } else { + await self.countdown(self.segmentSeconds) { !self.isIndexing && self.phase != .indexing } + if self.isIndexing { run.cancel() } + await run.value + } + guard !Task.isCancelled, self.indexedWindows > 0 else { return } + // 🔎 Search + self.segment = .search + let deadline = ContinuousClock.now + .seconds(self.segmentSeconds) + let ticker = Task { @MainActor [weak self] in + while !Task.isCancelled, let self { + self.segmentRemaining = max( + 0, Int((deadline - ContinuousClock.now).components.seconds)) + try? await Task.sleep(for: .milliseconds(250)) + } + } let queries = self.allSuggestions - guard !queries.isEmpty else { return } - for query in queries { - guard !Task.isCancelled else { return } + while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline { + let query = queries[nextQuery % queries.count] + nextQuery += 1 // Type it like a person would, then search. self.autoTyping = true + self.autoQuery = "" self.query = "" for character in query { + self.autoQuery.append(character) self.query.append(character) - try? await Task.sleep(for: .milliseconds(35)) + try? await Task.sleep(for: .milliseconds(30)) } self.autoTyping = false - guard !Task.isCancelled, let ranked = await self.rank(query) else { return } + guard !Task.isCancelled, let ranked = await self.rank(query) else { break } self.results = ranked for (rank, result) in ranked.prefix(3).enumerated() { - guard !Task.isCancelled else { return } + let left = (deadline - ContinuousClock.now).components.seconds + guard !Task.isCancelled, left > 0 else { break } DemoLog.line( String( format: " ▶ #%d %@ %@ @ %@", rank + 1, self.collections[result.entry.collection].name, result.entry.file.lastPathComponent, Self.clock(result.entry.start)), color: 120) - await self.playClip(result, seconds: min(self.clipSeconds, result.entry.duration)) + await self.playClip( + result, seconds: min(self.clipSeconds, result.entry.duration, Double(left))) } - try? await Task.sleep(for: .seconds(1.2)) + try? await Task.sleep(for: .milliseconds(800)) } + ticker.cancel() + self.stop() + } + } + } + + /// Counts the speed segment's seconds up (the first pass runs to completion, however long that is). + private func countUp() -> Task { + let start = ContinuousClock.now + return Task { @MainActor [weak self] in + while !Task.isCancelled, let self { + self.segmentRemaining = Int((ContinuousClock.now - start).components.seconds) + try? await Task.sleep(for: .milliseconds(250)) } } } + /// Waits `seconds` (or until `done()`), counting down `segmentRemaining`. + private func countdown(_ seconds: Double, until done: () -> Bool) async { + let deadline = ContinuousClock.now + .seconds(seconds) + while !Task.isCancelled, ContinuousClock.now < deadline, !done() { + segmentRemaining = max(0, Int((deadline - ContinuousClock.now).components.seconds)) + try? await Task.sleep(for: .milliseconds(250)) + } + } + private func stopAutoPlay() { guard autoPlay || autoTask != nil else { return } autoTask?.cancel() autoTask = nil autoTyping = false autoPlay = false + segment = nil + // A timed re-read stops with the show; the first pass finishes so the index is complete. + if pass > 1, isIndexing { runTask?.cancel() } stop() DemoLog.event("❚❚ hands-free off") } @@ -379,6 +499,20 @@ final class AudioSearchModel: ObservableObject { } } +/// Windows finished by the embedding tasks, waiting for the UI to add them to the index. +final class WindowInbox: Sendable { + private let pending = OSAllocatedUnfairLock(initialState: [AudioSearchModel.Entry]()) + + func append(_ entry: AudioSearchModel.Entry) { pending.withLock { $0.append(entry) } } + + func drain() -> [AudioSearchModel.Entry] { + pending.withLock { entries in + defer { entries = [] } + return entries + } + } +} + /// Window counts from the embedding tasks, read by the UI on a timer. final class IndexProgress: Sendable { private let counts = OSAllocatedUnfairLock(initialState: (done: 0, total: 0)) diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index 04912d1..25db0cd 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -30,21 +30,21 @@ struct Header: View { Label("Index", systemImage: "play.fill") } .keyboardShortcut(.return, modifiers: [.command]) - .disabled(model.phase != .ready) + .disabled(model.phase != .ready && !(model.phase == .done && !model.isIndexing)) Button { model.toggleAutoPlay() } label: { Label(model.autoPlay ? "Stop" : "Auto", systemImage: model.autoPlay ? "pause.fill" : "sparkles") } .keyboardShortcut(.space, modifiers: []) - .disabled(model.phase != .done) + .disabled(model.indexedWindows == 0) Button { model.reset() } label: { Label("Reset", systemImage: "arrow.counterclockwise") } .keyboardShortcut("r", modifiers: [.command]) - .disabled(model.phase != .done && model.phase != .indexing) + .disabled(model.indexedWindows == 0 && !model.isIndexing) } GeometryReader { proxy in Capsule().fill(Color.primary.opacity(0.08)) @@ -57,12 +57,14 @@ struct Header: View { } .frame(height: 6) HStack(spacing: 10) { - Tile(value: AudioSearchModel.clock(model.audioSeconds), caption: "audio indexed") + Tile( + value: AudioSearchModel.clock(model.audioSeconds), + caption: model.pass > 1 ? "audio read, pass \(model.pass)" : "audio read") Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") Tile( value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), caption: "faster than real time") - Tile(value: "\(model.windowsDone)", caption: "10 s windows") + Tile(value: "\(model.indexedWindows)", caption: "windows searchable") } HStack(spacing: 8) { ForEach(model.collections) { collection in @@ -83,12 +85,17 @@ struct Header: View { } private var status: String { - switch model.phase { - case .loading(let message): message - case .ready: "Ready · press Index (⌘↩)" - case .indexing: "Indexing \(model.windowsDone)/\(model.windowsTotal) windows…" - case .done: model.autoPlay ? "Hands-free · top 3 for each query" : "Indexed · search below" - case .failed(let message): message + switch (model.segment, model.phase) { + case (.speed, _): + model.pass > 1 && model.indexedWindows > 0 + ? "⚡ Reading everything again, as fast as it can · \(model.segmentRemaining) s" + : "⚡ Reading everything for the first time · \(model.segmentRemaining) s" + case (.search, _): "🔎 Searching · \(model.segmentRemaining) s left" + case (nil, .loading(let message)): message + case (nil, .ready): "Ready · press Index (⌘↩)" + case (nil, .indexing): "Indexing \(model.windowsDone)/\(model.windowsTotal) windows…" + case (nil, .done): "Indexed · search below" + case (nil, .failed(let message)): message } } } diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 590773c..3b4773a 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -14,9 +14,10 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from [FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles them once (a few minutes). Needs macOS 15+. -- **Hands-free by default:** it indexes on launch, then types each suggested query, plays the top three windows - (6 s each, `--clip=`), and moves on, round and round. **Space** (Stop/Auto) or typing yourself ends it; - `--manual` starts without it. +- **Hands-free by default**, alternating two 30 s segments (`--segment=`): **⚡ Speed** reads every file again as fast + as it can (the first pass runs to completion and is searchable as it grows), then **🔎 Search** types each suggested + query and plays its top three windows (4 s each, `--clip=`). Searches keep using the previous index while a speed + pass rebuilds it. **Space** (Stop/Auto) or typing yourself ends it; `--manual` starts without it. - **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. - Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. diff --git a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift index 6adeeff..b9ad3f4 100644 --- a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift +++ b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift @@ -99,8 +99,10 @@ public final class EmbeddingGemma2Audio: Sendable { /// Windows of several recordings, embedded together so short files still keep the GPU and the Neural Engine /// busy (one file at a time leaves a single window in flight). Result: one window list per recording, in order. + /// `onWindow` receives each window as soon as it is embedded (completion order), for indexes that grow live. public func embed( - recordings: [[Float]], progress: (@Sendable (_ done: Int, _ total: Int) -> Void)? = nil + recordings: [[Float]], progress: (@Sendable (_ done: Int, _ total: Int) -> Void)? = nil, + onWindow: (@Sendable (_ recording: Int, _ window: Window) -> Void)? = nil ) async throws -> [[Window]] { var jobs: [(recording: Int, start: Int)] = [] for (recording, samples) in recordings.enumerated() { @@ -135,6 +137,8 @@ public final class EmbeddingGemma2Audio: Sendable { for _ in 0.. Date: Fri, 9 Oct 2026 00:30:22 -0400 Subject: [PATCH 04/10] AudioSearchDemo: show search speed, not re-reading, in the hands-free loop After the first read, the show alternates 30 s of listening (type a query, play its top three) with 30 s of search speed: ~100 different queries (suggestions, earnings/news topics, the ESC-50 sound classes) back to back, 64 at a time, embedded eight per Neural Engine call (pack_256) and ranked against every window with one sgemm; each query gets its own top 10, the latest is shown. Tiles switch to searches run, searches/s (rolling 1 s), ms per search and windows ranked per second. ~700 searches/s, 1.4 ms each (embed 1.4 ms, rank 0.02 ms) over 2,786 windows. The packed path is warmed at load so the first second is not a cold start. The timed re-read segments are gone. --- .../AudioSearchDemo/AudioSearchModel.swift | 198 ++++++++++++++---- Sources/AudioSearchDemo/ContentView.swift | 31 +-- Sources/AudioSearchDemo/README.md | 10 +- 3 files changed, 178 insertions(+), 61 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index 34ee555..4a32882 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -57,10 +57,16 @@ final class AudioSearchModel: ObservableObject { @Published private(set) var pass = 0 @Published private(set) var isIndexing = false @Published private(set) var indexedWindows = 0 - /// Hands-free show: alternating `--segment=` seconds (default 30) of speed (reading everything again) and search. - enum Segment { case speed, search } + /// Hands-free show: read everything once (searchable as it grows), then alternate `--segment=` seconds (default + /// 30) of listening (type a query, play its top three) and of search speed (queries back to back, as fast as the + /// Neural Engine embeds them). + enum Segment { case reading, listen, burst } @Published private(set) var segment: Segment? @Published private(set) var segmentRemaining = 0 + /// Search-speed counters for the current burst. + @Published private(set) var burstQueries = 0 + @Published private(set) var burstPerSecond: Double = 0 + @Published private(set) var burstMilliseconds: Double = 0 private var entries: [Entry] = [] /// Every window's embedding, row after row, so a query is one matrix-vector product. @@ -108,6 +114,8 @@ final class AudioSearchModel: ObservableObject { let audio = try await EmbeddingGemma2Audio.load(text: text) _ = try await audio.embed(window: [Float](repeating: 0, count: 16_000)) _ = try await text.embed("warm up", prompt: .searchQuery) + // The search-speed segment embeds queries eight per call (pack_256): load that path now too. + _ = try await text.embed(Array(repeating: "warm up", count: 64), prompt: .searchQuery) self.text = text self.audio = audio DemoLog.model( @@ -297,49 +305,30 @@ final class AudioSearchModel: ObservableObject { if autoPlay { stopAutoPlay() } else if !entries.isEmpty { startAutoPlay() } } - /// The hands-free show: a speed segment (read every file again as fast as possible; the very first one runs - /// until the index is complete), then a search segment (type a query, play its top three, next query), each + /// The hands-free show: the first read (searchable as it grows), then listen / search-speed segments, each /// `segmentSeconds` long, round and round. private func startAutoPlay() { autoTask?.cancel() autoPlay = true DemoLog.event( - "▶ hands-free: \(Int(segmentSeconds)) s reading as fast as it can, then \(Int(segmentSeconds)) s of searches" + "▶ hands-free: \(Int(segmentSeconds)) s listening to the top 3, then \(Int(segmentSeconds)) s of searches as fast as it can" ) autoTask = Task { [weak self] in var nextQuery = 0 - while let self, !Task.isCancelled { - // ⚡ Speed - self.segment = .speed - self.stop() - self.results = [] - self.autoQuery = "" - self.query = "" - let firstPass = self.indexedWindows == 0 - // Mark it now: the countdown below polls `isIndexing` before the task gets to run. + if let self, self.indexedWindows == 0 { + self.segment = .reading self.isIndexing = true let run = Task { await self.index() } self.runTask = run - if firstPass { - let ticker = self.countUp() - await run.value - ticker.cancel() - } else { - await self.countdown(self.segmentSeconds) { !self.isIndexing && self.phase != .indexing } - if self.isIndexing { run.cancel() } - await run.value - } - guard !Task.isCancelled, self.indexedWindows > 0 else { return } - // 🔎 Search - self.segment = .search + let ticker = self.countUp() + await run.value + ticker.cancel() + } + while let self, !Task.isCancelled, self.indexedWindows > 0 { + // 🔎 Listen + self.segment = .listen let deadline = ContinuousClock.now + .seconds(self.segmentSeconds) - let ticker = Task { @MainActor [weak self] in - while !Task.isCancelled, let self { - self.segmentRemaining = max( - 0, Int((deadline - ContinuousClock.now).components.seconds)) - try? await Task.sleep(for: .milliseconds(250)) - } - } + let ticker = self.countDown(to: deadline) let queries = self.allSuggestions while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline { let query = queries[nextQuery % queries.count] @@ -371,10 +360,142 @@ final class AudioSearchModel: ObservableObject { } ticker.cancel() self.stop() + guard !Task.isCancelled else { return } + // ⚡ Search speed + self.segment = .burst + let burstEnd = ContinuousClock.now + .seconds(self.segmentSeconds) + let burstTicker = self.countDown(to: burstEnd) + await self.burst(until: burstEnd) + burstTicker.cancel() + } + } + } + + /// Queries back to back until `deadline`: 64 at a time, embedded eight per Neural Engine call, then ranked + /// against every window in one matrix multiply. The field and the results show the latest query of each batch. + private func burst(until deadline: ContinuousClock.Instant) async { + guard let text, !entries.isEmpty else { return } + let pool = queryPool + let windows = entries.count + let dimension = EmbeddingGemma2Manager.dimension + let batchSize = 64 + burstQueries = 0 + burstPerSecond = 0 + burstMilliseconds = 0 + DemoLog.event("⚡ search speed: \(pool.count) different queries, back to back, against \(windows) windows") + let start = DispatchTime.now().uptimeNanoseconds + var embedNanoseconds: UInt64 = 0 + var rankNanoseconds: UInt64 = 0 + var next = 0 + var lastLog = start + var recent: [(time: UInt64, count: Int)] = [(start, 0)] + while !Task.isCancelled, ContinuousClock.now < deadline { + let batch = (0.. 1_000_000_000 } + let window = Double(end - (recent.first.map { $0.time } ?? start)) / 1e9 + let recentQueries = recent.dropFirst().reduce(0) { $0 + $1.count } + if window > 0.2, recentQueries > 0 { + burstPerSecond = Double(recentQueries) / window + burstMilliseconds = 1000 / burstPerSecond + } + autoQuery = batch[batchSize - 1] + query = autoQuery + results = shown + queryMilliseconds = burstMilliseconds + if end - lastLog > 1_000_000_000 { + lastLog = end + DemoLog.line( + String( + format: + "⚡ %d queries · %.0f/s · %.2f ms each (embed %.2f + rank %.2f) · %.1f M window scores/s", + burstQueries, burstPerSecond, burstMilliseconds, + Double(embedNanoseconds) / 1e6 / Double(burstQueries), + Double(rankNanoseconds) / 1e6 / Double(burstQueries), + burstPerSecond * Double(windows) / 1e6), color: 81) + } + } + let total = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 + DemoLog.event( + String( + format: "⚡ %d searches in %.0f s = %.0f/s on average, %.0f/s at the end", burstQueries, total, + Double(burstQueries) / total, burstPerSecond)) + } + + /// Indices of the `count` highest scores in one row of a row-major score matrix, best first. + static func topIndices(_ scores: [Float], row: Int, width: Int, count: Int) -> [Int] { + var best: [(index: Int, score: Float)] = [] + best.reserveCapacity(count + 1) + for column in 0.. best[best.count - 1].score { + let position = best.firstIndex { score > $0.score } ?? best.count + best.insert((column, score), at: position) + if best.count > count { best.removeLast() } } } + return best.map(\.index) } + private func countDown(to deadline: ContinuousClock.Instant) -> Task { + Task { @MainActor [weak self] in + while !Task.isCancelled, let self { + self.segmentRemaining = max(0, Int((deadline - ContinuousClock.now).components.seconds)) + try? await Task.sleep(for: .milliseconds(250)) + } + } + } + + /// Everything the search-speed segment cycles through: the suggestions plus a spread of topics and sounds. + var queryPool: [String] { + var pool = allSuggestions + Self.extraQueries + let sounds = Set( + collections.flatMap(\.files).compactMap { file -> String? in + let name = file.deletingPathExtension().lastPathComponent + guard let range = name.range(of: "__") else { return nil } + return "the sound of " + name[.. Task { let start = ContinuousClock.now @@ -386,15 +507,6 @@ final class AudioSearchModel: ObservableObject { } } - /// Waits `seconds` (or until `done()`), counting down `segmentRemaining`. - private func countdown(_ seconds: Double, until done: () -> Bool) async { - let deadline = ContinuousClock.now + .seconds(seconds) - while !Task.isCancelled, ContinuousClock.now < deadline, !done() { - segmentRemaining = max(0, Int((deadline - ContinuousClock.now).components.seconds)) - try? await Task.sleep(for: .milliseconds(250)) - } - } - private func stopAutoPlay() { guard autoPlay || autoTask != nil else { return } autoTask?.cancel() @@ -402,8 +514,6 @@ final class AudioSearchModel: ObservableObject { autoTyping = false autoPlay = false segment = nil - // A timed re-read stops with the show; the first pass finishes so the index is complete. - if pass > 1, isIndexing { runTask?.cancel() } stop() DemoLog.event("❚❚ hands-free off") } diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index 25db0cd..8a1461c 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -57,14 +57,21 @@ struct Header: View { } .frame(height: 6) HStack(spacing: 10) { - Tile( - value: AudioSearchModel.clock(model.audioSeconds), - caption: model.pass > 1 ? "audio read, pass \(model.pass)" : "audio read") - Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") - Tile( - value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), - caption: "faster than real time") - Tile(value: "\(model.indexedWindows)", caption: "windows searchable") + if model.segment == .burst { + Tile(value: "\(model.burstQueries)", caption: "searches run") + Tile(value: String(format: "%.0f", model.burstPerSecond), caption: "searches / second") + Tile(value: String(format: "%.2f ms", model.burstMilliseconds), caption: "per search") + Tile( + value: String(format: "%.1f M", model.burstPerSecond * Double(model.indexedWindows) / 1e6), + caption: "audio windows ranked / s") + } else { + Tile(value: AudioSearchModel.clock(model.audioSeconds), caption: "audio indexed") + Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") + Tile( + value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), + caption: "faster than real time") + Tile(value: "\(model.indexedWindows)", caption: "windows searchable") + } } HStack(spacing: 8) { ForEach(model.collections) { collection in @@ -86,11 +93,9 @@ struct Header: View { private var status: String { switch (model.segment, model.phase) { - case (.speed, _): - model.pass > 1 && model.indexedWindows > 0 - ? "⚡ Reading everything again, as fast as it can · \(model.segmentRemaining) s" - : "⚡ Reading everything for the first time · \(model.segmentRemaining) s" - case (.search, _): "🔎 Searching · \(model.segmentRemaining) s left" + case (.reading, _): "📥 Reading everything · searchable as it goes · \(model.segmentRemaining) s" + case (.listen, _): "🔎 Listening to the top 3 · \(model.segmentRemaining) s left" + case (.burst, _): "⚡ Searching as fast as it can · \(model.segmentRemaining) s left" case (nil, .loading(let message)): message case (nil, .ready): "Ready · press Index (⌘↩)" case (nil, .indexing): "Indexing \(model.windowsDone)/\(model.windowsTotal) windows…" diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 3b4773a..95171b6 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -14,10 +14,11 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from [FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles them once (a few minutes). Needs macOS 15+. -- **Hands-free by default**, alternating two 30 s segments (`--segment=`): **⚡ Speed** reads every file again as fast - as it can (the first pass runs to completion and is searchable as it grows), then **🔎 Search** types each suggested - query and plays its top three windows (4 s each, `--clip=`). Searches keep using the previous index while a speed - pass rebuilds it. **Space** (Stop/Auto) or typing yourself ends it; `--manual` starts without it. +- **Hands-free by default.** It reads everything once (searchable while it reads), then alternates two 30 s segments + (`--segment=`): **🔎 Listen** types each suggested query and plays its top three windows (4 s each, `--clip=`); + **⚡ Search speed** runs ~100 different queries back to back (64 at a time, embedded eight per Neural Engine call, + ranked against every window with one matrix multiply) and shows searches per second live. **Space** (Stop/Auto) or + typing yourself ends it; `--manual` starts without it. - **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. - Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. @@ -33,6 +34,7 @@ Numbers (M5 Pro, macOS 27): |---|---| | 1 hour earnings call, 360 windows | 4.6–5.7 s, 626–777× real time | | 4 h 36 min (2,301 files incl. 2,000 five-second clips) | 34.7 s, 478× real time | +| Search speed over those 2,786 windows | ~700 searches/s, 1.4 ms each (query embedding 1.4 ms on the ANE, ranking 0.02 ms) | | Swift vs sentence-transformers (fp32) | cosine 0.9995 on a LibriSpeech clip | Quality: speech search works well when the query names something said ("forward-looking statements" finds the From 98faf123985205adf6366109d75ea97c5e31a7db Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 9 Oct 2026 00:34:14 -0400 Subject: [PATCH 05/10] AudioSearchDemo: indexing runs on its own; listening starts at once Indexing starts at launch with its own tile row; the listen / search-speed show starts as soon as 64 windows are searchable and searches whatever is indexed by then. Search speed waits for indexing to finish (both use the Neural Engine; during indexing it measured 75/s instead of ~700/s), so the first listening segment runs until then. Search tiles get their own row. --- .../AudioSearchDemo/AudioSearchModel.swift | 44 ++++++++----------- Sources/AudioSearchDemo/ContentView.swift | 41 ++++++++++------- Sources/AudioSearchDemo/README.md | 9 ++-- 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index 4a32882..2912f3f 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -60,7 +60,7 @@ final class AudioSearchModel: ObservableObject { /// Hands-free show: read everything once (searchable as it grows), then alternate `--segment=` seconds (default /// 30) of listening (type a query, play its top three) and of search speed (queries back to back, as fast as the /// Neural Engine embeds them). - enum Segment { case reading, listen, burst } + enum Segment { case listen, burst } @Published private(set) var segment: Segment? @Published private(set) var segmentRemaining = 0 /// Search-speed counters for the current burst. @@ -315,14 +315,14 @@ final class AudioSearchModel: ObservableObject { ) autoTask = Task { [weak self] in var nextQuery = 0 + // Indexing is its own thing: it runs in the background while the show goes on. The show starts as + // soon as there is something to search, and every search sees whatever is indexed by then. if let self, self.indexedWindows == 0 { - self.segment = .reading - self.isIndexing = true - let run = Task { await self.index() } - self.runTask = run - let ticker = self.countUp() - await run.value - ticker.cancel() + if !self.isIndexing { + self.isIndexing = true + self.runTask = Task { await self.index() } + } + while !Task.isCancelled, self.indexedWindows < 64 { try? await Task.sleep(for: .milliseconds(100)) } } while let self, !Task.isCancelled, self.indexedWindows > 0 { // 🔎 Listen @@ -330,7 +330,8 @@ final class AudioSearchModel: ObservableObject { let deadline = ContinuousClock.now + .seconds(self.segmentSeconds) let ticker = self.countDown(to: deadline) let queries = self.allSuggestions - while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline { + // Search speed shares the Neural Engine with indexing, so listening goes on until indexing is done. + while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline || self.isIndexing { let query = queries[nextQuery % queries.count] nextQuery += 1 // Type it like a person would, then search. @@ -346,7 +347,7 @@ final class AudioSearchModel: ObservableObject { guard !Task.isCancelled, let ranked = await self.rank(query) else { break } self.results = ranked for (rank, result) in ranked.prefix(3).enumerated() { - let left = (deadline - ContinuousClock.now).components.seconds + let left = self.isIndexing ? 60 : (deadline - ContinuousClock.now).components.seconds guard !Task.isCancelled, left > 0 else { break } DemoLog.line( String( @@ -376,13 +377,13 @@ final class AudioSearchModel: ObservableObject { private func burst(until deadline: ContinuousClock.Instant) async { guard let text, !entries.isEmpty else { return } let pool = queryPool - let windows = entries.count let dimension = EmbeddingGemma2Manager.dimension let batchSize = 64 burstQueries = 0 burstPerSecond = 0 burstMilliseconds = 0 - DemoLog.event("⚡ search speed: \(pool.count) different queries, back to back, against \(windows) windows") + DemoLog.event( + "⚡ search speed: \(pool.count) different queries, back to back, against \(entries.count)+ windows") let start = DispatchTime.now().uptimeNanoseconds var embedNanoseconds: UInt64 = 0 var rankNanoseconds: UInt64 = 0 @@ -395,11 +396,15 @@ final class AudioSearchModel: ObservableObject { let embedStart = DispatchTime.now().uptimeNanoseconds guard let vectors = try? await text.embed(batch, prompt: .searchQuery) else { return } let rankStart = DispatchTime.now().uptimeNanoseconds + // The index may still be growing: rank against this moment's windows. + let snapshot = entries + let index = matrix + let windows = snapshot.count let queryMatrix = vectors.flatMap { $0 } var scores = [Float](repeating: 0, count: batchSize * windows) cblas_sgemm( CblasRowMajor, CblasNoTrans, CblasTrans, Int32(batchSize), Int32(windows), Int32(dimension), 1, - queryMatrix, Int32(dimension), matrix, Int32(dimension), 0, &scores, Int32(windows)) + queryMatrix, Int32(dimension), index, Int32(dimension), 0, &scores, Int32(windows)) // Every query gets its own top 10, as a real search would; only the last one is shown. var shown: [Result] = [] for row in 0.. Task { - let start = ContinuousClock.now - return Task { @MainActor [weak self] in - while !Task.isCancelled, let self { - self.segmentRemaining = Int((ContinuousClock.now - start).components.seconds) - try? await Task.sleep(for: .milliseconds(250)) - } - } - } - private func stopAutoPlay() { guard autoPlay || autoTask != nil else { return } autoTask?.cancel() diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index 8a1461c..2c71bf6 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -57,22 +57,27 @@ struct Header: View { } .frame(height: 6) HStack(spacing: 10) { - if model.segment == .burst { - Tile(value: "\(model.burstQueries)", caption: "searches run") - Tile(value: String(format: "%.0f", model.burstPerSecond), caption: "searches / second") - Tile(value: String(format: "%.2f ms", model.burstMilliseconds), caption: "per search") - Tile( - value: String(format: "%.1f M", model.burstPerSecond * Double(model.indexedWindows) / 1e6), - caption: "audio windows ranked / s") - } else { - Tile(value: AudioSearchModel.clock(model.audioSeconds), caption: "audio indexed") - Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") - Tile( - value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), - caption: "faster than real time") - Tile(value: "\(model.indexedWindows)", caption: "windows searchable") - } + Tile(value: AudioSearchModel.clock(model.audioSeconds), caption: "audio indexed") + Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") + Tile( + value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), + caption: model.isIndexing ? "faster than real time (indexing…)" : "faster than real time") + Tile(value: "\(model.indexedWindows)", caption: "windows searchable") + } + HStack(spacing: 10) { + Tile(value: model.burstQueries == 0 ? "–" : "\(model.burstQueries)", caption: "searches run") + Tile( + value: model.burstPerSecond == 0 ? "–" : String(format: "%.0f", model.burstPerSecond), + caption: "searches / second") + Tile( + value: model.burstMilliseconds == 0 ? "–" : String(format: "%.2f ms", model.burstMilliseconds), + caption: "per search") + Tile( + value: model.burstPerSecond == 0 + ? "–" : String(format: "%.1f M", model.burstPerSecond * Double(model.indexedWindows) / 1e6), + caption: "audio windows ranked / s") } + .opacity(model.segment == .burst ? 1 : 0.55) HStack(spacing: 8) { ForEach(model.collections) { collection in HStack(spacing: 5) { @@ -93,8 +98,10 @@ struct Header: View { private var status: String { switch (model.segment, model.phase) { - case (.reading, _): "📥 Reading everything · searchable as it goes · \(model.segmentRemaining) s" - case (.listen, _): "🔎 Listening to the top 3 · \(model.segmentRemaining) s left" + case (.listen, _): + model.isIndexing && model.segmentRemaining == 0 + ? "🔎 Listening to the top 3 · search speed once indexing finishes" + : "🔎 Listening to the top 3 · \(model.segmentRemaining) s left" case (.burst, _): "⚡ Searching as fast as it can · \(model.segmentRemaining) s left" case (nil, .loading(let message)): message case (nil, .ready): "Ready · press Index (⌘↩)" diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 95171b6..521d639 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -14,11 +14,12 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from [FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles them once (a few minutes). Needs macOS 15+. -- **Hands-free by default.** It reads everything once (searchable while it reads), then alternates two 30 s segments - (`--segment=`): **🔎 Listen** types each suggested query and plays its top three windows (4 s each, `--clip=`); +- **Hands-free by default.** Indexing starts at launch and runs on its own (its tiles: audio indexed, time, × real + time); the show starts as soon as the first windows are searchable and alternates two 30 s segments + (`--segment=`): **🔎 Listen** types each suggested query and plays its top three windows (4 s each, `--clip=`), and **⚡ Search speed** runs ~100 different queries back to back (64 at a time, embedded eight per Neural Engine call, - ranked against every window with one matrix multiply) and shows searches per second live. **Space** (Stop/Auto) or - typing yourself ends it; `--manual` starts without it. + ranked against every window with one matrix multiply; second tile row). Search speed waits for indexing to finish, + since both use the Neural Engine. **Space** (Stop/Auto) or typing yourself ends it; `--manual` starts without it. - **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. - Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. From 06c12968494a2eb4eff928b037fa8218ceb51bc0 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 9 Oct 2026 00:35:55 -0400 Subject: [PATCH 06/10] AudioSearchDemo: index first, then the listen / search-speed show MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The show waits until every file is indexed (status: indexing · search starts when done), then alternates listen and search speed. --- Sources/AudioSearchDemo/AudioSearchModel.swift | 14 ++++++-------- Sources/AudioSearchDemo/ContentView.swift | 7 ++----- Sources/AudioSearchDemo/README.md | 11 +++++------ 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index 2912f3f..fa81539 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -315,14 +315,13 @@ final class AudioSearchModel: ObservableObject { ) autoTask = Task { [weak self] in var nextQuery = 0 - // Indexing is its own thing: it runs in the background while the show goes on. The show starts as - // soon as there is something to search, and every search sees whatever is indexed by then. - if let self, self.indexedWindows == 0 { - if !self.isIndexing { + // Indexing comes first, on its own; the show starts once every file is searchable. + if let self, self.indexedWindows == 0 || self.isIndexing { + if !self.isIndexing, self.indexedWindows == 0 { self.isIndexing = true self.runTask = Task { await self.index() } } - while !Task.isCancelled, self.indexedWindows < 64 { try? await Task.sleep(for: .milliseconds(100)) } + while !Task.isCancelled, self.isIndexing { try? await Task.sleep(for: .milliseconds(100)) } } while let self, !Task.isCancelled, self.indexedWindows > 0 { // 🔎 Listen @@ -330,8 +329,7 @@ final class AudioSearchModel: ObservableObject { let deadline = ContinuousClock.now + .seconds(self.segmentSeconds) let ticker = self.countDown(to: deadline) let queries = self.allSuggestions - // Search speed shares the Neural Engine with indexing, so listening goes on until indexing is done. - while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline || self.isIndexing { + while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline { let query = queries[nextQuery % queries.count] nextQuery += 1 // Type it like a person would, then search. @@ -347,7 +345,7 @@ final class AudioSearchModel: ObservableObject { guard !Task.isCancelled, let ranked = await self.rank(query) else { break } self.results = ranked for (rank, result) in ranked.prefix(3).enumerated() { - let left = self.isIndexing ? 60 : (deadline - ContinuousClock.now).components.seconds + let left = (deadline - ContinuousClock.now).components.seconds guard !Task.isCancelled, left > 0 else { break } DemoLog.line( String( diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index 2c71bf6..a7b6581 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -98,14 +98,11 @@ struct Header: View { private var status: String { switch (model.segment, model.phase) { - case (.listen, _): - model.isIndexing && model.segmentRemaining == 0 - ? "🔎 Listening to the top 3 · search speed once indexing finishes" - : "🔎 Listening to the top 3 · \(model.segmentRemaining) s left" + case (.listen, _): "🔎 Listening to the top 3 · \(model.segmentRemaining) s left" case (.burst, _): "⚡ Searching as fast as it can · \(model.segmentRemaining) s left" case (nil, .loading(let message)): message case (nil, .ready): "Ready · press Index (⌘↩)" - case (nil, .indexing): "Indexing \(model.windowsDone)/\(model.windowsTotal) windows…" + case (nil, .indexing): "📥 Indexing \(model.windowsDone)/\(model.windowsTotal) windows · search starts when done" case (nil, .done): "Indexed · search below" case (nil, .failed(let message)): message } diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 521d639..10a9091 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -14,12 +14,11 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from [FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles them once (a few minutes). Needs macOS 15+. -- **Hands-free by default.** Indexing starts at launch and runs on its own (its tiles: audio indexed, time, × real - time); the show starts as soon as the first windows are searchable and alternates two 30 s segments - (`--segment=`): **🔎 Listen** types each suggested query and plays its top three windows (4 s each, `--clip=`), and - **⚡ Search speed** runs ~100 different queries back to back (64 at a time, embedded eight per Neural Engine call, - ranked against every window with one matrix multiply; second tile row). Search speed waits for indexing to finish, - since both use the Neural Engine. **Space** (Stop/Auto) or typing yourself ends it; `--manual` starts without it. +- **Hands-free by default.** Indexing runs first, on its own (tiles: audio indexed, time, × real time). Once every + file is searchable the show alternates two 30 s segments (`--segment=`): **🔎 Listen** types each suggested query and + plays its top three windows (4 s each, `--clip=`), and **⚡ Search speed** runs ~100 different queries back to back + (64 at a time, embedded eight per Neural Engine call, ranked against every window with one matrix multiply; second + tile row). **Space** (Stop/Auto) or typing yourself ends it; `--manual` starts without it. - **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. - Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. From 02b6bc607f3631203e0b2dccf2a5161ad9bd629c Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 9 Oct 2026 00:43:20 -0400 Subject: [PATCH 07/10] AudioSearchDemo: run the show once (index, listen, search speed), then stop Ends with a summary line (audio indexed and its speed, searches run at their average rate); Auto (Space) runs it again. --- Sources/AudioSearchDemo/AudioSearchModel.swift | 17 ++++++++++++++--- Sources/AudioSearchDemo/ContentView.swift | 3 ++- Sources/AudioSearchDemo/README.md | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index fa81539..d190457 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -67,6 +67,7 @@ final class AudioSearchModel: ObservableObject { @Published private(set) var burstQueries = 0 @Published private(set) var burstPerSecond: Double = 0 @Published private(set) var burstMilliseconds: Double = 0 + private var burstAverage: Double = 0 private var entries: [Entry] = [] /// Every window's embedding, row after row, so a query is one matrix-vector product. @@ -305,8 +306,8 @@ final class AudioSearchModel: ObservableObject { if autoPlay { stopAutoPlay() } else if !entries.isEmpty { startAutoPlay() } } - /// The hands-free show: the first read (searchable as it grows), then listen / search-speed segments, each - /// `segmentSeconds` long, round and round. + /// The hands-free show, once: index everything, then `segmentSeconds` of listening, then `segmentSeconds` of + /// search speed, then stop. Auto (Space) runs it again. private func startAutoPlay() { autoTask?.cancel() autoPlay = true @@ -323,7 +324,8 @@ final class AudioSearchModel: ObservableObject { } while !Task.isCancelled, self.isIndexing { try? await Task.sleep(for: .milliseconds(100)) } } - while let self, !Task.isCancelled, self.indexedWindows > 0 { + // One run: listen, then search speed, then stop. + if let self, !Task.isCancelled, self.indexedWindows > 0 { // 🔎 Listen self.segment = .listen let deadline = ContinuousClock.now + .seconds(self.segmentSeconds) @@ -366,6 +368,14 @@ final class AudioSearchModel: ObservableObject { let burstTicker = self.countDown(to: burstEnd) await self.burst(until: burstEnd) burstTicker.cancel() + guard !Task.isCancelled else { return } + self.autoTask = nil + self.autoPlay = false + self.segment = nil + DemoLog.event( + String( + format: "■ done: %@ of audio indexed at %.0fx real time; %d searches at %.0f/s", + Self.clock(self.audioSeconds), self.realTimeFactor, self.burstQueries, self.burstAverage)) } } } @@ -441,6 +451,7 @@ final class AudioSearchModel: ObservableObject { } } let total = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 + burstAverage = Double(burstQueries) / max(total, 1e-6) DemoLog.event( String( format: "⚡ %d searches in %.0f s = %.0f/s on average, %.0f/s at the end", burstQueries, total, diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index a7b6581..fda4f1c 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -103,7 +103,8 @@ struct Header: View { case (nil, .loading(let message)): message case (nil, .ready): "Ready · press Index (⌘↩)" case (nil, .indexing): "📥 Indexing \(model.windowsDone)/\(model.windowsTotal) windows · search starts when done" - case (nil, .done): "Indexed · search below" + case (nil, .done): + model.burstQueries > 0 ? "Done · type to search, or Auto (Space) to run it again" : "Indexed · search below" case (nil, .failed(let message)): message } } diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 10a9091..b327d7c 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -15,10 +15,10 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from them once (a few minutes). Needs macOS 15+. - **Hands-free by default.** Indexing runs first, on its own (tiles: audio indexed, time, × real time). Once every - file is searchable the show alternates two 30 s segments (`--segment=`): **🔎 Listen** types each suggested query and + file is searchable it runs two 30 s segments, once (`--segment=`): **🔎 Listen** types each suggested query and plays its top three windows (4 s each, `--clip=`), and **⚡ Search speed** runs ~100 different queries back to back (64 at a time, embedded eight per Neural Engine call, ranked against every window with one matrix multiply; second - tile row). **Space** (Stop/Auto) or typing yourself ends it; `--manual` starts without it. + tile row), then stops. **Space** (Stop/Auto) stops or reruns it, typing yourself ends it; `--manual` starts without it. - **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. - Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. From bf402a82268711b72cec6abc0a09773515b251a0 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 9 Oct 2026 00:53:46 -0400 Subject: [PATCH 08/10] AudioSearchDemo: Play / Pause / Replay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three buttons replace Index / Auto / Reset. Play (↩) runs the show once: index every file, 30 s of listening (type each suggested query, play its top three), 30 s of search speed, done with a summary. Pause (Space) holds it anywhere: indexing (EmbeddingGemma2Audio.embed gains a beforeWindow hook used as a pause gate), typing, the playing clip, the speed loop, and the step timers and speeds, which count running time only. Replay (⌘R) clears the index and runs it again. Nothing starts on launch (--autostart presses Play); the search field, suggestions and row play buttons work when the show is not running. --- .../AudioSearchDemo/AudioSearchModel.swift | 575 +++++++++--------- Sources/AudioSearchDemo/ContentView.swift | 47 +- Sources/AudioSearchDemo/README.md | 16 +- Sources/AudioSearchDemo/demo.sh | 2 +- .../EmbeddingGemma2Audio.swift | 7 +- 5 files changed, 327 insertions(+), 320 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index d190457..5b74273 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -6,14 +6,18 @@ import SwiftUI import os /// Indexes audio collections with EmbeddingGemma 2 (10 s windows: audio model on the GPU, text model on the Neural -/// Engine) and searches every window with a text query. +/// Engine) and searches every window with a text query. Play runs the show once: index everything, listen to the +/// top three of a few queries, then run searches back to back as fast as they go. Pause holds it anywhere (timers +/// included); Replay starts over from an empty index. @MainActor final class AudioSearchModel: ObservableObject { - enum Phase: Equatable { + enum Step: Equatable { case loading(String) - case ready + case idle case indexing - case done + case listening + case speed + case finished case failed(String) } @@ -39,35 +43,22 @@ final class AudioSearchModel: ObservableObject { var id: String { "\(entry.file.path)#\(entry.start)" } } - @Published private(set) var phase: Phase = .loading("Starting…") + @Published private(set) var step: Step = .loading("Starting…") + @Published private(set) var paused = false @Published private(set) var collections: [Collection] = [] @Published private(set) var windowsDone = 0 @Published private(set) var windowsTotal = 0 @Published private(set) var audioSeconds: Double = 0 @Published private(set) var indexSeconds: Double = 0 + @Published private(set) var indexedWindows = 0 @Published private(set) var results: [Result] = [] @Published private(set) var queryMilliseconds: Double = 0 @Published private(set) var playing: String? - @Published var query = "" - /// Hands-free mode (default; `--manual` turns it off): index on launch, then type each suggested query and play - /// its top three windows, round and round. Typing or pressing play yourself switches it off. - @Published private(set) var autoPlay = !CommandLine.arguments.contains("--manual") - /// Indexing passes so far. The first grows the index live; later ones rebuild it in the background while the - /// current index keeps answering searches, to keep showing the indexing speed. - @Published private(set) var pass = 0 - @Published private(set) var isIndexing = false - @Published private(set) var indexedWindows = 0 - /// Hands-free show: read everything once (searchable as it grows), then alternate `--segment=` seconds (default - /// 30) of listening (type a query, play its top three) and of search speed (queries back to back, as fast as the - /// Neural Engine embeds them). - enum Segment { case listen, burst } - @Published private(set) var segment: Segment? - @Published private(set) var segmentRemaining = 0 - /// Search-speed counters for the current burst. + @Published private(set) var stepRemaining = 0 @Published private(set) var burstQueries = 0 @Published private(set) var burstPerSecond: Double = 0 @Published private(set) var burstMilliseconds: Double = 0 - private var burstAverage: Double = 0 + @Published var query = "" private var entries: [Entry] = [] /// Every window's embedding, row after row, so a query is one matrix-vector product. @@ -76,12 +67,14 @@ final class AudioSearchModel: ObservableObject { private var audio: EmbeddingGemma2Audio? private var player: AVAudioPlayer? private var searchTask: Task? - private var runTask: Task? - private var autoTask: Task? - private var autoTyping = false - /// The last query text hands-free mode put in the field; the field's change callback echoing it is not typing. - private var autoQuery = "" - /// Seconds of each result to play in hands-free mode (`--clip=`). + private var showTask: Task? + /// The last query the show put in the field; the field's change callback echoing it is not typing. + private var showQuery = "" + /// Paused time so far, so timers and speeds count only running time. + private var pausedNanoseconds: UInt64 = 0 + private var pauseBegan: UInt64? + private let gate = PauseGate() + /// Seconds of each result to play (`--clip=`) and length of the listen and speed steps (`--segment=`). private let clipSeconds = max( 1, CommandLine.arguments.first { $0.hasPrefix("--clip=") }.flatMap { Double($0.dropFirst(7)) } ?? 4) private let segmentSeconds = max( @@ -89,6 +82,9 @@ final class AudioSearchModel: ObservableObject { var realTimeFactor: Double { indexSeconds > 0 ? audioSeconds / indexSeconds : 0 } var allSuggestions: [String] { collections.flatMap(\.suggestions) } + var isRunning: Bool { [.indexing, .listening, .speed].contains(step) } + var canPlay: Bool { step == .idle || (isRunning && paused) } + var canReplay: Bool { isRunning || step == .finished } func prepare() async { do { @@ -104,8 +100,8 @@ final class AudioSearchModel: ObservableObject { while !Task.isCancelled { try? await Task.sleep(for: .seconds(1)) seconds += 1 - guard let self, case .loading = self.phase else { continue } - self.phase = .loading( + guard let self, case .loading = self.step else { continue } + self.step = .loading( "Loading EmbeddingGemma 2… \(seconds) s" + (seconds >= 8 ? " (first launch downloads and compiles the models once)" : "")) } @@ -115,7 +111,7 @@ final class AudioSearchModel: ObservableObject { let audio = try await EmbeddingGemma2Audio.load(text: text) _ = try await audio.embed(window: [Float](repeating: 0, count: 16_000)) _ = try await text.embed("warm up", prompt: .searchQuery) - // The search-speed segment embeds queries eight per call (pack_256): load that path now too. + // The speed step embeds queries eight per call (pack_256): load that path now too. _ = try await text.embed(Array(repeating: "warm up", count: 64), prompt: .searchQuery) self.text = text self.audio = audio @@ -123,60 +119,109 @@ final class AudioSearchModel: ObservableObject { String( format: "models loaded in %.1f s · audio encoder on the GPU, text model on the Neural Engine", Double(DispatchTime.now().uptimeNanoseconds - loadStart) / 1e9)) - phase = .ready - if autoPlay { - startAutoPlay() - } else if CommandLine.arguments.contains("--autostart") { - start() - } + step = .idle + if CommandLine.arguments.contains("--autostart") { play() } } catch { - phase = .failed(error.localizedDescription) + step = .failed(error.localizedDescription) DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) } } - func start() { - guard phase == .ready || phase == .done, !isIndexing else { return } - runTask = Task { await index() } + // MARK: Play / Pause / Replay + + /// Starts the show, or resumes it after Pause. + func play() { + if isRunning, paused { + if let began = pauseBegan { pausedNanoseconds += DispatchTime.now().uptimeNanoseconds - began } + pauseBegan = nil + paused = false + gate.set(paused: false) + player?.play() + DemoLog.event("▶ resume") + return + } + guard step == .idle else { return } + showTask = Task { await show() } + } + + func pause() { + guard isRunning, !paused else { return } + paused = true + pauseBegan = DispatchTime.now().uptimeNanoseconds + gate.set(paused: true) + player?.pause() + DemoLog.event("❚❚ pause") } - func reset() { - autoTask?.cancel() - autoTask = nil - runTask?.cancel() - runTask = nil + /// Clears the index and runs the show again from the start. + func replay() { + guard canReplay else { return } + showTask?.cancel() + showTask = nil stop() + paused = false + pauseBegan = nil + gate.set(paused: false) entries = [] matrix = [] indexedWindows = 0 - pass = 0 - isIndexing = false results = [] + query = "" + showQuery = "" windowsDone = 0 windowsTotal = 0 audioSeconds = 0 indexSeconds = 0 - if audio != nil { phase = .ready } - DemoLog.event("↺ reset") + burstQueries = 0 + burstPerSecond = 0 + burstMilliseconds = 0 + step = .idle + DemoLog.event("↺ replay") + showTask = Task { await show() } } - /// One indexing pass over every collection. Pass 1 adds windows to the live index as they finish (searchable at - /// once); later passes build a new index beside the current one and swap it in at the end. + /// Nanoseconds of running (unpaused) time on a monotonic clock. + private var activeNow: UInt64 { + let now = DispatchTime.now().uptimeNanoseconds + return now - pausedNanoseconds - (pauseBegan.map { now - $0 } ?? 0) + } + + private func waitWhilePaused() async { + while paused, !Task.isCancelled { try? await Task.sleep(for: .milliseconds(50)) } + } + + /// Index, listen, speed, done. + private func show() async { + DemoLog.event( + "▶ play: index everything, \(Int(segmentSeconds)) s of listening to the top 3, \(Int(segmentSeconds)) s of searches as fast as they go" + ) + await index() + guard !Task.isCancelled, indexedWindows > 0 else { return } + await listen() + guard !Task.isCancelled else { return } + stop() + await speed() + guard !Task.isCancelled else { return } + step = .finished + DemoLog.event( + String( + format: "■ done: %@ of audio indexed in %.1f s (%.0fx real time); %d searches at %.0f/s", + Self.clock(audioSeconds), indexSeconds, realTimeFactor, burstQueries, burstPerSecond)) + } + + // MARK: Index + private func index() async { guard let audio else { return } - pass += 1 - isIndexing = true - phase = .indexing - let live = pass == 1 - // Round-robin across collections, so a growing index covers every collection early. + step = .indexing + // Round-robin across collections. var queues = collections.map { collection in collection.files.map { (collection.id, $0) } } var ordered: [(Int, URL)] = [] while queues.contains(where: { !$0.isEmpty }) { for index in queues.indices where !queues[index].isEmpty { ordered.append(queues[index].removeFirst()) } } let files = ordered - DemoLog.event( - "▶ pass \(pass): reading \(files.count) files" + (live ? "" : " (searches keep using pass \(pass - 1))")) + DemoLog.event("📥 indexing \(files.count) files") do { let decodeStart = DispatchTime.now().uptimeNanoseconds let recordings = try await Task.detached(priority: .userInitiated) { @@ -186,21 +231,22 @@ final class AudioSearchModel: ObservableObject { String( format: "decoded %d files to 16 kHz in %.1f s", files.count, Double(DispatchTime.now().uptimeNanoseconds - decodeStart) / 1e9), color: 245) + await waitWhilePaused() let total = recordings.reduce(0.0) { $0 + Double($1.count) } / Double(EmbeddingGemma2Audio.sampleRate) - let start = DispatchTime.now().uptimeNanoseconds + let start = activeNow let progress = IndexProgress() let inbox = WindowInbox() - var staged: [Entry] = [] + let gate = gate // The embedding tasks report into `progress` and `inbox`; the UI drains them ten times a second. let watcher = Task { @MainActor [weak self] in while !Task.isCancelled { try? await Task.sleep(for: .milliseconds(100)) guard let self else { return } self.refresh(progress: progress, total: total, start: start) - let fresh = inbox.drain() - if live { self.add(fresh) } else { staged += fresh } + self.add(inbox.drain()) } } + defer { watcher.cancel() } _ = try await audio.embed( recordings: recordings, progress: { done, count in progress.update(done: done, total: count) }, onWindow: { recording, window in @@ -208,39 +254,18 @@ final class AudioSearchModel: ObservableObject { Entry( collection: files[recording].0, file: files[recording].1, start: window.start, duration: window.duration, embedding: window.embedding)) - }) - watcher.cancel() + }, + beforeWindow: { await gate.wait() }) try Task.checkCancellation() refresh(progress: progress, total: total, start: start) - if live { - add(inbox.drain()) - } else { - staged += inbox.drain() - entries = staged - matrix = staged.flatMap(\.embedding) - indexedWindows = staged.count - } - isIndexing = false - phase = .done + add(inbox.drain()) DemoLog.event( String( - format: "pass %d: indexed %@ of audio (%d windows) in %.1f s = %.0fx real time", pass, - Self.clock(total), indexedWindows, indexSeconds, realTimeFactor)) - if let preset = CommandLine.arguments.first(where: { $0.hasPrefix("--query=") }), pass == 1 { - query = String(preset.dropFirst(8)) - } - if !autoPlay, !query.isEmpty, pass == 1 { search() } + format: "📥 indexed %@ of audio (%d windows) in %.1f s = %.0fx real time", Self.clock(total), + indexedWindows, indexSeconds, realTimeFactor)) } catch is CancellationError { - // Cut short (a timed speed run, or Reset): the previous index stays in place. - isIndexing = false - if phase == .indexing { phase = indexedWindows > 0 ? .done : .ready } - DemoLog.event( - String( - format: "pass %d: read %@ of audio in %.1f s = %.0fx real time (stopped there)", pass, - Self.clock(audioSeconds), indexSeconds, realTimeFactor)) } catch { - isIndexing = false - phase = .failed(error.localizedDescription) + step = .failed(error.localizedDescription) DemoLog.line("failed: \(error.localizedDescription)", color: 196, bold: true) } } @@ -258,161 +283,114 @@ final class AudioSearchModel: ObservableObject { windowsDone = done windowsTotal = count audioSeconds = total * Double(done) / Double(count) - indexSeconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 + indexSeconds = Double(activeNow - start) / 1e9 } - /// Ranks every indexed window against the query (debounced while typing). Typing by hand ends hands-free mode. - func search() { - if autoTyping || query == autoQuery { return } - stopAutoPlay() - searchTask?.cancel() - let query = query.trimmingCharacters(in: .whitespacesAndNewlines) - guard !query.isEmpty, !entries.isEmpty else { - results = [] - return + // MARK: Listen + + /// Types each suggested query, then plays its top three, for `segmentSeconds` of running time. + private func listen() async { + step = .listening + let queries = allSuggestions + guard !queries.isEmpty else { return } + let start = activeNow + let length = UInt64(segmentSeconds * 1e9) + let ticker = tick { [weak self] in + guard let self else { return } + self.stepRemaining = max(0, Int((Double(length) - Double(self.activeNow - start)) / 1e9)) } - searchTask = Task { - try? await Task.sleep(for: .milliseconds(120)) - guard !Task.isCancelled, let ranked = await rank(query), !Task.isCancelled else { return } + defer { ticker.cancel() } + var next = 0 + func left() -> Double { (Double(length) - Double(activeNow - start)) / 1e9 } + while !Task.isCancelled, left() > 0 { + let query = queries[next % queries.count] + next += 1 + showQuery = "" + self.query = "" + for character in query { + await waitWhilePaused() + showQuery.append(character) + self.query.append(character) + try? await Task.sleep(for: .milliseconds(30)) + } + await waitWhilePaused() + guard !Task.isCancelled, let ranked = await rank(query) else { return } results = ranked + for (rank, result) in ranked.prefix(3).enumerated() { + guard !Task.isCancelled, left() > 0.5 else { break } + DemoLog.line( + String( + format: " ▶ #%d %@ %@ @ %@", rank + 1, collections[result.entry.collection].name, + result.entry.file.lastPathComponent, Self.clock(result.entry.start)), color: 120) + await playClip(result, seconds: min(clipSeconds, result.entry.duration, left())) + } + await waitWhilePaused() + try? await Task.sleep(for: .milliseconds(800)) } } - /// Top 25 windows for `query`: one text embedding, then one matrix-vector product over every window. - private func rank(_ query: String) async -> [Result]? { - guard let text else { return nil } - let begin = DispatchTime.now().uptimeNanoseconds - guard let vector = try? await text.embed(query, prompt: .searchQuery) else { return nil } - var scores = [Float](repeating: 0, count: entries.count) - cblas_sgemv( - CblasRowMajor, CblasNoTrans, Int32(entries.count), Int32(vector.count), 1, matrix, Int32(vector.count), - vector, 1, 0, &scores, 1) - let ranked = scores.indices.sorted { scores[$0] > scores[$1] }.prefix(25).map { - Result(entry: entries[$0], score: scores[$0]) - } - queryMilliseconds = Double(DispatchTime.now().uptimeNanoseconds - begin) / 1e6 - DemoLog.line( - String(format: "🔎 “%@” → %d windows ranked in %.0f ms", query, entries.count, queryMilliseconds), color: 81) - for result in ranked.prefix(3) { - DemoLog.line( - String( - format: " %.3f %@ %@ @ %@", result.score, collections[result.entry.collection].name, - result.entry.file.lastPathComponent, Self.clock(result.entry.start))) + /// Plays `seconds` of a result's window, holding while paused; returns when it ends or is stopped. + private func playClip(_ result: Result, seconds: TimeInterval) async { + stop() + guard let player = try? AVAudioPlayer(contentsOf: result.entry.file) else { return } + player.currentTime = result.entry.start + player.play() + self.player = player + playing = result.id + var played: UInt64 = 0 + var last = DispatchTime.now().uptimeNanoseconds + while !Task.isCancelled, playing == result.id, Double(played) / 1e9 < seconds { + try? await Task.sleep(for: .milliseconds(50)) + let now = DispatchTime.now().uptimeNanoseconds + if !paused { played += now - last } + last = now } - return ranked + if playing == result.id { stop() } } - func toggleAutoPlay() { - if autoPlay { stopAutoPlay() } else if !entries.isEmpty { startAutoPlay() } - } + // MARK: Speed - /// The hands-free show, once: index everything, then `segmentSeconds` of listening, then `segmentSeconds` of - /// search speed, then stop. Auto (Space) runs it again. - private func startAutoPlay() { - autoTask?.cancel() - autoPlay = true - DemoLog.event( - "▶ hands-free: \(Int(segmentSeconds)) s listening to the top 3, then \(Int(segmentSeconds)) s of searches as fast as it can" - ) - autoTask = Task { [weak self] in - var nextQuery = 0 - // Indexing comes first, on its own; the show starts once every file is searchable. - if let self, self.indexedWindows == 0 || self.isIndexing { - if !self.isIndexing, self.indexedWindows == 0 { - self.isIndexing = true - self.runTask = Task { await self.index() } - } - while !Task.isCancelled, self.isIndexing { try? await Task.sleep(for: .milliseconds(100)) } - } - // One run: listen, then search speed, then stop. - if let self, !Task.isCancelled, self.indexedWindows > 0 { - // 🔎 Listen - self.segment = .listen - let deadline = ContinuousClock.now + .seconds(self.segmentSeconds) - let ticker = self.countDown(to: deadline) - let queries = self.allSuggestions - while !Task.isCancelled, !queries.isEmpty, ContinuousClock.now < deadline { - let query = queries[nextQuery % queries.count] - nextQuery += 1 - // Type it like a person would, then search. - self.autoTyping = true - self.autoQuery = "" - self.query = "" - for character in query { - self.autoQuery.append(character) - self.query.append(character) - try? await Task.sleep(for: .milliseconds(30)) - } - self.autoTyping = false - guard !Task.isCancelled, let ranked = await self.rank(query) else { break } - self.results = ranked - for (rank, result) in ranked.prefix(3).enumerated() { - let left = (deadline - ContinuousClock.now).components.seconds - guard !Task.isCancelled, left > 0 else { break } - DemoLog.line( - String( - format: " ▶ #%d %@ %@ @ %@", rank + 1, - self.collections[result.entry.collection].name, - result.entry.file.lastPathComponent, Self.clock(result.entry.start)), color: 120) - await self.playClip( - result, seconds: min(self.clipSeconds, result.entry.duration, Double(left))) - } - try? await Task.sleep(for: .milliseconds(800)) - } - ticker.cancel() - self.stop() - guard !Task.isCancelled else { return } - // ⚡ Search speed - self.segment = .burst - let burstEnd = ContinuousClock.now + .seconds(self.segmentSeconds) - let burstTicker = self.countDown(to: burstEnd) - await self.burst(until: burstEnd) - burstTicker.cancel() - guard !Task.isCancelled else { return } - self.autoTask = nil - self.autoPlay = false - self.segment = nil - DemoLog.event( - String( - format: "■ done: %@ of audio indexed at %.0fx real time; %d searches at %.0f/s", - Self.clock(self.audioSeconds), self.realTimeFactor, self.burstQueries, self.burstAverage)) - } - } - } - - /// Queries back to back until `deadline`: 64 at a time, embedded eight per Neural Engine call, then ranked - /// against every window in one matrix multiply. The field and the results show the latest query of each batch. - private func burst(until deadline: ContinuousClock.Instant) async { + /// Queries back to back for `segmentSeconds` of running time: 64 at a time, embedded eight per Neural Engine + /// call, ranked against every window in one matrix multiply. The field shows the latest query of each batch. + private func speed() async { guard let text, !entries.isEmpty else { return } + step = .speed let pool = queryPool let dimension = EmbeddingGemma2Manager.dimension + let windows = entries.count let batchSize = 64 burstQueries = 0 burstPerSecond = 0 burstMilliseconds = 0 - DemoLog.event( - "⚡ search speed: \(pool.count) different queries, back to back, against \(entries.count)+ windows") - let start = DispatchTime.now().uptimeNanoseconds + DemoLog.event("⚡ search speed: \(pool.count) different queries, back to back, against \(windows) windows") + let start = activeNow + let length = UInt64(segmentSeconds * 1e9) + let ticker = tick { [weak self] in + guard let self else { return } + self.stepRemaining = max(0, Int((Double(length) - Double(self.activeNow - start)) / 1e9)) + } + defer { ticker.cancel() } var embedNanoseconds: UInt64 = 0 var rankNanoseconds: UInt64 = 0 var next = 0 var lastLog = start var recent: [(time: UInt64, count: Int)] = [(start, 0)] - while !Task.isCancelled, ContinuousClock.now < deadline { + while !Task.isCancelled, activeNow - start < length { + if paused { + await waitWhilePaused() + recent = [(activeNow, 0)] + continue + } let batch = (0.. 1_000_000_000 } - let window = Double(end - (recent.first.map { $0.time } ?? start)) / 1e9 + // Rate over the last second of running time, so the tiles show the current speed. + let now = activeNow + recent.append((now, batchSize)) + recent.removeAll { now - $0.time > 1_000_000_000 } + let span = Double(now - (recent.first?.time ?? start)) / 1e9 let recentQueries = recent.dropFirst().reduce(0) { $0 + $1.count } - if window > 0.2, recentQueries > 0 { - burstPerSecond = Double(recentQueries) / window + if span > 0.2, recentQueries > 0 { + burstPerSecond = Double(recentQueries) / span burstMilliseconds = 1000 / burstPerSecond } - autoQuery = batch[batchSize - 1] - query = autoQuery + showQuery = batch[batchSize - 1] + query = showQuery results = shown queryMilliseconds = burstMilliseconds - if end - lastLog > 1_000_000_000 { - lastLog = end + if now - lastLog > 1_000_000_000 { + lastLog = now DemoLog.line( String( format: - "⚡ %d queries · %.0f/s · %.2f ms each (embed %.2f + rank %.2f) · %.1f M window scores/s", + "⚡ %d searches · %.0f/s · %.2f ms each (embed %.2f + rank %.2f) · %.1f M window scores/s", burstQueries, burstPerSecond, burstMilliseconds, Double(embedNanoseconds) / 1e6 / Double(burstQueries), Double(rankNanoseconds) / 1e6 / Double(burstQueries), - burstPerSecond * Double(entries.count) / 1e6), color: 81) + burstPerSecond * Double(windows) / 1e6), color: 81) } } - let total = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 - burstAverage = Double(burstQueries) / max(total, 1e-6) - DemoLog.event( - String( - format: "⚡ %d searches in %.0f s = %.0f/s on average, %.0f/s at the end", burstQueries, total, - Double(burstQueries) / total, burstPerSecond)) + // The tiles keep the whole step's average. + let elapsed = Double(activeNow - start) / 1e9 + if elapsed > 0, burstQueries > 0 { + burstPerSecond = Double(burstQueries) / elapsed + burstMilliseconds = 1000 / burstPerSecond + } + } + + private func tick(_ update: @escaping @MainActor () -> Void) -> Task { + Task { @MainActor in + while !Task.isCancelled { + update() + try? await Task.sleep(for: .milliseconds(250)) + } + } + } + + // MARK: Search by hand (when the show is not running) + + /// Ranks every indexed window against the typed query (debounced). + func search() { + if query == showQuery || isRunning { return } + searchTask?.cancel() + let query = query.trimmingCharacters(in: .whitespacesAndNewlines) + guard !query.isEmpty, !entries.isEmpty else { + results = [] + return + } + searchTask = Task { + try? await Task.sleep(for: .milliseconds(120)) + guard !Task.isCancelled, let ranked = await rank(query), !Task.isCancelled else { return } + results = ranked + } + } + + /// Top 25 windows for `query`: one text embedding, then one matrix-vector product over every window. + private func rank(_ query: String) async -> [Result]? { + guard let text else { return nil } + let begin = DispatchTime.now().uptimeNanoseconds + guard let vector = try? await text.embed(query, prompt: .searchQuery) else { return nil } + var scores = [Float](repeating: 0, count: entries.count) + cblas_sgemv( + CblasRowMajor, CblasNoTrans, Int32(entries.count), Int32(vector.count), 1, matrix, Int32(vector.count), + vector, 1, 0, &scores, 1) + let ranked = scores.indices.sorted { scores[$0] > scores[$1] }.prefix(25).map { + Result(entry: entries[$0], score: scores[$0]) + } + queryMilliseconds = Double(DispatchTime.now().uptimeNanoseconds - begin) / 1e6 + DemoLog.line( + String(format: "🔎 “%@” → %d windows ranked in %.0f ms", query, entries.count, queryMilliseconds), color: 81) + return ranked + } + + /// Row play button (when the show is not running): plays or stops one window. + func play(_ result: Result) { + guard !isRunning else { return } + if playing == result.id { + stop() + return + } + Task { await playClip(result, seconds: result.entry.duration) } + } + + func stop() { + player?.stop() + player = nil + playing = nil + } + + static func clock(_ seconds: TimeInterval) -> String { + let total = Int(seconds.rounded()) + return total >= 3600 + ? String(format: "%d:%02d:%02d", total / 3600, total / 60 % 60, total % 60) + : String(format: "%d:%02d", total / 60, total % 60) } /// Indices of the `count` highest scores in one row of a row-major score matrix, best first. @@ -473,15 +520,6 @@ final class AudioSearchModel: ObservableObject { return best.map(\.index) } - private func countDown(to deadline: ContinuousClock.Instant) -> Task { - Task { @MainActor [weak self] in - while !Task.isCancelled, let self { - self.segmentRemaining = max(0, Int((deadline - ContinuousClock.now).components.seconds)) - try? await Task.sleep(for: .milliseconds(250)) - } - } - } - /// Everything the search-speed segment cycles through: the suggestions plus a spread of topics and sounds. var queryPool: [String] { var pool = allSuggestions + Self.extraQueries @@ -510,52 +548,6 @@ final class AudioSearchModel: ObservableObject { "a festival with fireworks", ] - private func stopAutoPlay() { - guard autoPlay || autoTask != nil else { return } - autoTask?.cancel() - autoTask = nil - autoTyping = false - autoPlay = false - segment = nil - stop() - DemoLog.event("❚❚ hands-free off") - } - - /// Plays `seconds` of a result's window and returns when it ends (or is stopped). - private func playClip(_ result: Result, seconds: TimeInterval) async { - stop() - guard let player = try? AVAudioPlayer(contentsOf: result.entry.file) else { return } - player.currentTime = result.entry.start - player.play() - self.player = player - playing = result.id - try? await Task.sleep(for: .seconds(seconds)) - if playing == result.id { stop() } - } - - /// Play button: plays (or stops) one window; ends hands-free mode. - func play(_ result: Result) { - stopAutoPlay() - if playing == result.id { - stop() - return - } - Task { await playClip(result, seconds: result.entry.duration) } - } - - func stop() { - player?.stop() - player = nil - playing = nil - } - - static func clock(_ seconds: TimeInterval) -> String { - let total = Int(seconds.rounded()) - return total >= 3600 - ? String(format: "%d:%02d:%02d", total / 3600, total / 60 % 60, total % 60) - : String(format: "%d:%02d", total / 60, total % 60) - } - /// `--audio=` (repeatable), else the public datasets already on this Mac. static func defaultCollections() -> [Collection] { let palette: [Color] = [.blue, .orange, .green, .pink, .purple, .teal] @@ -626,6 +618,17 @@ final class WindowInbox: Sendable { } } +/// Holds embedding tasks while the show is paused. +final class PauseGate: Sendable { + private let state = OSAllocatedUnfairLock(initialState: false) + + func set(paused: Bool) { state.withLock { $0 = paused } } + + func wait() async { + while state.withLock({ $0 }), !Task.isCancelled { try? await Task.sleep(for: .milliseconds(50)) } + } +} + /// Window counts from the embedding tasks, read by the UI on a timer. final class IndexProgress: Sendable { private let counts = OSAllocatedUnfairLock(initialState: (done: 0, total: 0)) diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index fda4f1c..241cfc8 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -25,26 +25,26 @@ struct Header: View { Spacer() Text(status).font(.callout).foregroundStyle(.secondary) Button { - model.start() + model.play() } label: { - Label("Index", systemImage: "play.fill") + Label(model.paused ? "Resume" : "Play", systemImage: "play.fill") } - .keyboardShortcut(.return, modifiers: [.command]) - .disabled(model.phase != .ready && !(model.phase == .done && !model.isIndexing)) + .keyboardShortcut(.return, modifiers: []) + .disabled(!model.canPlay) Button { - model.toggleAutoPlay() + model.pause() } label: { - Label(model.autoPlay ? "Stop" : "Auto", systemImage: model.autoPlay ? "pause.fill" : "sparkles") + Label("Pause", systemImage: "pause.fill") } .keyboardShortcut(.space, modifiers: []) - .disabled(model.indexedWindows == 0) + .disabled(!model.isRunning || model.paused) Button { - model.reset() + model.replay() } label: { - Label("Reset", systemImage: "arrow.counterclockwise") + Label("Replay", systemImage: "arrow.counterclockwise") } .keyboardShortcut("r", modifiers: [.command]) - .disabled(model.indexedWindows == 0 && !model.isIndexing) + .disabled(!model.canReplay) } GeometryReader { proxy in Capsule().fill(Color.primary.opacity(0.08)) @@ -61,7 +61,7 @@ struct Header: View { Tile(value: String(format: "%.1f s", model.indexSeconds), caption: "indexing time") Tile( value: model.realTimeFactor == 0 ? "–" : String(format: "%.0f×", model.realTimeFactor), - caption: model.isIndexing ? "faster than real time (indexing…)" : "faster than real time") + caption: "faster than real time") Tile(value: "\(model.indexedWindows)", caption: "windows searchable") } HStack(spacing: 10) { @@ -77,7 +77,7 @@ struct Header: View { ? "–" : String(format: "%.1f M", model.burstPerSecond * Double(model.indexedWindows) / 1e6), caption: "audio windows ranked / s") } - .opacity(model.segment == .burst ? 1 : 0.55) + .opacity(model.step == .speed || model.step == .finished ? 1 : 0.55) HStack(spacing: 8) { ForEach(model.collections) { collection in HStack(spacing: 5) { @@ -97,15 +97,15 @@ struct Header: View { } private var status: String { - switch (model.segment, model.phase) { - case (.listen, _): "🔎 Listening to the top 3 · \(model.segmentRemaining) s left" - case (.burst, _): "⚡ Searching as fast as it can · \(model.segmentRemaining) s left" - case (nil, .loading(let message)): message - case (nil, .ready): "Ready · press Index (⌘↩)" - case (nil, .indexing): "📥 Indexing \(model.windowsDone)/\(model.windowsTotal) windows · search starts when done" - case (nil, .done): - model.burstQueries > 0 ? "Done · type to search, or Auto (Space) to run it again" : "Indexed · search below" - case (nil, .failed(let message)): message + let pause = model.paused ? " · paused" : "" + switch model.step { + case .loading(let message): return message + case .idle: return "Ready · press Play" + case .indexing: return "📥 Indexing \(model.windowsDone)/\(model.windowsTotal) windows" + pause + case .listening: return "🔎 Listening to the top 3 · \(model.stepRemaining) s left" + pause + case .speed: return "⚡ Searching as fast as it can · \(model.stepRemaining) s left" + pause + case .finished: return "Done · Replay to run it again" + case .failed(let message): return message } } } @@ -121,6 +121,7 @@ struct SearchBar: View { .textFieldStyle(.plain).font(.title3) .onSubmit { model.search() } .onChange(of: model.query) { model.search() } + .disabled(model.isRunning) if model.queryMilliseconds > 0, !model.results.isEmpty { Text(String(format: "%.0f ms", model.queryMilliseconds)).font(.caption.monospacedDigit()) .foregroundStyle(.secondary) @@ -134,6 +135,7 @@ struct SearchBar: View { model.search() } .buttonStyle(.bordered).controlSize(.small) + .disabled(model.isRunning) } } } @@ -156,7 +158,7 @@ struct Results: View { } .overlay { if model.results.isEmpty { - Text(model.phase == .done ? "Type a query or pick a suggestion" : "Index the audio, then search it") + Text(model.step == .finished ? "Type a query or pick a suggestion" : "Press Play") .foregroundStyle(.secondary) } } @@ -179,6 +181,7 @@ struct ResultRow: View { .font(.system(size: 26)).foregroundStyle(collection.color) } .buttonStyle(.plain) + .disabled(model.isRunning) VStack(alignment: .leading, spacing: 3) { HStack(spacing: 6) { Text(collection.name).font(.caption.bold()).foregroundStyle(collection.color) diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index b327d7c..28a1488 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -14,15 +14,13 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from [FluidInference/embeddinggemma-2-coreml](https://huggingface.co/FluidInference/embeddinggemma-2-coreml) and compiles them once (a few minutes). Needs macOS 15+. -- **Hands-free by default.** Indexing runs first, on its own (tiles: audio indexed, time, × real time). Once every - file is searchable it runs two 30 s segments, once (`--segment=`): **🔎 Listen** types each suggested query and - plays its top three windows (4 s each, `--clip=`), and **⚡ Search speed** runs ~100 different queries back to back - (64 at a time, embedded eight per Neural Engine call, ranked against every window with one matrix multiply; second - tile row), then stops. **Space** (Stop/Auto) stops or reruns it, typing yourself ends it; `--manual` starts without it. -- **Index** (⌘↩) decodes every file to 16 kHz mono, then embeds 10 s windows across all files with four in flight, so - the GPU (audio encoder) and the Neural Engine (text model) work at the same time. **Reset** (⌘R) clears the index. -- Type a query (or pick a suggestion): every window is ranked in one matrix-vector product; ▶ plays that window. -- `--query="…"` runs a search right after indexing; `--autostart` indexes without waiting. +- **Play** (↩) runs the show once, in order: **📥 index** every file (16 kHz mono, 10 s windows across all files with + four in flight, so the GPU and the Neural Engine work at the same time), then **🔎 listen** for 30 s (type each + suggested query, play its top three windows, 4 s each), then **⚡ search speed** for 30 s (~100 different queries + back to back, 64 at a time, embedded eight per Neural Engine call, ranked against every window with one matrix + multiply). **Pause** (Space) holds it anywhere, timers included; **Replay** (⌘R) starts over from an empty index. +- When it is done, type a query (or pick a suggestion); ▶ plays a window. `--segment=` and `--clip=` change the step + and clip lengths; `--autostart` presses Play on launch Default audio, when present: the 1-hour Earnings-22 sample and FLEURS English/French used by FluidAudio's benchmarks (`~/Library/Application Support/FluidAudio`), and ESC-50 sound clips diff --git a/Sources/AudioSearchDemo/demo.sh b/Sources/AudioSearchDemo/demo.sh index 1566e88..1b8ec43 100755 --- a/Sources/AudioSearchDemo/demo.sh +++ b/Sources/AudioSearchDemo/demo.sh @@ -31,4 +31,4 @@ fi pkill -f "release/AudioSearchDemo" 2>/dev/null || true cd "$ROOT" "$ROOT/.build/release/AudioSearchDemo" "$@" > "$LOG" 2> "$LOG.stderr" & -echo "demo pid $! · log $LOG · indexes and plays on its own (Space stops hands-free mode)" +echo "demo pid $! · log $LOG · press Play (↩) · Pause (Space) · Replay (⌘R)" diff --git a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift index b9ad3f4..33c5fe1 100644 --- a/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift +++ b/Sources/FluidUse/EmbeddingGemma2/EmbeddingGemma2Audio.swift @@ -99,10 +99,12 @@ public final class EmbeddingGemma2Audio: Sendable { /// Windows of several recordings, embedded together so short files still keep the GPU and the Neural Engine /// busy (one file at a time leaves a single window in flight). Result: one window list per recording, in order. - /// `onWindow` receives each window as soon as it is embedded (completion order), for indexes that grow live. + /// `onWindow` receives each window as soon as it is embedded (completion order), for indexes that grow live; + /// `beforeWindow` is awaited before each window starts (a pause gate, for example). public func embed( recordings: [[Float]], progress: (@Sendable (_ done: Int, _ total: Int) -> Void)? = nil, - onWindow: (@Sendable (_ recording: Int, _ window: Window) -> Void)? = nil + onWindow: (@Sendable (_ recording: Int, _ window: Window) -> Void)? = nil, + beforeWindow: (@Sendable () async -> Void)? = nil ) async throws -> [[Window]] { var jobs: [(recording: Int, start: Int)] = [] for (recording, samples) in recordings.enumerated() { @@ -125,6 +127,7 @@ public final class EmbeddingGemma2Audio: Sendable { let samples = recordings[job.recording] let slice = Array(samples[job.start.. Date: Fri, 9 Oct 2026 00:58:02 -0400 Subject: [PATCH 09/10] AudioSearchDemo: listening step plays the top 3 of four example searches forward-looking statements disclaimer, a baby crying, the operator opens the line for questions, someone laughing (two earnings-call phrases, two sounds; all four return three right hits). Clips are 3 s; the step is no longer cut at 30 s. --- .../AudioSearchDemo/AudioSearchModel.swift | 38 +++++++++---------- Sources/AudioSearchDemo/ContentView.swift | 5 ++- Sources/AudioSearchDemo/README.md | 4 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index 5b74273..f825b78 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -76,7 +76,7 @@ final class AudioSearchModel: ObservableObject { private let gate = PauseGate() /// Seconds of each result to play (`--clip=`) and length of the listen and speed steps (`--segment=`). private let clipSeconds = max( - 1, CommandLine.arguments.first { $0.hasPrefix("--clip=") }.flatMap { Double($0.dropFirst(7)) } ?? 4) + 1, CommandLine.arguments.first { $0.hasPrefix("--clip=") }.flatMap { Double($0.dropFirst(7)) } ?? 3) private let segmentSeconds = max( 5, CommandLine.arguments.first { $0.hasPrefix("--segment=") }.flatMap { Double($0.dropFirst(10)) } ?? 30) @@ -193,7 +193,7 @@ final class AudioSearchModel: ObservableObject { /// Index, listen, speed, done. private func show() async { DemoLog.event( - "▶ play: index everything, \(Int(segmentSeconds)) s of listening to the top 3, \(Int(segmentSeconds)) s of searches as fast as they go" + "▶ play: index everything, top 3 of \(Self.showcase.count) example searches, \(Int(segmentSeconds)) s of searches as fast as they go" ) await index() guard !Task.isCancelled, indexedWindows > 0 else { return } @@ -288,23 +288,23 @@ final class AudioSearchModel: ObservableObject { // MARK: Listen - /// Types each suggested query, then plays its top three, for `segmentSeconds` of running time. + /// The listening step's examples: each is typed, then its top three play. Picked to land: two phrases from the + /// earnings call, two everyday sounds. + static let showcase = [ + "forward-looking statements disclaimer", "a baby crying", "the operator opens the line for questions", + "someone laughing", + ] + + /// Types each showcase query (those whose collection is present), then plays its top three. private func listen() async { step = .listening - let queries = allSuggestions + let available = Set(allSuggestions) + var queries = Self.showcase.filter { available.contains($0) } + if queries.isEmpty { queries = Array(allSuggestions.prefix(4)) } guard !queries.isEmpty else { return } - let start = activeNow - let length = UInt64(segmentSeconds * 1e9) - let ticker = tick { [weak self] in - guard let self else { return } - self.stepRemaining = max(0, Int((Double(length) - Double(self.activeNow - start)) / 1e9)) - } - defer { ticker.cancel() } - var next = 0 - func left() -> Double { (Double(length) - Double(activeNow - start)) / 1e9 } - while !Task.isCancelled, left() > 0 { - let query = queries[next % queries.count] - next += 1 + for (number, query) in queries.enumerated() { + guard !Task.isCancelled else { return } + stepRemaining = queries.count - number showQuery = "" self.query = "" for character in query { @@ -317,15 +317,15 @@ final class AudioSearchModel: ObservableObject { guard !Task.isCancelled, let ranked = await rank(query) else { return } results = ranked for (rank, result) in ranked.prefix(3).enumerated() { - guard !Task.isCancelled, left() > 0.5 else { break } + guard !Task.isCancelled else { return } DemoLog.line( String( format: " ▶ #%d %@ %@ @ %@", rank + 1, collections[result.entry.collection].name, result.entry.file.lastPathComponent, Self.clock(result.entry.start)), color: 120) - await playClip(result, seconds: min(clipSeconds, result.entry.duration, left())) + await playClip(result, seconds: min(clipSeconds, result.entry.duration)) } await waitWhilePaused() - try? await Task.sleep(for: .milliseconds(800)) + try? await Task.sleep(for: .milliseconds(700)) } } diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index 241cfc8..b069080 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -102,7 +102,10 @@ struct Header: View { case .loading(let message): return message case .idle: return "Ready · press Play" case .indexing: return "📥 Indexing \(model.windowsDone)/\(model.windowsTotal) windows" + pause - case .listening: return "🔎 Listening to the top 3 · \(model.stepRemaining) s left" + pause + case .listening: + return + "🔎 Top 3 for each example search · \(AudioSearchModel.showcase.count - model.stepRemaining + 1) of \(AudioSearchModel.showcase.count)" + + pause case .speed: return "⚡ Searching as fast as it can · \(model.stepRemaining) s left" + pause case .finished: return "Done · Replay to run it again" case .failed(let message): return message diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 28a1488..640efb2 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -15,8 +15,8 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from them once (a few minutes). Needs macOS 15+. - **Play** (↩) runs the show once, in order: **📥 index** every file (16 kHz mono, 10 s windows across all files with - four in flight, so the GPU and the Neural Engine work at the same time), then **🔎 listen** for 30 s (type each - suggested query, play its top three windows, 4 s each), then **⚡ search speed** for 30 s (~100 different queries + four in flight, so the GPU and the Neural Engine work at the same time), then **🔎 listen** to four example searches (each + typed, then its top three windows play, 3 s each), then **⚡ search speed** for 30 s (~100 different queries back to back, 64 at a time, embedded eight per Neural Engine call, ranked against every window with one matrix multiply). **Pause** (Space) holds it anywhere, timers included; **Replay** (⌘R) starts over from an empty index. - When it is done, type a query (or pick a suggestion); ▶ plays a window. `--segment=` and `--clip=` change the step From 52870932c63b545277ebe95f12df699f1a03ff2e Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 9 Oct 2026 01:06:50 -0400 Subject: [PATCH 10/10] AudioSearchDemo: eight example searches; sound clips under random file names The listening step plays the top 3 of eight searches: two earnings-call phrases (forward-looking statements, the operator opening the line) and six sounds (baby crying, church bells, siren, laughing, cow, birds). With the ESC-50 clips renamed to random ids (clip_3f9a1c.wav; classes kept in esc50/labels.json, outside the app's view) all 24 hits are right, so the file name plays no part; the header says so. Search-speed queries for the sound classes come from labels.json. Search speed stays 30 s (21,312 searches at 710/s over 2,786 windows). --- .../AudioSearchDemo/AudioSearchModel.swift | 46 +++++++++++++------ Sources/AudioSearchDemo/ContentView.swift | 5 ++ Sources/AudioSearchDemo/README.md | 5 +- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/Sources/AudioSearchDemo/AudioSearchModel.swift b/Sources/AudioSearchDemo/AudioSearchModel.swift index f825b78..baec102 100644 --- a/Sources/AudioSearchDemo/AudioSearchModel.swift +++ b/Sources/AudioSearchDemo/AudioSearchModel.swift @@ -288,19 +288,26 @@ final class AudioSearchModel: ObservableObject { // MARK: Listen - /// The listening step's examples: each is typed, then its top three play. Picked to land: two phrases from the - /// earnings call, two everyday sounds. + /// The listening step's examples: each is typed, then its top three play. Two phrases from the earnings call, + /// six everyday sounds; every one returns three right hits. static let showcase = [ - "forward-looking statements disclaimer", "a baby crying", "the operator opens the line for questions", - "someone laughing", + "forward-looking statements disclaimer", "a baby crying", "church bells ringing", + "the operator opens the line for questions", "a siren", "someone laughing", "a cow mooing", "birds chirping", + ] + + static let earningsExamples: Set = [ + "forward-looking statements disclaimer", "the operator opens the line for questions", ] /// Types each showcase query (those whose collection is present), then plays its top three. private func listen() async { step = .listening - let available = Set(allSuggestions) - var queries = Self.showcase.filter { available.contains($0) } - if queries.isEmpty { queries = Array(allSuggestions.prefix(4)) } + // Earnings-call examples need that collection; sound examples need the sound clips. + let names = Set(collections.map(\.name)) + var queries = Self.showcase.filter { query in + Self.earningsExamples.contains(query) ? names.contains("Earnings calls") : names.contains("Sounds (ESC-50)") + } + if queries.isEmpty { queries = Array(allSuggestions.prefix(8)) } guard !queries.isEmpty else { return } for (number, query) in queries.enumerated() { guard !Task.isCancelled else { return } @@ -522,15 +529,24 @@ final class AudioSearchModel: ObservableObject { /// Everything the search-speed segment cycles through: the suggestions plus a spread of topics and sounds. var queryPool: [String] { - var pool = allSuggestions + Self.extraQueries - let sounds = Set( - collections.flatMap(\.files).compactMap { file -> String? in - let name = file.deletingPathExtension().lastPathComponent - guard let range = name.range(of: "__") else { return nil } - return "the sound of " + name[..() + for collection in collections { + guard let folder = collection.files.first?.deletingLastPathComponent(), + let data = try? Data( + contentsOf: folder.deletingLastPathComponent().appendingPathComponent("labels.json")), + let labels = try? JSONSerialization.jsonObject(with: data) as? [String: [String: String]] + else { continue } + for label in labels.values { + if let category = label["category"] { + sounds.insert("the sound of " + category.replacingOccurrences(of: "_", with: " ")) + } + } + } pool += sounds.sorted() - return pool + var seen = Set() + return pool.filter { seen.insert($0).inserted } } static let extraQueries = [ diff --git a/Sources/AudioSearchDemo/ContentView.swift b/Sources/AudioSearchDemo/ContentView.swift index b069080..b2cb8ee 100644 --- a/Sources/AudioSearchDemo/ContentView.swift +++ b/Sources/AudioSearchDemo/ContentView.swift @@ -88,6 +88,11 @@ struct Header: View { .padding(.horizontal, 8).padding(.vertical, 3) .background(Capsule().fill(collection.color.opacity(0.12))) } + Spacer() + Text("Sound clips have random file names: the model only hears the audio") + .font(.caption.bold()).foregroundStyle(.secondary) + } + HStack { Spacer() Text("EmbeddingGemma 2 · audio on the GPU, text on the Neural Engine · nothing leaves this Mac") .font(.caption).foregroundStyle(.secondary) diff --git a/Sources/AudioSearchDemo/README.md b/Sources/AudioSearchDemo/README.md index 640efb2..cfafc31 100644 --- a/Sources/AudioSearchDemo/README.md +++ b/Sources/AudioSearchDemo/README.md @@ -15,7 +15,7 @@ The first run downloads the models (about 1.1 GB, checksum-verified) from them once (a few minutes). Needs macOS 15+. - **Play** (↩) runs the show once, in order: **📥 index** every file (16 kHz mono, 10 s windows across all files with - four in flight, so the GPU and the Neural Engine work at the same time), then **🔎 listen** to four example searches (each + four in flight, so the GPU and the Neural Engine work at the same time), then **🔎 listen** to eight example searches (each typed, then its top three windows play, 3 s each), then **⚡ search speed** for 30 s (~100 different queries back to back, 64 at a time, embedded eight per Neural Engine call, ranked against every window with one matrix multiply). **Pause** (Space) holds it anywhere, timers included; **Replay** (⌘R) starts over from an empty index. @@ -24,7 +24,8 @@ them once (a few minutes). Needs macOS 15+. Default audio, when present: the 1-hour Earnings-22 sample and FLEURS English/French used by FluidAudio's benchmarks (`~/Library/Application Support/FluidAudio`), and ESC-50 sound clips -(`~/Library/Application Support/FluidUse/Datasets/esc50/wav`). Nothing is bundled. +(`~/Library/Application Support/FluidUse/Datasets/esc50/wav`), renamed to random ids (`clip_3f9a1c.wav`) so the file +name cannot give the answer away; the classes live in `esc50/labels.json`. Nothing is bundled. Numbers (M5 Pro, macOS 27):