diff --git a/.gitignore b/.gitignore index 48f411d..9020097 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,6 @@ .swiftpm/ xcuserdata/ DerivedData/ -# Your voice recordings and run output stay local. -Eval/clips/*.wav -Eval/clips/*.m4a -Eval/results/ - # Personal signing settings (Team ID) App/Signing.local.xcconfig diff --git a/CLAUDE.md b/CLAUDE.md index 2945f49..40047db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,5 +4,6 @@ Push-to-talk dictation: hold a key, speak, and cleaned-up text is pasted at the - **Murmur is the Rust workspace in `rust/`**, for macOS (the menu-bar app) and Linux (the Hyprland daemon). The crates and the interfaces between them are in [rust/CONTRACTS.md](rust/CONTRACTS.md); change it together with its callers. `cd rust && cargo test --workspace` runs the tests; CI (`.github/workflows/rust.yml`) also runs `cargo fmt --check` and `cargo clippy --workspace --all-targets -- -D warnings` on macOS and Linux. - **Linux (Hyprland/Omarchy):** if you're running on Linux, start with [rust/linux/README.md](rust/linux/README.md): setup, system packages, `rust/linux/install.sh`, the API key, Hyprland bindings, and a testing checklist. -- **Swift, being retired:** the original macOS app (`App/`, `Core/`) and the eval (`Eval/`, macOS only) stay until the eval and the release scripts are ported. Don't add features there. -- The cleanup prompt drives most of the quality. It lives in `rust/crates/murmur-core/src/prompt.rs` and, byte for byte, in `Core/Sources/MurmurKit/CleanupPrompt.swift`; a Rust test fails if they differ, so edit both. Check changes with the eval. +- **Eval:** `murmur-eval` (`rust/crates/murmur-eval`) scores speech-to-text and the cleanup prompt against recordings in `rust/eval/clips/`; see "Eval harness" in the README. +- **Swift, being retired:** the original macOS app (`App/`, `Core/`) stays until it's deleted. Don't add features there. +- The cleanup prompt drives most of the quality. It lives in `rust/crates/murmur-core/src/prompt.rs` and, byte for byte, in `Core/Sources/MurmurKit/CleanupPrompt.swift`; a Rust test fails if they differ, so edit both. Check changes with `murmur-eval run` (and `--prompt ` for a draft). diff --git a/Core/Sources/MurmurKit/CleanupPrompt.swift b/Core/Sources/MurmurKit/CleanupPrompt.swift index 4af1031..0176a14 100644 --- a/Core/Sources/MurmurKit/CleanupPrompt.swift +++ b/Core/Sources/MurmurKit/CleanupPrompt.swift @@ -1,7 +1,7 @@ import Foundation /// System prompt for the cleanup pass. Murmur's quality lives here, so treat edits like code changes and -/// check them against the eval (`MurmurEval run`). `MurmurEval --prompt ` tries a draft without a rebuild. +/// check them against the eval (`murmur-eval run`, in rust/). `murmur-eval --prompt ` tries a draft without a rebuild. public enum CleanupPrompt { /// What the model returns when there's nothing to type (see the Output section below). public static let emptyMarker = "" diff --git a/Eval/Package.resolved b/Eval/Package.resolved deleted file mode 100644 index cfd8e9e..0000000 --- a/Eval/Package.resolved +++ /dev/null @@ -1,15 +0,0 @@ -{ - "originHash" : "cd4dfd3df94ce26bc8aef1d393cb9ef95bad3227a1cafd44a76d6827ec7b9602", - "pins" : [ - { - "identity" : "fluidaudio", - "kind" : "remoteSourceControl", - "location" : "https://github.com/FluidInference/FluidAudio.git", - "state" : { - "revision" : "503b4bd1bbf7220882de39fe8ae6716aae4132da", - "version" : "0.17.7" - } - } - ], - "version" : 3 -} diff --git a/Eval/Package.swift b/Eval/Package.swift deleted file mode 100644 index 872c63c..0000000 --- a/Eval/Package.swift +++ /dev/null @@ -1,25 +0,0 @@ -// swift-tools-version: 6.2 -import PackageDescription - -let package = Package( - name: "MurmurEval", - platforms: [.macOS(.v26)], - dependencies: [ - .package(path: "../Core"), - .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.17.7"), - ], - targets: [ - .executableTarget( - name: "MurmurEval", - dependencies: [ - .product(name: "MurmurCore", package: "Core"), - .product(name: "FluidAudio", package: "FluidAudio"), - ], - linkerSettings: [ - // Embed an Info.plist so macOS shows the mic / speech permission prompts for this CLI. - .unsafeFlags(["-Xlinker", "-sectcreate", "-Xlinker", "__TEXT", "-Xlinker", "__info_plist", - "-Xlinker", "Support/Info.plist"]), - ] - ), - ] -) diff --git a/Eval/Sources/MurmurEval/Clip.swift b/Eval/Sources/MurmurEval/Clip.swift deleted file mode 100644 index 13a3e34..0000000 --- a/Eval/Sources/MurmurEval/Clip.swift +++ /dev/null @@ -1,52 +0,0 @@ -import Foundation -import MurmurCore - -/// Where the eval keeps its inputs and outputs. Resolved from this source file's location so the -/// CLI works no matter which directory `swift run` is invoked from. -enum Paths { - static let evalRoot = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // MurmurEval - .deletingLastPathComponent() // Sources - .deletingLastPathComponent() // Eval - static let clips = evalRoot.appending(path: "clips") - static let results = evalRoot.appending(path: "results") -} - -/// Sidecar metadata stored next to each recording as `.json`. -struct ClipMeta: Codable { - /// Exactly what was said, word for word. Used to score raw STT accuracy (WER). - var reference: String? - /// The text you would want typed. Used to eyeball (and later grade) the cleanup output. - var intended: String? - var context: TargetContext = .plain -} - -struct Clip { - let name: String - let audio: URL - var meta: ClipMeta - - static let audioExtensions: Set = ["wav", "m4a", "caf", "aiff", "mp3"] - - /// Every audio file in `clips/`, paired with its sidecar if one exists. - static func loadAll(from dir: URL = Paths.clips) throws -> [Clip] { - let files = try FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil) - return try files - .filter { audioExtensions.contains($0.pathExtension.lowercased()) } - .sorted { $0.lastPathComponent < $1.lastPathComponent } - .map { audio in - let name = audio.deletingPathExtension().lastPathComponent - let sidecar = dir.appending(path: "\(name).json") - let meta = FileManager.default.fileExists(atPath: sidecar.path) - ? try JSONDecoder().decode(ClipMeta.self, from: Data(contentsOf: sidecar)) - : ClipMeta() - return Clip(name: name, audio: audio, meta: meta) - } - } - - static func saveMeta(_ meta: ClipMeta, name: String, in dir: URL = Paths.clips) throws { - let encoder = JSONEncoder() - encoder.outputFormatting = [.prettyPrinted, .sortedKeys] - try encoder.encode(meta).write(to: dir.appending(path: "\(name).json")) - } -} diff --git a/Eval/Sources/MurmurEval/Recorder.swift b/Eval/Sources/MurmurEval/Recorder.swift deleted file mode 100644 index 42a6ce9..0000000 --- a/Eval/Sources/MurmurEval/Recorder.swift +++ /dev/null @@ -1,41 +0,0 @@ -import AVFoundation -import Foundation - -/// Records the default input device to a WAV file until the user presses Return. -/// This is a stand-in for the app's push-to-talk capture: same engine, no hotkey yet. -enum Recorder { - static func record(to url: URL) async throws -> TimeInterval { - guard await AVAudioApplication.requestRecordPermission() else { - throw EvalError("Microphone access was denied. Allow it for your terminal app in System Settings › Privacy & Security › Microphone.") - } - - let engine = AVAudioEngine() - let input = engine.inputNode - let format = input.outputFormat(forBus: 0) - let file = try AVAudioFile(forWriting: url, settings: format.settings) - - input.installTap(onBus: 0, bufferSize: 4096, format: format) { buffer, _ in - try? file.write(from: buffer) - } - try engine.start() - let started = Date() - - print("● Recording… press Return to stop.") - await withCheckedContinuation { (done: CheckedContinuation) in - Thread.detachNewThread { - _ = readLine() - done.resume() - } - } - - input.removeTap(onBus: 0) - engine.stop() - return Date().timeIntervalSince(started) - } -} - -/// Duration of any audio file AVFoundation can read. -func audioDuration(_ url: URL) -> TimeInterval { - guard let file = try? AVAudioFile(forReading: url) else { return 0 } - return Double(file.length) / file.processingFormat.sampleRate -} diff --git a/Eval/Sources/MurmurEval/Report.swift b/Eval/Sources/MurmurEval/Report.swift deleted file mode 100644 index 5101b0b..0000000 --- a/Eval/Sources/MurmurEval/Report.swift +++ /dev/null @@ -1,122 +0,0 @@ -import Foundation -import MurmurCore - -struct EngineResult: Codable { - let engine: String - let raw: String - let sttSeconds: Double - let wer: Double? - var cleaned: String? - var cleanupSeconds: Double? - var error: String? -} - -struct ClipResult: Codable { - let clip: String - let context: TargetContext - let audioSeconds: Double - let reference: String? - let intended: String? - var engines: [EngineResult] -} - -// MARK: - Word error rate - -/// Lowercase, drop punctuation (keeping apostrophes inside words), split on whitespace. -func normalizedWords(_ text: String) -> [String] { - let kept = text.lowercased().unicodeScalars.map { scalar -> Character in - CharacterSet.alphanumerics.contains(scalar) || scalar == "'" ? Character(scalar) : " " - } - return String(kept).split(separator: " ").map { $0.trimmingCharacters(in: CharacterSet(charactersIn: "'")) } - .filter { !$0.isEmpty } -} - -/// (substitutions + deletions + insertions) / reference length, via word-level edit distance. -func wordErrorRate(hypothesis: String, reference: String) -> Double { - let ref = normalizedWords(reference) - let hyp = normalizedWords(hypothesis) - guard !ref.isEmpty else { return hyp.isEmpty ? 0 : 1 } - var previous = Array(0...hyp.count) - for i in 1...ref.count { - var current = [i] + Array(repeating: 0, count: hyp.count) - for j in stride(from: 1, through: hyp.count, by: 1) { - let cost = ref[i - 1] == hyp[j - 1] ? 0 : 1 - current[j] = min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost) - } - previous = current - } - return Double(previous[hyp.count]) / Double(ref.count) -} - -// MARK: - Output - -enum Report { - /// Writes `results/.md` (for reading) and `.json` (for diffing runs later). - static func write(_ results: [ClipResult], engines: [String]) throws -> URL { - try FileManager.default.createDirectory(at: Paths.results, withIntermediateDirectories: true) - let stamp = ISO8601DateFormatter().string(from: Date()).replacingOccurrences(of: ":", with: "-") - let base = Paths.results.appending(path: stamp) - - let encoder = JSONEncoder() - encoder.outputFormatting = [.prettyPrinted, .sortedKeys] - try encoder.encode(results).write(to: base.appendingPathExtension("json")) - - let markdown = render(results, engines: engines) - let mdURL = base.appendingPathExtension("md") - try markdown.write(to: mdURL, atomically: true, encoding: .utf8) - return mdURL - } - - static func summaryTable(_ results: [ClipResult], engines: [String]) -> String { - var lines = ["| Engine | Mean WER | Mean STT latency | Mean cleanup latency |", - "|---|---|---|---|"] - for engine in engines { - let rows = results.flatMap(\.engines).filter { $0.engine == engine && $0.error == nil } - let wers = rows.compactMap(\.wer) - let stt = rows.map(\.sttSeconds) - let cleanup = rows.compactMap(\.cleanupSeconds) - lines.append("| \(engine) | \(wers.isEmpty ? "–" : pct(mean(wers))) | \(stt.isEmpty ? "–" : ms(mean(stt))) | \(cleanup.isEmpty ? "–" : ms(mean(cleanup))) |") - } - return lines.joined(separator: "\n") - } - - private static func render(_ results: [ClipResult], engines: [String]) -> String { - var out = "# Murmur eval: \(Date().formatted(date: .abbreviated, time: .shortened))\n\n" - out += "WER is measured on the **raw** STT output against each clip's verbatim `reference`.\n\n" - out += summaryTable(results, engines: engines) + "\n" - - for clip in results { - out += "\n---\n\n## \(clip.clip)\n\n" - out += "`\(clip.context.rawValue)` · \(String(format: "%.1f", clip.audioSeconds))s audio\n\n" - if let reference = clip.reference { out += "**Reference (verbatim):** \(reference)\n\n" } - if let intended = clip.intended { out += "**Intended:**\n\n\(quote(intended))\n\n" } - for engine in clip.engines { - out += "### \(engine.engine)\n\n" - if let error = engine.error { - out += "> ⚠️ \(error)\n\n" - continue - } - let wer = engine.wer.map { " · WER \(pct($0))" } ?? "" - out += "Raw, \(ms(engine.sttSeconds))\(wer):\n\n\(quote(engine.raw))\n\n" - if let cleaned = engine.cleaned { - out += "Cleaned, \(ms(engine.cleanupSeconds ?? 0)):\n\n\(quote(cleaned))\n\n" - } - } - } - return out - } - - private static func quote(_ text: String) -> String { - text.isEmpty ? "> _(empty)_" : text.split(separator: "\n", omittingEmptySubsequences: false) - .map { "> \($0)" }.joined(separator: "\n") - } -} - -func mean(_ values: [Double]) -> Double { values.reduce(0, +) / Double(max(values.count, 1)) } -func pct(_ value: Double) -> String { String(format: "%.1f%%", value * 100) } -func ms(_ seconds: Double) -> String { String(format: "%.0f ms", seconds * 1000) } - -struct EvalError: Error, CustomStringConvertible { - let description: String - init(_ description: String) { self.description = description } -} diff --git a/Eval/Sources/MurmurEval/STTEngines.swift b/Eval/Sources/MurmurEval/STTEngines.swift deleted file mode 100644 index 4e3a69e..0000000 --- a/Eval/Sources/MurmurEval/STTEngines.swift +++ /dev/null @@ -1,21 +0,0 @@ -import MurmurCore - -/// Engines the eval can compare, by command-line name. -enum Engines { - static let all: [String: @Sendable () -> any STTEngine] = [ - "parakeet": { ParakeetEngine(id: "parakeet", version: .v3) }, - "parakeet-v2": { ParakeetEngine(id: "parakeet-v2", version: .v2) }, - "parakeet-ultra": { ParakeetEngine(id: "parakeet-ultra", version: .ultra) }, - "apple": { AppleSpeechEngine() }, - ] - static let defaults = ["parakeet", "apple"] - - static func make(_ ids: [String]) throws -> [any STTEngine] { - try ids.map { id in - guard let factory = all[id] else { - throw EvalError("Unknown engine '\(id)'. Available: \(all.keys.sorted().joined(separator: ", "))") - } - return factory() - } - } -} diff --git a/Eval/Sources/MurmurEval/main.swift b/Eval/Sources/MurmurEval/main.swift deleted file mode 100644 index 53b520b..0000000 --- a/Eval/Sources/MurmurEval/main.swift +++ /dev/null @@ -1,263 +0,0 @@ -import Foundation -import FluidAudio -import MurmurCore - -let usage = """ -murmur-eval: compare speech-to-text engines and the Claude cleanup pass on your own voice. - -USAGE - swift run MurmurEval record [--context plain|chat|email|notes|code] - Record a clip from the mic into clips/.wav, then type what you said. - - swift run MurmurEval run [--engines parakeet,apple] [--no-cleanup] [--thinking] [--cleaner claude|ollama:] [--prompt draft.md] - Transcribe every clip in clips/ with each engine, clean each transcript with Claude, - and write a report to results/. - - swift run MurmurEval clean "" [--context notes] - Run only the cleanup pass on a string. Add --prompt draft.md to try a prompt edit before - moving it into Core/Sources/MurmurKit/CleanupPrompt.swift. - - swift run MurmurEval bench [--cleaner claude|ollama:] [--runs 3] - Time a cleaner on built-in test dictations and show its output. - - swift run MurmurEval vad