diff --git a/Package.swift b/Package.swift index ba5ead5..dd9e47f 100644 --- a/Package.swift +++ b/Package.swift @@ -67,7 +67,9 @@ let package = Package( .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), .executableTarget(name: "KevCheck", dependencies: ["FluidUse"]), .executableTarget(name: "ClefVisionCheck", dependencies: ["FluidUse"]), - .executableTarget(name: "ClefFlashDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh"]), + .executableTarget(name: "ClefFlashDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh", "bars.py"]), + .executableTarget(name: "ClefTextCheck", dependencies: ["FluidUse"]), + .executableTarget(name: "ClefCompareDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh"]), .executableTarget(name: "InternDecisionCheck", dependencies: ["FluidUse"]), .executableTarget(name: "Decision2Check", dependencies: ["FluidUse"]), .executableTarget( diff --git a/Sources/ClefCompareDemo/ClefCompareDemoApp.swift b/Sources/ClefCompareDemo/ClefCompareDemoApp.swift new file mode 100644 index 0000000..c6d4a60 --- /dev/null +++ b/Sources/ClefCompareDemo/ClefCompareDemoApp.swift @@ -0,0 +1,27 @@ +import AppKit +import SwiftUI + +@main +struct ClefCompareDemoApp: App { + @StateObject private var model = CompareModel() + + init() { + setvbuf(stdout, nil, _IOLBF, 0) + for key in UserDefaults.standard.dictionaryRepresentation().keys where key.hasPrefix("NSWindow Frame") { + UserDefaults.standard.removeObject(forKey: key) // open at the default size, not a remembered one + } + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("clef-flash 9B vs clef-text 0.6B") { + ContentView() + .environmentObject(model) + .frame(minWidth: 1000, minHeight: 640) + .task { await model.start() } + } + .defaultSize(width: 1320, height: 840) + .windowResizability(.contentMinSize) + } +} diff --git a/Sources/ClefCompareDemo/CompareModel.swift b/Sources/ClefCompareDemo/CompareModel.swift new file mode 100644 index 0000000..6d9d912 --- /dev/null +++ b/Sources/ClefCompareDemo/CompareModel.swift @@ -0,0 +1,242 @@ +import Foundation +import FluidUse +import SwiftUI + +/// Side by side on the same tickets: Cloudflare clef-flash (9B, Core ML on the GPU) and clef-text-0.6b (distilled +/// from it, Core ML on the Neural Engine). Both answer the same three typed questions per ticket, concurrently. +/// Tickets are fictional mock data. +@MainActor +final class CompareModel: ObservableObject { + struct Decision: Equatable { + let team: String + let urgency: Int + let refund: Bool + let teamConfidence: Float + let milliseconds: Double + } + + struct Ticket: Identifiable { + let id = UUID() + let text: String + let big: Decision + let small: Decision + let typed: Bool + var teamAgrees: Bool { big.team == small.team } + var urgencyAgrees: Bool { big.urgency == small.urgency } + var refundAgrees: Bool { big.refund == small.refund } + var agreements: Int { [teamAgrees, urgencyAgrees, refundAgrees].filter { $0 }.count } + } + + struct Incoming: Identifiable, Equatable { + let id = UUID() + let text: String + let typed: Bool + } + + static let urgencyLabels = ["Low", "Normal", "High", "Critical"] + + static let questions: [(id: String, question: ClefQuestion)] = [ + ( + "team", + .choice( + instructions: "Which team should handle this ticket?", + criteria: [ + "billing": "Charges, refunds, invoices", "engineering": "Bugs, outages, API errors", + "account": "Login, security, account changes", "product": "Feature requests and questions", + ]) + ), + ("urgency", .score(instructions: "How urgent is this ticket?", criteria: urgencyLabels)), + ("refund", .noul(instructions: "Is the customer asking for money back?")), + ] + + // Fictional tickets (mock data, not real customers). + static let mockTickets = [ + "Hi, I was charged twice for my Pro plan this month. Can you refund the duplicate?", + "The dashboard has been throwing 500 errors since this morning and none of our team can log in.", + "Would love a dark mode for the mobile app, the white screen is brutal at night.", + "I forgot my password and the reset email never arrives. Checked spam already.", + "Our checkout is down and customers can't pay. This is costing us thousands per hour!!", + "Can I get an invoice with our VAT number on it for last quarter?", + "Your latest update deleted all my saved drafts. Two weeks of work gone. I want my money back.", + "Quick question: do you support exporting reports to CSV?", + "Please cancel my subscription and close my account, I'm moving to another tool.", + "API requests started timing out after we upgraded to v3 of the SDK.", + "Someone logged into my account from another country. I didn't do that. Please lock it.", + "Is there a discount for nonprofits? We're a small animal shelter.", + "Webhooks stopped firing for new orders about an hour ago, nothing in the logs.", + "I was promised a refund three weeks ago and still haven't seen it on my card.", + "Can you add Spanish to the language options? Half our team would use it.", + "How do I change the email address on my account? I left my old job.", + "The iOS app crashes every time I open the calendar tab.", + "We were billed for 50 seats but we only have 12 users. Please fix and credit us.", + "Two-factor codes aren't arriving by SMS, I'm locked out before a client meeting.", + "Would be great if the search bar supported filters by date.", + ] + + @Published var status = "Loading both models…" + @Published var ready = false + @Published var tickets: [Ticket] = [] + @Published var running = false + @Published var draft = "" + @Published var incoming: [Incoming] = [] + @Published var current: Incoming? + + /// Loaded models as closures (the managers need macOS 15; the package targets 14): answers + total ms. + private var bigAnswer: (@Sendable (String) async throws -> ([ClefAnswer], Double))? + private var smallAnswer: (@Sendable (String) async throws -> ([ClefAnswer], Double))? + private var next = 0 + private var generation = 0 + private var streamTask: Task? + + var agreement: Double? { + let decisions = tickets.count * 3 + return decisions == 0 ? nil : Double(tickets.reduce(0) { $0 + $1.agreements }) / Double(decisions) + } + func median(_ values: [Double]) -> Double? { + let sorted = values.sorted() + return sorted.isEmpty ? nil : sorted[sorted.count / 2] + } + var bigMedian: Double? { median(tickets.map(\.big.milliseconds)) } + var smallMedian: Double? { median(tickets.map(\.small.milliseconds)) } + + static func directory(_ variable: String) -> URL? { + guard let path = ProcessInfo.processInfo.environment[variable], !path.isEmpty else { return nil } + return URL(fileURLWithPath: path) + } + + func start() async { + guard bigAnswer == nil else { return } + do { + guard #available(macOS 15.0, *) else { + status = "These Core ML models need macOS 15" + return + } + status = "Loading clef-text-0.6b (Neural Engine)…" + let smallDirectory: URL + if let local = Self.directory("CLEF_TEXT_BUNDLE") { + smallDirectory = local + } else { + status = "Downloading clef-text-0.6b (FluidInference/clef-text-0.6b-coreml, ~1.3 GB)…" + smallDirectory = try await ClefTextModelStore.ensure() + } + let small = try await ClefTextManager.load(from: smallDirectory, buckets: [512]) + try await small.warm() + let bigDirectory: URL + if let local = Self.directory("CLEF_FLASH_BUNDLE") { + bigDirectory = local + } else { + status = "Downloading clef-flash Core ML (FluidInference/clef-flash-coreml, ~11 GB)…" + bigDirectory = try await ClefFlashModelStore.ensure() + } + status = "Loading clef-flash 9B (GPU)…" + let big = try await ClefFlashManager.load(from: bigDirectory, bucket: 512) + try await big.warm() + for text in Self.mockTickets.prefix(2) { + _ = try await big.answer(state: text, questions: Self.questions) + _ = try await small.answer(state: text, questions: Self.questions) + } + bigAnswer = { text in + let r = try await big.answer(state: text, questions: Self.questions) + return (r.answers, r.totalMilliseconds) + } + smallAnswer = { text in + let r = try await small.answer(state: text, questions: Self.questions) + return (r.answers, r.totalMilliseconds) + } + ready = true + status = "" + toggleStream() + } catch { + status = "Failed to load: \(error.localizedDescription)" + } + } + + func toggleStream() { + if running { + streamTask?.cancel() + running = false + return + } + running = true + streamTask = Task { [weak self] in + while let self, !Task.isCancelled { + self.refill() + let ticket = withAnimation(.easeOut(duration: 0.2)) { self.incoming.removeFirst() } + withAnimation(.easeOut(duration: 0.2)) { self.current = ticket } + await self.triage(ticket) + await Task.yield() + } + } + } + + private func refill() { + while incoming.count < 8 { + incoming.append(Incoming(text: Self.mockTickets[next % Self.mockTickets.count], typed: false)) + next += 1 + } + } + + func submitDraft() { + let text = draft.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return } + draft = "" + withAnimation(.easeOut(duration: 0.2)) { incoming.insert(Incoming(text: text, typed: true), at: 0) } + if !running { + Task { + let ticket = incoming.removeFirst() + current = ticket + await triage(ticket) + } + } + } + + func restart() { + streamTask?.cancel() + running = false + generation += 1 + withAnimation(.easeOut(duration: 0.2)) { + tickets.removeAll() + incoming.removeAll() + current = nil + } + next = 0 + toggleStream() + } + + private static func decision(_ answers: [ClefAnswer], _ ms: Double) -> Decision { + let team = answers[0] + let urgency = answers[1] + let refund = answers[2] + let teamIndex = team.probabilities.indices.max { team.probabilities[$0] < team.probabilities[$1] } ?? 0 + let urgencyIndex = + urgency.probabilities.indices.max { urgency.probabilities[$0] < urgency.probabilities[$1] } ?? 0 + return Decision( + team: team.optionIDs[teamIndex], urgency: urgencyIndex, refund: (refund.noul ?? 0) > 0.5, + teamConfidence: team.probabilities[teamIndex], milliseconds: ms) + } + + private func triage(_ ticket: Incoming) async { + guard let bigAnswer, let smallAnswer else { return } + let generation = generation + do { + // different hardware (GPU vs Neural Engine), so the two run at the same time + async let big = bigAnswer(ticket.text) + async let small = smallAnswer(ticket.text) + let (b, s) = try await (big, small) + guard generation == self.generation else { return } + let row = Ticket( + text: ticket.text, big: Self.decision(b.0, b.1), small: Self.decision(s.0, s.1), typed: ticket.typed) + print( + String( + format: "9B %4.0f ms 0.6B %4.0f ms agree %d/3 %@", row.big.milliseconds, + row.small.milliseconds, row.agreements, ticket.text)) + withAnimation(.spring(duration: 0.35)) { + tickets.append(row) + if tickets.count > 200 { tickets.removeFirst(tickets.count - 200) } + current = nil + } + } catch { + status = "Error: \(error.localizedDescription)" + } + } +} diff --git a/Sources/ClefCompareDemo/ContentView.swift b/Sources/ClefCompareDemo/ContentView.swift new file mode 100644 index 0000000..79e4376 --- /dev/null +++ b/Sources/ClefCompareDemo/ContentView.swift @@ -0,0 +1,217 @@ +import SwiftUI + +struct ContentView: View { + @EnvironmentObject var model: CompareModel + + var body: some View { + VStack(spacing: 0) { + header + Divider() + if model.ready { + HStack(alignment: .top, spacing: 16) { + IncomingLane().frame(width: 300) + ComparedFeed().frame(maxWidth: .infinity) + } + .padding(18) + Divider() + composer + } else { + Spacer() + ProgressView(model.status).controlSize(.large) + Spacer() + } + } + .background(Color(nsColor: .windowBackgroundColor)) + } + + private var header: some View { + HStack(alignment: .center, spacing: 28) { + VStack(alignment: .leading, spacing: 4) { + Text("clef-flash 9B vs clef-text 0.6B").font(.system(size: 26, weight: .bold)) + Text("Same tickets, same questions · 9B on the GPU · 0.6B distilled student on the Neural Engine") + .font(.system(size: 14)).foregroundStyle(.secondary) + } + Spacer() + if let agreement = model.agreement { + Stat(value: String(format: "%.0f%%", agreement * 100), label: "decisions match") + } + if let big = model.bigMedian, let small = model.smallMedian, small > 0 { + Stat(value: String(format: "%.0f vs %.0f ms", big, small), label: "per ticket · 9B vs 0.6B") + } + } + .padding(.horizontal, 28).padding(.vertical, 16) + } + + private var composer: some View { + HStack(spacing: 12) { + TextField("Type your own ticket and press Return…", text: $model.draft) + .textFieldStyle(.roundedBorder).font(.system(size: 15)) + .onSubmit { model.submitDraft() } + Button(model.running ? "Pause stream" : "Resume stream") { model.toggleStream() } + Button("Restart") { model.restart() } + } + .padding(.horizontal, 28).padding(.vertical, 14) + } +} + +struct Stat: View { + let value: String + let label: String + + var body: some View { + VStack(alignment: .trailing, spacing: 0) { + Text(value).font(.system(size: 34, weight: .heavy, design: .rounded)).monospacedDigit() + Text(label).font(.system(size: 12)).foregroundStyle(.secondary) + } + } +} + +struct IncomingLane: View { + @EnvironmentObject var model: CompareModel + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack { + Image(systemName: "tray.and.arrow.down.fill").foregroundStyle(.blue) + Text("Incoming").font(.system(size: 17, weight: .semibold)) + Spacer() + Text("\(model.tickets.count) triaged").font(.system(size: 13)).foregroundStyle(.secondary) + .monospacedDigit() + } + if let current = model.current { + VStack(alignment: .leading, spacing: 8) { + HStack(spacing: 6) { + ProgressView().controlSize(.small) + Text("both models reading…").font(.system(size: 12, weight: .semibold)).foregroundStyle(.blue) + } + Text(current.text).font(.system(size: 14)).fixedSize(horizontal: false, vertical: true) + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(RoundedRectangle(cornerRadius: 10).fill(Color.blue.opacity(0.12))) + .overlay(RoundedRectangle(cornerRadius: 10).stroke(Color.blue, lineWidth: 2)) + .id(current.id) + } + ForEach(model.incoming) { ticket in + Text(ticket.text).font(.system(size: 13)).foregroundStyle(.secondary).lineLimit(2) + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background(RoundedRectangle(cornerRadius: 10).fill(Color(nsColor: .controlBackgroundColor))) + .overlay( + RoundedRectangle(cornerRadius: 10).stroke( + ticket.typed ? Color.accentColor : .clear, lineWidth: 2) + ) + .transition(.move(edge: .bottom).combined(with: .opacity)) + } + Spacer(minLength: 0) + } + .padding(14) + .frame(maxHeight: .infinity, alignment: .top) + .background(RoundedRectangle(cornerRadius: 14).fill(Color.blue.opacity(0.05))) + .clipped() + } +} + +struct ComparedFeed: View { + @EnvironmentObject var model: CompareModel + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack(spacing: 10) { + Image(systemName: "checkmark.circle.fill").foregroundStyle(.green) + Text("Decided").font(.system(size: 17, weight: .semibold)) + Spacer() + Label("9B · GPU", systemImage: "cpu").font(.system(size: 12)).foregroundStyle(.secondary) + Label("0.6B · Neural Engine", systemImage: "brain").font(.system(size: 12)).foregroundStyle(.secondary) + } + ScrollView { + LazyVStack(spacing: 8) { + ForEach(model.tickets.reversed()) { ticket in + CompareRow(ticket: ticket).transition(.move(edge: .top).combined(with: .opacity)) + } + } + } + } + .padding(14) + .frame(maxHeight: .infinity, alignment: .top) + .background(RoundedRectangle(cornerRadius: 14).fill(Color.green.opacity(0.04))) + } +} + +struct CompareRow: View { + let ticket: CompareModel.Ticket + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text(ticket.text).font(.system(size: 14)).lineLimit(2) + Spacer() + if ticket.agreements == 3 { + Image(systemName: "equal.circle.fill").foregroundStyle(.green) + } else { + Text("\(ticket.agreements)/3 match").font(.system(size: 12, weight: .semibold)).foregroundStyle( + .orange) + } + } + DecisionLine(label: "9B", detail: "GPU", decision: ticket.big, ticket: ticket) + DecisionLine(label: "0.6B", detail: "ANE", decision: ticket.small, ticket: ticket) + } + .padding(12) + .background(RoundedRectangle(cornerRadius: 10).fill(Color(nsColor: .controlBackgroundColor))) + .overlay( + RoundedRectangle(cornerRadius: 10).stroke( + ticket.typed ? Color.accentColor : (ticket.agreements < 3 ? Color.orange.opacity(0.6) : .clear), + lineWidth: 2)) + } +} + +struct DecisionLine: View { + let label: String + let detail: String + let decision: CompareModel.Decision + let ticket: CompareModel.Ticket + + private static let teamColors: [String: Color] = [ + "billing": .orange, "engineering": .red, "account": .purple, "product": .teal, + ] + private var urgencyColor: Color { [Color.green, .blue, .orange, .red][decision.urgency] } + + var body: some View { + HStack(spacing: 10) { + Text(label).font(.system(size: 12, weight: .bold)).frame(width: 36, alignment: .leading) + Text(detail).font(.system(size: 11)).foregroundStyle(.secondary).frame(width: 30, alignment: .leading) + Text(decision.team.capitalized) + .font(.system(size: 12, weight: .semibold)).lineLimit(1) + .frame(width: 100).padding(.vertical, 4) + .background( + RoundedRectangle(cornerRadius: 6).fill((Self.teamColors[decision.team] ?? .gray).opacity(0.2)) + ) + .foregroundStyle(Self.teamColors[decision.team] ?? .gray) + .overlay(mismatch(ticket.teamAgrees)) + Text(CompareModel.urgencyLabels[decision.urgency]) + .font(.system(size: 11, weight: .semibold)).frame(width: 64).padding(.vertical, 3) + .background(Capsule().fill(urgencyColor.opacity(0.18))).foregroundStyle(urgencyColor) + .overlay(mismatch(ticket.urgencyAgrees, capsule: true)) + Text(decision.refund ? "Refund" : "No refund") + .font(.system(size: 11, weight: .semibold)).frame(width: 72).padding(.vertical, 3) + .background(Capsule().fill((decision.refund ? Color.pink : .gray).opacity(0.15))) + .foregroundStyle(decision.refund ? Color.pink : .secondary) + .overlay(mismatch(ticket.refundAgrees, capsule: true)) + Spacer() + Text(String(format: "%.0f%%", decision.teamConfidence * 100)) + .font(.system(size: 12)).foregroundStyle(.secondary).monospacedDigit() + Text(String(format: "%.0f ms", decision.milliseconds)) + .font(.system(size: 13, weight: .semibold)).monospacedDigit().frame(width: 64, alignment: .trailing) + } + } + + @ViewBuilder private func mismatch(_ agrees: Bool, capsule: Bool = false) -> some View { + if !agrees { + if capsule { + Capsule().stroke(Color.orange, lineWidth: 2) + } else { + RoundedRectangle(cornerRadius: 6).stroke(Color.orange, lineWidth: 2) + } + } + } +} diff --git a/Sources/ClefCompareDemo/README.md b/Sources/ClefCompareDemo/README.md new file mode 100644 index 0000000..a5ddd03 --- /dev/null +++ b/Sources/ClefCompareDemo/README.md @@ -0,0 +1,15 @@ +# ClefCompareDemo + +The same support tickets through two Core ML decision models at once: + +- **clef-flash 9B** ([FluidInference/clef-flash-coreml](https://huggingface.co/FluidInference/clef-flash-coreml), + Cloudflare's model, 8-bit) on the GPU +- **clef-text-0.6b** ([FluidInference/clef-text-0.6b-coreml](https://huggingface.co/FluidInference/clef-text-0.6b-coreml), + Qwen3-0.6B distilled from clef-flash) on the Neural Engine + +Each ticket gets three typed decisions (team / urgency / refund) from each model; mismatches are outlined, the header +shows the share of decisions that match and both models' median time per ticket. Tickets are fictional mock data. + +```bash +Sources/ClefCompareDemo/demo.sh # downloads both bundles on first run (~12 GB), or set CLEF_FLASH_BUNDLE / CLEF_TEXT_BUNDLE +``` diff --git a/Sources/ClefCompareDemo/demo.sh b/Sources/ClefCompareDemo/demo.sh new file mode 100755 index 0000000..e6cf308 --- /dev/null +++ b/Sources/ClefCompareDemo/demo.sh @@ -0,0 +1,24 @@ +#!/bin/zsh +# clef-flash 9B vs clef-text 0.6B comparison demo, plus one Ghostty window with macmon (GPU / power) on top and the live decision log below. +# Sources/ClefCompareDemo/demo.sh (CLEF_FLASH_BUNDLE / CLEF_TEXT_BUNDLE for local bundles; default: download both) +set -e +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +BUNDLE=${CLEF_FLASH_BUNDLE:-} +LOG=${TMPDIR:-/tmp}/clef-compare-demo.log + +[[ -z "$BUNDLE" || -f "$BUNDLE/config.json" ]] || { echo "no clef-flash bundle at $BUNDLE"; exit 1; } +# the 9B weights need the GPU and ~7 GB of RAM to themselves; other model jobs make every ticket page +if pgrep -f "reference.py|run_coreml.py|export.py" >/dev/null; then + echo "warning: a model-lab job is running and will slow the demo (pgrep -fl 'reference.py|run_coreml.py|export.py')" +fi +swift build -c release --product ClefCompareDemo --package-path "$ROOT" +: > "$LOG" +if ! tmux has-session -t clef-compare 2>/dev/null; then + tmux new-session -d -s clef-compare -x 160 -y 60 macmon + tmux set-option -t clef-compare remain-on-exit on + tmux split-window -v -l 60% -t clef-compare "tail -n 300 -F '$LOG'" + open -na /Applications/Ghostty.app --args -e "$(command -v tmux)" attach -t clef-compare +fi +pkill -x ClefCompareDemo 2>/dev/null || true +CLEF_FLASH_BUNDLE="$BUNDLE" CLEF_TEXT_BUNDLE="${CLEF_TEXT_BUNDLE:-}" "$ROOT/.build/release/ClefCompareDemo" > "$LOG" 2> "$LOG.stderr" & +echo "demo pid $! · log $LOG · first ticket after ~40 s of model loading (plus ~12 GB of downloads on first run)" diff --git a/Sources/ClefFlashDemo/ClefFlashDemoApp.swift b/Sources/ClefFlashDemo/ClefFlashDemoApp.swift index 66c19fe..0b10530 100644 --- a/Sources/ClefFlashDemo/ClefFlashDemoApp.swift +++ b/Sources/ClefFlashDemo/ClefFlashDemoApp.swift @@ -16,7 +16,7 @@ struct ClefFlashDemoApp: App { } var body: some Scene { - WindowGroup("clef-flash 9B — on this Mac") { + WindowGroup(TriageModel.model == .text ? "clef-text 0.6B — on this Mac" : "clef-flash 9B — on this Mac") { ContentView() .environmentObject(model) .frame(minWidth: 1000, minHeight: 640) diff --git a/Sources/ClefFlashDemo/ContentView.swift b/Sources/ClefFlashDemo/ContentView.swift index c78efb1..080c815 100644 --- a/Sources/ClefFlashDemo/ContentView.swift +++ b/Sources/ClefFlashDemo/ContentView.swift @@ -23,17 +23,21 @@ struct ContentView: View { private var header: some View { HStack(alignment: .firstTextBaseline, spacing: 18) { VStack(alignment: .leading, spacing: 4) { - Text("Cloudflare clef-flash · 9B decision model") + Text(TriageModel.model.title) .font(.system(size: 26, weight: .bold)) - Text("Running on this Mac · Core ML on the GPU · no network") + Text(TriageModel.model.subtitle) .font(.system(size: 15)).foregroundStyle(.secondary) } Spacer() if let median = model.medianMilliseconds { VStack(alignment: .trailing, spacing: 0) { - Text(String(format: "%.2f s", median / 1000)) - .font(.system(size: 44, weight: .heavy, design: .rounded)).monospacedDigit() - Text("per ticket · 3 decisions in one pass") + Text( + model.finished + ? "\(model.backlogDone) tickets in \(String(format: "%.1f", model.activeSeconds)) s" + : "\(model.backlogDone) / \(TriageModel.backlog.count) · \(String(format: "%.1f", model.activeSeconds)) s" + ) + .font(.system(size: 36, weight: .heavy, design: .rounded)).monospacedDigit() + Text(String(format: "%.0f ms per ticket · 3 decisions each", median)) .font(.system(size: 13)).foregroundStyle(.secondary) } } @@ -58,7 +62,6 @@ struct ContentView: View { .onSubmit { model.submitDraft() } Button(model.running ? "Pause stream" : "Resume stream") { model.toggleStream() } Button("Restart") { model.restart() } - if model.busy { ProgressView().controlSize(.small) } } .padding(.horizontal, 28).padding(.vertical, 14) } @@ -79,16 +82,18 @@ struct IncomingLane: View { if let current = model.current { VStack(alignment: .leading, spacing: 8) { HStack(spacing: 6) { - ProgressView().controlSize(.small) - Text("clef-flash is reading…").font(.system(size: 12, weight: .semibold)).foregroundStyle(.blue) + ProgressView().controlSize(.small).opacity(model.running ? 1 : 0) + Text("\(TriageModel.model.short) is reading…").font(.system(size: 12, weight: .semibold)) + .foregroundStyle(.blue) } - Text(current.text).font(.system(size: 14)).fixedSize(horizontal: false, vertical: true) + // fixed two-line slot: the text swaps in place every ~80 ms without the card resizing + Text(current.text).font(.system(size: 14)).lineLimit(2) + .frame(maxWidth: .infinity, minHeight: 38, alignment: .topLeading) } .padding(12) .frame(maxWidth: .infinity, alignment: .leading) .background(RoundedRectangle(cornerRadius: 10).fill(Color.blue.opacity(0.12))) .overlay(RoundedRectangle(cornerRadius: 10).stroke(Color.blue, lineWidth: 2)) - .id(current.id) } ForEach(model.incoming) { ticket in Text(ticket.text).font(.system(size: 13)).foregroundStyle(.secondary).lineLimit(2) @@ -99,10 +104,10 @@ struct IncomingLane: View { RoundedRectangle(cornerRadius: 10).stroke( ticket.typed ? Color.accentColor : .clear, lineWidth: 2) ) - .transition(.move(edge: .bottom).combined(with: .opacity)) } Spacer(minLength: 0) } + .transaction { $0.animation = nil } // the queue advances faster than any animation could finish .padding(14) .frame(maxHeight: .infinity, alignment: .top) .background(RoundedRectangle(cornerRadius: 14).fill(Color.blue.opacity(0.05))) @@ -122,7 +127,7 @@ struct DecidedFeed: View { ForEach(TriageModel.Team.allCases) { team in HStack(spacing: 5) { Circle().fill(team.color).frame(width: 8, height: 8) - Text(team.title).font(.system(size: 13)) + Text(team.title).font(.system(size: 13)).lineLimit(1).fixedSize() Text("\(model.count(for: team))").font(.system(size: 13, weight: .semibold)).monospacedDigit() } .padding(.horizontal, 9).padding(.vertical, 4) @@ -133,7 +138,7 @@ struct DecidedFeed: View { LazyVStack(spacing: 8) { ForEach(model.tickets.reversed()) { ticket in TicketRow(ticket: ticket) - .transition(.move(edge: .top).combined(with: .opacity)) + .transition(.opacity) } } } diff --git a/Sources/ClefFlashDemo/MockTickets.swift b/Sources/ClefFlashDemo/MockTickets.swift new file mode 100644 index 0000000..44423d6 --- /dev/null +++ b/Sources/ClefFlashDemo/MockTickets.swift @@ -0,0 +1,93 @@ +import Foundation + +/// A deterministic backlog of fictional support tickets (mock data, no real customers): issue templates × products × +/// details × openers / closers, shuffled with a fixed seed so every run shows the same queue. +enum MockTickets { + static let plans = ["Pro plan", "Team plan", "Starter plan", "Business plan"] + static let products = ["mobile app", "desktop app", "dashboard", "web app", "browser extension"] + static let features = [ + "dark mode", "CSV export", "Spanish language support", "date filters in search", "a Slack integration", + "bulk editing", "keyboard shortcuts", "an offline mode", "custom report templates", + "two-factor via authenticator apps", + ] + static let amounts = ["$12", "$29", "$49", "$99", "$240", "$1,200"] + static let times = [ + "since this morning", "for the last hour", "since yesterday", "since the last update", "for two days", + ] + + static let openers = ["", "Hi, ", "Hello team, ", "Hey, ", "Urgent: ", "Quick one: "] + static let closers = [ + "", " Thanks.", " Please help.", " Can someone look at this today?", " This is really frustrating.", + ] + + /// Templates: {plan} {product} {feature} {amount} {time}. Templates starting with "+" are positive: no complaint closer. + static let templates = [ + "I was charged twice for my {plan} this month. Can you refund the duplicate {amount}?", + "Our invoice shows {amount} but we downgraded to the {plan} weeks ago.", + "Can I get an invoice with our VAT number for last quarter?", + "I cancelled my {plan} but was still billed {amount}. I want my money back.", + "Is there a discount for nonprofits on the {plan}?", + "We were billed for 50 seats but only have 12 users. Please fix and credit us.", + "The {product} has been throwing 500 errors {time} and nobody on our team can log in.", + "Our checkout is down {time} and customers can't pay.", + "Webhooks stopped firing for new orders {time}, nothing in the logs.", + "The {product} crashes every time I open the calendar tab.", + "API requests started timing out {time} after we upgraded the SDK.", + "Exports from the {product} come out empty {time}.", + "Sync between the {product} and our calendar has been broken {time}.", + "I forgot my password and the reset email never arrives.", + "Someone logged into my account from another country last night. Please lock it.", + "How do I change the email address on my account? I left my old job.", + "Two-factor codes aren't arriving by SMS and I'm locked out.", + "Please delete my account and all my data.", + "I need to transfer ownership of our workspace to a colleague.", + "+Would love {feature} in the {product}.", + "Do you support {feature}?", + "Any plans to add {feature}? Half our team would use it.", + "Is {feature} on the roadmap for the {product}?", + "Your latest update deleted my saved drafts. Two weeks of work gone.", + "+Thanks for the fast fix yesterday, everything works great now!", + ] + + static func make(count: Int = 1000, seed: UInt64 = 7) -> [String] { + var generator = SplitMix64(seed: seed) + var tickets: [String] = [] + var seen: Set = [] + while tickets.count < count { + var text = templates[Int(generator.next() % UInt64(templates.count))] + let positive = text.hasPrefix("+") + if positive { text.removeFirst() } + for (slot, values) in [ + ("{plan}", plans), ("{product}", products), ("{feature}", features), ("{amount}", amounts), + ("{time}", times), + ] { + text = text.replacingOccurrences(of: slot, with: values[Int(generator.next() % UInt64(values.count))]) + } + var opener = openers[Int(generator.next() % UInt64(openers.count))] + if !opener.isEmpty, let first = text.first { + // "Hi, the dashboard…" but keep "API…" and "I…" capitalised + let second = text.dropFirst().first + if second?.isLowercase == true { text = first.lowercased() + text.dropFirst() } + } else { + opener = "" + } + let closer = closers[Int(generator.next() % UInt64(closers.count))] + let full = opener + text + (positive ? "" : closer) + if seen.insert(full).inserted { tickets.append(full) } + } + return tickets + } +} + +/// Small deterministic PRNG (the mock queue must be identical across runs). +struct SplitMix64 { + var state: UInt64 + init(seed: UInt64) { state = seed } + mutating func next() -> UInt64 { + state &+= 0x9E37_79B9_7F4A_7C15 + var z = state + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB + return z ^ (z >> 31) + } +} diff --git a/Sources/ClefFlashDemo/README.md b/Sources/ClefFlashDemo/README.md index e7be844..54a9c28 100644 --- a/Sources/ClefFlashDemo/README.md +++ b/Sources/ClefFlashDemo/README.md @@ -1,18 +1,21 @@ # ClefFlashDemo -Cloudflare [clef-flash](https://huggingface.co/Cloudflare/clef-flash) (Qwen3.5-9B decision model, Apache-2.0) triaging -support tickets on this Mac through Core ML. Each ticket gets three typed decisions in one forward pass: team (choice), -urgency (score), refund (noul). Tickets are fictional mock data; typed tickets jump the queue. +Support-ticket triage with a Clef decision model on this Mac. Each ticket gets three typed decisions in one forward +pass — team (choice), urgency (score), refund (noul) — while the board works through a fixed backlog of 1,000 +fictional tickets (`MockTickets.swift`, deterministic); typed tickets jump the queue. ```bash -Sources/ClefFlashDemo/demo.sh # downloads FluidInference/clef-flash-coreml (~11 GB) on first run -Sources/ClefFlashDemo/demo.sh /path/to/bundle # or a local bundle (CLEF_FLASH_BUNDLE) +CLEF_MODEL=text Sources/ClefFlashDemo/demo.sh # clef-text-0.6b, GPU: ~48 ms / ticket, 1,000 tickets in ~57 s +Sources/ClefFlashDemo/demo.sh # Cloudflare clef-flash 9B, GPU: ~0.5 s / ticket ``` -- Runtime: `ClefFlashManager` — 8 decoder packages (4 layers each, 8-bit weights, 6.5 GB) chained on the GPU + the - joint schema head (fp32), 512-token bucket. ~0.50 s per ticket on an M5 Pro; ~40 s to load (first launch also - compiles the packages into `/compiled/`). -- Bundle: [FluidInference/clef-flash-coreml](https://huggingface.co/FluidInference/clef-flash-coreml), pinned in - `ClefFlashModelStore` with per-file SHA-256. -- GPU only: the Qwen3.5 decoder puts 0 ops on the Neural Engine (every op falls back to CPU under `CPU_AND_NE`). -- Same answers as the Python Core ML pipeline on the mock tickets; 8-bit vs fp32 reference: 1 flip / 611 ARC questions. +`demo.sh` downloads the pinned bundle on first run (`CLEF_TEXT_BUNDLE` / `CLEF_FLASH_BUNDLE` for a local one) and +opens Ghostty with live CPU / GPU bar charts (`bars.py` over `macmon pipe`, no sudo) above the decision log. + +- **clef-text-0.6b** — [FluidInference/clef-text-0.6b-coreml](https://huggingface.co/FluidInference/clef-text-0.6b-coreml), + `ClefTextManager`: Qwen3-0.6B + Clef joint head distilled from clef-flash; runs on the GPU or 100% on the Neural + Engine (`.cpuAndNeuralEngine`, ~77 ms / ticket). +- **clef-flash 9B** — [FluidInference/clef-flash-coreml](https://huggingface.co/FluidInference/clef-flash-coreml), + `ClefFlashManager`: 8 decoder packages (8-bit) chained on the GPU; ~7 GB of RAM. + +`ClefCompareDemo` runs both side by side on the same tickets (9B on the GPU, 0.6B on the Neural Engine). diff --git a/Sources/ClefFlashDemo/TriageModel.swift b/Sources/ClefFlashDemo/TriageModel.swift index 8b1a96c..3db80ea 100644 --- a/Sources/ClefFlashDemo/TriageModel.swift +++ b/Sources/ClefFlashDemo/TriageModel.swift @@ -77,7 +77,7 @@ final class TriageModel: ObservableObject { "Would be great if the search bar supported filters by date.", ] - @Published var status = "Loading clef-flash 9B…" + @Published var status = "Loading…" @Published var ready = false @Published var tickets: [Ticket] = [] @Published var running = false @@ -90,6 +90,11 @@ final class TriageModel: ObservableObject { /// The loaded model as a closure (ClefFlashManager needs macOS 15; the package targets 14): answers + total ms. private var answer: ((String) async throws -> ([ClefAnswer], Double))? private var next = 0 + /// The fixed 1,000-ticket backlog the stream works through (fictional). + static let backlog = MockTickets.make(count: 1000) + /// Wall time spent triaging (pauses excluded), and whether the backlog is done. + @Published var activeSeconds: Double = 0 + @Published var finished = false /// Bumped by `restart()`; a decision that finishes for an older generation is dropped. private var generation = 0 private var streamTask: Task? @@ -101,31 +106,68 @@ final class TriageModel: ObservableObject { func count(for team: Team) -> Int { tickets.reduce(0) { $0 + ($1.team == team ? 1 : 0) } } + /// Which model the board runs: `CLEF_MODEL=text` for clef-text-0.6b on the Neural Engine, else clef-flash 9B. + enum Model { + case flash, text + var title: String { + self == .text ? "clef-text · 0.6B decision model" : "Cloudflare clef-flash · 9B decision model" + } + var subtitle: String { + self == .text + ? "Distilled from Cloudflare clef-flash · Core ML on the GPU · no network" + : "Running on this Mac · Core ML on the GPU · no network" + } + var short: String { self == .text ? "clef-text 0.6B" : "clef-flash" } + } + + static let model: Model = ProcessInfo.processInfo.environment["CLEF_MODEL"] == "text" ? .text : .flash + func start() async { guard answer == nil else { return } do { guard #available(macOS 15.0, *) else { - status = "clef-flash's Core ML path needs macOS 15" + status = "These Core ML models need macOS 15" return } - let directory: URL - if let path = ProcessInfo.processInfo.environment["CLEF_FLASH_BUNDLE"], !path.isEmpty { - directory = URL(fileURLWithPath: path) - } else { - status = "Downloading clef-flash Core ML (FluidInference/clef-flash-coreml, ~11 GB)…" - directory = try await ClefFlashModelStore.ensure() - } let started = Date() - status = "Loading clef-flash 9B (first launch compiles the packages)…" - let manager = try await ClefFlashManager.load(from: directory, bucket: 512) - try await manager.warm() - // a couple of full-length tickets page every part's weights in before the board starts timing - for text in Self.mockTickets.prefix(2) { - _ = try await manager.answer(state: text, questions: Self.questions) - } - answer = { text in - let result = try await manager.answer(state: text, questions: Self.questions) - return (result.answers, result.totalMilliseconds) + switch Self.model { + case .flash: + let directory: URL + if let path = ProcessInfo.processInfo.environment["CLEF_FLASH_BUNDLE"], !path.isEmpty { + directory = URL(fileURLWithPath: path) + } else { + status = "Downloading clef-flash Core ML (FluidInference/clef-flash-coreml, ~11 GB)…" + directory = try await ClefFlashModelStore.ensure() + } + status = "Loading clef-flash 9B (first launch compiles the packages)…" + let manager = try await ClefFlashManager.load(from: directory, bucket: 512) + try await manager.warm() + // a couple of full-length tickets page every part's weights in before the board starts timing + for text in Self.mockTickets.prefix(2) { + _ = try await manager.answer(state: text, questions: Self.questions) + } + answer = { text in + let result = try await manager.answer(state: text, questions: Self.questions) + return (result.answers, result.totalMilliseconds) + } + case .text: + let directory: URL + if let path = ProcessInfo.processInfo.environment["CLEF_TEXT_BUNDLE"], !path.isEmpty { + directory = URL(fileURLWithPath: path) + } else { + status = "Downloading clef-text-0.6b Core ML (FluidInference/clef-text-0.6b-coreml, ~1.3 GB)…" + directory = try await ClefTextModelStore.ensure() + } + status = "Loading clef-text 0.6B…" + let manager = try await ClefTextManager.load(from: directory, buckets: [512], computeUnits: .cpuAndGPU) + try await manager.warm() + for text in Self.mockTickets.prefix(2) { + _ = try await manager.answer(state: text, questions: Self.questions) + } + answer = { text in + let result = try await manager.answer(state: text, questions: Self.questions) + return (result.answers, result.totalMilliseconds) + } } print("loaded in \(String(format: "%.1f", Date().timeIntervalSince(started))) s") ready = true @@ -147,29 +189,39 @@ final class TriageModel: ObservableObject { // back to back: the next ticket starts the moment the previous one is decided while let self, !Task.isCancelled { self.refill() - guard !self.incoming.isEmpty else { break } - let ticket = withAnimation(.easeOut(duration: 0.2)) { self.incoming.removeFirst() } - withAnimation(.easeOut(duration: 0.2)) { self.current = ticket } + guard !self.incoming.isEmpty else { + self.finished = true + self.running = false + self.current = nil + break + } + let ticket = self.incoming.removeFirst() // no animation: a ticket takes ~50 ms on the 0.6B + self.current = ticket + let started = Date() await self.triage(ticket.text, typed: ticket.typed) + self.activeSeconds += Date().timeIntervalSince(started) await Task.yield() } } } - /// Keep the visible queue full of mock arrivals. + /// Keep the visible queue topped up from the backlog. private func refill() { - while incoming.count < 8 { - incoming.append(Incoming(text: Self.mockTickets[next % Self.mockTickets.count], typed: false)) + while incoming.count < 8, next < Self.backlog.count { + incoming.append(Incoming(text: Self.backlog[next], typed: false)) next += 1 } } + /// Backlog tickets decided so far (typed ones not counted). + var backlogDone: Int { tickets.filter { !$0.typed }.count } + func submitDraft() { let text = draft.trimmingCharacters(in: .whitespacesAndNewlines) guard !text.isEmpty else { return } draft = "" // typed tickets jump the queue - withAnimation(.easeOut(duration: 0.2)) { incoming.insert(Incoming(text: text, typed: true), at: 0) } + incoming.insert(Incoming(text: text, typed: true), at: 0) if !running { Task { let ticket = incoming.removeFirst() @@ -190,6 +242,8 @@ final class TriageModel: ObservableObject { current = nil } next = 0 + activeSeconds = 0 + finished = false toggleStream() } @@ -217,10 +271,9 @@ final class TriageModel: ObservableObject { String( format: "%5.0f ms %-11@ %-8@ refund %.2f %@", milliseconds, ticket.team.rawValue, Self.urgencyLabels[urgencyIndex], ticket.refund, text)) - withAnimation(.spring(duration: 0.35)) { + withAnimation(.easeOut(duration: 0.12)) { tickets.append(ticket) - if tickets.count > 200 { tickets.removeFirst(tickets.count - 200) } - current = nil + if tickets.count > 1200 { tickets.removeFirst(tickets.count - 1200) } } } catch { status = "Error: \(error.localizedDescription)" diff --git a/Sources/ClefFlashDemo/bars.py b/Sources/ClefFlashDemo/bars.py new file mode 100755 index 0000000..4d1969e --- /dev/null +++ b/Sources/ClefFlashDemo/bars.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Live bar charts for the demo's top pane: CPU / GPU utilisation and GPU power from `macmon pipe` (no sudo). + +On M5 Pro / macOS 27, macmon's CPU and ANE power read 0 W and powermetrics' GPU power reads milliwatts while the GPU +is busy, so this shows the counters that do track load: P-/E-cluster usage, GPU usage and GPU watts. +""" +import json +import shutil +import subprocess +import sys + +BAR = "█" +COLORS = {"P-CPU": "\033[38;5;75m", "E-CPU": "\033[38;5;111m", "GPU": "\033[38;5;214m", "GPU W": "\033[38;5;203m"} +RESET, DIM, BOLD = "\033[0m", "\033[2m", "\033[1m" + + +def bar(label: str, fraction: float, text: str, width: int) -> str: + fraction = max(0.0, min(1.0, fraction)) + filled = int(round(fraction * width)) + return (f" {BOLD}{label:<6}{RESET} {COLORS[label]}{BAR * filled}{RESET}{DIM}{'·' * (width - filled)}{RESET} " + f"{text}") + + +def main() -> None: + process = subprocess.Popen(["macmon", "pipe", "-i", "500"], stdout=subprocess.PIPE, text=True) + gpu_watts_max = 30.0 + sys.stdout.write("\033[?25l") # hide cursor + try: + for line in process.stdout: + try: + d = json.loads(line) + except json.JSONDecodeError: + continue + width = max(10, shutil.get_terminal_size().columns - 30) + p_mhz, p_use = d["pcpu_usage"] + e_mhz, e_use = d["ecpu_usage"] + g_mhz, g_use = d["gpu_usage"] + g_watts = d.get("gpu_power", 0.0) + gpu_watts_max = max(gpu_watts_max, g_watts) + ram = d.get("memory", {}) + rows = [ + f" {BOLD}Apple silicon load{RESET} {DIM}macmon · 0.5 s{RESET}", + "", + bar("P-CPU", p_use, f"{p_use * 100:5.1f}% {p_mhz:4d} MHz", width), + bar("E-CPU", e_use, f"{e_use * 100:5.1f}% {e_mhz:4d} MHz", width), + bar("GPU", g_use, f"{g_use * 100:5.1f}% {g_mhz:4d} MHz", width), + bar("GPU W", g_watts / gpu_watts_max, f"{g_watts:5.1f} W", width), + ] + if ram: + rows += ["", f" {DIM}RAM {ram.get('ram_usage', 0) / 2**30:.1f} / {ram.get('ram_total', 0) / 2**30:.0f} GB" + f" swap {ram.get('swap_usage', 0) / 2**30:.1f} GB{RESET}"] + sys.stdout.write("\033[H\033[2J" + "\n".join(rows) + "\n") + sys.stdout.flush() + except KeyboardInterrupt: + pass + finally: + sys.stdout.write("\033[?25h") + process.terminate() + + +if __name__ == "__main__": + main() diff --git a/Sources/ClefFlashDemo/demo.sh b/Sources/ClefFlashDemo/demo.sh index 5410bae..f118ec8 100755 --- a/Sources/ClefFlashDemo/demo.sh +++ b/Sources/ClefFlashDemo/demo.sh @@ -1,6 +1,8 @@ #!/bin/zsh -# clef-flash triage demo, plus one Ghostty window with macmon (GPU / power) on top and the live decision log below. +# clef-flash triage demo, plus one Ghostty window: live CPU / GPU bar charts (bars.py over macmon) on top and the +# decision log below. (asitop crashes on M5 Pro; CPU / ANE power counters read 0 W on macOS 27, so bars show usage.) # Sources/ClefFlashDemo/demo.sh [bundle dir] (or CLEF_FLASH_BUNDLE; default: download FluidInference/clef-flash-coreml) +# CLEF_MODEL=text Sources/ClefFlashDemo/demo.sh clef-text-0.6b on the Neural Engine (CLEF_TEXT_BUNDLE or download) set -e ROOT=$(cd "$(dirname "$0")/../.." && pwd) BUNDLE=${1:-${CLEF_FLASH_BUNDLE:-}} @@ -14,11 +16,14 @@ fi swift build -c release --product ClefFlashDemo --package-path "$ROOT" : > "$LOG" if ! tmux has-session -t clef-flash 2>/dev/null; then - tmux new-session -d -s clef-flash -x 160 -y 60 macmon + tmux new-session -d -s clef-flash -x 160 -y 60 "python3 '$ROOT/Sources/ClefFlashDemo/bars.py'" tmux set-option -t clef-flash remain-on-exit on - tmux split-window -v -l 60% -t clef-flash "tail -n 300 -F '$LOG'" + tmux split-window -v -t clef-flash "tail -n 300 -F '$LOG'" + # the bars need 10 rows whatever size the terminal window attaches with + tmux set-hook -t clef-flash client-attached 'resize-pane -t clef-flash:0.0 -y 10' + tmux set-hook -t clef-flash client-resized 'resize-pane -t clef-flash:0.0 -y 10' open -na /Applications/Ghostty.app --args -e "$(command -v tmux)" attach -t clef-flash fi pkill -x ClefFlashDemo 2>/dev/null || true -CLEF_FLASH_BUNDLE="$BUNDLE" "$ROOT/.build/release/ClefFlashDemo" > "$LOG" 2> "$LOG.stderr" & +CLEF_MODEL="${CLEF_MODEL:-}" CLEF_TEXT_BUNDLE="${CLEF_TEXT_BUNDLE:-}" CLEF_FLASH_BUNDLE="$BUNDLE" "$ROOT/.build/release/ClefFlashDemo" > "$LOG" 2> "$LOG.stderr" & echo "demo pid $! · log $LOG · first ticket after ~40 s of model loading (plus an ~11 GB download on first run)" diff --git a/Sources/ClefTextCheck/main.swift b/Sources/ClefTextCheck/main.swift new file mode 100644 index 0000000..665bd38 --- /dev/null +++ b/Sources/ClefTextCheck/main.swift @@ -0,0 +1,52 @@ +import Foundation +import FluidUse + +// ClefTextCheck (string states): token count + per-question probabilities for each request (one JSON line +// out per request), so the Python Core ML / PyTorch pipeline can be diffed against the Swift host. +guard CommandLine.arguments.count == 3 else { + print("usage: ClefTextCheck ") + exit(2) +} +@available(macOS 15.0, *) +func run() async throws { + let bundle = URL(fileURLWithPath: CommandLine.arguments[1]) + let manager = try await ClefTextManager.load(from: bundle, buckets: [256, 512, 1024, 2048]) + let lines = try String(contentsOfFile: CommandLine.arguments[2], encoding: .utf8).split(separator: "\n") + for line in lines { + guard let request = try JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any], + let state = request["state"] as? String, let raw = request["questions"] as? [[Any]] + else { continue } + var questions: [(id: String, question: ClefQuestion)] = [] + for item in raw { + guard let id = item[0] as? String, let q = item[1] as? [String: Any], let type = q["type"] as? String else { + continue + } + let instructions = q["instructions"] as? String + switch type { + case "choice": + questions.append( + (id, .choice(instructions: instructions, criteria: q["criteria"] as? [String: String] ?? [:]))) + case "score": + questions.append((id, .score(instructions: instructions, criteria: q["criteria"] as? [String] ?? []))) + default: questions.append((id, .noul(instructions: instructions))) + } + } + let record = try await manager.encode(state: state, questions: questions) + let result = try await manager.answer(state: state, questions: questions) + let answers = Dictionary( + uniqueKeysWithValues: result.answers.map { + ($0.questionID, Dictionary(uniqueKeysWithValues: zip($0.optionIDs, $0.probabilities.map(Double.init)))) + }) + let out: [String: Any] = [ + "tokens": record.inputIDs.count, "ids_head": Array(record.inputIDs.prefix(40)), "answers": answers, + "ms": result.totalMilliseconds, + ] + print(String(data: try JSONSerialization.data(withJSONObject: out, options: [.sortedKeys]), encoding: .utf8)!) + } +} + +if #available(macOS 15.0, *) { + try await run() +} else { + fatalError("needs macOS 15") +} diff --git a/Sources/FluidUse/ClefText/ClefTextManager.swift b/Sources/FluidUse/ClefText/ClefTextManager.swift new file mode 100644 index 0000000..d9edf3a --- /dev/null +++ b/Sources/FluidUse/ClefText/ClefTextManager.swift @@ -0,0 +1,232 @@ +import CoreML +import Foundation + +/// clef-text-0.6b on Core ML: a Qwen3-0.6B text decision model distilled from Cloudflare clef-flash (9B), same +/// Clef / SystemOne contract, running on the Neural Engine. One decoder package (a function per sequence bucket) +/// and the joint schema head, both fp16. An actor: Core ML's synchronous `prediction` is not thread-safe. +@available(macOS 15.0, iOS 18.0, *) +public actor ClefTextManager { + struct Config: Sendable { + let hidden: Int + let headDim: Int + let ropeTheta: Double + let padID: Int + let vocab: Int + let buckets: [Int] + let maxQuestions: Int + let maxOptions: Int + let decoder: String + let head: String + + init(json: [String: Any]) throws { + func int(_ key: String) throws -> Int { + guard let v = (json[key] as? NSNumber)?.intValue else { + throw ClefVisionError.invalidAsset("config.json: \(key)") + } + return v + } + guard let buckets = json["buckets"] as? [NSNumber], let decoder = json["decoder"] as? String, + let head = json["head"] as? String, let theta = (json["rope_theta"] as? NSNumber)?.doubleValue + else { throw ClefVisionError.invalidAsset("config.json is missing sections") } + hidden = try int("hidden_size") + headDim = try int("head_dim") + ropeTheta = theta + padID = try int("pad_id") + vocab = try int("vocab_size") + self.buckets = buckets.map(\.intValue).sorted() + maxQuestions = try int("max_questions") + maxOptions = try int("max_options") + self.decoder = decoder + self.head = head + } + } + + public nonisolated let buckets: [Int] + let config: Config + let tokenizer: QwenBPETokenizer + private let decoders: [Int: MLModel] + private let heads: [Int: MLModel] + private let embeddings: Data // fp16 [vocab, hidden]; tied, so also the head's lexical option rows + private var ropeCache: [Int: (MLMultiArray, MLMultiArray)] = [:] + + /// Loads the given buckets (records up to the largest one). Packages are compiled once into `compiled/`. + public static func load( + from directory: URL, buckets: [Int] = [256, 512], computeUnits: MLComputeUnits = .cpuAndNeuralEngine + ) async throws -> ClefTextManager { + let json = try JSONSerialization.jsonObject( + with: Data(contentsOf: directory.appendingPathComponent("config.json"))) + guard let dict = json as? [String: Any] else { throw ClefVisionError.invalidAsset("config.json") } + let config = try Config(json: dict) + let tokenizer = try QwenBPETokenizer(tokenizerJsonURL: directory.appendingPathComponent("tokenizer.json")) + let embeddings = try Data( + contentsOf: directory.appendingPathComponent("embeddings.f16"), options: .alwaysMapped) + guard embeddings.count == config.vocab * config.hidden * 2 else { + throw ClefVisionError.invalidAsset("embeddings.f16 size") + } + let compiledDir = directory.appendingPathComponent("compiled") + try FileManager.default.createDirectory(at: compiledDir, withIntermediateDirectories: true) + func compiled(_ path: String) async throws -> URL { + let source = directory.appendingPathComponent(path) + let target = compiledDir.appendingPathComponent( + source.deletingPathExtension().lastPathComponent + ".mlmodelc") + if !FileManager.default.fileExists(atPath: target.path) { + let temporary = try await MLModel.compileModel(at: source) + try FileManager.default.moveItem(at: temporary, to: target) + } + return target + } + let decoderURL = try await compiled(config.decoder) + let headURL = try await compiled(config.head) + var decoders: [Int: MLModel] = [:] + var heads: [Int: MLModel] = [:] + for bucket in buckets { + guard config.buckets.contains(bucket) else { + throw ClefVisionError.invalidInput("no \(bucket)-token bucket") + } + let configuration = MLModelConfiguration() + configuration.computeUnits = computeUnits + configuration.functionName = "L\(bucket)" + decoders[bucket] = try await MLModel.load(contentsOf: decoderURL, configuration: configuration) + heads[bucket] = try await MLModel.load(contentsOf: headURL, configuration: configuration) + } + return ClefTextManager( + config: config, tokenizer: tokenizer, decoders: decoders, heads: heads, embeddings: embeddings) + } + + init(config: Config, tokenizer: QwenBPETokenizer, decoders: [Int: MLModel], heads: [Int: MLModel], embeddings: Data) + { + self.config = config + self.tokenizer = tokenizer + self.decoders = decoders + self.heads = heads + self.embeddings = embeddings + self.buckets = decoders.keys.sorted() + } + + /// Plain RoPE tables [L, head_dim] fp16 (rotate-half layout: the two halves repeat). + static func ropeTables(length: Int, headDim: Int, theta: Double) throws -> (MLMultiArray, MLMultiArray) { + let half = headDim / 2 + let cosArray = try MLMultiArray(shape: [length as NSNumber, headDim as NSNumber], dataType: .float16) + let sinArray = try MLMultiArray(shape: [length as NSNumber, headDim as NSNumber], dataType: .float16) + let c = cosArray.dataPointer.assumingMemoryBound(to: Float16.self) + let s = sinArray.dataPointer.assumingMemoryBound(to: Float16.self) + for position in 0.. MLMultiArray { + let array = try MLMultiArray(shape: [1, 1, length as NSNumber, length as NSNumber], dataType: .float16) + let m = array.dataPointer.assumingMemoryBound(to: Float16.self) + let blocked = Float16(-1e4) + for query in 0..= n && key <= query)) + m[query * length + key] = visible ? 0 : blocked + } + } + return array + } + + public func warm() throws { + for bucket in buckets { + let filler = String(repeating: "warm ", count: max(1, bucket / 4)) + _ = try? answer(state: filler, questions: [("ready", .noul(instructions: "Ready?"))]) + } + } + + public func encode(state: Any, questions: [(id: String, question: ClefQuestion)]) throws -> ClefEncodedRecord { + try ClefRecordEncoder.encode( + state: state, questions: questions, imageTokenCounts: [], tokenizer: tokenizer, imageTokenID: -1, + visionStartID: -1, visionEndID: -1, maxLength: buckets.max() ?? 0) + } + + /// Decide `questions` about `state`. Answers follow the question order; probabilities follow Clef's option order. + public func answer(state: Any, questions: [(id: String, question: ClefQuestion)]) throws -> ClefFlashManager.Result + { + let started = Date() + let record = try encode(state: state, questions: questions) + let n = record.inputIDs.count + guard let L = buckets.first(where: { $0 >= n }), let decoder = decoders[L], let head = heads[L] else { + throw ClefVisionError.tooLong(tokens: n, bucket: buckets.max() ?? 0) + } + let D = config.hidden + if ropeCache[L] == nil { + ropeCache[L] = try Self.ropeTables(length: L, headDim: config.headDim, theta: config.ropeTheta) + } + let (cos, sin) = ropeCache[L]! + let row = try MLMultiArray(shape: [1, L as NSNumber, D as NSNumber], dataType: .float16) + let padID = config.padID + embeddings.withUnsafeBytes { raw in + let table = raw.bindMemory(to: Float16.self).baseAddress! + let out = row.dataPointer.assumingMemoryBound(to: Float16.self) + for position in 0..