From 07ce1e9a77a5882d8eec58d6aa807c501cff314c Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Wed, 16 Sep 2026 16:53:36 +0800 Subject: [PATCH 1/5] chore(release): prepare 2.0.0-Beta.2 fork release --- .github/workflows/android-apk.yml | 2 +- .github/workflows/release-tauri.yml | 2 +- openless-all/app/package-lock.json | 4 ++-- openless-all/app/package.json | 2 +- openless-all/app/src-tauri/Cargo.lock | 2 +- openless-all/app/src-tauri/Cargo.toml | 2 +- openless-all/app/src-tauri/tauri.conf.json | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/android-apk.yml b/.github/workflows/android-apk.yml index ff1bf6d6e..30b784a5e 100644 --- a/.github/workflows/android-apk.yml +++ b/.github/workflows/android-apk.yml @@ -348,7 +348,7 @@ jobs: working-directory: openless-all/app env: OPENLESS_UPDATE_APK_DIR: ${{ steps.apk.outputs.out_dir }} - OPENLESS_UPDATE_REPO: Open-Less/openless + OPENLESS_UPDATE_REPO: ${{ github.repository }} OPENLESS_UPDATE_MIRROR_BASE_URL: https://fastgit.cc/https://github.com OPENLESS_RELEASE_CHANNEL: ${{ env.OPENLESS_RELEASE_CHANNEL }} OPENLESS_RELEASE_TAG: ${{ github.ref_name }} diff --git a/.github/workflows/release-tauri.yml b/.github/workflows/release-tauri.yml index f161811e3..1503aac40 100644 --- a/.github/workflows/release-tauri.yml +++ b/.github/workflows/release-tauri.yml @@ -387,7 +387,7 @@ jobs: TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }} OPENLESS_UPDATE_TARGET: ${{ matrix.updater-target }} OPENLESS_UPDATE_ARCH: ${{ matrix.updater-arch }} - OPENLESS_UPDATE_REPO: Open-Less/openless + OPENLESS_UPDATE_REPO: ${{ github.repository }} OPENLESS_UPDATE_MIRROR_BASE_URL: https://fastgit.cc/https://github.com # beta 渠道时输出 latest-{tgt}-{arch}-beta.json，stable 沿用旧文件名。 OPENLESS_RELEASE_CHANNEL: ${{ env.OPENLESS_RELEASE_CHANNEL }} diff --git a/openless-all/app/package-lock.json b/openless-all/app/package-lock.json index 31fde046d..120132464 100644 --- a/openless-all/app/package-lock.json +++ b/openless-all/app/package-lock.json @@ -1,12 +1,12 @@ { "name": "openless-app", - "version": "2.0.0-Beta.1", + "version": "2.0.0-Beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openless-app", - "version": "2.0.0-Beta.1", + "version": "2.0.0-Beta.2", "dependencies": { "@base-ui/react": "^1.6.0", "@shadcn/react": "^0.2.1", diff --git a/openless-all/app/package.json b/openless-all/app/package.json index bbe7f4a17..594113b27 100644 --- a/openless-all/app/package.json +++ b/openless-all/app/package.json @@ -1,7 +1,7 @@ { "name": "openless-app", "private": true, - "version": "2.0.0-Beta.1", + "version": "2.0.0-Beta.2", "type": "module", "scripts": { "pretest": "npm run build", diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index 6cacfb09a..021319e72 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -4257,7 +4257,7 @@ dependencies = [ [[package]] name = "openless" -version = "2.0.0-Beta.1" +version = "2.0.0-Beta.2" dependencies = [ "anyhow", "arboard", diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index 13fc610a6..ae3023ba8 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "openless" -version = "2.0.0-Beta.1" +version = "2.0.0-Beta.2" license = "AGPL-3.0-only" description = "OpenLess — local voice input that types where your cursor is" authors = ["OpenLess"] diff --git a/openless-all/app/src-tauri/tauri.conf.json b/openless-all/app/src-tauri/tauri.conf.json index 0005e47f7..f27f11742 100644 --- a/openless-all/app/src-tauri/tauri.conf.json +++ b/openless-all/app/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "OpenLess", - "version": "2.0.0-Beta.1", + "version": "2.0.0-Beta.2", "identifier": "com.openless.app", "build": { "beforeDevCommand": "npm run dev", From b0196a9ed9bb1358c4d4bb1fb9b7cbf8ca4c66b3 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Wed, 14 Oct 2026 15:06:18 +0800 Subject: [PATCH 2/5] feat: add configurable voice activation Co-Authored-By: Claude Fable 5 --- .../app/crates/openless-core/src/api.rs | 988 ++++++++++++++++-- .../src/cloud_sync_e2ee_documents/registry.rs | 49 + .../app/crates/openless-core/src/config.rs | 2 + .../app/crates/openless-core/src/events.rs | 1 + .../app/crates/openless-core/src/lib.rs | 13 +- .../app/crates/openless-core/src/ports.rs | 80 ++ .../crates/openless-core/src/shared_types.rs | 65 ++ .../openless-core/src/voice_activation.rs | 507 +++++++++ .../app/linux-egui/examples/headless_host.rs | 1 + openless-all/app/linux-egui/src/backend.rs | 1 + openless-all/app/linux-egui/src/runtime.rs | 1 + .../app/linux-egui/tests/host_contract.rs | 2 + .../backend-tests/tests/core_contract.rs | 1 + .../backend-tests/tests/voice_edit_host.rs | 1 + .../src-tauri/src/android/native_bridge.rs | 1 + .../src/asr/local/sherpa_provider.rs | 10 +- .../src-tauri/src/asr/local/sherpa_runtime.rs | 43 +- .../app/src-tauri/src/commands/misc.rs | 33 +- .../app/src-tauri/src/commands/mod.rs | 1 + .../app/src-tauri/src/commands/settings.rs | 55 +- .../app/src-tauri/src/core_adapters.rs | 758 +++++++++++++- openless-all/app/src-tauri/src/lib.rs | 16 +- openless-all/app/src-tauri/src/recorder.rs | 359 ++++++- .../app/src-tauri/src/tauri_events.rs | 3 + openless-all/app/src/i18n/de.ts | 9 + openless-all/app/src/i18n/en.ts | 9 + openless-all/app/src/i18n/es.ts | 9 + openless-all/app/src/i18n/fr.ts | 9 + openless-all/app/src/i18n/ja.ts | 9 + openless-all/app/src/i18n/ko.ts | 9 + openless-all/app/src/i18n/zh-CN.ts | 10 + openless-all/app/src/i18n/zh-TW.ts | 9 + openless-all/app/src/lib/ipc/mock-data.ts | 7 + openless-all/app/src/lib/types.ts | 8 + .../pages/settings/RecordingInputSection.tsx | 101 +- 35 files changed, 2992 insertions(+), 188 deletions(-) create mode 100644 openless-all/app/crates/openless-core/src/voice_activation.rs diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 99c3ed816..47e060597 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -38,7 +38,7 @@ use crate::style_packs::StylePack; use crate::types::{ CorrectionRule, DictationPhase, DictationResult, DictationSession, DictationStateSnapshot, DictionaryEntry, HistoryChange, HistoryInsertStatus, HistorySource, PreferencesChange, - SessionId, StylePackChange, VocabPresetStore, VocabularyChange, + SessionId, StylePackChange, TranscriptAccumulator, VocabPresetStore, VocabularyChange, }; use crate::vocabulary::DictionaryStore; use crate::{ActivityStore, CorrectionRuleStore, HistoryStore, PreferencesStore, StylePackStore}; @@ -127,12 +127,18 @@ struct DictationReservation { pending_capture: Option>, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum PendingAudioCaptureKey { + Hotkey(u64), + VoiceActivation(u64), +} + /// Removes a provisional capture if the async hotkey dispatch is aborted before /// it transfers ownership to a session reservation. struct PendingAudioCaptureAbortGuard { - captures: Arc>>>, + captures: Arc>>>, task_spawner: Arc, - press_id: u64, + key: PendingAudioCaptureKey, } impl Drop for PendingAudioCaptureAbortGuard { @@ -141,7 +147,7 @@ impl Drop for PendingAudioCaptureAbortGuard { .captures .lock() .expect("pending audio capture lock poisoned") - .remove(&self.press_id); + .remove(&self.key); let Some(capture) = capture else { return; }; @@ -2062,6 +2068,8 @@ struct BackendEngineProgress { state: Arc>, phase_changed: Arc, text_insertions: Arc>>, + voice_activation_session: Arc>>, + raw_transcripts: Mutex>, } impl EngineProgressSink for BackendEngineProgress { @@ -2185,7 +2193,14 @@ impl EngineProgressSink for BackendEngineProgress { self.phase_changed.notify_waiters(); } } - EngineProgress::TranscriptDelta(delta) => { + EngineProgress::TranscriptDelta(mut delta) => { + let finish_phrase = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .filter(|session| session.finish_enabled && session.session_id == session_id) + .map(|session| session.finish_phrase.clone()); let mut state = self.state.write().expect("backend state lock poisoned"); ensure_active_session(&state, session_id)?; if !matches!( @@ -2199,6 +2214,34 @@ impl EngineProgressSink for BackendEngineProgress { "transcript delta arrived after transcription completed", )); } + if let Some(finish_phrase) = finish_phrase { + let mut raw_transcripts = self + .raw_transcripts + .lock() + .expect("raw transcript lock poisoned"); + let raw_transcript = raw_transcripts.entry(session_id).or_insert_with(|| { + let mut transcript = TranscriptAccumulator::default(); + if let Some(existing) = state.transcripts.get(&session_id) { + let seed = crate::types::TranscriptDelta { + text: existing.text().to_string(), + offset: 0, + is_final: false, + }; + let _ = transcript.apply(&seed); + } + transcript + }); + raw_transcript.apply(&delta)?; + delta = crate::types::TranscriptDelta { + text: crate::voice_activation::strip_phrase_suffix( + raw_transcript.text(), + &finish_phrase, + ) + .unwrap_or_else(|| raw_transcript.text().to_string()), + offset: 0, + is_final: delta.is_final, + }; + } state .transcripts .entry(session_id) @@ -2247,7 +2290,8 @@ pub struct OpenLessBackend { phase_changed: Arc, hotkey: Mutex, hotkey_dispatch_gate: tokio::sync::Mutex<()>, - pending_audio_captures: Arc>>>, + pending_audio_captures: + Arc>>>, less_computer_hotkey_press_at: Mutex>, vocabulary: Arc, correction_rules: Arc, @@ -2269,6 +2313,10 @@ pub struct OpenLessBackend { voice_sessions: Arc, runtime_start_work: Arc, less_computer_voice_controls: Arc>>>, + voice_activation_handle: Arc>>>, + voice_activation_generation: Arc, + voice_activation_configure_gate: tokio::sync::Mutex<()>, + voice_activation_session: Arc>>, } #[path = "cloud_sync_e2ee/api.rs"] @@ -2434,6 +2482,78 @@ impl HistoryProviderAttribution { } } +struct CoreVoiceActivationSink { + events: Arc, +} + +impl crate::ports::VoiceActivationSink for CoreVoiceActivationSink { + fn publish( + &self, + signal: crate::voice_activation::VoiceActivationSignal, + ) -> Result<(), BackendError> { + use crate::voice_activation::{VoiceActivationPhase, VoiceActivationSignal}; + let (generation, phase, session_id) = match signal { + VoiceActivationSignal::WakeDetected { + activation_generation, + } => ( + activation_generation, + VoiceActivationPhase::WakeDetected, + None, + ), + VoiceActivationSignal::FinishDetected { + activation_generation, + session_id, + } => ( + activation_generation, + VoiceActivationPhase::Finishing, + Some(session_id), + ), + VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + phase, + } => (activation_generation, phase, None), + }; + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase, + activation_generation: generation, + session_id, + message: None, + }, + ), + ); + Ok(()) + } +} + +fn map_voice_activation_error(error: crate::ports::VoiceActivationError) -> BackendError { + let (code, message) = match error { + crate::ports::VoiceActivationError::Unsupported => ( + BackendErrorCode::Unsupported, + "voice activation is unsupported".into(), + ), + crate::ports::VoiceActivationError::ModelMissing => ( + BackendErrorCode::InvalidState, + "voice activation model is not installed".into(), + ), + crate::ports::VoiceActivationError::PermissionDenied => ( + BackendErrorCode::PermissionDenied, + "microphone permission was denied".into(), + ), + crate::ports::VoiceActivationError::MicBusy => ( + BackendErrorCode::Busy, + "microphone is already in use".into(), + ), + crate::ports::VoiceActivationError::InvalidConfig(message) + | crate::ports::VoiceActivationError::Runtime(message) => { + (BackendErrorCode::InvalidState, message) + } + }; + BackendError::new(code, message) +} + impl OpenLessBackend { /// Construct an explicitly unavailable Core so the native shell can show /// its startup error instead of panicking before any window is visible. @@ -2868,6 +2988,10 @@ impl OpenLessBackend { voice_sessions, runtime_start_work, less_computer_voice_controls: Arc::new(Mutex::new(HashMap::new())), + voice_activation_handle: Arc::new(Mutex::new(None)), + voice_activation_generation: Arc::new(AtomicU64::new(0)), + voice_activation_configure_gate: tokio::sync::Mutex::new(()), + voice_activation_session: Arc::new(Mutex::new(None)), }) } @@ -2889,6 +3013,534 @@ impl OpenLessBackend { Ok(()) } + /// Configure the opt-in local voice activation listener from the current + /// preference snapshot. Reconfiguration replaces the previous native + /// handle only after it has been stopped, so one process owns one mic. + pub async fn configure_voice_activation(&self) -> Result<(), BackendError> { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + if self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .is_some() + { + return Ok(()); + } + let preferences = self.get_preferences(); + // Invalidate callbacks before awaiting the old runtime. Sherpa may have + // queued a final callback while its cancellation is being observed. + let generation = self + .voice_activation_generation + .fetch_add(1, Ordering::AcqRel) + + 1; + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + let previous = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + if let Some(previous) = previous { + let _ = previous + .stop(crate::voice_activation::VoiceActivationStopReason::Disabled) + .await; + } + if !preferences.voice_activation_enabled { + return Ok(()); + } + let Some(runtime) = self.deps.voice_activation_runtime.as_ref() else { + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Unavailable, + activation_generation: generation, + session_id: None, + message: Some("voice activation host runtime is unavailable".into()), + }, + ), + ); + return if preferences.voice_activation_enabled { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "voice activation host runtime is unavailable", + )) + } else { + Ok(()) + }; + }; + + let config = crate::voice_activation::VoiceActivationConfig { + enabled: preferences.voice_activation_enabled, + wake_enabled: preferences.voice_activation_wake_enabled, + finish_enabled: preferences.voice_activation_finish_enabled, + wake_phrase: preferences.voice_activation_wake_phrase, + finish_phrase: preferences.voice_activation_finish_phrase, + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + microphone_device_name: (!preferences.microphone_device_name.trim().is_empty()) + .then_some(preferences.microphone_device_name), + activation_generation: generation, + } + .normalized() + .map_err(|error| BackendError::new(BackendErrorCode::InvalidArgument, error.to_string()))?; + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Starting, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + let sink: Arc = Arc::new(CoreVoiceActivationSink { + events: Arc::clone(&self.events), + }); + match runtime.configure(config, sink).await { + Ok(handle) => { + let stale_handle = { + let mut slot = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned"); + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + Some(handle) + } else { + *slot = Some(handle.into()); + None + } + }; + if let Some(handle) = stale_handle { + let _ = handle + .stop(crate::voice_activation::VoiceActivationStopReason::Cancelled) + .await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation configuration became stale", + )); + } + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForWake, + activation_generation: generation, + session_id: None, + message: None, + }, + ), + ); + Ok(()) + } + Err(error) => { + let error = map_voice_activation_error(error); + self.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::Failed, + activation_generation: generation, + session_id: None, + message: Some(error.message.clone()), + }, + ), + ); + Err(error) + } + } + } + + pub async fn stop_voice_activation( + &self, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> Result<(), BackendError> { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + self.voice_activation_generation + .fetch_add(1, Ordering::AcqRel); + if let Some(handle) = handle { + handle + .stop(reason) + .await + .map_err(map_voice_activation_error)?; + } + Ok(()) + } + + pub async fn stop_voice_activation_if_current( + &self, + generation: u64, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> Result { + let _configure_gate = self.voice_activation_configure_gate.lock().await; + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + return Ok(false); + } + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .take(); + self.voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .take(); + self.voice_activation_generation + .fetch_add(1, Ordering::AcqRel); + if let Some(handle) = handle { + handle + .stop(reason) + .await + .map_err(map_voice_activation_error)?; + } + Ok(true) + } + + pub fn voice_activation_generation_is_current(&self, generation: u64) -> bool { + self.voice_activation_generation.load(Ordering::Acquire) == generation + } + + /// Called by the host runtime after it recognizes a semantic command. The + /// host callback is synchronous, so the actual handoff/finalization is + /// owned by Core's task spawner and survives a dropped IPC request. + pub fn handle_voice_activation_signal( + self: &Arc, + signal: crate::voice_activation::VoiceActivationSignal, + ) { + use crate::voice_activation::VoiceActivationSignal; + let generation = self.voice_activation_generation.load(Ordering::Acquire); + let signal_generation = match signal { + VoiceActivationSignal::WakeDetected { + activation_generation, + } + | VoiceActivationSignal::FinishDetected { + activation_generation, + .. + } + | VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + .. + } => activation_generation, + }; + if signal_generation != generation { + return; + } + match signal { + VoiceActivationSignal::WakeDetected { .. } => { + if self.snapshot().dictation.session_id.is_some() + || self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .is_some() + { + return; + } + let handle = self + .voice_activation_handle + .lock() + .expect("voice activation handle lock poisoned") + .clone(); + let Some(handle) = handle else { + return; + }; + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + return; + } + let handoff = match handle + .handoff_to_dictation( + crate::ports::VoiceDictationHandoffRequest { + activation_generation: signal_generation, + }, + ) + .await + { + Ok(handoff) => handoff, + Err(error) => { + log::warn!("voice activation handoff failed: {error}"); + match backend + .stop_voice_activation_if_current( + signal_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + return; + } + }; + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + let _ = handoff.capture.discard().await; + return; + } + match backend + .start_voice_dictation_with_capture(handoff.capture, signal_generation) + .await + { + Ok(session_id) => { + if backend + .voice_activation_generation + .load(Ordering::Acquire) + != signal_generation + { + let _ = backend.cancel_dictation(Some(session_id)).await; + return; + } + let preferences = backend.get_preferences(); + let timeout = crate::voice_activation::VoiceActivationConfig { + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + ..crate::voice_activation::VoiceActivationConfig::default() + } + .normalized() + .ok() + .and_then(|config| config.timeout()); + let voice_session = crate::voice_activation::VoiceActivationSession { + generation: crate::voice_activation::VoiceActivationGeneration( + signal_generation, + ), + session_id, + started_at: std::time::Instant::now(), + timeout, + finish_enabled: preferences.voice_activation_finish_enabled, + finish_phrase: preferences.voice_activation_finish_phrase, + }; + let committed = { + let _configure_gate = + backend.voice_activation_configure_gate.lock().await; + let session_is_current = backend + .state + .read() + .expect("backend state lock poisoned") + .dictation + .session_id + == Some(session_id); + if backend.voice_activation_generation.load(Ordering::Acquire) + != signal_generation + || !session_is_current + { + false + } else { + *backend + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") = + Some(voice_session); + handle.bind_dictation_session(session_id); + true + } + }; + if !committed { + let _ = backend.cancel_dictation(Some(session_id)).await; + return; + } + backend.events.publish( + None, + BackendEventKind::VoiceActivationStateChanged( + crate::voice_activation::VoiceActivationStateSnapshot { + phase: crate::voice_activation::VoiceActivationPhase::ListeningForFinish, + activation_generation: signal_generation, + session_id: Some(session_id), + message: None, + }, + ), + ); + if let Some(timeout) = timeout { + let timeout_backend = Arc::clone(&backend); + backend.deps.task_spawner.spawn(Box::pin(async move { + tokio::time::sleep(timeout).await; + timeout_backend + .finish_voice_activation( + signal_generation, + session_id, + crate::voice_activation::VoiceActivationStopReason::Timeout, + ) + .await; + })); + } + } + Err(error) => { + log::warn!("voice dictation start failed: {error}"); + match backend + .stop_voice_activation_if_current( + signal_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + } + } + })); + } + VoiceActivationSignal::FinishDetected { + session_id, + activation_generation, + } => { + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + backend + .finish_voice_activation( + activation_generation, + session_id, + crate::voice_activation::VoiceActivationStopReason::FinishPhrase, + ) + .await; + })); + } + VoiceActivationSignal::RuntimeStateChanged { + activation_generation, + phase: crate::voice_activation::VoiceActivationPhase::Failed, + } => { + let backend = Arc::clone(self); + self.deps.task_spawner.spawn(Box::pin(async move { + match backend + .stop_voice_activation_if_current( + activation_generation, + crate::voice_activation::VoiceActivationStopReason::RuntimeFailure, + ) + .await + { + Ok(true) if backend.get_preferences().voice_activation_enabled => { + let _ = backend.configure_voice_activation().await; + } + Ok(_) => {} + Err(error) => { + log::warn!("voice activation recovery stop failed: {error}"); + } + } + })); + } + VoiceActivationSignal::RuntimeStateChanged { .. } => {} + } + } + + async fn start_voice_dictation_with_capture( + &self, + capture: Box, + generation: u64, + ) -> Result { + if self.voice_activation_generation.load(Ordering::Acquire) != generation { + let _ = capture.discard().await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "voice activation generation is stale", + )); + } + self.pending_audio_captures + .lock() + .expect("pending audio capture lock poisoned") + .insert(PendingAudioCaptureKey::VoiceActivation(generation), capture); + let reservation = match self.reserve_dictation_session( + true, + DictationOutputTarget::ForegroundApp, + Some(PendingAudioCaptureKey::VoiceActivation(generation)), + ) { + Ok(reservation) => reservation, + Err(error) => { + self.discard_pending_audio_capture(PendingAudioCaptureKey::VoiceActivation( + generation, + )) + .await?; + return Err(error); + } + }; + self.start_reserved_dictation(reservation, DictationStartOptions::default()) + .await + } + + async fn cleanup_voice_activation_after_dictation(&self, session_id: SessionId) { + let is_voice_session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| session.session_id == session_id); + if !is_voice_session { + return; + } + let enabled = self.get_preferences().voice_activation_enabled; + let _ = self + .stop_voice_activation(crate::voice_activation::VoiceActivationStopReason::Cancelled) + .await; + if enabled { + let _ = self.configure_voice_activation().await; + } + } + + async fn finish_voice_activation( + &self, + generation: u64, + session_id: SessionId, + reason: crate::voice_activation::VoiceActivationStopReason, + ) { + let accepted = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| { + session.accepts( + crate::voice_activation::VoiceActivationGeneration(generation), + session_id, + ) + }); + if !accepted { + return; + } + if self.stop_dictation_session(session_id).await.is_err() { + return; + } + let still_voice_session = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .is_some_and(|session| session.session_id == session_id); + if still_voice_session { + let _ = self.stop_voice_activation(reason).await; + if self.get_preferences().voice_activation_enabled { + let _ = self.configure_voice_activation().await; + } + } + } + pub fn repositories(&self) -> BackendRepositories { BackendRepositories { preferences: Arc::clone(&self.preferences), @@ -3781,6 +4433,8 @@ impl OpenLessBackend { state: Arc::clone(&self.state), phase_changed: Arc::clone(&self.phase_changed), text_insertions: Arc::clone(&self.text_insertions), + voice_activation_session: Arc::clone(&self.voice_activation_session), + raw_transcripts: Mutex::new(HashMap::new()), }) } @@ -3828,10 +4482,26 @@ impl OpenLessBackend { } } }; - let mut state = self.state.write().expect("backend state lock poisoned"); - state.credentials = credentials; - if state.running { - return Ok(StartupSnapshot { + let snapshot = { + let mut state = self.state.write().expect("backend state lock poisoned"); + state.credentials = credentials; + if state.running { + return Ok(StartupSnapshot { + contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), + backend: BackendSnapshot { + running: true, + dictation: state.dictation.clone(), + vocabulary_revision: self.vocabulary_revision.load(Ordering::Acquire), + history_revision: self.history_revision.load(Ordering::Acquire), + style_pack_revision: self.style_pack_revision.load(Ordering::Acquire), + preferences_revision: self.preferences_revision.load(Ordering::Acquire), + credentials: state.credentials.clone(), + }, + }); + } + state.running = true; + self.events.publish(None, BackendEventKind::BackendStarted); + StartupSnapshot { contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), backend: BackendSnapshot { running: true, @@ -3842,25 +4512,18 @@ impl OpenLessBackend { preferences_revision: self.preferences_revision.load(Ordering::Acquire), credentials: state.credentials.clone(), }, - }); + } + }; + if let Err(error) = self.configure_voice_activation().await { + log::warn!("[voice-activation] startup configuration failed: {error}"); } - state.running = true; - self.events.publish(None, BackendEventKind::BackendStarted); - Ok(StartupSnapshot { - contract_version: crate::BACKEND_CONTRACT_VERSION.to_string(), - backend: BackendSnapshot { - running: true, - dictation: state.dictation.clone(), - vocabulary_revision: self.vocabulary_revision.load(Ordering::Acquire), - history_revision: self.history_revision.load(Ordering::Acquire), - style_pack_revision: self.style_pack_revision.load(Ordering::Acquire), - preferences_revision: self.preferences_revision.load(Ordering::Acquire), - credentials: state.credentials.clone(), - }, - }) + Ok(snapshot) } pub async fn shutdown(&self) -> Result<(), BackendError> { + let _ = self + .stop_voice_activation(crate::voice_activation::VoiceActivationStopReason::Disabled) + .await; if let Some(sync) = &self.encrypted_sync { sync.shutdown().await; } @@ -4115,7 +4778,7 @@ impl OpenLessBackend { .pending_audio_captures .lock() .expect("pending audio capture lock poisoned") - .insert(press_id, capture); + .insert(PendingAudioCaptureKey::Hotkey(press_id), capture); if let Some(replaced) = replaced { self.deps.task_spawner.spawn(Box::pin(async move { if let Err(error) = replaced.discard().await { @@ -4133,15 +4796,21 @@ impl OpenLessBackend { } } - fn take_pending_audio_capture(&self, press_id: u64) -> Option> { + fn take_pending_audio_capture( + &self, + key: PendingAudioCaptureKey, + ) -> Option> { self.pending_audio_captures .lock() .expect("pending audio capture lock poisoned") - .remove(&press_id) + .remove(&key) } - async fn discard_pending_audio_capture(&self, press_id: u64) -> Result<(), BackendError> { - let capture = self.take_pending_audio_capture(press_id); + async fn discard_pending_audio_capture( + &self, + key: PendingAudioCaptureKey, + ) -> Result<(), BackendError> { + let capture = self.take_pending_audio_capture(key); match capture { Some(capture) => capture.discard().await, None => Ok(()), @@ -4256,7 +4925,7 @@ impl OpenLessBackend { Some(PendingAudioCaptureAbortGuard { captures: Arc::clone(&self.pending_audio_captures), task_spawner: Arc::clone(&self.deps.task_spawner), - press_id, + key: PendingAudioCaptureKey::Hotkey(press_id), }) } else { None @@ -4265,7 +4934,9 @@ impl OpenLessBackend { None }; if let DictationHotkeyEdge::Combined { press_id, .. } = edge { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; } let (intent, reservation) = if let HotkeyIntent::WaitForModifierGrace { press_id } = intent { @@ -4285,7 +4956,7 @@ impl OpenLessBackend { Some(self.reserve_dictation_session( options.start.insert_text, options.start.output_target, - Some(press_id), + Some(PendingAudioCaptureKey::Hotkey(press_id)), )) } else { None @@ -4293,7 +4964,9 @@ impl OpenLessBackend { (intent, reservation) }; if !matches!(intent, HotkeyIntent::Start { .. }) { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; } (intent, reservation) } else { @@ -4308,14 +4981,16 @@ impl OpenLessBackend { self.reserve_dictation_session( options.start.insert_text, options.start.output_target, - Some(press_id), + Some(PendingAudioCaptureKey::Hotkey(press_id)), ) }) }; if reservation.is_some() { (intent, reservation) } else { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; self.hotkey .lock() .expect("hotkey interpreter lock poisoned") @@ -4338,7 +5013,9 @@ impl OpenLessBackend { .await } Err(error) => { - let _ = self.discard_pending_audio_capture(press_id).await; + let _ = self + .discard_pending_audio_capture(PendingAudioCaptureKey::Hotkey(press_id)) + .await; Err(error) } }; @@ -4919,6 +5596,25 @@ impl OpenLessBackend { } }; + let voice_activation_config = crate::voice_activation::VoiceActivationConfig { + enabled: preferences.voice_activation_enabled, + wake_enabled: preferences.voice_activation_wake_enabled, + finish_enabled: preferences.voice_activation_finish_enabled, + wake_phrase: preferences.voice_activation_wake_phrase.clone(), + finish_phrase: preferences.voice_activation_finish_phrase.clone(), + timeout_enabled: preferences.voice_activation_timeout_enabled, + timeout_seconds: preferences.voice_activation_timeout_seconds, + microphone_device_name: (!preferences.microphone_device_name.trim().is_empty()) + .then(|| preferences.microphone_device_name.clone()), + activation_generation: 0, + }; + voice_activation_config.normalized().map_err(|error| { + BackendError::new( + BackendErrorCode::InvalidArgument, + format!("invalid voice activation settings: {error}"), + ) + })?; + let effects = crate::SettingsEffectPlan::between(&previous, &preferences); let mut receipt = match runtime.prepare(&effects) { Ok(receipt) => receipt, @@ -5749,7 +6445,7 @@ impl OpenLessBackend { &self, insert_text: bool, output_target: DictationOutputTarget, - capture_generation: Option, + capture_key: Option, ) -> Result { { let state = self.state.read().expect("backend state lock poisoned"); @@ -5807,11 +6503,14 @@ impl OpenLessBackend { ); self.phase_changed.notify_waiters(); } - let pending_capture = - capture_generation.and_then(|generation| self.take_pending_audio_capture(generation)); + let capture_generation = capture_key.map(|key| match key { + PendingAudioCaptureKey::Hotkey(generation) + | PendingAudioCaptureKey::VoiceActivation(generation) => generation, + }); + let pending_capture = capture_key.and_then(|key| self.take_pending_audio_capture(key)); let capture_generation = pending_capture .as_ref() - .map(|_| capture_generation.expect("capture generation was checked")); + .map(|_| capture_generation.expect("capture key was checked")); Ok(DictationReservation { session_id, resources: starting_resources, @@ -6266,6 +6965,8 @@ impl OpenLessBackend { let _ = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } } @@ -6333,10 +7034,41 @@ impl OpenLessBackend { let _ = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } }; + let finish_phrase = self + .voice_activation_session + .lock() + .expect("voice activation session lock poisoned") + .as_ref() + .filter(|session| session.finish_enabled && session.session_id == session_id) + .map(|session| session.finish_phrase.clone()); + if let Some(finish_phrase) = finish_phrase { + if let Some(stripped) = crate::voice_activation::strip_phrase_suffix( + &engine_result.raw_text, + &finish_phrase, + ) { + engine_result.raw_text = stripped; + } + if let Some(stripped) = crate::voice_activation::strip_phrase_suffix( + &engine_result.polished_text, + &finish_phrase, + ) { + engine_result.polished_text = stripped; + } + if let Some(asr_transcript) = engine_result.asr_transcript.as_mut() { + if let Some(stripped) = + crate::voice_activation::strip_phrase_suffix(asr_transcript, &finish_phrase) + { + *asr_transcript = stripped; + } + } + } + if context.output_target == DictationOutputTarget::CloudNote && engine_result.polish_failed { let error = @@ -6344,6 +7076,8 @@ impl OpenLessBackend { self.mark_dictation_failed(session_id, &error); let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; self.cleanup_cloud_note(session_id, true)?; return Err(error); } @@ -6372,6 +7106,8 @@ impl OpenLessBackend { let _ = self.cancel_text_insertion(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } @@ -6403,19 +7139,21 @@ impl OpenLessBackend { // Cancellation may happen while ASR/LLM work is in flight. Never // insert a result after the session has been cancelled or replaced. - { + let cancelled_before_insertion = { let state = self.state.read().expect("backend state lock poisoned"); - if state.dictation.session_id != Some(session_id) + state.dictation.session_id != Some(session_id) || !matches!( state.dictation.phase, DictationPhase::Transcribing | DictationPhase::Polishing ) - { - return Err(BackendError::new( - BackendErrorCode::Cancelled, - "dictation session was cancelled before insertion", - )); - } + }; + if cancelled_before_insertion { + self.cleanup_voice_activation_after_dictation(session_id) + .await; + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "dictation session was cancelled before insertion", + )); } { @@ -6459,6 +7197,8 @@ impl OpenLessBackend { let _ = self.cancel_text_insertion(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(error); } } @@ -6476,46 +7216,52 @@ impl OpenLessBackend { .unwrap_or(crate::types::InsertStatus::NotRequested), }; - let mut state = self.state.write().expect("backend state lock poisoned"); - if state.dictation.session_id != Some(session_id) - || state.dictation.phase - != if context.insertion.enabled { - DictationPhase::Inserting - } else { - DictationPhase::Completed + let session_replaced = { + let mut state = self.state.write().expect("backend state lock poisoned"); + let replaced = state.dictation.session_id != Some(session_id) + || state.dictation.phase + != if context.insertion.enabled { + DictationPhase::Inserting + } else { + DictationPhase::Completed + }; + if !replaced { + state.dictation.phase = DictationPhase::Completed; + state.dictation.message = Some(match insert_outcome { + Some(InsertOutcome::Inserted) => "inserted".to_string(), + Some(InsertOutcome::PasteSent) => "paste_sent".to_string(), + Some(InsertOutcome::CopiedFallback) => "copied_fallback".to_string(), + None => "insertion_not_requested".to_string(), + }); + if matches!(insert_outcome, Some(InsertOutcome::CopiedFallback)) { + self.events.publish( + Some(session_id), + BackendEventKind::InsertFallback(crate::types::InsertFallbackPayload { + reason: "clipboard_fallback".to_string(), + copied_text: Some(engine_result.polished_text.clone()), + }), + ); } - { + self.events.publish( + Some(session_id), + BackendEventKind::DictationCompleted(result.clone()), + ); + self.events.publish( + Some(session_id), + BackendEventKind::DictationStateChanged(state.dictation.clone()), + ); + self.phase_changed.notify_waiters(); + } + replaced + }; + if session_replaced { + self.cleanup_voice_activation_after_dictation(session_id) + .await; return Err(BackendError::new( BackendErrorCode::Cancelled, "dictation session was replaced before completion", )); } - state.dictation.phase = DictationPhase::Completed; - state.dictation.message = Some(match insert_outcome { - Some(InsertOutcome::Inserted) => "inserted".to_string(), - Some(InsertOutcome::PasteSent) => "paste_sent".to_string(), - Some(InsertOutcome::CopiedFallback) => "copied_fallback".to_string(), - None => "insertion_not_requested".to_string(), - }); - if matches!(insert_outcome, Some(InsertOutcome::CopiedFallback)) { - self.events.publish( - Some(session_id), - BackendEventKind::InsertFallback(crate::types::InsertFallbackPayload { - reason: "clipboard_fallback".to_string(), - copied_text: Some(engine_result.polished_text.clone()), - }), - ); - } - self.events.publish( - Some(session_id), - BackendEventKind::DictationCompleted(result.clone()), - ); - self.events.publish( - Some(session_id), - BackendEventKind::DictationStateChanged(state.dictation.clone()), - ); - self.phase_changed.notify_waiters(); - drop(state); self.arm_edit_observation( session_id, context.insertion.observe_edits, @@ -6527,6 +7273,8 @@ impl OpenLessBackend { // delayed successful completion must not clear its successor's state, // transcript, silence detector or physical-hotkey generation. self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; self.persist_completed_dictation(&context, &result, insert_outcome, &engine_result); self.cleanup_cloud_note(session_id, true)?; host_result?; @@ -6986,6 +7734,8 @@ impl OpenLessBackend { let cancel_result = self.cancel_session_adapters(session_id).await; let _ = self.hide_dictation_feedback(session_id); self.reset_dictation_session(session_id); + self.cleanup_voice_activation_after_dictation(session_id) + .await; cancel_result } @@ -7046,6 +7796,7 @@ impl OpenLessBackend { Ok(()) }; let cancel_result = self.cancel_session_adapters(active).await; + self.cleanup_voice_activation_after_dictation(active).await; // The state can already display cancellation, but native audio/input // cleanup still owns the shared resource. Reject new capture until that // cleanup finishes, including on its error path. @@ -7768,6 +8519,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -8391,6 +9143,7 @@ mod tests { let runtime = Arc::new(LessComputerCaptureRuntime::default()); let host = Arc::new(crate::testing::RecordingHostActions::default()); let dependencies = BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), dictation_engine: Arc::new(VoiceOnlyEngine(Arc::clone(&transcription))), ..BackendDependencies::unsupported() @@ -8551,6 +9304,7 @@ mod tests { }); let runtime = Arc::new(LessComputerCaptureRuntime::default()); let dependencies = BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(crate::testing::RecordingHostActions::default()), dictation_engine: Arc::new(DictationOnlyEngine(Arc::clone(&transcription))), ..BackendDependencies::unsupported() @@ -8845,6 +9599,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -8918,6 +9673,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -9388,6 +10144,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, dictation_engine: Arc::new(engine), ..BackendDependencies::unsupported() }, @@ -9460,6 +10217,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -9519,6 +10277,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -9645,6 +10404,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -9778,6 +10538,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -9928,6 +10689,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -10285,6 +11047,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host, text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -10433,6 +11196,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -10502,6 +11266,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -10611,6 +11376,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -10791,6 +11557,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -10827,6 +11594,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, services, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), @@ -10945,6 +11713,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -11005,6 +11774,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -11136,6 +11906,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host.clone(), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -11235,6 +12006,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -11963,11 +12735,18 @@ mod tests { Box::pin(async { Ok(crate::ports::PolishOutput::text("polished words")) }) } - fn cancel(&self, _session_id: SessionId) -> BoxFuture<'static, Result<(), BackendError>> { + fn cancel( + &self, + _session_id: SessionId, + ) -> BoxFuture<'static, Result<(), BackendError>> { Box::pin(async { Ok(()) }) } } - fn files_containing(dir: &std::path::Path, needle: &str, found: &mut Vec) { + fn files_containing( + dir: &std::path::Path, + needle: &str, + found: &mut Vec, + ) { for entry in std::fs::read_dir(dir).unwrap().flatten() { let path = entry.path(); if path.is_dir() { @@ -12030,7 +12809,9 @@ mod tests { let seen = seen.lock().unwrap().clone(); assert_eq!(seen.len(), 1, "{target}"); assert!( - seen[0].as_deref().is_some_and(|context| context.contains(HOST_TEXT)), + seen[0] + .as_deref() + .is_some_and(|context| context.contains(HOST_TEXT)), "{target}: polish must still see the cursor context" ); assert_eq!( @@ -12041,7 +12822,10 @@ mod tests { backend.shutdown().await.unwrap(); let mut leaked = Vec::new(); files_containing(&data_dir, HOST_TEXT, &mut leaked); - assert!(leaked.is_empty(), "{target}: cursor context persisted in {leaked:?}"); + assert!( + leaked.is_empty(), + "{target}: cursor context persisted in {leaked:?}" + ); // The scan is meaningful: what does get stored is readable the same way. let mut stored = Vec::new(); files_containing(&data_dir, "polished words", &mut stored); @@ -12115,6 +12899,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12256,6 +13041,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12335,6 +13121,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(inserter.clone()), dictation_engine: Arc::new(crate::testing::FixtureDictationEngine::successful( @@ -12434,6 +13221,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12500,6 +13288,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12566,6 +13355,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12622,6 +13412,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12672,6 +13463,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12728,6 +13520,7 @@ mod tests { ..Default::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12943,6 +13736,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( InsertOutcome::Inserted, @@ -12986,6 +13780,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(inserter.clone()), dictation_engine: Arc::new(StartFailingEngine), @@ -13040,6 +13835,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(crate::testing::FixtureTextInserter::failing( BackendError::new(BackendErrorCode::Platform, "fixture insertion failure"), @@ -13164,6 +13960,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(inserter.clone()), dictation_engine: Arc::new(engine), @@ -13518,6 +14315,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(inserter.clone()), dictation_engine: Arc::new(engine), @@ -13655,6 +14453,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -13781,6 +14580,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host, text_inserter: Arc::new(FakeInserter), dictation_engine: engine.clone(), @@ -13846,6 +14646,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: host, text_inserter: Arc::new(crate::testing::FixtureTextInserter::with_outcome( outcome, @@ -13992,6 +14793,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(DelayedStartEngine { @@ -14055,6 +14857,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(FakeEngine), @@ -14167,6 +14970,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(DelayedStartEngine { @@ -14233,6 +15037,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(completed_inserter.clone()), dictation_engine: Arc::new(completed_engine.clone()), @@ -14277,6 +15082,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(cancelled_inserter.clone()), dictation_engine: Arc::new(cancelled_engine.clone()), @@ -14318,6 +15124,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(shutdown_inserter.clone()), dictation_engine: Arc::new(shutdown_engine.clone()), @@ -14361,6 +15168,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(engine), @@ -14453,6 +15261,8 @@ mod tests { state: Arc::clone(&backend.state), phase_changed: Arc::clone(&backend.phase_changed), text_insertions: Arc::clone(&backend.text_insertions), + voice_activation_session: Arc::clone(&backend.voice_activation_session), + raw_transcripts: Mutex::new(HashMap::new()), }; backend.cancel_dictation(Some(session)).await.unwrap(); @@ -14481,6 +15291,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { @@ -14577,6 +15388,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(DelayedInserter { preparing: preparing.clone(), @@ -14652,6 +15464,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { @@ -14732,6 +15545,7 @@ mod tests { ..BackendConfig::default() }, BackendDependencies { + voice_activation_runtime: None, host_actions: Arc::new(FakeHost::default()), text_inserter: Arc::new(FakeInserter), dictation_engine: Arc::new(BlockingStartEngine { diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index a9489e274..fdebb804b 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -165,6 +165,55 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Boolean, reason: "local_only_microphone_capture", }, + PreferenceField { + rust_name: "voice_activation_enabled", + key: "voiceActivationEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_wake_enabled", + key: "voiceActivationWakeEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_finish_enabled", + key: "voiceActivationFinishEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_wake_phrase", + key: "voiceActivationWakePhrase", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Text, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_finish_phrase", + key: "voiceActivationFinishPhrase", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Text, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_timeout_enabled", + key: "voiceActivationTimeoutEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, + PreferenceField { + rust_name: "voice_activation_timeout_seconds", + key: "voiceActivationTimeoutSeconds", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Unsigned32, + reason: "local_only_microphone_capture", + }, PreferenceField { rust_name: "active_asr_provider", key: "activeAsrProvider", diff --git a/openless-all/app/crates/openless-core/src/config.rs b/openless-all/app/crates/openless-core/src/config.rs index 096dc9fd2..75527de1b 100644 --- a/openless-all/app/crates/openless-core/src/config.rs +++ b/openless-all/app/crates/openless-core/src/config.rs @@ -72,6 +72,7 @@ pub struct BackendDependencies { pub dictation_engine: Arc, pub task_spawner: Arc, pub credential_store: Arc, + pub voice_activation_runtime: Option>, pub services: BackendServices, pub local_asr_runtime: Option>, pub marketplace_config: Option, @@ -91,6 +92,7 @@ impl BackendDependencies { dictation_engine: Arc::new(UnsupportedDictationEngine), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(crate::credentials::UnsupportedCredentialStore), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/crates/openless-core/src/events.rs b/openless-all/app/crates/openless-core/src/events.rs index b118992d7..23d7aa50c 100644 --- a/openless-all/app/crates/openless-core/src/events.rs +++ b/openless-all/app/crates/openless-core/src/events.rs @@ -407,6 +407,7 @@ pub enum BackendEventKind { BackendStarted, BackendStopping, DictationStateChanged(DictationStateSnapshot), + VoiceActivationStateChanged(crate::voice_activation::VoiceActivationStateSnapshot), TranscriptDelta(TranscriptDelta), PolishDelta(PolishDelta), DictationCompleted(DictationResult), diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 261eed540..74f8d3b29 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -73,6 +73,7 @@ pub mod style_packs; pub mod testing; pub mod types; pub mod vocabulary; +pub mod voice_activation; pub mod voice_edit_session; mod voice_session; @@ -297,7 +298,9 @@ pub use ports::{ PendingAudioCapture, PolishOutput, RecordingArchive, RecordingControlSink, RecordingEvent, RecordingProgressSink, ResourceResolver, TextInserter, TextInsertionSession, TextPolisher, TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, - UnsupportedTextInserter, VoiceCapture, + UnsupportedTextInserter, UnsupportedVoiceActivationRuntime, VoiceActivationError, + VoiceActivationHandle, VoiceActivationRuntime, VoiceActivationSink, VoiceCapture, + VoiceDictationHandoff, VoiceDictationHandoffRequest, }; pub use preferences::PreferencesStore; pub use prompt_compose::{ @@ -371,6 +374,14 @@ pub use vocabulary::{ builtin_vocab_presets, list_vocab_presets, resolve_vocab_presets, save_vocab_presets, DictionaryStore, }; +pub use voice_activation::{ + normalize_phrase, validate_phrase, PhraseValidationError, VoiceActivationCommand, + VoiceActivationConfig, VoiceActivationDecision, VoiceActivationGeneration, + VoiceActivationPhase, VoiceActivationSession, VoiceActivationSignal, + VoiceActivationStateSnapshot, VoiceActivationStopReason, VoiceCommandMatch, + VoiceCommandRecognizer, VoicePhraseMatcher, DEFAULT_FINISH_PHRASE, DEFAULT_TIMEOUT_SECONDS, + DEFAULT_WAKE_PHRASE, MAX_PHRASE_CHARS, MAX_TIMEOUT_SECONDS, MIN_TIMEOUT_SECONDS, +}; pub use voice_edit_session::{ TextSelection, VoiceEditCommit, VoiceEditContext, VoiceEditError, VoiceEditPhase, VoiceEditSession, VoiceEditSnapshot, VoiceEditTarget, VoiceEditTurn, diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 1f9f937ba..d2e4f4ad1 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -7,6 +7,7 @@ use futures_util::future::BoxFuture; use crate::dictation_context::DictationContext; use crate::errors::{BackendError, BackendErrorCode}; use crate::types::{InsertStatus, PolishDelta, SessionId, TranscriptDelta}; +use crate::voice_activation::{VoiceActivationConfig, VoiceActivationSignal}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum HostAction { @@ -72,6 +73,85 @@ impl HostActions for NoopHostActions { } } +/// Host-owned sink for semantic voice activation signals. Standby recognition +/// must only publish wake/finish commands and runtime state; it must never +/// publish the recognized standby text as a normal transcript delta. +pub trait VoiceActivationSink: Send + Sync { + fn publish(&self, signal: VoiceActivationSignal) -> Result<(), BackendError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceActivationError { + Unsupported, + ModelMissing, + PermissionDenied, + MicBusy, + InvalidConfig(String), + Runtime(String), +} + +impl std::fmt::Display for VoiceActivationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unsupported => formatter.write_str("voice activation is unsupported"), + Self::ModelMissing => formatter.write_str("voice activation model is not installed"), + Self::PermissionDenied => formatter.write_str("microphone permission was denied"), + Self::MicBusy => formatter.write_str("microphone is already in use"), + Self::InvalidConfig(message) | Self::Runtime(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for VoiceActivationError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceDictationHandoffRequest { + pub activation_generation: u64, +} + +pub struct VoiceDictationHandoff { + pub capture: Box, +} + +/// A single host-owned microphone/runtime seam for voice activation. The Core +/// decides lifecycle and stale-event policy; the host decides how local ASR and +/// the native capture stream are implemented. +pub trait VoiceActivationRuntime: Send + Sync + 'static { + fn configure( + &self, + config: VoiceActivationConfig, + sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>>; +} + +pub trait VoiceActivationHandle: Send + Sync { + /// Bind the Core-owned dictation session so finish signals can carry the + /// same session identity even though the host performs recognition. + fn bind_dictation_session(&self, _session_id: SessionId) {} + + fn handoff_to_dictation( + &self, + request: VoiceDictationHandoffRequest, + ) -> BoxFuture<'static, Result>; + + fn stop( + &self, + reason: crate::voice_activation::VoiceActivationStopReason, + ) -> BoxFuture<'static, Result<(), VoiceActivationError>>; +} + +pub struct UnsupportedVoiceActivationRuntime; + +impl VoiceActivationRuntime for UnsupportedVoiceActivationRuntime { + fn configure( + &self, + _config: VoiceActivationConfig, + _sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>> { + Box::pin(async { Err(VoiceActivationError::Unsupported) }) + } +} + /// Resolve a packaged resource without exposing a framework-specific resource /// directory object to the core or UI. pub trait ResourceResolver: Send + Sync { diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 5bb02b083..f0bb86474 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -360,6 +360,18 @@ fn default_silence_auto_stop_seconds() -> f32 { 3.0 } +fn default_voice_activation_wake_phrase() -> String { + crate::voice_activation::DEFAULT_WAKE_PHRASE.into() +} + +fn default_voice_activation_finish_phrase() -> String { + crate::voice_activation::DEFAULT_FINISH_PHRASE.into() +} + +fn default_voice_activation_timeout_seconds() -> u32 { + crate::voice_activation::DEFAULT_TIMEOUT_SECONDS +} + fn resolve_windows_insertion_mode( mode: WindowsInsertionMode, legacy_sendinput_only: bool, @@ -432,6 +444,21 @@ pub struct UserPreferences { /// Recording input device name. Empty string = system default microphone. #[serde(default)] pub microphone_device_name: String, + /// Opt-in local voice command listener. Standby audio is never sent to a cloud provider. + #[serde(default)] + pub voice_activation_enabled: bool, + #[serde(default = "default_true")] + pub voice_activation_wake_enabled: bool, + #[serde(default = "default_true")] + pub voice_activation_finish_enabled: bool, + #[serde(default = "default_voice_activation_wake_phrase")] + pub voice_activation_wake_phrase: String, + #[serde(default = "default_voice_activation_finish_phrase")] + pub voice_activation_finish_phrase: String, + #[serde(default = "default_true")] + pub voice_activation_timeout_enabled: bool, + #[serde(default = "default_voice_activation_timeout_seconds")] + pub voice_activation_timeout_seconds: u32, pub active_asr_provider: String, // "volcengine" | "apple-speech" | ... pub active_llm_provider: String, // "ark" | "openai" | ... /// Recognition pipeline mode (experimental, issue #902). With @@ -961,6 +988,20 @@ struct UserPreferencesWire { silence_auto_stop_seconds: f32, #[serde(default)] microphone_device_name: String, + #[serde(default)] + voice_activation_enabled: bool, + #[serde(default = "default_true")] + voice_activation_wake_enabled: bool, + #[serde(default = "default_true")] + voice_activation_finish_enabled: bool, + #[serde(default = "default_voice_activation_wake_phrase")] + voice_activation_wake_phrase: String, + #[serde(default = "default_voice_activation_finish_phrase")] + voice_activation_finish_phrase: String, + #[serde(default = "default_true")] + voice_activation_timeout_enabled: bool, + #[serde(default = "default_voice_activation_timeout_seconds")] + voice_activation_timeout_seconds: u32, active_asr_provider: String, active_llm_provider: String, #[serde(default = "default_pipeline_mode")] @@ -1211,6 +1252,13 @@ impl Default for UserPreferencesWire { silence_auto_stop_enabled: prefs.silence_auto_stop_enabled, silence_auto_stop_seconds: prefs.silence_auto_stop_seconds, microphone_device_name: prefs.microphone_device_name, + voice_activation_enabled: prefs.voice_activation_enabled, + voice_activation_wake_enabled: prefs.voice_activation_wake_enabled, + voice_activation_finish_enabled: prefs.voice_activation_finish_enabled, + voice_activation_wake_phrase: prefs.voice_activation_wake_phrase, + voice_activation_finish_phrase: prefs.voice_activation_finish_phrase, + voice_activation_timeout_enabled: prefs.voice_activation_timeout_enabled, + voice_activation_timeout_seconds: prefs.voice_activation_timeout_seconds, active_asr_provider: prefs.active_asr_provider, active_llm_provider: prefs.active_llm_provider, pipeline_mode: prefs.pipeline_mode, @@ -1411,6 +1459,16 @@ impl<'de> Deserialize<'de> for UserPreferences { silence_auto_stop_enabled: wire.silence_auto_stop_enabled, silence_auto_stop_seconds: wire.silence_auto_stop_seconds, microphone_device_name: wire.microphone_device_name, + voice_activation_enabled: wire.voice_activation_enabled, + voice_activation_wake_enabled: wire.voice_activation_wake_enabled, + voice_activation_finish_enabled: wire.voice_activation_finish_enabled, + voice_activation_wake_phrase: wire.voice_activation_wake_phrase, + voice_activation_finish_phrase: wire.voice_activation_finish_phrase, + voice_activation_timeout_enabled: wire.voice_activation_timeout_enabled, + voice_activation_timeout_seconds: wire.voice_activation_timeout_seconds.clamp( + crate::voice_activation::MIN_TIMEOUT_SECONDS, + crate::voice_activation::MAX_TIMEOUT_SECONDS, + ), active_asr_provider: wire.active_asr_provider, active_llm_provider: wire.active_llm_provider, pipeline_mode: wire.pipeline_mode, @@ -1788,6 +1846,13 @@ impl Default for UserPreferences { silence_auto_stop_enabled: false, silence_auto_stop_seconds: default_silence_auto_stop_seconds(), microphone_device_name: String::new(), + voice_activation_enabled: false, + voice_activation_wake_enabled: true, + voice_activation_finish_enabled: true, + voice_activation_wake_phrase: default_voice_activation_wake_phrase(), + voice_activation_finish_phrase: default_voice_activation_finish_phrase(), + voice_activation_timeout_enabled: true, + voice_activation_timeout_seconds: default_voice_activation_timeout_seconds(), active_asr_provider: default_active_asr_provider(), active_llm_provider: "ark".into(), pipeline_mode: PipelineMode::Traditional, diff --git a/openless-all/app/crates/openless-core/src/voice_activation.rs b/openless-all/app/crates/openless-core/src/voice_activation.rs new file mode 100644 index 000000000..12f21f699 --- /dev/null +++ b/openless-all/app/crates/openless-core/src/voice_activation.rs @@ -0,0 +1,507 @@ +//! Pure policy and matching primitives for voice-activated dictation. +//! +//! The host owns the microphone and local recognizer. This module deliberately +//! only consumes revised recognition text and produces command decisions, so a +//! standby listener can never accidentally publish normal transcript events. + +use std::time::{Duration, Instant}; + +use unicode_normalization::UnicodeNormalization; + +use crate::types::SessionId; + +pub const DEFAULT_WAKE_PHRASE: &str = "开始语音录入"; +pub const DEFAULT_FINISH_PHRASE: &str = "完成语音录入"; +pub const DEFAULT_TIMEOUT_SECONDS: u32 = 30; +pub const MIN_TIMEOUT_SECONDS: u32 = 5; +pub const MAX_TIMEOUT_SECONDS: u32 = 300; +pub const MAX_PHRASE_CHARS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PhraseValidationError { + Empty, + TooLong { max_chars: usize }, + ControlCharacter, +} + +impl std::fmt::Display for PhraseValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Empty => write!(f, "voice activation phrase must not be empty"), + Self::TooLong { max_chars } => { + write!( + f, + "voice activation phrase must be at most {max_chars} characters" + ) + } + Self::ControlCharacter => { + write!( + f, + "voice activation phrase must not contain control characters" + ) + } + } + } +} + +impl std::error::Error for PhraseValidationError {} + +/// Normalize ASR output and user phrases before suffix matching. +/// +/// NFKC handles full-width punctuation and compatibility forms. Whitespace is +/// intentionally removed because streaming ASR commonly inserts spaces between +/// Chinese characters while English recognition may emit multiple spaces. +pub fn normalize_phrase(text: &str) -> String { + text.nfkc() + .filter(|character| !character.is_whitespace()) + .flat_map(char::to_lowercase) + .collect() +} + +/// Remove a recognized command from the end of a transcript while preserving +/// the spacing and punctuation in the spoken body. +pub fn strip_phrase_suffix(text: &str, phrase: &str) -> Option { + let normalized_phrase = validate_phrase(phrase).ok()?; + let phrase_ends_punctuation = normalized_phrase + .chars() + .last() + .is_some_and(is_terminal_punctuation); + let phrase_chars = normalized_phrase.chars().rev(); + let mut cursor = text.len(); + while cursor > 0 { + let previous = text[..cursor].char_indices().next_back()?; + if previous.1.is_whitespace() + || (!phrase_ends_punctuation && is_terminal_punctuation(previous.1)) + { + cursor = previous.0; + } else { + break; + } + } + let mut cut_at = cursor; + for expected in phrase_chars { + while cursor > 0 { + let previous = text[..cursor].char_indices().next_back()?; + if !previous.1.is_whitespace() { + break; + } + cursor = previous.0; + } + let (start, character) = text[..cursor].char_indices().next_back()?; + if normalize_phrase(&character.to_string()) != expected.to_string() { + return None; + } + cut_at = start; + cursor = start; + } + Some(text[..cut_at].trim_end().to_string()) +} + +fn is_terminal_punctuation(character: char) -> bool { + character.is_ascii_punctuation() + || matches!( + character, + '。' | '！' + | '？' + | '，' + | '、' + | '；' + | '：' + | '“' + | '”' + | '‘' + | '’' + | '（' + | '）' + | '【' + | '】' + | '《' + | '》' + | '…' + | '～' + | '—' + | '–' + ) +} + +pub fn validate_phrase(text: &str) -> Result { + let normalized = normalize_phrase(text); + if normalized.is_empty() { + return Err(PhraseValidationError::Empty); + } + if normalized.chars().any(char::is_control) { + return Err(PhraseValidationError::ControlCharacter); + } + if normalized.chars().count() > MAX_PHRASE_CHARS { + return Err(PhraseValidationError::TooLong { + max_chars: MAX_PHRASE_CHARS, + }); + } + Ok(normalized) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceActivationConfig { + pub enabled: bool, + pub wake_enabled: bool, + pub finish_enabled: bool, + pub wake_phrase: String, + pub finish_phrase: String, + pub timeout_enabled: bool, + pub timeout_seconds: u32, + /// Runtime-only selected input; never serialized as a user preference field. + pub microphone_device_name: Option, + /// Runtime-owned generation; this is not persisted in preferences. + pub activation_generation: u64, +} + +impl Default for VoiceActivationConfig { + fn default() -> Self { + Self { + enabled: false, + wake_enabled: true, + finish_enabled: true, + wake_phrase: DEFAULT_WAKE_PHRASE.into(), + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + timeout_enabled: true, + timeout_seconds: DEFAULT_TIMEOUT_SECONDS, + microphone_device_name: None, + activation_generation: 0, + } + } +} + +impl VoiceActivationConfig { + pub fn normalized(&self) -> Result { + let wake_phrase = validate_phrase(&self.wake_phrase)?; + let finish_phrase = validate_phrase(&self.finish_phrase)?; + Ok(Self { + enabled: self.enabled, + wake_enabled: self.wake_enabled, + finish_enabled: self.finish_enabled, + wake_phrase, + finish_phrase, + timeout_enabled: self.timeout_enabled, + timeout_seconds: self + .timeout_seconds + .clamp(MIN_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS), + microphone_device_name: self.microphone_device_name.clone(), + activation_generation: self.activation_generation, + }) + } + + pub fn timeout(&self) -> Option { + self.timeout_enabled + .then(|| Duration::from_secs(u64::from(self.timeout_seconds))) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationCommand { + Wake, + Finish, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VoiceCommandMatch { + pub command: VoiceActivationCommand, + pub text_without_command: String, +} + +/// Matches only the end of the revised ASR text. A phrase in the middle of a +/// sentence must never trigger activation or stop an active dictation session. +#[derive(Debug, Clone)] +pub struct VoicePhraseMatcher { + normalized_phrase: String, + command: VoiceActivationCommand, +} + +impl VoicePhraseMatcher { + pub fn new( + phrase: impl AsRef, + command: VoiceActivationCommand, + ) -> Result { + Ok(Self { + normalized_phrase: validate_phrase(phrase.as_ref())?, + command, + }) + } + + pub fn matches_suffix(&self, text: &str) -> bool { + normalize_phrase(text).ends_with(&self.normalized_phrase) + } + + pub fn strip_suffix(&self, text: &str) -> Option { + let normalized = normalize_phrase(text); + normalized + .strip_suffix(&self.normalized_phrase) + .map(str::to_string) + } + + pub fn command(&self) -> VoiceActivationCommand { + self.command + } +} + +/// Handles ASR revisions. A revised result may replace a previous result, +/// therefore a command is reported at most once for each observed text value. +#[derive(Debug, Clone)] +pub struct VoiceCommandRecognizer { + wake: VoicePhraseMatcher, + finish: VoicePhraseMatcher, + last_text: String, + wake_reported: bool, + finish_reported: bool, +} + +impl VoiceCommandRecognizer { + pub fn new(config: &VoiceActivationConfig) -> Result { + let config = config.normalized()?; + Ok(Self { + wake: VoicePhraseMatcher::new(config.wake_phrase, VoiceActivationCommand::Wake)?, + finish: VoicePhraseMatcher::new(config.finish_phrase, VoiceActivationCommand::Finish)?, + last_text: String::new(), + wake_reported: false, + finish_reported: false, + }) + } + + pub fn observe( + &mut self, + text: &str, + command: VoiceActivationCommand, + ) -> Option { + let normalized = normalize_phrase(text); + if normalized == self.last_text { + return None; + } + self.last_text = normalized.clone(); + let matcher = match command { + VoiceActivationCommand::Wake => &self.wake, + VoiceActivationCommand::Finish => &self.finish, + }; + let already_reported = match command { + VoiceActivationCommand::Wake => self.wake_reported, + VoiceActivationCommand::Finish => self.finish_reported, + }; + if already_reported || !normalized.ends_with(&matcher.normalized_phrase) { + return None; + } + match command { + VoiceActivationCommand::Wake => self.wake_reported = true, + VoiceActivationCommand::Finish => self.finish_reported = true, + } + Some(VoiceCommandMatch { + command, + text_without_command: normalized + .strip_suffix(&matcher.normalized_phrase) + .unwrap_or_default() + .to_string(), + }) + } + + pub fn reset(&mut self) { + self.last_text.clear(); + self.wake_reported = false; + self.finish_reported = false; + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VoiceActivationSignal { + WakeDetected { + activation_generation: u64, + }, + FinishDetected { + activation_generation: u64, + session_id: SessionId, + }, + RuntimeStateChanged { + activation_generation: u64, + phase: VoiceActivationPhase, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VoiceActivationPhase { + Disabled, + Starting, + ListeningForWake, + WakeDetected, + HandingOff, + ListeningForFinish, + Finishing, + Unavailable, + PermissionDenied, + MicBusy, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VoiceActivationStateSnapshot { + pub phase: VoiceActivationPhase, + pub activation_generation: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationStopReason { + Disabled, + DictationStarted, + FinishPhrase, + Timeout, + Cancelled, + RuntimeFailure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VoiceActivationGeneration(pub u64); + +impl VoiceActivationGeneration { + pub fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +#[derive(Debug, Clone)] +pub struct VoiceActivationSession { + pub generation: VoiceActivationGeneration, + pub session_id: SessionId, + pub started_at: Instant, + pub timeout: Option, + pub finish_enabled: bool, + pub finish_phrase: String, +} + +impl VoiceActivationSession { + pub fn timed_out_at(&self, now: Instant) -> bool { + self.timeout + .is_some_and(|timeout| now.saturating_duration_since(self.started_at) >= timeout) + } + + pub fn accepts(&self, generation: VoiceActivationGeneration, session_id: SessionId) -> bool { + self.generation == generation && self.session_id == session_id + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VoiceActivationDecision { + Ignore, + StartDictation, + FinishDictation, + StopAfterTimeout, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalization_handles_spaces_width_and_case() { + assert_eq!(normalize_phrase(" Ａ 开始 语音录入 "), "a开始语音录入"); + assert_eq!(normalize_phrase("完成\n语音\t录入"), DEFAULT_FINISH_PHRASE); + } + + #[test] + fn strip_phrase_suffix_preserves_spoken_body() { + assert_eq!( + strip_phrase_suffix("你好，世界 完成 语音录入", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_suffix("你好，世界 完成语音录入。", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert!(strip_phrase_suffix("完成语音录入之前", DEFAULT_FINISH_PHRASE).is_none()); + } + + #[test] + fn phrase_validation_rejects_empty_control_and_long_values() { + assert_eq!(validate_phrase(" \n"), Err(PhraseValidationError::Empty)); + assert_eq!( + validate_phrase("开始\u{0000}录入"), + Err(PhraseValidationError::ControlCharacter) + ); + assert!(matches!( + validate_phrase(&"a".repeat(MAX_PHRASE_CHARS + 1)), + Err(PhraseValidationError::TooLong { .. }) + )); + } + + #[test] + fn matcher_only_accepts_suffix_and_strips_command() { + let matcher = + VoicePhraseMatcher::new(DEFAULT_FINISH_PHRASE, VoiceActivationCommand::Finish).unwrap(); + assert!(!matcher.matches_suffix("完成语音录入之前的内容")); + assert_eq!( + matcher.strip_suffix("正文 完成语音录入").as_deref(), + Some( + "正文完成语音录入" + .strip_suffix(DEFAULT_FINISH_PHRASE) + .unwrap() + ) + ); + } + + #[test] + fn recognizer_does_not_repeat_a_command_for_asr_revisions() { + let config = VoiceActivationConfig::default(); + let mut recognizer = VoiceCommandRecognizer::new(&config).unwrap(); + assert!(recognizer + .observe("开始语音", VoiceActivationCommand::Wake) + .is_none()); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_some()); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_none()); + recognizer.reset(); + assert!(recognizer + .observe("开始语音录入", VoiceActivationCommand::Wake) + .is_some()); + } + + #[test] + fn generation_and_session_guards_reject_stale_events() { + let session_id = SessionId::new(); + let session = VoiceActivationSession { + generation: VoiceActivationGeneration(7), + session_id, + started_at: Instant::now(), + timeout: Some(Duration::from_secs(1)), + finish_enabled: true, + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + }; + assert!(session.accepts(VoiceActivationGeneration(7), session_id)); + assert!(!session.accepts(VoiceActivationGeneration(6), session_id)); + assert!(!session.accepts(VoiceActivationGeneration(7), SessionId::new())); + } + + #[test] + fn timeout_is_bounded_and_only_applies_after_the_voice_session_starts() { + let config = VoiceActivationConfig { + timeout_seconds: 1, + ..VoiceActivationConfig::default() + }; + assert_eq!( + config.normalized().unwrap().timeout_seconds, + MIN_TIMEOUT_SECONDS + ); + let session = VoiceActivationSession { + generation: VoiceActivationGeneration(1), + session_id: SessionId::new(), + started_at: Instant::now(), + timeout: Some(Duration::from_secs(1)), + finish_enabled: true, + finish_phrase: DEFAULT_FINISH_PHRASE.into(), + }; + assert!(!session.timed_out_at(Instant::now())); + assert!(session.timed_out_at(session.started_at + Duration::from_secs(1))); + } +} diff --git a/openless-all/app/linux-egui/examples/headless_host.rs b/openless-all/app/linux-egui/examples/headless_host.rs index 7e7e6bf04..507411dd4 100644 --- a/openless-all/app/linux-egui/examples/headless_host.rs +++ b/openless-all/app/linux-egui/examples/headless_host.rs @@ -74,6 +74,7 @@ async fn main() -> Result<(), BackendError> { )), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/linux-egui/src/backend.rs b/openless-all/app/linux-egui/src/backend.rs index d01c3c4d1..419049464 100644 --- a/openless-all/app/linux-egui/src/backend.rs +++ b/openless-all/app/linux-egui/src/backend.rs @@ -284,6 +284,7 @@ impl LinuxBackendBuilder { dictation_engine, task_spawner, credential_store, + voice_activation_runtime: None, services, // Linux ships no local inference runtime. Leaving this unset // makes every local-ASR call resolve to Core's unsupported diff --git a/openless-all/app/linux-egui/src/runtime.rs b/openless-all/app/linux-egui/src/runtime.rs index 1f1fc0ae7..7786f0af6 100644 --- a/openless-all/app/linux-egui/src/runtime.rs +++ b/openless-all/app/linux-egui/src/runtime.rs @@ -245,6 +245,7 @@ mod tests { )), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services, local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/linux-egui/tests/host_contract.rs b/openless-all/app/linux-egui/tests/host_contract.rs index e05fe9d1e..8bb4b7df0 100644 --- a/openless-all/app/linux-egui/tests/host_contract.rs +++ b/openless-all/app/linux-egui/tests/host_contract.rs @@ -66,6 +66,7 @@ async fn linux_host_exposes_snapshot_and_non_blocking_event_subscription() { dictation_engine: Arc::new(FixtureDictationEngine::successful("raw", "polished")), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, @@ -168,6 +169,7 @@ async fn forwarded_launch_intents_use_core_state_and_semantic_host_actions() { dictation_engine: Arc::new(engine.clone()), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: BackendServices::unsupported(), local_asr_runtime: None, marketplace_config: None, diff --git a/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs b/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs index 148966681..222b9852c 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/core_contract.rs @@ -28,6 +28,7 @@ async fn backend_tests_can_use_the_framework_independent_core_contract() { dictation_engine: Arc::new(FixtureDictationEngine::successful("raw", "polished")), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: openless_core::BackendServices::unsupported(), local_asr_runtime: None, selection_runtime: None, diff --git a/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs b/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs index ee91c9cf8..2d8d8ed5f 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/voice_edit_host.rs @@ -237,6 +237,7 @@ fn fixture_with_transcript(transcript: &str) -> (tempfile::TempDir, Arc, dictation_engine: engine.clone(), task_spawner: Arc::new(TokioTaskSpawner), credential_store: Arc::new(InMemoryCredentialStore::default()), + voice_activation_runtime: None, services: openless_core::BackendServices::unsupported(), local_asr_runtime: None, selection_runtime: None, diff --git a/openless-all/app/src-tauri/src/android/native_bridge.rs b/openless-all/app/src-tauri/src/android/native_bridge.rs index c881f66e0..9248c41aa 100644 --- a/openless-all/app/src-tauri/src/android/native_bridge.rs +++ b/openless-all/app/src-tauri/src/android/native_bridge.rs @@ -1461,6 +1461,7 @@ mod tests { credential_store: Arc::new(InMemoryCredentialStore::default()), services: BackendServices::unsupported(), local_asr_runtime: None, + voice_activation_runtime: None, selection_runtime: None, selection_polisher: None, qa_runtime: None, diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs index 3d0004956..3d85ba37e 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_provider.rs @@ -24,7 +24,7 @@ use parking_lot::Mutex; use crate::asr::RawTranscript; -use super::sherpa_runtime::{SherpaOnlineSession, SherpaOnnxRuntime}; +use super::sherpa_runtime::{SherpaOnlineSession, SherpaOnnxRuntime, SherpaTokenEmission}; pub struct SherpaOnnxAsr { runtime: Arc, @@ -54,7 +54,7 @@ enum OnlineWorkerMessage { Cancel, } -pub type SherpaTokenHandler = Arc; +pub type SherpaTokenHandler = Arc; impl SherpaOnnxAsr { pub fn new( @@ -281,12 +281,12 @@ impl OnlineWorker { let join_handle = std::thread::Builder::new() .name(format!("openless-sherpa-online-{alias}")) .spawn(move || { - let emit = |piece: &str| { - if piece.is_empty() || worker_cancelled.load(Ordering::SeqCst) { + let emit = |emission: &SherpaTokenEmission| { + if emission.text.is_empty() || worker_cancelled.load(Ordering::SeqCst) { return; } if let Some(handler) = token_handler.as_ref() { - handler(piece.to_string()); + handler(emission.clone()); } }; let result = loop { diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs index 7de3b00f7..747c09cbc 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs @@ -26,6 +26,13 @@ use crate::asr::local::sherpa::{ }; const QWEN3_ASR_CHUNK_LIMIT_MS: u64 = 30_000; +#[derive(Clone, Debug)] +pub struct SherpaTokenEmission { + pub text: String, + pub is_delta: bool, + pub is_new_segment: bool, +} + const QWEN3_ASR_MAX_NEW_TOKENS: i32 = 256; #[cfg(target_os = "windows")] @@ -649,6 +656,7 @@ pub struct SherpaOnlineSession { committed_text: String, last_partial_text: String, last_emitted_text: String, + pending_new_segment: bool, } impl SherpaOnlineSession { @@ -658,7 +666,7 @@ impl SherpaOnlineSession { pub fn accept_pcm_chunk(&mut self, pcm: &[u8], on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { if pcm.is_empty() { return Ok(()); @@ -669,7 +677,7 @@ impl SherpaOnlineSession { pub fn finish(&mut self, on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.finish_inner(on_token) } @@ -677,7 +685,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn accept_samples(&mut self, samples: &[f32], on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.stream.accept_waveform(16_000, samples); self.drain_ready(on_token); @@ -687,7 +695,7 @@ impl SherpaOnlineSession { #[cfg(not(target_os = "windows"))] fn accept_samples(&mut self, _samples: &[f32], _on_token: &F) -> Result<()> where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { Ok(()) } @@ -695,7 +703,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn finish_inner(&mut self, on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { self.stream.input_finished(); self.drain_ready(on_token); @@ -706,7 +714,7 @@ impl SherpaOnlineSession { #[cfg(not(target_os = "windows"))] fn finish_inner(&mut self, _on_token: &F) -> Result where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { Ok(self.final_text()) } @@ -714,7 +722,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn drain_ready(&mut self, on_token: &F) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { while self.recognizer.is_ready(&self.stream) { self.recognizer.decode(&self.stream); @@ -725,7 +733,7 @@ impl SherpaOnlineSession { #[cfg(target_os = "windows")] fn capture_result(&mut self, on_token: &F, allow_endpoint_reset: bool) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { let Some(result) = self.recognizer.get_result(&self.stream) else { return; @@ -740,24 +748,35 @@ impl SherpaOnlineSession { append_segment(&mut self.committed_text, &text); self.last_partial_text.clear(); self.last_emitted_text.clear(); + self.pending_new_segment = true; self.recognizer.reset(&self.stream); } } fn emit_delta(&mut self, text: &str, on_token: &F) where - F: Fn(&str), + F: Fn(&SherpaTokenEmission), { if text == self.last_emitted_text { return; } + let is_new_segment = self.pending_new_segment; if let Some(delta) = text.strip_prefix(&self.last_emitted_text) { if !delta.is_empty() { - on_token(delta); + on_token(&SherpaTokenEmission { + text: delta.to_string(), + is_delta: true, + is_new_segment, + }); } } else { - on_token(text); + on_token(&SherpaTokenEmission { + text: text.to_string(), + is_delta: false, + is_new_segment, + }); } + self.pending_new_segment = false; self.last_emitted_text = text.to_string(); } @@ -791,6 +810,7 @@ fn create_online_session_from_loaded(loaded: LoadedOnlineModel) -> Result Result Result<(), String> { let warm_hub = crate::recorder::warm_capture_hub(); + if !warm_hub.begin_preview() { + return Err("dictation is using the microphone".into()); + } + if let Some(backend) = app.try_state::>() { + if let Err(error) = backend + .stop_voice_activation(openless_core::VoiceActivationStopReason::Disabled) + .await + { + warm_hub.end_preview(); + return Err(error.to_string()); + } + } let warm_hub_for_error = Arc::clone(&warm_hub); let app_for_error = app.clone(); + let backend_for_resume = app + .try_state::>() + .map(|state| Arc::clone(&*state)); let result = match tauri::async_runtime::spawn_blocking(move || { let state = app.state::(); let _operation = state.operation.lock(); - if !warm_hub.begin_preview() { - return Err("dictation is using the microphone".into()); - } if let Some(existing) = state.recorder.lock().take() { existing.stop(); } @@ -176,9 +188,14 @@ pub async fn start_microphone_level_monitor( if state.recorder.lock().is_none() { warm_hub_for_error.end_preview(); } - return Err(format!("start microphone monitor task failed: {error}")); + Err(format!("start microphone monitor task failed: {error}")) } }; + if result.is_err() { + if let Some(backend) = backend_for_resume { + let _ = backend.configure_voice_activation().await; + } + } result } @@ -196,6 +213,10 @@ pub async fn stop_microphone_level_monitor(app: AppHandle) { #[cfg(not(mobile))] let app_for_error = app.clone(); #[cfg(not(mobile))] + let backend_for_resume = app + .try_state::>() + .map(|state| Arc::clone(&*state)); + #[cfg(not(mobile))] let result = tauri::async_runtime::spawn_blocking(move || { let state = app.state::(); let _operation = state.operation.lock(); @@ -216,6 +237,10 @@ pub async fn stop_microphone_level_monitor(app: AppHandle) { } #[cfg(not(mobile))] let _ = result; + #[cfg(not(mobile))] + if let Some(backend) = backend_for_resume { + let _ = backend.configure_voice_activation().await; + } } /// Copy the current session's openless.log to a user-chosen location (the frontend obtains diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index 76c8efe6e..6133a17fc 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -107,6 +107,7 @@ pub use selection_polish_preview::*; #[cfg(all(not(mobile), target_os = "windows"))] pub use selection_voice::*; pub use settings::*; +pub(crate) use settings::sync_low_latency_capture_preferences; #[cfg(not(mobile))] // Sherpa commands are registered on Windows only; other desktop targets keep // an empty re-export. diff --git a/openless-all/app/src-tauri/src/commands/settings.rs b/openless-all/app/src-tauri/src/commands/settings.rs index d386175fb..47c35051d 100644 --- a/openless-all/app/src-tauri/src/commands/settings.rs +++ b/openless-all/app/src-tauri/src/commands/settings.rs @@ -219,16 +219,48 @@ pub(crate) fn persist_strict_settings( } #[cfg(target_os = "windows")] -fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { - crate::recorder::warm_capture_hub().configure( - prefs.low_latency_dictation_enabled, +pub(crate) fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { + let warm_hub = crate::recorder::warm_capture_hub(); + warm_hub.configure( + prefs.low_latency_dictation_enabled || warm_hub.activation_is_active(), (!prefs.microphone_device_name.trim().is_empty()) .then(|| prefs.microphone_device_name.clone()), ); } #[cfg(not(target_os = "windows"))] -fn sync_low_latency_capture_preferences(_prefs: &UserPreferences) {} +pub(crate) fn sync_low_latency_capture_preferences(_prefs: &UserPreferences) {} + +fn voice_activation_preferences_changed(previous: &UserPreferences, next: &UserPreferences) -> bool { + previous.voice_activation_enabled != next.voice_activation_enabled + || previous.voice_activation_wake_enabled != next.voice_activation_wake_enabled + || previous.voice_activation_finish_enabled != next.voice_activation_finish_enabled + || previous.voice_activation_wake_phrase != next.voice_activation_wake_phrase + || previous.voice_activation_finish_phrase != next.voice_activation_finish_phrase + || previous.voice_activation_timeout_enabled != next.voice_activation_timeout_enabled + || previous.voice_activation_timeout_seconds != next.voice_activation_timeout_seconds + || previous.microphone_device_name != next.microphone_device_name +} + +fn validate_voice_activation_preferences( + prefs: &UserPreferences, +) -> Result<(), String> { + openless_core::VoiceActivationConfig { + enabled: prefs.voice_activation_enabled, + wake_enabled: prefs.voice_activation_wake_enabled, + finish_enabled: prefs.voice_activation_finish_enabled, + wake_phrase: prefs.voice_activation_wake_phrase.clone(), + finish_phrase: prefs.voice_activation_finish_phrase.clone(), + timeout_enabled: prefs.voice_activation_timeout_enabled, + timeout_seconds: prefs.voice_activation_timeout_seconds, + microphone_device_name: (!prefs.microphone_device_name.trim().is_empty()) + .then(|| prefs.microphone_device_name.clone()), + activation_generation: 0, + } + .normalized() + .map(|_| ()) + .map_err(|error| format!("invalid voice activation settings: {error}")) +} async fn invalidate_llm_tests_if_thinking_changed( coord: &Coordinator, @@ -266,6 +298,7 @@ pub async fn set_settings( .map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); + validate_voice_activation_preferences(&prefs)?; invalidate_llm_tests_if_thinking_changed(&coord, &remote_prev, &prefs).await?; // Persist changed fields and notify every WebView through the shared settings path. if let Some(edits) = edits { @@ -275,6 +308,11 @@ pub async fn set_settings( } let prefs = coord.backend().get_preferences(); sync_low_latency_capture_preferences(&prefs); + if voice_activation_preferences_changed(&remote_prev, &prefs) { + if let Err(error) = coord.backend().configure_voice_activation().await { + log::warn!("[voice-activation] settings reconfiguration failed: {error}"); + } + } // Sync the capsule-style atom on save: the next recording's entrance frame carries the // new style instead of depending on emit_capsule's ~30Hz main-thread closure sync (a // congested Windows main thread delays the closure → the whole session shows the old @@ -342,6 +380,7 @@ pub async fn set_settings( .map_err(|e| e.to_string())?; sync_style_pack_preferences(&mut prefs, &packs); prefs.android_overlay_trigger = prefs.android_overlay_trigger.normalized(); + validate_voice_activation_preferences(&prefs)?; invalidate_llm_tests_if_thinking_changed(&coord, &previous, &prefs).await?; if let Some(edits) = edits { persist_setting_fields(&coord, &edits)?; @@ -350,8 +389,12 @@ pub async fn set_settings( } let prefs = coord.backend().get_preferences(); sync_low_latency_capture_preferences(&prefs); - // Sync the capsule-style atom on save (same source as the Android notification capsule - // payload; see emit_capsule). + if voice_activation_preferences_changed(&previous, &prefs) { + if let Err(error) = coord.backend().configure_voice_activation().await { + log::warn!("[voice-activation] settings reconfiguration failed: {error}"); + } + } + // Sync the capsule-style atom on save (same source as the Android notification capsule payload; see emit_capsule). coord.sync_capsule_style_from_preferences(); // Rebuild the client connection pool immediately when the system-proxy switch changes (issue #869). if previous.use_system_proxy != prefs.use_system_proxy { diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index 49da035b6..f2c072e82 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -15,12 +15,16 @@ use openless_core::{ RecordingArchive, RecordingProgressSink, SessionId, TextInserter as CoreTextInserter, TextInsertionSession, TextPolisher, TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, + VoiceActivationCommand, VoiceActivationConfig, VoiceActivationError, VoiceActivationPhase, + VoiceActivationSignal, VoiceActivationStopReason, VoiceCommandRecognizer, + VoiceDictationHandoff, VoiceDictationHandoffRequest, }; use parking_lot::Mutex; use tauri::{AppHandle, Emitter, Manager}; use crate::recorder::{ - warm_capture_hub, AudioConsumer as LegacyAudioConsumer, Recorder, RecorderError, + warm_capture_hub, AudioConsumer as LegacyAudioConsumer, AudioConsumerFanout, Recorder, + RecorderError, StagedRecorder, WarmActiveCapture, WarmCaptureHub, WarmStagedRecorder, STAGED_CAPTURE_MAX_BYTES, }; @@ -125,6 +129,15 @@ pub(crate) fn backend_dependencies( ); let warm_hub = warm_capture_hub(); #[cfg(target_os = "windows")] + let voice_activation_runtime = Arc::new(TauriVoiceActivationRuntime::new( + Arc::clone(&app), + Arc::clone(&backend), + Arc::clone(&native_asr_dependencies.sherpa), + model_store.clone(), + Arc::clone(&warm_hub), + Arc::clone(&preferences), + )); + #[cfg(target_os = "windows")] { let prefs = preferences.get(); warm_hub.configure( @@ -240,9 +253,14 @@ pub(crate) fn backend_dependencies( .expect("built-in Omni provider ids are non-empty"); } let mut dependencies = openless_core::BackendDependencies::unsupported(); - if let Some(model_store) = model_store { + if let Some(model_store) = model_store.clone() { dependencies.services.configure_model_store(model_store); } + #[cfg(target_os = "windows")] + { + dependencies.voice_activation_runtime = + Some(voice_activation_runtime as Arc); + } dependencies .services .configure_auxiliary_runtime(auxiliary_polisher, auxiliary_transcription); @@ -1569,6 +1587,543 @@ impl openless_core::PlatformApi for TauriPlatformApi { } } +#[cfg(target_os = "windows")] +const VOICE_ACTIVATION_MODEL_ALIAS: &str = "zipformer-bilingual-zh-en-streaming"; + +#[cfg(target_os = "windows")] +const MAX_VOICE_RECOGNIZER_CONTEXT_CHARS: usize = 256; + +#[cfg(target_os = "windows")] +fn map_voice_activation_recorder_error(error: RecorderError) -> VoiceActivationError { + match error { + RecorderError::PermissionDenied => VoiceActivationError::PermissionDenied, + RecorderError::NoInputDevice => { + VoiceActivationError::Runtime("no microphone input device detected".into()) + } + RecorderError::EngineFailed(message) + if message.to_ascii_lowercase().contains("busy") => VoiceActivationError::MicBusy, + RecorderError::EngineFailed(message) => VoiceActivationError::Runtime(message), + } +} + +#[cfg(target_os = "windows")] +struct VoiceActivationEmitter { + backend: BackendSlot, + sink: Arc, + generation: u64, + command: VoiceActivationCommand, + session_id: Arc>>, + binding_gate: Mutex<()>, + state: Mutex, +} + +#[cfg(target_os = "windows")] +struct VoiceActivationEmitterState { + recognizer: VoiceCommandRecognizer, + text: String, + segment_text: String, + pending_finish: bool, + pending_finish_match: bool, +} + +#[cfg(target_os = "windows")] +impl VoiceActivationEmitter { + fn new( + backend: BackendSlot, + sink: Arc, + generation: u64, + command: VoiceActivationCommand, + config: &VoiceActivationConfig, + session_id: Arc>>, + ) -> Result { + let recognizer = VoiceCommandRecognizer::new(config) + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + Ok(Self { + backend, + sink, + generation, + command, + session_id, + binding_gate: Mutex::new(()), + state: Mutex::new(VoiceActivationEmitterState { + recognizer, + text: String::new(), + segment_text: String::new(), + pending_finish: false, + pending_finish_match: false, + }), + }) + } + + fn observe(&self, piece: String, is_delta: bool, is_new_segment: bool) { + if piece.trim().is_empty() { + return; + } + let _binding = self.binding_gate.lock(); + let bound_session_id = if self.command == VoiceActivationCommand::Finish { + *self.session_id.lock() + } else { + None + }; + let signal = { + let mut state = self.state.lock(); + if is_new_segment { + state.text.push_str(&piece); + state.segment_text = piece; + } else if is_delta { + state.text.push_str(&piece); + state.segment_text.push_str(&piece); + } else { + let prefix_len = state + .text + .chars() + .count() + .saturating_sub(state.segment_text.chars().count()); + state.text = state.text.chars().take(prefix_len).collect(); + state.text.push_str(&piece); + state.segment_text = piece; + } + if state.text.chars().count() > MAX_VOICE_RECOGNIZER_CONTEXT_CHARS { + state.text = state + .text + .chars() + .rev() + .take(MAX_VOICE_RECOGNIZER_CONTEXT_CHARS) + .collect::() + .chars() + .rev() + .collect(); + state.segment_text = state.text.clone(); + state.recognizer.reset(); + } + let text = state.text.clone(); + let matched = state.recognizer.observe(&text, self.command); + if self.command == VoiceActivationCommand::Finish && bound_session_id.is_none() { + state.pending_finish = true; + state.pending_finish_match |= matched.is_some(); + return; + } + let Some(_) = matched else { + state.pending_finish = false; + return; + }; + state.pending_finish = false; + state.pending_finish_match = false; + match self.command { + VoiceActivationCommand::Wake => { + VoiceActivationSignal::WakeDetected { + activation_generation: self.generation, + } + } + VoiceActivationCommand::Finish => VoiceActivationSignal::FinishDetected { + activation_generation: self.generation, + session_id: bound_session_id.expect("finish session was checked above"), + }, + } + }; + let _ = self.sink.publish(signal.clone()); + if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + backend.handle_voice_activation_signal(signal); + } + } + + fn bind_session(&self, session_id: SessionId) { + { + let _binding = self.binding_gate.lock(); + *self.session_id.lock() = Some(session_id); + } + self.retry_pending(); + } + + fn retry_pending(&self) { + let (text, pending_match) = { + let _binding = self.binding_gate.lock(); + let mut state = self.state.lock(); + if !state.pending_finish { + return; + } + let pending_match = state.pending_finish_match; + state.pending_finish = false; + state.pending_finish_match = false; + (state.text.clone(), pending_match) + }; + if pending_match { + let Some(session_id) = *self.session_id.lock() else { + return; + }; + let signal = VoiceActivationSignal::FinishDetected { + activation_generation: self.generation, + session_id, + }; + let _ = self.sink.publish(signal.clone()); + if let Some(backend) = self.backend.lock().as_ref().and_then(std::sync::Weak::upgrade) { + backend.handle_voice_activation_signal(signal); + } + } else if !text.is_empty() { + self.observe(text, false, false); + } + } +} + +#[cfg(target_os = "windows")] +struct TauriVoiceActivationRuntime { + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_store: Option>, + warm_hub: Arc, + preferences: Arc, +} + +#[cfg(target_os = "windows")] +impl TauriVoiceActivationRuntime { + fn new( + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_store: Option>, + warm_hub: Arc, + preferences: Arc, + ) -> Self { + Self { + app, + backend, + sherpa, + model_store, + warm_hub, + preferences, + } + } +} + +#[cfg(target_os = "windows")] +impl openless_core::VoiceActivationRuntime for TauriVoiceActivationRuntime { + fn configure( + &self, + config: VoiceActivationConfig, + sink: Arc, + ) -> BoxFuture<'static, Result, VoiceActivationError>> { + let app = Arc::clone(&self.app); + let backend = Arc::clone(&self.backend); + let sherpa = Arc::clone(&self.sherpa); + let model_store = self.model_store.clone(); + let warm_hub = Arc::clone(&self.warm_hub); + let preferences = Arc::clone(&self.preferences); + Box::pin(async move { + let config = config + .normalized() + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + let target = openless_core::LocalAsrTarget::parse( + openless_core::LocalAsrRuntime::SherpaOnnx, + VOICE_ACTIVATION_MODEL_ALIAS, + ) + .map_err(|error| VoiceActivationError::InvalidConfig(error.to_string()))?; + let Some(model_store) = model_store else { + return Err(VoiceActivationError::ModelMissing); + }; + if !model_store + .is_installed(&target) + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))? + { + return Err(VoiceActivationError::ModelMissing); + } + let model_dir = model_store + .runtime_model_dir(&target) + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?; + let session_id = Arc::new(Mutex::new(None)); + let emitter = Arc::new(VoiceActivationEmitter::new( + Arc::clone(&backend), + Arc::clone(&sink), + config.activation_generation, + VoiceActivationCommand::Wake, + &config, + Arc::clone(&session_id), + )?); + let wake_asr = if config.wake_enabled { + let emitter_for_handler = Arc::clone(&emitter); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + emitter_for_handler.observe( + emission.text, + emission.is_delta, + emission.is_new_segment, + ) + }, + ); + Some(Arc::new( + crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + VOICE_ACTIVATION_MODEL_ALIAS.to_string(), + model_dir.clone(), + Some("zh".into()), + Some(token_handler), + ) + .await + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?, + )) + } else { + None + }; + let fallback_warm_capture_enabled = preferences.get().low_latency_dictation_enabled; + warm_hub.configure( + wake_asr.is_some() || fallback_warm_capture_enabled, + config.microphone_device_name.clone(), + ); + if wake_asr.is_some() && !warm_hub.is_cold_capture_active() { + let probe = match warm_hub.arm_without_preroll( + config.activation_generation, + config.microphone_device_name.clone(), + ) { + Ok(probe) => probe, + Err(error) => { + warm_hub.configure( + fallback_warm_capture_enabled, + config.microphone_device_name.clone(), + ); + return Err(map_voice_activation_recorder_error(error)); + } + }; + drop(probe); + } + if let Some(wake_asr) = wake_asr.as_ref() { + warm_hub.set_activation_generation(config.activation_generation); + if !warm_hub.set_activation_consumer_for_generation( + config.activation_generation, + Some(Arc::clone(wake_asr) as Arc), + false, + ) { + return Err(VoiceActivationError::Runtime( + "voice activation generation became stale".into(), + )); + } + let fault_backend = Arc::clone(&backend); + let fault_sink = Arc::clone(&sink); + let fault_generation = config.activation_generation; + warm_hub.set_activation_fault_handler( + fault_generation, + Some(Arc::new(move |error| { + log::warn!("[voice-activation] warm microphone restart failed: {error}"); + let signal = VoiceActivationSignal::RuntimeStateChanged { + activation_generation: fault_generation, + phase: VoiceActivationPhase::Failed, + }; + let _ = fault_sink.publish(signal.clone()); + if let Some(backend) = fault_backend + .lock() + .as_ref() + .and_then(std::sync::Weak::upgrade) + { + backend.handle_voice_activation_signal(signal); + } + })), + ); + } + let handle = TauriVoiceActivationHandle { + app, + backend, + sherpa, + model_dir, + model_alias: VOICE_ACTIVATION_MODEL_ALIAS.to_string(), + warm_hub, + session_id, + sink: Arc::clone(&sink), + preferences, + config: config.clone(), + wake_asr: Arc::new(Mutex::new(wake_asr)), + finish_asr: Arc::new(Mutex::new(None)), + finish_emitter: Arc::new(Mutex::new(None)), + }; + let _ = sink.publish(VoiceActivationSignal::RuntimeStateChanged { + activation_generation: config.activation_generation, + phase: VoiceActivationPhase::ListeningForWake, + }); + Ok(Box::new(handle) as Box) + }) + } +} + +#[cfg(target_os = "windows")] +struct TauriVoiceActivationHandle { + app: AppHandleSlot, + backend: BackendSlot, + sherpa: Arc, + model_dir: PathBuf, + model_alias: String, + warm_hub: Arc, + session_id: Arc>>, + sink: Arc, + preferences: Arc, + config: VoiceActivationConfig, + wake_asr: Arc>>>, + finish_asr: Arc>>>, + finish_emitter: Arc>>>, +} + +#[cfg(target_os = "windows")] +impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { + fn bind_dictation_session(&self, session_id: SessionId) { + if let Some(emitter) = self.finish_emitter.lock().clone() { + emitter.bind_session(session_id); + } else { + *self.session_id.lock() = Some(session_id); + } + } + + fn handoff_to_dictation( + &self, + request: VoiceDictationHandoffRequest, + ) -> BoxFuture<'static, Result> { + let app = Arc::clone(&self.app); + let backend = Arc::clone(&self.backend); + let backend_for_generation = Arc::clone(&backend); + let requested_generation = request.activation_generation; + let sherpa = Arc::clone(&self.sherpa); + let model_dir = self.model_dir.clone(); + let model_alias = self.model_alias.clone(); + let warm_hub = Arc::clone(&self.warm_hub); + let session_id = Arc::clone(&self.session_id); + let sink = Arc::clone(&self.sink); + let config = self.config.clone(); + let wake_asr = Arc::clone(&self.wake_asr); + let finish_asr = Arc::clone(&self.finish_asr); + let finish_emitter_slot = Arc::clone(&self.finish_emitter); + Box::pin(async move { + let generation_is_current = || { + backend_for_generation + .lock() + .as_ref() + .and_then(std::sync::Weak::upgrade) + .is_some_and(|backend| { + backend.voice_activation_generation_is_current(requested_generation) + }) + }; + if !generation_is_current() { + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + let staged = warm_hub + .arm_without_preroll( + config.activation_generation, + config.microphone_device_name.clone(), + ) + .map_err(map_voice_activation_recorder_error)?; + if !generation_is_current() { + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if !warm_hub.set_activation_consumer_for_generation( + requested_generation, + None, + false, + ) { + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if let Some(wake_asr) = wake_asr.lock().take() { + wake_asr.cancel(); + } + if config.finish_enabled { + let emitter = Arc::new(VoiceActivationEmitter::new( + Arc::clone(&backend), + sink, + config.activation_generation, + VoiceActivationCommand::Finish, + &config, + Arc::clone(&session_id), + )?); + let emitter_for_handler = Arc::clone(&emitter); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + emitter_for_handler.observe( + emission.text, + emission.is_delta, + emission.is_new_segment, + ) + }, + ); + let asr = Arc::new( + crate::asr::local::SherpaOnnxAsr::new_for_model( + Arc::clone(&sherpa), + model_alias, + model_dir, + Some("zh".into()), + Some(token_handler), + ) + .await + .map_err(|error| VoiceActivationError::Runtime(error.to_string()))?, + ); + if !generation_is_current() { + asr.cancel(); + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + if !warm_hub.set_activation_consumer_for_generation( + requested_generation, + Some(Arc::clone(&asr) as Arc), + true, + ) { + asr.cancel(); + drop(staged); + return Err(VoiceActivationError::Runtime( + "voice activation generation is stale".into(), + )); + } + *finish_asr.lock() = Some(asr); + *finish_emitter_slot.lock() = Some(emitter); + } + Ok(VoiceDictationHandoff { + capture: Box::new(TauriPendingAudioCapture { + app, + backend, + warm_hub, + staged: Some(TauriStagedCapture::Warm(staged)), + cold_capture_reservation: None, + }), + }) + }) + } + + fn stop( + &self, + _reason: VoiceActivationStopReason, + ) -> BoxFuture<'static, Result<(), VoiceActivationError>> { + let warm_hub = Arc::clone(&self.warm_hub); + let wake_asr = Arc::clone(&self.wake_asr); + let finish_asr = Arc::clone(&self.finish_asr); + let preferences = Arc::clone(&self.preferences); + let activation_generation = self.config.activation_generation; + Box::pin(async move { + let owns_activation = warm_hub.clear_activation_generation_if(activation_generation); + if owns_activation { + warm_hub.clear_activation_fault_handler_if(activation_generation); + warm_hub.set_activation_consumer(None); + } + if let Some(asr) = wake_asr.lock().take() { + asr.cancel(); + } + if let Some(asr) = finish_asr.lock().take() { + asr.cancel(); + } + if owns_activation { + let prefs = preferences.get(); + warm_hub.configure( + prefs.low_latency_dictation_enabled, + (!prefs.microphone_device_name.trim().is_empty()) + .then(|| prefs.microphone_device_name.clone()), + ); + } + Ok(()) + }) + } +} + #[cfg(any(target_os = "windows", target_os = "macos"))] struct TauriNativeTranscriptionEngine { dependencies: TauriNativeAsrDependencies, @@ -1739,17 +2294,31 @@ impl TranscriptionEngine for TauriNativeTranscriptionEngine { }); let token_sink = Arc::clone(&partials); let token_offset = Arc::new(AtomicU64::new(0)); + let segment_start = Arc::new(AtomicU64::new(0)); let handler_offset = Arc::clone(&token_offset); - let token_handler = Arc::new(move |piece: String| { - let offset = - handler_offset.fetch_add(piece.chars().count() as u64, Ordering::AcqRel); - if let Err(error) = token_sink.publish(TextStreamChunk { - text: piece, - offset, - }) { - log::warn!("[core-adapter] publish sherpa partial failed: {error}"); - } - }); + let handler_segment_start = Arc::clone(&segment_start); + let token_handler = Arc::new( + move |emission: crate::asr::local::sherpa_runtime::SherpaTokenEmission| { + let text_len = emission.text.chars().count() as u64; + let offset = if emission.is_new_segment { + let offset = handler_offset.load(Ordering::Acquire); + handler_segment_start.store(offset, Ordering::Release); + handler_offset.fetch_add(text_len, Ordering::AcqRel) + } else if emission.is_delta { + handler_offset.fetch_add(text_len, Ordering::AcqRel) + } else { + let offset = handler_segment_start.load(Ordering::Acquire); + handler_offset.store(offset + text_len, Ordering::Release); + offset + }; + if let Err(error) = token_sink.publish(TextStreamChunk { + text: emission.text, + offset, + }) { + log::warn!("[core-adapter] publish sherpa partial failed: {error}"); + } + }, + ); let target = openless_core::LocalAsrTarget::parse( openless_core::LocalAsrRuntime::SherpaOnnx, model.clone(), @@ -2468,11 +3037,54 @@ impl TauriStagedCapture { } } +fn release_cold_capture_and_resume(warm_hub: &WarmCaptureHub) { + warm_hub.end_cold_capture(); + warm_hub.resume_after_cold_capture(); +} + +struct ColdCaptureReservation { + warm_hub: Arc, + resume_warm_hub: bool, + active: bool, +} + +impl ColdCaptureReservation { + fn new(warm_hub: Arc, resume_warm_hub: bool) -> Self { + Self { + warm_hub, + resume_warm_hub, + active: true, + } + } + + fn release(&mut self) { + if !self.active { + return; + } + self.active = false; + self.warm_hub.end_cold_capture(); + if self.resume_warm_hub { + self.warm_hub.resume_after_cold_capture(); + } + } + + fn disarm(&mut self) { + self.active = false; + } +} + +impl Drop for ColdCaptureReservation { + fn drop(&mut self) { + self.release(); + } +} + enum TauriCommittedCapture { Cold { recorder: Recorder, runtime_errors: std::sync::mpsc::Receiver, archive_active: bool, + reconfigure_voice_activation: bool, }, Warm { capture: WarmActiveCapture, @@ -2496,6 +3108,7 @@ impl TauriStagedCapture { recorder, runtime_errors, archive_active, + reconfigure_voice_activation: false, }) } Self::Warm(staged) => { @@ -2519,25 +3132,32 @@ impl TauriStagedCapture { if message == "warm provisional capture was not armed" ) => { - fallback_hub.begin_cold_capture()?; - fallback_hub.suspend_for_cold_capture(); + let finish_consumer = fallback_hub.activation_consumer_for_capture(); + fallback_hub.begin_cold_capture_and_suspend()?; let cold = match Recorder::start_staged(fallback_device, STAGED_CAPTURE_MAX_BYTES) { Ok(cold) => cold, Err(error) => { - fallback_hub.end_cold_capture(); + release_cold_capture_and_resume(&fallback_hub); return Err(error); } }; let (recorder, runtime_errors, archive_active) = match cold.commit( - fallback_consumer, + finish_consumer + .map(|finish| { + AudioConsumerFanout::new(vec![ + Arc::clone(&fallback_consumer), + finish, + ]) as Arc + }) + .unwrap_or(fallback_consumer), fallback_level_handler, fallback_archive_path, ) { Ok(committed) => committed, Err(error) => { - fallback_hub.end_cold_capture(); + release_cold_capture_and_resume(&fallback_hub); return Err(error); } }; @@ -2545,6 +3165,7 @@ impl TauriStagedCapture { recorder, runtime_errors, archive_active, + reconfigure_voice_activation: true, }) } Err(error) => Err(error), @@ -2566,6 +3187,7 @@ struct TauriPendingAudioCapture { backend: BackendSlot, warm_hub: Arc, staged: Option, + cold_capture_reservation: Option, } struct TauriActiveRecording { @@ -2573,6 +3195,8 @@ struct TauriActiveRecording { warm_capture: Option, cold_capture_hub: Option>, resume_warm_hub: Option>, + backend: BackendSlot, + reconfigure_voice_activation: bool, archive: Option>, _mute: Option, } @@ -2590,6 +3214,7 @@ impl PendingAudioCapture for TauriPendingAudioCapture { backend, warm_hub, staged, + cold_capture_reservation, } = *self; let Some(staged) = staged else { return Box::pin(async { @@ -2599,6 +3224,8 @@ impl PendingAudioCapture for TauriPendingAudioCapture { )) }); }; + let cold_capture = staged.is_cold(); + let has_cold_capture_reservation = cold_capture_reservation.is_some(); Box::pin(async move { let permanent_archive = matches!( context.output_target, @@ -2623,7 +3250,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { #[cfg(not(mobile))] let monitor_app = app.lock().clone(); let warm_hub_for_join = Arc::clone(&warm_hub); + let backend_for_recording = Arc::clone(&backend); let joined = tauri::async_runtime::spawn_blocking(move || { + let mut cold_capture_reservation = cold_capture_reservation; let archive_path = archive_path .map_err(|error| { log::warn!("[recordings] archive unavailable; capture continues: {error}") @@ -2679,7 +3308,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Ok(committed) => committed, Err(error) => { drop(mute); - warm_hub.end_cold_capture(); + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.release(); + } return Err(error); } }; @@ -2690,11 +3321,13 @@ impl PendingAudioCapture for TauriPendingAudioCapture { runtime_errors, archive_active, resume_warm_hub, + reconfigure_voice_activation, ) = match committed { TauriCommittedCapture::Cold { recorder, runtime_errors, archive_active, + reconfigure_voice_activation, } => ( Some(recorder), None, @@ -2702,6 +3335,7 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Some(runtime_errors), archive_active, Some(Arc::clone(&warm_hub)), + reconfigure_voice_activation, ), TauriCommittedCapture::Warm { capture, @@ -2714,21 +3348,21 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Some(runtime_errors), archive_active, None, + false, ), }; if recording_plan.archive_required && !archive_active { - let has_warm_capture = warm_capture.is_some(); if let Some(recorder) = recorder { recorder.stop(); } if let Some(capture) = warm_capture { capture.stop(); } - if let Some(cold_capture_hub) = cold_capture_hub.as_ref() { + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.release(); + } else if let Some(cold_capture_hub) = cold_capture_hub.as_ref() { cold_capture_hub.end_cold_capture(); - } - if !has_warm_capture { - if let Some(warm_hub) = resume_warm_hub { + if let Some(warm_hub) = resume_warm_hub.as_ref() { warm_hub.resume_after_cold_capture(); } } @@ -2740,6 +3374,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { "速记录音文件无法创建，已阻止开始录音以避免丢失内容", )); } + if let Some(reservation) = cold_capture_reservation.as_mut() { + reservation.disarm(); + } if recording_plan.archive_enabled { if let Err(error) = crate::persistence::prune_recordings( recording_plan.retention_days, @@ -2753,6 +3390,8 @@ impl PendingAudioCapture for TauriPendingAudioCapture { warm_capture, cold_capture_hub, resume_warm_hub, + backend: Arc::clone(&backend_for_recording), + reconfigure_voice_activation, archive: archive_path .map(|path| Arc::new(TauriRecordingArchive::new(path, archive_active))), _mute: mute, @@ -2763,7 +3402,9 @@ impl PendingAudioCapture for TauriPendingAudioCapture { Ok(Ok(result)) => result, Ok(Err(error)) => return Err(error), Err(error) => { - warm_hub_for_join.end_cold_capture(); + if cold_capture && !has_cold_capture_reservation { + release_cold_capture_and_resume(&warm_hub_for_join); + } return Err(BackendError::new( BackendErrorCode::Internal, format!("join promoted Tauri recorder task: {error}"), @@ -2801,26 +3442,32 @@ impl PendingAudioCapture for TauriPendingAudioCapture { fn discard(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { let TauriPendingAudioCapture { - warm_hub, staged, .. + staged, + cold_capture_reservation, + .. } = *self; Box::pin(async move { let Some(staged) = staged else { + if let Some(reservation) = cold_capture_reservation { + drop(reservation); + } return Ok(()); }; - let cold_capture = staged.is_cold(); - let result = tauri::async_runtime::spawn_blocking(move || { - staged.discard().map_err(map_recorder_error) + let result = match tauri::async_runtime::spawn_blocking(move || { + let result = staged.discard().map_err(map_recorder_error); + if let Some(mut reservation) = cold_capture_reservation { + reservation.release(); + } + result }) .await - .map_err(|error| { - BackendError::new( + { + Ok(result) => result, + Err(error) => Err(BackendError::new( BackendErrorCode::Internal, format!("join provisional Tauri recorder discard task: {error}"), - ) - })?; - if cold_capture { - warm_hub.end_cold_capture(); - } + )), + }; result }) } @@ -2974,13 +3621,15 @@ impl ActiveRecording for TauriActiveRecording { let warm_capture = self.warm_capture.take(); let cold_capture_hub = self.cold_capture_hub.take(); let resume_warm_hub = self.resume_warm_hub.take(); + let backend = Arc::clone(&self.backend); + let reconfigure_voice_activation = self.reconfigure_voice_activation; if recorder.is_none() && warm_capture.is_none() { return Err(BackendError::new( BackendErrorCode::InvalidState, "Tauri recorder was already stopped", )); } - tauri::async_runtime::spawn_blocking(move || { + let stop_result = tauri::async_runtime::spawn_blocking(move || { if let Some(recorder) = recorder { recorder.stop(); } @@ -3004,7 +3653,21 @@ impl ActiveRecording for TauriActiveRecording { BackendErrorCode::Internal, format!("join Tauri recorder stop task: {error}"), ) - })? + })?; + if reconfigure_voice_activation { + let backend = { + let slot = backend.lock(); + slot.as_ref().and_then(std::sync::Weak::upgrade) + }; + if let Some(backend) = backend { + if let Err(error) = backend.configure_voice_activation().await { + log::warn!( + "[voice-activation] reconfigure after warm fallback release failed: {error}" + ); + } + } + } + stop_result }) } } @@ -3049,24 +3712,28 @@ impl TauriAudioRecorder { backend: Arc::clone(&self.backend), warm_hub: Arc::clone(&self.warm_hub), staged: Some(TauriStagedCapture::Warm(staged)), + cold_capture_reservation: None, })); } log::warn!("[recorder] warm capture unavailable; falling back to press-time staging"); } if !request.low_latency { - self.warm_hub.stop(); + self.warm_hub + .begin_cold_capture_and_suspend() + .map_err(map_recorder_error)?; + } else { + self.warm_hub + .begin_cold_capture() + .map_err(map_recorder_error)?; } let max_buffer_bytes = (request.pre_roll_ms as usize) .saturating_mul(32) .clamp(2, 16_000); - self.warm_hub - .begin_cold_capture() - .map_err(map_recorder_error)?; let staged = match Recorder::start_staged(request.microphone_device_name, max_buffer_bytes) { Ok(staged) => staged, Err(error) => { - self.warm_hub.end_cold_capture(); + release_cold_capture_and_resume(&self.warm_hub); return Err(map_recorder_error(error)); } }; @@ -3075,6 +3742,10 @@ impl TauriAudioRecorder { backend: Arc::clone(&self.backend), warm_hub: Arc::clone(&self.warm_hub), staged: Some(TauriStagedCapture::Cold(staged)), + cold_capture_reservation: Some(ColdCaptureReservation::new( + Arc::clone(&self.warm_hub), + !request.low_latency, + )), })) } } @@ -3161,6 +3832,7 @@ impl AudioRecorder for TauriAudioRecorder { || context.output_target == openless_core::DictationOutputTarget::Undecided); let fault_progress = Arc::clone(&progress); let worker_resume_warm_hub = Arc::clone(&resume_warm_hub); + let backend_for_recording = Arc::clone(&backend); let joined = tauri::async_runtime::spawn_blocking(move || { if prune_recordings_before_capture { if let Err(error) = crate::persistence::prune_recordings( @@ -3217,6 +3889,8 @@ impl AudioRecorder for TauriAudioRecorder { warm_capture: None, cold_capture_hub: Some(Arc::clone(&warm_hub)), resume_warm_hub: Some(worker_resume_warm_hub), + backend: Arc::clone(&backend_for_recording), + reconfigure_voice_activation: true, archive: archive_path .map(|path| Arc::new(TauriRecordingArchive::new(path, archive_active))), _mute: mute, diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index a441b94e6..182212e02 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -1494,13 +1494,21 @@ fn handle_microphone_tray_menu_event(app: &AppHandle, id: &str) { }; let coord = app.state::>(); - if let Err(err) = coord - .backend() - .select_microphone_device(selected.device_name.clone()) - { + let backend = coord.backend(); + if let Err(err) = backend.select_microphone_device(selected.device_name.clone()) { log::warn!("[tray] save microphone preference failed: {err}"); return; } + commands::sync_low_latency_capture_preferences(&backend.get_preferences()); + let selected_device_name = selected.device_name.clone(); + let backend_for_reconfigure = Arc::clone(&backend); + tauri::async_runtime::spawn(async move { + if let Err(err) = backend_for_reconfigure.configure_voice_activation().await { + log::warn!( + "[tray] reconfigure microphone-dependent voice runtime for {selected_device_name:?} failed: {err}" + ); + } + }); commands::sync_tray_microphone_selection(&items, &selected.device_name); } diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index b1c6485fe..68e3eb69b 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -37,6 +37,13 @@ const LEVEL_RMS_GAIN: f32 = 4.0; const WAV_ARCHIVE_QUEUE_CAPACITY: usize = 256; /// The provisional capture cap is 500 ms at the canonical 16 kHz mono i16 format. pub(crate) const STAGED_CAPTURE_MAX_BYTES: usize = TARGET_SAMPLE_RATE as usize; +/// Voice handoff staging keeps up to 30 seconds while the formal dictation session is prepared. +const VOICE_STAGED_CAPTURE_MAX_BYTES: usize = TARGET_SAMPLE_RATE as usize * 30 * 2; +const VOICE_CAPTURE_KEY_PREFIX: u64 = 1 << 63; + +fn voice_capture_key(generation: u64) -> u64 { + generation | VOICE_CAPTURE_KEY_PREFIX +} /// Downstream that receives resampled Int16 PCM bytes (little-endian). pub trait AudioConsumer: Send + Sync { @@ -45,6 +52,24 @@ pub trait AudioConsumer: Send + Sync { fn consume_pcm_chunk(&self, pcm: &[u8]); } +pub(crate) struct AudioConsumerFanout { + consumers: Vec>, +} + +impl AudioConsumerFanout { + pub(crate) fn new(consumers: Vec>) -> Arc { + Arc::new(Self { consumers }) + } +} + +impl AudioConsumer for AudioConsumerFanout { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + for consumer in &self.consumers { + consumer.consume_pcm_chunk(pcm); + } + } +} + /// Compatibility bridge for legacy recorder call sites while cloud ASR /// implementations live in the framework-independent core. impl AudioConsumer for T @@ -293,6 +318,12 @@ impl CaptureSink { struct WarmCaptureSink { state: Mutex, delivery_gate: Mutex<()>, + activation_consumer: Mutex>>, + activation_consumer_during_capture: AtomicBool, + activation_generation: Mutex>, + activation_fault_handler: Mutex< + Option<(u64, Arc)>, + >, max_bytes: usize, } @@ -307,6 +338,8 @@ enum WarmCaptureClaim { Staging { pcm: VecDeque, pre_roll_bytes: usize, + allow_postroll_eviction: bool, + max_bytes: usize, max_level: f32, }, Promoted(ForwardTarget), @@ -321,10 +354,32 @@ impl WarmCaptureSink { claims: HashMap::new(), }), delivery_gate: Mutex::new(()), + activation_consumer: Mutex::new(None), + activation_consumer_during_capture: AtomicBool::new(false), + activation_generation: Mutex::new(None), + activation_fault_handler: Mutex::new(None), max_bytes, }) } + fn activation_consumer_during_capture(&self) -> Option> { + self.activation_consumer_during_capture + .load(Ordering::Acquire) + .then(|| self.activation_consumer.lock().clone()) + .flatten() + } + + fn set_activation_consumer(&self, consumer: Option>) { + self.activation_consumer_during_capture.store(false, Ordering::Release); + *self.activation_consumer.lock() = consumer; + } + + fn set_activation_consumer_during_capture(&self, consumer: Option>) { + self.activation_consumer_during_capture + .store(consumer.is_some(), Ordering::Release); + *self.activation_consumer.lock() = consumer; + } + fn append_bounded(buffer: &mut VecDeque, pcm: &[u8], max_bytes: usize) { let even_len = pcm.len() & !1; buffer.extend(pcm[..even_len].iter().copied()); @@ -337,23 +392,42 @@ impl WarmCaptureSink { fn append_staged( buffer: &mut VecDeque, pre_roll_bytes: &mut usize, + allow_postroll_eviction: bool, pcm: &[u8], max_bytes: usize, ) { let even_len = pcm.len() & !1; let incoming = &pcm[..even_len]; let excess = buffer.len().saturating_add(incoming.len()).saturating_sub(max_bytes); - let evictable = excess.min(*pre_roll_bytes); - for _ in 0..(evictable / 2) { + let evictable_preroll = excess.min(*pre_roll_bytes); + for _ in 0..(evictable_preroll / 2) { + let _ = buffer.pop_front(); + let _ = buffer.pop_front(); + } + *pre_roll_bytes -= evictable_preroll; + let evictable_postroll = if allow_postroll_eviction { + excess.saturating_sub(evictable_preroll) + } else { + 0 + }; + for _ in 0..(evictable_postroll / 2) { let _ = buffer.pop_front(); let _ = buffer.pop_front(); } - *pre_roll_bytes -= evictable; let available = max_bytes.saturating_sub(buffer.len()) & !1; buffer.extend(incoming[..incoming.len().min(available)].iter().copied()); } - fn arm(&self, press_id: u64) -> Result<(), RecorderError> { + fn arm(&self, press_id: u64, include_preroll: bool) -> Result<(), RecorderError> { + self.arm_with_limit(press_id, include_preroll, self.max_bytes) + } + + fn arm_with_limit( + &self, + press_id: u64, + include_preroll: bool, + max_bytes: usize, + ) -> Result<(), RecorderError> { let _delivery = self.delivery_gate.lock(); let mut state = self.state.lock(); let WarmCaptureSinkState::Ring { pcm, claims } = &mut *state; @@ -362,11 +436,15 @@ impl WarmCaptureSink { "warm capture generation is already armed".into(), )); } + let max_bytes = max_bytes.max(2) & !1; + let staged = include_preroll.then(|| pcm.clone()).unwrap_or_default(); claims.insert( press_id, WarmCaptureClaim::Staging { - pcm: pcm.clone(), - pre_roll_bytes: pcm.len(), + pre_roll_bytes: staged.len(), + allow_postroll_eviction: !include_preroll, + max_bytes, + pcm: staged, max_level: 0.0, }, ); @@ -416,8 +494,18 @@ impl WarmCaptureSink { Some(WarmCaptureClaim::Staging { .. }) | None => None, } }; + let activation = self + .activation_consumer_during_capture + .load(Ordering::Acquire) + .then(|| self.activation_consumer.lock().clone()) + .flatten(); if let Some(target) = target { - CaptureSink::deliver(&target, &prefix); + if !prefix.is_empty() { + CaptureSink::deliver(&target, &prefix); + if let Some(activation) = activation { + activation.consume_pcm_chunk(&prefix); + } + } (target.level_handler)(staged_level); } Ok(()) @@ -498,30 +586,55 @@ impl AudioConsumer for WarmCaptureSink { return; } let _delivery = self.delivery_gate.lock(); - let mut targets = Vec::new(); - { + let (targets, has_promoted_claim, has_staging_claim) = { let mut state = self.state.lock(); let WarmCaptureSinkState::Ring { pcm: ring, claims, } = &mut *state; Self::append_bounded(ring, pcm, self.max_bytes); + let mut targets = Vec::new(); + let mut has_promoted_claim = false; + let mut has_staging_claim = false; for claim in claims.values_mut() { match claim { WarmCaptureClaim::Staging { pcm: staged, pre_roll_bytes, + allow_postroll_eviction, + max_bytes, .. } => { - Self::append_staged(staged, pre_roll_bytes, pcm, self.max_bytes); + has_staging_claim = true; + Self::append_staged( + staged, + pre_roll_bytes, + *allow_postroll_eviction, + pcm, + *max_bytes, + ); + } + WarmCaptureClaim::Promoted(target) => { + has_promoted_claim = true; + targets.push(target.clone()); } - WarmCaptureClaim::Promoted(target) => targets.push(target.clone()), } } - } + (targets, has_promoted_claim, has_staging_claim) + }; + let activation = ((!has_promoted_claim && !has_staging_claim) + || (has_promoted_claim + && self + .activation_consumer_during_capture + .load(Ordering::Acquire))) + .then(|| self.activation_consumer.lock().clone()) + .flatten(); for target in targets { CaptureSink::deliver(&target, pcm); } + if let Some(activation) = activation { + activation.consume_pcm_chunk(pcm); + } } } @@ -679,6 +792,77 @@ impl WarmCaptureHub { } } + pub(crate) fn activation_consumer_for_capture(&self) -> Option> { + self.sink.activation_consumer_during_capture() + } + + pub(crate) fn set_activation_consumer(&self, consumer: Option>) { + self.sink.set_activation_consumer(consumer); + } + + pub(crate) fn set_activation_consumer_during_capture( + &self, + consumer: Option>, + ) { + self.sink.set_activation_consumer_during_capture(consumer); + } + + pub(crate) fn set_activation_consumer_for_generation( + &self, + generation: u64, + consumer: Option>, + during_capture: bool, + ) -> bool { + let active_generation = self.sink.activation_generation.lock(); + if *active_generation != Some(generation) { + return false; + } + if during_capture { + self.sink.set_activation_consumer_during_capture(consumer); + } else { + self.sink.set_activation_consumer(consumer); + } + true + } + + pub(crate) fn set_activation_generation(&self, generation: u64) { + *self.sink.activation_generation.lock() = Some(generation); + } + + pub(crate) fn clear_activation_generation_if(&self, generation: u64) -> bool { + let mut active_generation = self.sink.activation_generation.lock(); + if *active_generation == Some(generation) { + *active_generation = None; + true + } else { + false + } + } + + pub(crate) fn activation_is_active(&self) -> bool { + self.sink.activation_generation.lock().is_some() + } + + pub(crate) fn is_cold_capture_active(&self) -> bool { + self.cold_capture_active.load(Ordering::Acquire) + } + + pub(crate) fn set_activation_fault_handler( + &self, + generation: u64, + handler: Option>, + ) { + *self.sink.activation_fault_handler.lock() = + handler.map(|handler| (generation, handler)); + } + + pub(crate) fn clear_activation_fault_handler_if(&self, generation: u64) { + let mut handler = self.sink.activation_fault_handler.lock(); + if handler.as_ref().is_some_and(|(current, _)| *current == generation) { + *handler = None; + } + } + pub(crate) fn configure(&self, enabled: bool, device_name: Option) { let _lifecycle = self.lifecycle_gate.lock(); self.configure_locked(enabled, device_name); @@ -811,6 +995,20 @@ impl WarmCaptureHub { if let Some(recorder) = recorder { recorder.stop(); } + let restart_error = if self.enabled.load(Ordering::Acquire) + && !self.preview_active.load(Ordering::Acquire) + && !self.cold_capture_active.load(Ordering::Acquire) + { + let device_name = self.device_name.lock().clone(); + self.ensure_started_locked(device_name).err() + } else { + None + }; + if let Some(error) = restart_error { + if let Some((_, handler)) = self.sink.activation_fault_handler.lock().clone() { + handler(error); + } + } } fn promote( @@ -849,20 +1047,59 @@ impl WarmCaptureHub { self: &Arc, press_id: u64, device_name: Option, + ) -> Result { + self.arm_internal(press_id, device_name, true, STAGED_CAPTURE_MAX_BYTES) + } + + pub(crate) fn arm_without_preroll( + self: &Arc, + press_id: u64, + device_name: Option, + ) -> Result { + self.arm_internal( + voice_capture_key(press_id), + device_name, + false, + VOICE_STAGED_CAPTURE_MAX_BYTES, + ) + } + + fn arm_internal( + self: &Arc, + press_id: u64, + device_name: Option, + include_preroll: bool, + staged_max_bytes: usize, ) -> Result { if !self.enabled.load(Ordering::Acquire) { return Err(RecorderError::EngineFailed( "warm capture is disabled".into(), )); } + if self.cold_capture_active.load(Ordering::Acquire) { + return Err(RecorderError::EngineFailed( + "warm capture is unavailable during cold capture".into(), + )); + } loop { let lifecycle = self.lifecycle_gate.lock(); + if self.cold_capture_active.load(Ordering::Acquire) { + return Err(RecorderError::EngineFailed( + "warm capture is unavailable during cold capture".into(), + )); + } if *self.device_name.lock() != device_name { + if self.sink.has_promoted_claims() { + return Err(RecorderError::EngineFailed( + "warm capture device change is deferred while capture is active".into(), + )); + } drop(lifecycle); self.configure(true, device_name.clone()); continue; } - self.sink.arm(press_id)?; + self.sink + .arm_with_limit(press_id, include_preroll, staged_max_bytes)?; if let Err(error) = self.ensure_started_locked(device_name.clone()) { self.sink.discard(press_id); return Err(error); @@ -896,6 +1133,22 @@ impl WarmCaptureHub { pub(crate) fn begin_cold_capture(&self) -> Result<(), RecorderError> { let _lifecycle = self.lifecycle_gate.lock(); + self.begin_cold_capture_locked() + } + + /// Atomically reserve the microphone for a cold recorder and suspend the warm stream. + /// + /// The reservation is acquired before the stream is stopped while holding the same lifecycle + /// gate, so a concurrent warm arm cannot slip into the interval between admission and + /// suspension. + pub(crate) fn begin_cold_capture_and_suspend(&self) -> Result<(), RecorderError> { + let _lifecycle = self.lifecycle_gate.lock(); + self.begin_cold_capture_locked()?; + self.suspend_for_cold_capture_locked(); + Ok(()) + } + + fn begin_cold_capture_locked(&self) -> Result<(), RecorderError> { if self.cold_capture_active.swap(true, Ordering::AcqRel) { return Err(RecorderError::EngineFailed( "another microphone capture is already active".into(), @@ -977,6 +1230,10 @@ impl WarmCaptureHub { /// preference enabled so the stream can be rebuilt when that recorder stops. pub(crate) fn suspend_for_cold_capture(&self) { let _lifecycle = self.lifecycle_gate.lock(); + self.suspend_for_cold_capture_locked(); + } + + fn suspend_for_cold_capture_locked(&self) { self.stream_generation.fetch_add(1, Ordering::AcqRel); let recorder = self.recorder.lock().take(); let archives = self.sink.clear(); @@ -2074,6 +2331,28 @@ mod tests { .collect() } + #[test] + fn warm_promotion_replays_staged_prefix_to_finish_consumer() { + let sink = WarmCaptureSink::new(STAGED_CAPTURE_MAX_BYTES); + let formal = Arc::new(RecordingConsumer::default()); + let finish = Arc::new(RecordingConsumer::default()); + sink.arm(7, false).unwrap(); + sink.set_activation_consumer_during_capture(Some( + Arc::clone(&finish) as Arc, + )); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + + sink.promote( + 7, + Arc::clone(&formal) as Arc, + Arc::new(|_| {}), + None, + ) + .unwrap(); + + assert_eq!(formal.chunks.lock().unwrap().as_slice(), &[vec![1, 2, 3, 4]]); + assert_eq!(finish.chunks.lock().unwrap().as_slice(), &[vec![1, 2, 3, 4]]); + } #[test] fn native_alternatives_preserve_channels_format_and_advertised_rates() { let range = cpal::SupportedStreamConfigRange::new( @@ -2320,7 +2599,7 @@ mod tests { fn warm_sink_captures_bounded_preroll_and_post_press_audio_in_order() { let sink = WarmCaptureSink::new(8); sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); - sink.arm(17).unwrap(); + sink.arm(17, true).unwrap(); sink.consume_pcm_chunk(&[9, 10]); sink.publish_level(0.75); @@ -2345,10 +2624,34 @@ mod tests { } #[test] - fn warm_sink_preserves_earliest_post_press_audio_after_preroll_eviction() { + fn warm_sink_activation_consumer_receives_standby_pcm_without_claiming_archive() { + let sink = WarmCaptureSink::new(16); + let consumer = Arc::new(RecordingConsumer::default()); + sink.set_activation_consumer(Some(consumer.clone() as Arc)); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + sink.set_activation_consumer(None); + sink.consume_pcm_chunk(&[5, 6]); + + assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![1, 2, 3, 4]]); + assert!(matches!(&*sink.state.lock(), WarmCaptureSinkState::Ring { .. })); + } + + #[test] + fn warm_sink_activation_arm_excludes_wake_preroll() { + let sink = WarmCaptureSink::new(16); + sink.consume_pcm_chunk(&[1, 2, 3, 4]); + sink.arm(19, false).unwrap(); + sink.consume_pcm_chunk(&[5, 6]); + let consumer = Arc::new(RecordingConsumer::default()); + sink.promote(19, consumer.clone(), Arc::new(|_| {}), None) + .unwrap(); + assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![5, 6]]); + } + #[test] + fn warm_sink_staging_is_bounded_to_the_capture_capacity() { let sink = WarmCaptureSink::new(8); sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); - sink.arm(17).unwrap(); + sink.arm(17, true).unwrap(); sink.consume_pcm_chunk(&[9, 10, 11, 12, 13, 14, 15, 16]); sink.consume_pcm_chunk(&[17, 18, 19, 20, 21, 22]); @@ -2362,12 +2665,30 @@ mod tests { ); } + #[test] + fn warm_sink_without_preroll_keeps_latest_post_wake_audio() { + let sink = WarmCaptureSink::new(8); + sink.arm(17, false).unwrap(); + sink.consume_pcm_chunk(&[1, 2, 3, 4, 5, 6, 7, 8]); + sink.consume_pcm_chunk(&[9, 10, 11, 12, 13, 14, 15, 16]); + sink.consume_pcm_chunk(&[17, 18, 19, 20, 21, 22]); + + let consumer = Arc::new(RecordingConsumer::default()); + sink.promote(17, consumer.clone(), Arc::new(|_| {}), None) + .unwrap(); + + assert_eq!( + consumer.chunks.lock().unwrap().as_slice(), + &[vec![15, 16, 17, 18, 19, 20, 21, 22]] + ); + } + #[test] fn warm_sink_discard_removes_only_matching_generation() { let sink = WarmCaptureSink::new(16); sink.consume_pcm_chunk(&[1, 2]); - sink.arm(17).unwrap(); - sink.arm(18).unwrap(); + sink.arm(17, true).unwrap(); + sink.arm(18, true).unwrap(); sink.discard(17); sink.consume_pcm_chunk(&[3, 4]); @@ -2422,7 +2743,7 @@ mod tests { let hub = WarmCaptureHub::new(); hub.enabled.store(true, Ordering::Release); for press_id in 1..=claim_count { - hub.sink.arm(press_id).unwrap(); + hub.sink.arm(press_id, true).unwrap(); hub.sink .promote( press_id, diff --git a/openless-all/app/src-tauri/src/tauri_events.rs b/openless-all/app/src-tauri/src/tauri_events.rs index 2307d67ec..47cf8b7b6 100644 --- a/openless-all/app/src-tauri/src/tauri_events.rs +++ b/openless-all/app/src-tauri/src/tauri_events.rs @@ -186,6 +186,9 @@ async fn forward_legacy_event( } emit_dictation_state(app, backend, snapshot) } + BackendEventKind::VoiceActivationStateChanged(state) => { + let _ = app.emit("voice-activation:state-changed", state); + } BackendEventKind::TranscriptDelta(_) => {} BackendEventKind::DictationCompleted(result) => { capsule_owners.transcription_notice = None; diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index f551209b9..0706a3c76 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1593,6 +1593,15 @@ export const de: typeof zhCN = { 'Speichert das rohe Mikrofonaudio als WAV, um Erkennungsprobleme zu untersuchen.', audioRecordingMaxEntriesLabel: 'Maximale Rohaufnahmen', audioRecordingMaxEntriesDesc: 'Maximale Anzahl lokal gespeicherter WAV-Dateien. Leer = 200.', + voiceActivationLabel: 'Sprachaktivierung', + voiceActivationDesc: 'Lokal auf eine Aktivierungsphrase warten, um das Diktat zu starten.', + voiceActivationLocalOnly: 'Die Erkennung bleibt auf diesem Gerät; Bereitschaftsaudio wird weder hochgeladen noch archiviert.', + voiceActivationWakeEnabled: 'Aktivierungsphrase', + voiceActivationWakePhrase: 'Aktivierungsphrase', + voiceActivationFinishEnabled: 'Abschlussphrase', + voiceActivationFinishPhrase: 'Abschlussphrase', + voiceActivationTimeoutEnabled: 'Nach Zeitüberschreitung stoppen', + voiceActivationTimeoutSeconds: 'Zeitüberschreitung (Sekunden)', startupGroupTitle: 'Startverhalten', startMinimizedLabel: 'Minimiert starten (ohne Hauptfenster)', startMinimizedDesc: diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 48b2acfbc..efee704ff 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1562,6 +1562,15 @@ export const en: typeof zhCN = { 'Save raw microphone audio as wav for diagnosing recognition issues.', audioRecordingMaxEntriesLabel: 'Max raw recordings', audioRecordingMaxEntriesDesc: 'Max wav files retained locally. Blank = 200.', + voiceActivationLabel: 'Voice activation', + voiceActivationDesc: 'Listen locally for a wake phrase to start dictation.', + voiceActivationLocalOnly: 'Recognition stays on this device; standby audio is not uploaded or archived.', + voiceActivationWakeEnabled: 'Wake phrase', + voiceActivationWakePhrase: 'Wake phrase', + voiceActivationFinishEnabled: 'Finish phrase', + voiceActivationFinishPhrase: 'Finish phrase', + voiceActivationTimeoutEnabled: 'Stop after timeout', + voiceActivationTimeoutSeconds: 'Timeout (seconds)', startupGroupTitle: 'Startup', startMinimizedLabel: 'Start minimized (no main window)', startMinimizedDesc: 'No main window on any launch path — menu bar / tray only.', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 4cef678d6..57a87f3a0 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1586,6 +1586,15 @@ export const es: typeof zhCN = { audioRecordingMaxEntriesLabel: 'Máximo de grabaciones originales', audioRecordingMaxEntriesDesc: 'Máximo de archivos WAV guardados en el dispositivo. Vacío = 200.', + voiceActivationLabel: 'Activación por voz', + voiceActivationDesc: 'Escucha localmente una frase de activación para iniciar el dictado.', + voiceActivationLocalOnly: 'El reconocimiento permanece en este dispositivo; el audio de espera no se sube ni se archiva.', + voiceActivationWakeEnabled: 'Frase de activación', + voiceActivationWakePhrase: 'Frase de activación', + voiceActivationFinishEnabled: 'Frase de finalización', + voiceActivationFinishPhrase: 'Frase de finalización', + voiceActivationTimeoutEnabled: 'Detener al agotarse el tiempo', + voiceActivationTimeoutSeconds: 'Tiempo de espera (segundos)', startupGroupTitle: 'Inicio', startMinimizedLabel: 'Iniciar minimizado (sin ventana principal)', startMinimizedDesc: diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index aeb04a729..b6333f7dd 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1607,6 +1607,15 @@ export const fr: typeof zhCN = { audioRecordingMaxEntriesLabel: 'Nombre maximal d’enregistrements bruts', audioRecordingMaxEntriesDesc: 'Nombre maximal de fichiers WAV conservés localement. Vide = 200.', + voiceActivationLabel: 'Activation vocale', + voiceActivationDesc: 'Écouter localement une phrase d’activation pour démarrer la dictée.', + voiceActivationLocalOnly: 'La reconnaissance reste sur cet appareil ; l’audio en veille n’est ni envoyé ni archivé.', + voiceActivationWakeEnabled: 'Phrase d’activation', + voiceActivationWakePhrase: 'Phrase d’activation', + voiceActivationFinishEnabled: 'Phrase de fin', + voiceActivationFinishPhrase: 'Phrase de fin', + voiceActivationTimeoutEnabled: 'Arrêter après expiration', + voiceActivationTimeoutSeconds: 'Délai (secondes)', startupGroupTitle: 'Démarrage', startMinimizedLabel: 'Démarrer réduit (sans fenêtre principale)', startMinimizedDesc: diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 53f76de25..5dde59d12 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1549,6 +1549,15 @@ export const ja: typeof zhCN = { recordAudioForDebugDesc: '生のマイク音声を wav で保存し、認識問題の診断に利用。', audioRecordingMaxEntriesLabel: '元音声の保持件数', audioRecordingMaxEntriesDesc: 'ローカル保持 wav ファイル上限。空欄 = 200。', + voiceActivationLabel: '音声アクティベーション', + voiceActivationDesc: '起動フレーズをローカルで待ち受けて音声入力を開始します。', + voiceActivationLocalOnly: '認識はこの端末内で行われます。待機中の音声は送信・保存されません。', + voiceActivationWakeEnabled: '起動フレーズ', + voiceActivationWakePhrase: '起動フレーズ', + voiceActivationFinishEnabled: '終了フレーズ', + voiceActivationFinishPhrase: '終了フレーズ', + voiceActivationTimeoutEnabled: 'タイムアウトで停止', + voiceActivationTimeoutSeconds: 'タイムアウト（秒）', startupGroupTitle: '起動', startMinimizedLabel: '起動時にメインウィンドウを表示しない', startMinimizedDesc: diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index c30dc03ba..dd0b08598 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1537,6 +1537,15 @@ export const ko: typeof zhCN = { recordAudioForDebugDesc: '원시 마이크 오디오를 wav 로 저장하여 인식 문제 진단.', audioRecordingMaxEntriesLabel: '원본 녹음 보관 개수', audioRecordingMaxEntriesDesc: '로컬 보관 wav 파일 상한. 빈칸 = 200.', + voiceActivationLabel: '음성 활성화', + voiceActivationDesc: '받아쓰기 시작 구문을 로컬에서 감지합니다.', + voiceActivationLocalOnly: '인식은 이 기기에서만 수행되며 대기 오디오는 업로드하거나 보관하지 않습니다.', + voiceActivationWakeEnabled: '시작 구문', + voiceActivationWakePhrase: '시작 구문', + voiceActivationFinishEnabled: '종료 구문', + voiceActivationFinishPhrase: '종료 구문', + voiceActivationTimeoutEnabled: '시간 초과 후 중지', + voiceActivationTimeoutSeconds: '시간 초과(초)', startupGroupTitle: '시작', startMinimizedLabel: '시작 시 메인 창 숨기기', startMinimizedDesc: diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index d510df98a..e772ee48d 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1418,6 +1418,16 @@ export const zhCN = { lowLatencyDictationLabel: '低延迟录音', lowLatencyDictationDesc: '保持麦克风流就绪，并在内存中保留最多 500 毫秒音频，减少按下热键后第一句话被漏识别的情况。确认录音前不会保存或上传音频；此模式会增加功耗并保持系统麦克风指示灯亮起。', + voiceActivationLabel: '语音唤醒听写', + voiceActivationDesc: + '仅在 Windows 桌面端启用本地语音命令唤醒听写。待机音频只在内存中处理，不会上传或保存；会增加功耗并保持麦克风指示灯亮起。', + voiceActivationLocalOnly: '本地识别：说“开始语音录入”开始，说“完成语音录入”结束。需要已安装本地 Zipformer 模型。', + voiceActivationWakeEnabled: '启用开始命令', + voiceActivationWakePhrase: '开始命令', + voiceActivationFinishEnabled: '启用完成命令', + voiceActivationFinishPhrase: '完成命令', + voiceActivationTimeoutEnabled: '超时自动结束', + voiceActivationTimeoutSeconds: '超时秒数', muteDuringRecordingLabel: '录音时静音', muteDuringRecordingDesc: '录音期间临时静音系统输出，避免扬声器回音。', audioCueLabel: '录音提示音', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index d910a3d87..5367f7b55 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1484,6 +1484,15 @@ export const zhTW: typeof zhCN = { recordAudioForDebugDesc: '儲存原始麥克風音訊為 wav，便於排查識別問題。', audioRecordingMaxEntriesLabel: '原始錄音保留條數', audioRecordingMaxEntriesDesc: '本地保留 wav 檔案數上限，留空 = 200。', + voiceActivationLabel: '語音啟用', + voiceActivationDesc: '在本機等待喚醒詞以開始聽寫。', + voiceActivationLocalOnly: '辨識只在此裝置上進行；待機音訊不會上傳或封存。', + voiceActivationWakeEnabled: '喚醒詞', + voiceActivationWakePhrase: '喚醒詞', + voiceActivationFinishEnabled: '完成詞', + voiceActivationFinishPhrase: '完成詞', + voiceActivationTimeoutEnabled: '逾時後停止', + voiceActivationTimeoutSeconds: '逾時（秒）', startupGroupTitle: '啟動', startMinimizedLabel: '啟動時靜默執行', startMinimizedDesc: '所有啟動路徑都不彈主視窗，僅選單欄 / 託盤執行。', diff --git a/openless-all/app/src/lib/ipc/mock-data.ts b/openless-all/app/src/lib/ipc/mock-data.ts index e5ad41d24..a7ac34e97 100644 --- a/openless-all/app/src/lib/ipc/mock-data.ts +++ b/openless-all/app/src/lib/ipc/mock-data.ts @@ -52,6 +52,13 @@ export let mockSettings: UserPreferences = { muteDuringRecording: false, stableTranscriptionEnabled: false, lowLatencyDictationEnabled: false, + voiceActivationEnabled: false, + voiceActivationWakeEnabled: true, + voiceActivationFinishEnabled: true, + voiceActivationWakePhrase: '开始语音录入', + voiceActivationFinishPhrase: '完成语音录入', + voiceActivationTimeoutEnabled: true, + voiceActivationTimeoutSeconds: 30, audioCueOnRecord: true, silenceAutoStopEnabled: false, silenceAutoStopSeconds: 3, diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index 80b0a083e..e3268c2bc 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -360,6 +360,14 @@ export interface UserPreferences { stableTranscriptionEnabled: boolean; /** Keep a bounded microphone pre-roll in memory to reduce the delay before first speech is captured. */ lowLatencyDictationEnabled: boolean; + /** Enable local-only voice activation while idle (Windows desktop only). */ + voiceActivationEnabled: boolean; + voiceActivationWakeEnabled: boolean; + voiceActivationFinishEnabled: boolean; + voiceActivationWakePhrase: string; + voiceActivationFinishPhrase: string; + voiceActivationTimeoutEnabled: boolean; + voiceActivationTimeoutSeconds: number; /** When the recording hotkey enters the recording state, plays a synthesized cue that recording has started. * On by default; synthesized with the Web Audio API in the capsule window, independent of showCapsule. */ audioCueOnRecord: boolean; diff --git a/openless-all/app/src/pages/settings/RecordingInputSection.tsx b/openless-all/app/src/pages/settings/RecordingInputSection.tsx index 3cd1875f3..fedbb2019 100644 --- a/openless-all/app/src/pages/settings/RecordingInputSection.tsx +++ b/openless-all/app/src/pages/settings/RecordingInputSection.tsx @@ -506,15 +506,98 @@ export function RecordingInputSection() { /> {showDesktopHotkey && os === 'win' && ( - - - + <> + + + + + savePrefs({ ...prefs, voiceActivationEnabled: next })} + /> + + {prefs.voiceActivationEnabled && ( +
+
+ {t('settings.recording.voiceActivationLocalOnly')} +
+ + savePrefs({ ...prefs, voiceActivationWakeEnabled: next })} + /> + + {prefs.voiceActivationWakeEnabled && ( + + savePrefs({ ...prefs, voiceActivationWakePhrase: event.currentTarget.value }) + } + style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} + /> + )} + + savePrefs({ ...prefs, voiceActivationFinishEnabled: next })} + /> + + {prefs.voiceActivationFinishEnabled && ( + + savePrefs({ ...prefs, voiceActivationFinishPhrase: event.currentTarget.value }) + } + style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} + /> + )} + + savePrefs({ ...prefs, voiceActivationTimeoutEnabled: next })} + /> + + {prefs.voiceActivationTimeoutEnabled && ( + + savePrefs({ + ...prefs, + voiceActivationTimeoutSeconds: Math.max( + 5, + Math.min(300, Number(event.currentTarget.value) || 30), + ), + }) + } + style={{ ...inputStyle, width: 90, marginTop: 2 }} + /> + )} +
+ )} + )} Date: Wed, 14 Oct 2026 18:28:19 +0800 Subject: [PATCH 3/5] fix: guide voice activation model setup Co-Authored-By: Claude Fable 5 --- .../app/src/components/SettingsModal.tsx | 15 ++- openless-all/app/src/i18n/de.ts | 5 +- openless-all/app/src/i18n/en.ts | 5 +- openless-all/app/src/i18n/es.ts | 5 +- openless-all/app/src/i18n/fr.ts | 5 +- openless-all/app/src/i18n/ja.ts | 5 +- openless-all/app/src/i18n/ko.ts | 5 +- openless-all/app/src/i18n/zh-CN.ts | 5 +- openless-all/app/src/i18n/zh-TW.ts | 2 + .../app/src/lib/voiceActivation.test.ts | 43 +++++++ openless-all/app/src/lib/voiceActivation.ts | 30 +++++ .../pages/settings/RecordingInputSection.tsx | 111 +++++++++++++++++- openless-all/app/src/pages/settings/tabs.tsx | 13 +- 13 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 openless-all/app/src/lib/voiceActivation.test.ts create mode 100644 openless-all/app/src/lib/voiceActivation.ts diff --git a/openless-all/app/src/components/SettingsModal.tsx b/openless-all/app/src/components/SettingsModal.tsx index 0d8ded3ff..21eedf11e 100644 --- a/openless-all/app/src/components/SettingsModal.tsx +++ b/openless-all/app/src/components/SettingsModal.tsx @@ -30,6 +30,7 @@ import { visibleSettingsSections, visibleAdvancedPages, type AdvancedPageId, + type ServiceViewId, type SettingsSectionId, } from '../pages/settings/navigation'; import { ChannelEditorHostContext } from '../pages/settings/ChannelEditorHostContext'; @@ -64,6 +65,7 @@ export function SettingsModal({ const mobile = useMobileLayout(); const conservative = useConservativeLayout(); const [section, setSection] = useState(initialSettingsSection ?? 'general'); + const [requestedServiceView, setRequestedServiceView] = useState(); const [advancedPage, setAdvancedPage] = useState(null); const [query, setQuery] = useState(''); const [platformCaps, setPlatformCaps] = useState(getCachedPlatformCapabilities); @@ -205,16 +207,23 @@ export function SettingsModal({ }); }; - const selectSection = (next: SettingsSectionId, fromSearch = false) => { + const selectSection = ( + next: SettingsSectionId, + fromSearch = false, + serviceView?: ServiceViewId, + ) => { void providerForm.finish(() => { channelCloseRef.current?.(); setSection(next); + setRequestedServiceView(next === 'services' ? serviceView : undefined); setAdvancedPage(null); setQuery(''); if (fromSearch) window.requestAnimationFrame(() => headingRef.current?.focus()); }); }; + const openLocalModels = () => selectSection('services', false, 'models'); + const handleKeyDown = (event: KeyboardEvent) => { // Nested selectors and shortcut recorders own their keys before the dialog does. if (event.defaultPrevented || event.nativeEvent.isComposing) return; @@ -689,11 +698,11 @@ export function SettingsModal({ gap: 16, }} > - {section === 'general' && } + {section === 'general' && } {section === 'inputMethod' && } {section === 'shortcuts' && } {section === 'appearance' && } - {section === 'services' && } + {section === 'services' && } {section === 'privacy' && } {section === 'advanced' && ( { await invoke('plugin:autostart|disable'); } -export function RecordingInputSection() { +export function RecordingInputSection({ + onOpenLocalModels, +}: { + onOpenLocalModels?: () => void; +} = {}) { const { t } = useTranslation(); const os = detectOS(); const { prefs, capability, updatePrefs: savePrefs, refresh } = useHotkeySettings(); @@ -59,6 +69,8 @@ export function RecordingInputSection() { const [microphoneDevices, setMicrophoneDevices] = useState([]); const [microphoneDevicesLoaded, setMicrophoneDevicesLoaded] = useState(false); const [microphoneDevicesError, setMicrophoneDevicesError] = useState(null); + const [voiceActivationModelMissing, setVoiceActivationModelMissing] = useState(false); + const voiceActivationStateObservedRef = useRef(false); useEffect(() => { void getPlatformCapabilities().then(setPlatformCaps); @@ -129,6 +141,62 @@ export function RecordingInputSection() { }; }, [loadMicrophoneDevices]); + useEffect(() => { + if (!isTauri || os !== 'win') { + setVoiceActivationModelMissing(false); + return; + } + let cancelled = false; + let unlisten: (() => void) | undefined; + async function listenForVoiceActivationChanges() { + const { listen } = await import('@tauri-apps/api/event'); + if (cancelled) return; + const stopListening = await listen( + 'voice-activation:state-changed', + (event) => { + voiceActivationStateObservedRef.current = true; + setVoiceActivationModelMissing(isVoiceActivationModelMissing(event.payload)); + }, + ); + if (cancelled) { + stopListening(); + return; + } + unlisten = stopListening; + } + void listenForVoiceActivationChanges(); + return () => { + cancelled = true; + unlisten?.(); + }; + }, [os]); + + useEffect(() => { + if (!isTauri || os !== 'win' || !prefs?.voiceActivationEnabled) { + voiceActivationStateObservedRef.current = false; + setVoiceActivationModelMissing(false); + return; + } + voiceActivationStateObservedRef.current = false; + let cancelled = false; + void getSherpaOnnxAsrCatalog() + .then((catalog) => { + if (cancelled || voiceActivationStateObservedRef.current) return; + const model = catalog.find(({ alias }) => alias === VOICE_ACTIVATION_MODEL_ALIAS); + setVoiceActivationModelMissing(model?.cached === false); + }) + .catch(() => { + // The runtime event remains authoritative when the catalog cannot be read. + }); + return () => { + cancelled = true; + }; + }, [os, prefs?.voiceActivationEnabled]); + + useEffect(() => { + if (!prefs?.voiceActivationEnabled) setVoiceActivationModelMissing(false); + }, [prefs?.voiceActivationEnabled]); + const saveKeyboardListAffectingPrefs = useCallback( async (nextPrefs: UserPreferences) => { try { @@ -538,6 +606,36 @@ export function RecordingInputSection() {
{t('settings.recording.voiceActivationLocalOnly')}
+ {voiceActivationModelMissing && ( +
+ + {t('settings.recording.voiceActivationModelMissing')} + + {onOpenLocalModels && ( + + {t('settings.recording.voiceActivationDownloadZipformer')} + + )} +
+ )} - savePrefs({ ...prefs, voiceActivationFinishPhrase: event.currentTarget.value }) + savePrefs({ + ...prefs, + voiceActivationFinishPhrase: event.currentTarget.value, + }) } style={{ ...inputStyle, width: '100%', margin: '2px 0 8px' }} /> @@ -573,7 +674,9 @@ export function RecordingInputSection() { savePrefs({ ...prefs, voiceActivationTimeoutEnabled: next })} + onToggle={(next) => + savePrefs({ ...prefs, voiceActivationTimeoutEnabled: next }) + } /> {prefs.voiceActivationTimeoutEnabled && ( diff --git a/openless-all/app/src/pages/settings/tabs.tsx b/openless-all/app/src/pages/settings/tabs.tsx index c04d6d5e1..2e2695027 100644 --- a/openless-all/app/src/pages/settings/tabs.tsx +++ b/openless-all/app/src/pages/settings/tabs.tsx @@ -56,13 +56,13 @@ function usePlatformCaps(): PlatformCapabilities | null { } // Recording & input: from recording to text insertion, plus phone input. -export function GeneralTab() { +export function GeneralTab({ onOpenLocalModels }: { onOpenLocalModels?: () => void } = {}) { const platformCaps = usePlatformCaps(); const showRemoteInput = platformCaps?.platform === 'desktop'; return ( <> - + {showRemoteInput && } ); @@ -113,16 +113,19 @@ export function AppearanceTab() { } // AI services & models: organizes channels, local models, and network settings by current pipeline and platform capabilities. -export function ServicesTab() { +export function ServicesTab({ initialView }: { initialView?: ServiceViewId } = {}) { const { t } = useTranslation(); const { prefs, updatePrefs } = useHotkeySettings(); const platformCaps = usePlatformCaps(); const showLocalModel = platformCaps?.supportsLocalAsr === true; const multimodal = prefs?.multimodalPipelineEnabled === true && prefs.pipelineMode === 'multimodal'; - const [view, setView] = useState('llm'); + const [view, setView] = useState(initialView ?? 'llm'); const views = availableServiceViews(showLocalModel); - const selectedView = resolveServiceView(view, views, multimodal); + const selectedView = + initialView === 'models' && view === 'models' && showLocalModel + ? 'models' + : resolveServiceView(view, views, multimodal); const contentRef = useRef(null); useContentMotion(contentRef, selectedView); const pipelineRef = useRef(null); From 37e4665d78a39011431401393264002a4879c197 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Fri, 16 Oct 2026 23:12:24 +0800 Subject: [PATCH 4/5] fix: tolerate punctuation in voice finish phrase Co-Authored-By: Claude Fable 5 --- .../openless-core/src/voice_activation.rs | 90 +++++++++---------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/voice_activation.rs b/openless-all/app/crates/openless-core/src/voice_activation.rs index 12f21f699..8c04983fe 100644 --- a/openless-all/app/crates/openless-core/src/voice_activation.rs +++ b/openless-all/app/crates/openless-core/src/voice_activation.rs @@ -48,12 +48,13 @@ impl std::error::Error for PhraseValidationError {} /// Normalize ASR output and user phrases before suffix matching. /// -/// NFKC handles full-width punctuation and compatibility forms. Whitespace is -/// intentionally removed because streaming ASR commonly inserts spaces between -/// Chinese characters while English recognition may emit multiple spaces. +/// NFKC handles full-width punctuation and compatibility forms. Whitespace +/// and punctuation are intentionally removed because streaming ASR may insert +/// them between Chinese command characters while English recognition may emit +/// multiple spaces or hyphens. pub fn normalize_phrase(text: &str) -> String { text.nfkc() - .filter(|character| !character.is_whitespace()) + .filter(|character| !character.is_whitespace() && !is_terminal_punctuation(*character)) .flat_map(char::to_lowercase) .collect() } @@ -62,30 +63,17 @@ pub fn normalize_phrase(text: &str) -> String { /// the spacing and punctuation in the spoken body. pub fn strip_phrase_suffix(text: &str, phrase: &str) -> Option { let normalized_phrase = validate_phrase(phrase).ok()?; - let phrase_ends_punctuation = normalized_phrase - .chars() - .last() - .is_some_and(is_terminal_punctuation); let phrase_chars = normalized_phrase.chars().rev(); let mut cursor = text.len(); - while cursor > 0 { - let previous = text[..cursor].char_indices().next_back()?; - if previous.1.is_whitespace() - || (!phrase_ends_punctuation && is_terminal_punctuation(previous.1)) - { - cursor = previous.0; - } else { - break; - } - } let mut cut_at = cursor; for expected in phrase_chars { while cursor > 0 { let previous = text[..cursor].char_indices().next_back()?; - if !previous.1.is_whitespace() { - break; + if previous.1.is_whitespace() || is_terminal_punctuation(previous.1) { + cursor = previous.0; + continue; } - cursor = previous.0; + break; } let (start, character) = text[..cursor].char_indices().next_back()?; if normalize_phrase(&character.to_string()) != expected.to_string() { @@ -98,30 +86,32 @@ pub fn strip_phrase_suffix(text: &str, phrase: &str) -> Option { } fn is_terminal_punctuation(character: char) -> bool { - character.is_ascii_punctuation() - || matches!( - character, - '。' | '！' - | '？' - | '，' - | '、' - | '；' - | '：' - | '“' - | '”' - | '‘' - | '’' - | '（' - | '）' - | '【' - | '】' - | '《' - | '》' - | '…' - | '～' - | '—' - | '–' - ) + character.to_string().nfkc().any(|normalized| { + normalized.is_ascii_punctuation() + || matches!( + normalized, + '。' | '！' + | '？' + | '，' + | '、' + | '；' + | '：' + | '“' + | '”' + | '‘' + | '’' + | '（' + | '）' + | '【' + | '】' + | '《' + | '》' + | '…' + | '～' + | '—' + | '–' + ) + }) } pub fn validate_phrase(text: &str) -> Result { @@ -402,9 +392,10 @@ mod tests { use super::*; #[test] - fn normalization_handles_spaces_width_and_case() { + fn normalization_handles_spaces_width_case_and_punctuation() { assert_eq!(normalize_phrase(" Ａ 开始 语音录入 "), "a开始语音录入"); assert_eq!(normalize_phrase("完成\n语音\t录入"), DEFAULT_FINISH_PHRASE); + assert_eq!(normalize_phrase("完成，语音录入。"), DEFAULT_FINISH_PHRASE); } #[test] @@ -414,7 +405,11 @@ mod tests { Some("你好，世界".into()) ); assert_eq!( - strip_phrase_suffix("你好，世界 完成语音录入。", DEFAULT_FINISH_PHRASE), + strip_phrase_suffix("你好，世界 完成，语音录入。", DEFAULT_FINISH_PHRASE), + Some("你好，世界".into()) + ); + assert_eq!( + strip_phrase_suffix("你好，世界 完成语音录入．", DEFAULT_FINISH_PHRASE), Some("你好，世界".into()) ); assert!(strip_phrase_suffix("完成语音录入之前", DEFAULT_FINISH_PHRASE).is_none()); @@ -438,6 +433,7 @@ mod tests { let matcher = VoicePhraseMatcher::new(DEFAULT_FINISH_PHRASE, VoiceActivationCommand::Finish).unwrap(); assert!(!matcher.matches_suffix("完成语音录入之前的内容")); + assert!(matcher.matches_suffix("正文 完成，语音录入。")); assert_eq!( matcher.strip_suffix("正文 完成语音录入").as_deref(), Some( From 9270b43c7e79ad9d462ad8bec78250de5f499093 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sat, 17 Oct 2026 01:47:49 +0800 Subject: [PATCH 5/5] fix: update warm capture test for preroll flag Co-Authored-By: Claude Fable 5 --- openless-all/app/src-tauri/src/recorder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index 0f506a89c..84c445e42 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -2998,7 +2998,7 @@ mod tests { fn warm_config_change_keeps_staged_claim_on_cold_fallback() { let hub = WarmCaptureHub::new(); hub.enabled.store(true, Ordering::Release); - hub.sink.arm(42).unwrap(); + hub.sink.arm(42, true).unwrap(); hub.configure( false,