diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f330acb4d..1a5b847e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,11 @@ on: type: choice options: [all, macos] default: all + macos_runner: + description: macOS runner (Intel fallback when hosted ARM capacity is unavailable) + type: choice + options: [macos-latest, macos-15-intel] + default: macos-latest # Cancel old runs when the same PR is pushed repeatedly; workflow_dispatch is isolated by run_id. concurrency: @@ -599,7 +604,8 @@ jobs: fail-fast: false matrix: include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.platform == 'macos' && '[{"os":"macos-latest","label":"macOS","preflight":false}]' || '[{"os":"macos-latest","label":"macOS","preflight":false},{"os":"windows-latest","label":"Windows","preflight":true}]') }} - runs-on: ${{ matrix.os }} + # Dispatch-only fallback; push/PR runs keep the default matrix and all test gates. + runs-on: ${{ matrix.label == 'macOS' && github.event_name == 'workflow_dispatch' && inputs.macos_runner || matrix.os }} timeout-minutes: 60 env: # The shared Core and full Tauri test binary compile together; limit concurrency @@ -811,7 +817,7 @@ jobs: # Run the declared minimum toolchain even when stable has the same version. if: needs.changes.outputs.msrv != 'false' name: macOS Rust 1.99 MSRV - runs-on: macos-latest + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.macos_runner || 'macos-latest' }} timeout-minutes: 60 env: CARGO_BUILD_JOBS: 2 diff --git a/docs/issue-1223-aec-feasibility.md b/docs/issue-1223-aec-feasibility.md new file mode 100644 index 000000000..7d6b178a4 --- /dev/null +++ b/docs/issue-1223-aec-feasibility.md @@ -0,0 +1,159 @@ +# Issue #1223: playback-preserving acoustic echo cancellation + +**Status:** implementation and feasibility note; the baseline research below is supplemented by the feature-branch implementation. +**Baseline revision examined:** `de072a88373a222280a8c22948092a260880196c` (`upstream/beta`, 2.0.2 Beta.1). +**Issue:** [Open-Less/openless#1223](https://github.com/Open-Less/openless/issues/1223) + +## Executive conclusion + +Issue #1223 is technically feasible, but not as a platform-neutral switch around the existing microphone normalizer. Acoustic echo cancellation (AEC) needs two time-related signals: the microphone/capture signal and a render/playback reference, plus a usable estimate of the delay between them. The official WebRTC API exposes this explicitly through `ProcessReverseStream()` for render audio, `ProcessStream()` for capture audio, and `set_stream_delay_ms()` for their timing relationship ([WebRTC Audio Processing API](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h)). A microphone-only RMS gate or gain stage can reduce quiet noise, but it cannot reliably identify and subtract speech from a video. + +The practical recommendation is an **optional, capability-reported, best-effort feature** that leaves playback and volume untouched. Keep “Mute while recording” as the explicit fallback. Implement and validate one platform at a time rather than promising universal AEC: + +1. Add a capability probe, diagnostics, fallback behavior, and an offline/fixture test harness. +2. Prototype Windows with WASAPI loopback plus a proven AEC implementation such as WebRTC APM/AEC3. +3. Add a PipeWire-specific Linux path where the echo-cancel virtual sink/source can own the playback reference. +4. Investigate macOS VoiceProcessingIO or a supported Core Audio render tap; do not claim that VoiceProcessingIO cancels arbitrary applications whose playback is outside that unit. +5. Add Android’s native `AudioRecord`/`AcousticEchoCanceler` path only where the device reports the effect as available. + +## 1. Issue and GitHub history + +### User request + +The issue asks for an optional playback-suppression/AEC setting so a video can keep playing while OpenLess recognizes the user’s speech. It explicitly accepts device- and environment-dependent results and asks that supported platforms and limitations be documented ([issue body](https://github.com/Open-Less/openless/issues/1223)). + +The two issue comments are: + +- On 2026-10-17, `HKLHaoBin` explained that AEC was omitted from the earlier audio-enhancement work because of CPU cost, additional dependencies, and implementation/maintenance complexity, and offered to try an implementation ([comment 6237559767](https://github.com/Open-Less/openless/issues/1223#issuecomment-6237559767)). +- On 2026-10-17, the reporter said a closed-source competitor appears to preserve video playback while preventing the video voice from entering recognition ([comment 6237637338](https://github.com/Open-Less/openless/issues/1223#issuecomment-6237637338)). This is a user observation, not an independently verified technical claim. + +`gh api --paginate repos/Open-Less/openless/issues/1223/events` returned an empty list. The issue timeline contained only the two `commented` entries above; it contained no cross-reference, linked-commit, or pull-request event. `gh search prs 1223 --repo Open-Less/openless` and `gh search commits 1223 --repo Open-Less/openless` returned no results at investigation time. The issue body does explicitly reference [#269](https://github.com/Open-Less/openless/issues/269), so that issue and its implementation PR are relevant context, but they are not direct links from the #1223 timeline. + +### Related repository history + +- [#269](https://github.com/Open-Less/openless/issues/269) requested pausing and restoring background media. It is closed. +- [PR #274](https://github.com/Open-Less/openless/pull/274) implemented the current “Mute while recording” behavior. Its description says the implementation mutes the system output instead of pausing each media application. Its automated review recorded the important limitation: media continues advancing while muted, and macOS/Linux needed real-device verification. The resulting commits were `9511eb6`, `bb0fefc`, and `78a3846`. +- [PR #1194](https://github.com/Open-Less/openless/pull/1194), merged for #1026, added opt-in RMS/downmix microphone enhancement and explicitly documented that it does **not** include echo cancellation. Its merge commit is `44972015349cac3a52c4a28190675554a8986fcb`. + +## 2. Current audio architecture + +The following bullets describe the committed code at the baseline revision above; they explain why the feature branch adds both a shared DSP seam and host-specific reference capture: + +- `openless-all/app/crates/openless-core/src/audio.rs` owns the canonical 16 kHz, mono, signed-16-bit PCM normalization contract. The baseline `AudioProcessingConfig`/`AudioProcessor` path implemented optional microphone enhancement/noise gating but had no render-reference input or AEC state ([file at baseline revision](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/crates/openless-core/src/audio.rs)). +- `openless-all/app/src-tauri/src/recorder.rs` opens a `cpal` input stream, converts the selected microphone format to `f32`, then sends microphone callback data through the processor/normalizer. The baseline had no corresponding render/loopback stream in this adapter ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/src-tauri/src/recorder.rs)). +- `openless-all/app/linux-egui/src/audio.rs` likewise owned a `cpal` microphone input stream and forwarded normalized chunks. The baseline had no playback-monitor/reference capture path ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/linux-egui/src/audio.rs)). +- `openless-all/app/src-tauri/src/audio_mute.rs` is an output-mute guard: Windows uses `IAudioEndpointVolume`, macOS invokes `osascript` to set output mute, and Linux tries `wpctl` then `pactl`. It records and restores mute state, but it does not observe or process the render signal ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/src-tauri/src/audio_mute.rs)). +- `openless-all/app/crates/openless-core/src/ports.rs` states that each host owns device selection, permissions, resampling, and native stream implementation. This is the seam used by the feature branch for host-specific render-reference capture while keeping cancellation inside the existing PCM pipeline ([file](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/crates/openless-core/src/ports.rs)). + +### Feature-branch implementation boundary + +The feature branch adds a session-frozen, opt-in `echo_cancellation_enabled` preference, excluded from cloud sync because it is host/device dependent. `ReferenceAudioBuffer` keeps at most two seconds of downmixed render samples and bypasses cancellation on startup starvation or underrun. `AudioProcessor::process_with_reference` resamples microphone and reference data to the canonical rate, applies a bounded adaptive canceller before the existing microphone enhancement, and preserves the old microphone-only path when the switch is off or a reference is unavailable. + +The Tauri adapter uses cpal 0.18’s render-endpoint input path on desktop targets where the backend supplies loopback (WASAPI and cpal’s macOS CoreAudio loopback support). The Linux egui adapter attempts PipeWire sink capture and PulseAudio monitor sources; ALSA remains microphone-only. All reference samples remain transient and are not archived or transmitted. Any unavailable, invalidated, or runtime-failed reference leaves ordinary recording running without changing system volume or media playback state. The independent `mute_during_recording` option remains unchanged and is still the explicit output-mute fallback. + +## 3. What AEC requires + +### Render reference and timing are mandatory inputs + +The official WebRTC Audio Processing API describes approximately 10 ms linear PCM frames, sends the reverse/render stream to `ProcessReverseStream()`, sends microphone/capture audio to `ProcessStream()`, and exposes a stream-delay setting for the timing relationship. Its AEC configuration is therefore fundamentally different from a capture-only gain or noise gate ([WebRTC API header](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h)). WebRTC’s AEC3 interface similarly describes 10 ms band-split frames, a render-analysis path that stores the render signal, and a capture-processing path that removes estimated echo ([AEC3 header](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/audio_processing/aec3/echo_canceller3.h)). + +A FIFO of samples is not automatically a valid reference. The render and capture clocks can differ, the acoustic path has an unknown delay and impulse response, and output routing can change. The implementation must either use a platform voice-processing unit or maintain timestamps/delay and bounded drift handling in a software AEC path. + +### Why the existing enhancement is insufficient + +PR #1194’s enhancement is a stateful RMS/noise-floor gate and gain/soft-clip stage. It has no render input, delay estimate, adaptive echo filter, or double-talk reference. It should remain separately configurable; enabling it must not be presented as enabling AEC ([PR #1194](https://github.com/Open-Less/openless/pull/1194)). + +## 4. Platform feasibility and constraints + +| Platform | Official capability | Feasibility for OpenLess | Main limitation / required adapter | +| --- | --- | --- | --- | +| Windows | WASAPI loopback captures the stream played by a render endpoint; Microsoft says loopback exists primarily to support AEC. It is shared-mode only, and endpoint/hardware/DRM behavior matters ([Loopback Recording](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording)). Windows also exposes Communications/Speech processing modes and effect discovery, but modes are driver-defined and optional ([Audio Signal Processing Modes](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-signal-processing-modes); [APO architecture](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-processing-object-architecture)). | **Best first desktop prototype.** Passive WASAPI loopback can preserve audible playback while supplying a reference to WebRTC APM/AEC3. | The current `cpal` input path does not expose this render-reference lifecycle. Use a Windows-specific WASAPI adapter, select/track the actual render endpoint, handle shared-mode and protected-content gaps, and avoid assuming a driver APO is present. | +| macOS | Apple’s VoiceProcessingIO unit provides voice processing such as echo cancellation and AGC ([VoiceProcessingIO](https://developer.apple.com/documentation/audiotoolbox/kaudiounitsubtype_voiceprocessingio)). | **Feasible for a controlled voice-I/O route; uncertain for arbitrary system playback.** | VoiceProcessingIO expects application playback through its output path. OpenLess currently mutes the system endpoint and does not own every application’s playback graph. A separate supported Core Audio render tap/monitor path and permissions would need investigation; otherwise report unsupported and retain fallback. Voice processing can change speech-oriented audio characteristics, so compare near-end fidelity. | +| Linux (PipeWire) | PipeWire’s official echo-cancel module creates virtual echo-cancel capture/playback nodes and streams. Playback sent to the echo-cancel sink supplies the reference; `monitor.mode` can capture a default sink monitor. The documented backend is `aec/libspa-aec-webrtc` ([PipeWire Echo Cancel](https://docs.pipewire.org/page_module_echo_cancel.html)). | **Feasible when PipeWire can own or monitor the playback route.** | The current `linux-egui` recorder is a generic `cpal` input path. Add a PipeWire-specific node/monitor adapter or use the virtual source/sink; do not assume an ALSA/Pulse/default-device capture has a synchronized monitor. Device/route changes and non-PipeWire systems require capability fallback. | +| Android | `AcousticEchoCanceler` was added in API 16, is device-dependent, must be checked with `isAvailable()`, and attaches to the `AudioRecord` audio session; `create()` may return `null` ([Android API reference](https://developer.android.com/reference/android/media/audiofx/AcousticEchoCanceler)). | **Feasible only through a native Android capture path on supported devices.** | The current shared recorder uses `cpal`; the repository’s Kotlin code handles microphone permission but does not expose an `AudioRecord` session/effect adapter. Add native session ownership, route-specific tests, and a clear unavailable state. | +| Portable software fallback | WebRTC APM/AEC3 is a cross-platform algorithmic option, but its official API still requires render and capture frames plus timing ([APM header](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h)). | **Feasible only where each host can provide a reliable render/monitor reference.** | A portable library does not create a system loopback source. Adding it without a platform reference would produce an apparent setting with weak or misleading results. It also adds binary size, CPU, latency, and real-time integration work; these are consistent with the complexity concerns recorded in #1223’s first comment. | + +The W3C `echoCancellation` constraint is a useful model for product wording, not a guarantee that every source supports AEC. The specification says `true` must attempt at least remote-only cancellation, while `all` must attempt all system-played sound; it also requires capabilities to report only `false` when cancellation is unavailable ([Media Capture and Streams, `echoCancellation`](https://w3c.github.io/mediacapture-main/#def-constraint-echoCancellation)). OpenLess should expose an equivalent capability/unsupported result instead of claiming universal support. + +## 5. Risks and mitigations + +1. **Wrong or missing reference.** A default render monitor may not correspond to the selected output, a Bluetooth route may add variable delay, and applications may use another sink. Mitigate with endpoint identity, timestamps, route-change invalidation, reference starvation bypass, and a visible capability/diagnostic state. +2. **Double-talk damage.** The user and video speaker can talk simultaneously. AEC must protect near-end speech; use a proven double-talk strategy and measure near-end word recall, not only echo reduction. +3. **Clock drift and buffering.** Separate render/capture clocks eventually move out of alignment. Bound queues, monitor underflow/overflow, and resample or adapt delay rather than draining an un-timestamped FIFO indefinitely. +4. **Real-time callback safety.** Audio callbacks must not block on a long DSP operation, allocate without bounds, or wait on a worker. Keep callback work bounded and use a lock-free or carefully bounded handoff where the platform requires it. +5. **Native processing conflicts.** Windows driver APOs and Android/macOS voice processing may already apply AEC/AGC/NS. Discover/record active effects where the platform permits and avoid unintentionally stacking processors ([Windows effect discovery guidance](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-signal-processing-modes)). +6. **Protected or inaccessible playback.** Microsoft documents that loopback behavior depends on endpoint capabilities and that protected content may not be capturable ([WASAPI loopback](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording)). Treat reference loss as unsupported/bypass, never as silence that could train the canceller incorrectly. +7. **Privacy.** A render reference can contain all system audio, not only the video selected by the user. Keep it in-process, do not archive or transmit it, and document that it is used only transiently to process microphone PCM. +8. **Failure cleanup.** Existing mute behavior is best-effort and uses a guard restored on drop ([`audio_mute.rs`](https://github.com/Open-Less/openless/blob/de072a88373a222280a8c22948092a260880196c/openless-all/app/src-tauri/src/audio_mute.rs)). AEC startup, route changes, cancellation, recorder errors, and shutdown must preserve the same cleanup guarantees without muting playback when AEC is selected. + +## 6. Phased implementation recommendation + +### Phase 0: contract and observability + +- Add a session-frozen AEC mode separate from `mute_during_recording` and microphone enhancement: `off`, `native`, or `software` is preferable to a boolean if the UI needs to explain fallback. +- Define a host capability response containing platform, reference source, endpoint identity, availability, and reason for unavailability. +- Preserve current mute as an explicit fallback, never silently mute when the user selected playback-preserving AEC. +- Add deterministic tests for PCM framing, reference underflow/overflow, reset, cancellation, and “no reference means bypass.” +- Add an offline fixture containing far-end audio, a known delayed/filtered echo, near-end speech, and double-talk. Test both echo reduction and preservation of near-end samples. + +### Phase 1: Windows proof of value + +- Implement a Windows WASAPI shared-mode loopback capture on the render endpoint used for playback. +- Feed 10 ms render and microphone frames into WebRTC APM/AEC3, preserving stream delay and route identity. +- Measure speaker, wired, Bluetooth, default-device change, device unplug, Remote Desktop, and protected-content cases before exposing the setting broadly. +- Keep AEC off by default until real-device results meet the acceptance gates below. + +### Phase 2: Linux PipeWire + +- Detect PipeWire and create/use the documented echo-cancel source/sink or monitor path. Keep the generic `cpal` path as fallback. +- Test default sink monitor, per-application routing, route changes, PulseAudio compatibility mode, and systems without PipeWire. +- Do not advertise “Linux support” as a single guarantee; report the backend and route capability. + +### Phase 3: macOS and Android + +- macOS: prototype VoiceProcessingIO only for a controlled voice-processing route; separately investigate a supported system render tap and permission requirements. If arbitrary app playback cannot be referenced reliably, mark the feature unavailable rather than muting or claiming cancellation. +- Android: add a Kotlin/native `AudioRecord` adapter, attach `AcousticEchoCanceler` to the correct session, check both availability and `create()` result, and test speaker/headset/Bluetooth/device-vendor combinations. + +### Phase 4: product hardening + +- Add per-platform support text, diagnostic logging without raw audio, route-change handling, power/CPU telemetry, and a one-click AEC sample test. +- Keep the existing mute option and clarify that it is the deterministic fallback; keep microphone enhancement separately described as noise suppression, not AEC. + +## 7. Proposed acceptance and test matrix + +The following are proposed product gates, not guarantees stated by platform vendors. Record the exact device, OS, audio route, sample rates, output endpoint, and AEC mode for every run. + +| Test area | Required cases | Evidence / gate | +| --- | --- | --- | +| Baseline and toggle semantics | AEC off; enhancement off/on; mute off/on; AEC setting changed between sessions | Existing behavior is unchanged when AEC is off. AEC does not call endpoint mute or change system volume. Mute restores the user’s original state on normal stop, cancel, startup failure, runtime fault, and shutdown. | +| Far-end-only | Video speech, music, tones, silence at microphone; several levels and delays | Compare processed and unprocessed microphone recordings. False transcription rate decreases materially; report ERLE/SNR or another reproducible echo metric. | +| Near-end-only | User speech with no playback; quiet/normal/loud speech; short words and sustained speech | Near-end word recall/WER remains within an agreed regression budget; no audible clipping or gate-induced loss in the archived processed PCM. | +| Double talk | Video speech overlaps user speech at beginning, middle, and end of utterances | Near-end speech remains intelligible and transcribes; cancellation does not treat the user as echo. | +| Route/device matrix | Built-in speaker + mic, USB mic, wired headset, Bluetooth headset/speaker, output and input changes during capture | Each combination is either validated or reports unsupported/bypassed. No stale reference after endpoint change. | +| Platform matrix | Windows WASAPI loopback; PipeWire source/sink and monitor mode; macOS VoiceProcessingIO/tap candidate; Android supported and unsupported devices | Capability result identifies backend and reason. Unsupported platforms keep working without a crash and without unexpected mute. | +| Faults and permissions | Missing microphone permission, missing render permission/device, protected content, route unplug, stream underflow/overflow | Recording remains usable through documented bypass/fallback; no raw render audio is persisted or transmitted; cleanup is verified. | +| Timing and performance | Start/stop latency, reference starvation, clock drift, 10-minute capture, repeated sessions | Bounded queue memory, no callback watchdog failures, agreed p95 CPU/latency/power budget, and no unbounded growth. | +| Regression | History playback/retranscription, streaming ASR, QA/selection voice, archive retention, concurrent lifecycle races | Existing recorder/core contract tests pass; processed archive format remains canonical 16 kHz mono PCM WAV where enabled. | + +## Current implementation decision + +The feature branch follows the best-effort design above: AEC is opt-in, session-frozen, capability-by-fallback, and independent from output muting. The shared software canceller is covered by deterministic signal tests, while host-specific loopback and monitor paths remain subject to platform/device validation. The setting must not be presented as a guarantee of perfect cancellation. Windows WASAPI and macOS CoreAudio loopback are the primary desktop paths; PipeWire/Pulse monitor capture is attempted on Linux; unsupported routes, ALSA-only capture, mobile builds, protected content, and unavailable monitors continue with ordinary microphone processing. A route changed during an active session is not dynamically rebound by this phase; users must stop and start a new dictation session for the new output route to be selected, and the route-change acceptance cases remain hardware-validation work. + +The remaining acceptance work is real-device validation of endpoint identity, route changes, Bluetooth latency, double-talk speech preservation, and long-running clock drift. Those hardware-only results should be recorded per platform rather than inferred from the offline tests. + +## Primary sources + +- [Issue #1223 and comments](https://github.com/Open-Less/openless/issues/1223) +- [PR #274: mute system output](https://github.com/Open-Less/openless/pull/274) +- [PR #1194: lightweight microphone enhancement](https://github.com/Open-Less/openless/pull/1194) +- [WebRTC Audio Processing API](https://webrtc.googlesource.com/src/+/refs/heads/main/api/audio/audio_processing.h) +- [WebRTC EchoCanceller3](https://webrtc.googlesource.com/src/+/refs/heads/main/modules/audio_processing/aec3/echo_canceller3.h) +- [Microsoft WASAPI Loopback Recording](https://learn.microsoft.com/en-us/windows/win32/coreaudio/loopback-recording) +- [Microsoft Audio Signal Processing Modes](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-signal-processing-modes) +- [Microsoft Audio Processing Object Architecture](https://learn.microsoft.com/en-us/windows-hardware/drivers/audio/audio-processing-object-architecture) +- [Apple VoiceProcessingIO](https://developer.apple.com/documentation/audiotoolbox/kaudiounitsubtype_voiceprocessingio) +- [PipeWire Echo Cancel module](https://docs.pipewire.org/page_module_echo_cancel.html) +- [Android AcousticEchoCanceler](https://developer.android.com/reference/android/media/audiofx/AcousticEchoCanceler) +- [W3C `echoCancellation` constraint](https://w3c.github.io/mediacapture-main/#def-constraint-echoCancellation) + +*The baseline history and primary-source citations were collected during the read-only investigation; the feature-branch implementation notes above describe the changes now under test.* diff --git a/openless-all/app/crates/openless-core/examples/enhance_audio.rs b/openless-all/app/crates/openless-core/examples/enhance_audio.rs index 7acf34d98..7da52a73a 100644 --- a/openless-all/app/crates/openless-core/examples/enhance_audio.rs +++ b/openless-all/app/crates/openless-core/examples/enhance_audio.rs @@ -29,6 +29,7 @@ fn main() -> Result<(), Box> { .collect(); let mut processor = AudioProcessor::new(AudioProcessingConfig { enhancement_enabled: true, + echo_cancellation_enabled: false, dsp: AudioDspConfig { noise_reduction: algorithm, ..Default::default() diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 85d00be1d..14d05219a 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -5321,6 +5321,7 @@ impl OpenLessBackend { pre_roll_ms: 500, audio_processing: crate::audio::AudioProcessingConfig { enhancement_enabled: preferences.audio_enhancement_enabled, + echo_cancellation_enabled: preferences.echo_cancellation_enabled, dsp: preferences.audio_dsp, }, }; diff --git a/openless-all/app/crates/openless-core/src/audio.rs b/openless-all/app/crates/openless-core/src/audio.rs index 46233a7ab..3b680888d 100644 --- a/openless-all/app/crates/openless-core/src/audio.rs +++ b/openless-all/app/crates/openless-core/src/audio.rs @@ -1,5 +1,9 @@ //! Framework-independent audio normalization used by native recorder adapters. +#[path = "audio/echo.rs"] +mod echo; +pub use echo::ReferenceAudioBuffer; + pub const DICTATION_SAMPLE_RATE: u32 = 16_000; const LEVEL_RMS_GAIN: f32 = 4.0; @@ -13,6 +17,7 @@ pub struct NormalizedPcmChunk { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct AudioProcessingConfig { pub enhancement_enabled: bool, + pub echo_cancellation_enabled: bool, pub dsp: crate::audio_dsp::AudioDspConfig, } @@ -24,6 +29,7 @@ pub struct AudioProcessor { config: AudioProcessingConfig, normalizer: PcmNormalizer, enhanced: Option, + echo: Option, finished: bool, } @@ -41,6 +47,9 @@ impl AudioProcessor { config, normalizer: Default::default(), enhanced: None, + echo: config + .echo_cancellation_enabled + .then(echo::EchoProcessor::default), finished: false, } } @@ -97,6 +106,21 @@ impl AudioProcessor { pcm_chunk(&processed) } + /// A playback gap bypasses only cancellation; enhancement keeps its format and tail. + pub fn process_with_reference( + &mut self, + input: &[f32], + channels: usize, + rate: u32, + reference: Option<&[f32]>, + ) -> Option { + let Some(echo) = self.echo.as_mut() else { + return self.process(input, channels, rate); + }; + let samples = echo.process(input, channels, rate, reference); + self.process(&samples, 1, DICTATION_SAMPLE_RATE) + } + /// Idempotent end-of-stream barrier. Flushes fractional resampling state, /// a partial 10 ms frame and model lookahead without adding padding to PCM. pub fn finish(&mut self) -> Option { diff --git a/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs b/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs index bd04ee7be..59e82629d 100644 --- a/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs +++ b/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs @@ -5,6 +5,7 @@ use crate::{AudioDspConfig, NoiseReduction}; fn config(algorithm: NoiseReduction) -> AudioProcessingConfig { AudioProcessingConfig { enhancement_enabled: true, + echo_cancellation_enabled: false, dsp: AudioDspConfig { noise_reduction: algorithm, agc_enabled: false, @@ -174,6 +175,7 @@ fn default_agc_does_not_amplify_denoised_room_noise_back_to_original_level() { for algorithm in [NoiseReduction::Rnnoise, NoiseReduction::Purevox] { let config = AudioProcessingConfig { enhancement_enabled: true, + echo_cancellation_enabled: false, dsp: AudioDspConfig { noise_reduction: algorithm, ..Default::default() diff --git a/openless-all/app/crates/openless-core/src/audio/echo.rs b/openless-all/app/crates/openless-core/src/audio/echo.rs new file mode 100644 index 000000000..c2a03c4ab --- /dev/null +++ b/openless-all/app/crates/openless-core/src/audio/echo.rs @@ -0,0 +1,537 @@ +//! Timestamped playback reference and optional software cancellation. +use super::{PcmNormalizer, DICTATION_SAMPLE_RATE}; +use std::collections::VecDeque; +use std::sync::{Condvar, Mutex}; +use std::time::{Duration, Instant}; + +struct ReferenceChunk { + captured_at: Instant, + samples: Vec, +} +struct ReferenceState { + chunks: VecDeque, + wait_for_callback: bool, +} + +/// Two seconds of transient playback. Hosts map hardware capture timestamps to +/// `Instant`; callback arrival order must never determine audio alignment. +pub struct ReferenceAudioBuffer { + sample_rate: u32, + state: Mutex, + ready: Condvar, +} +impl ReferenceAudioBuffer { + pub fn new(sample_rate: u32) -> Self { + Self { + sample_rate, + state: Mutex::new(ReferenceState { + chunks: VecDeque::new(), + wait_for_callback: true, + }), + ready: Condvar::new(), + } + } + pub fn push_interleaved_at(&self, samples: &[f32], channels: usize, captured_at: Instant) { + if channels == 0 || self.sample_rate == 0 || samples.len() < channels { + return; + } + let mut mono = samples + .chunks_exact(channels) + .map(|frame| frame.iter().map(|&s| finite_sample(s)).sum::() / channels as f32) + .collect::>(); + let discard = mono.len().saturating_sub(self.sample_rate as usize * 2); + mono.drain(..discard); + let captured_at = captured_at + sample_duration(discard, self.sample_rate); + let end = captured_at + sample_duration(mono.len(), self.sample_rate); + let Ok(mut state) = self.state.try_lock() else { + return; + }; + if state + .chunks + .back() + .is_some_and(|last| captured_at < last.captured_at) + { + state.chunks.clear(); + } + state.chunks.push_back(ReferenceChunk { + captured_at, + samples: mono, + }); + while state.chunks.len() > 1 + && state.chunks.front().is_some_and(|first| { + end.saturating_duration_since(first.captured_at) > Duration::from_secs(2) + }) + { + state.chunks.pop_front(); + } + state.wait_for_callback = true; + self.ready.notify_one(); + } + /// A discontinuity invalidates the learned path; fresh callbacks can resume it. + pub fn invalidate(&self) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.chunks.clear(); + state.wait_for_callback = false; + self.ready.notify_one(); + } + /// Sample playback at microphone frame times, retaining fractional positions + /// rather than rounding up and consuming an extra source sample per callback. + pub fn take_for_input_at( + &self, + frames: usize, + rate: u32, + captured_at: Instant, + ) -> Option> { + if frames == 0 || rate == 0 || self.sample_rate == 0 { + return None; + } + let last_time = captured_at + sample_duration(frames - 1, rate); + let deadline = Instant::now() + Duration::from_millis(100); + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + if state + .chunks + .front() + .is_some_and(|first| captured_at < first.captured_at) + { + return None; + } + if state.chunks.back().is_some_and(|last| { + last_time < last.captured_at + sample_duration(last.samples.len(), self.sample_rate) + }) { + break; + } + if !state.wait_for_callback { + return None; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.wait_for_callback = false; + return None; + } + let (next, timeout) = self + .ready + .wait_timeout(state, remaining) + .unwrap_or_else(std::sync::PoisonError::into_inner); + state = next; + if timeout.timed_out() { + state.wait_for_callback = false; + return None; + } + } + let mut chunk_index = 0; + let mut output = Vec::with_capacity(frames); + for frame in 0..frames { + let time = captured_at + sample_duration(frame, rate); + loop { + let chunk = state.chunks.get(chunk_index)?; + let offset = time + .checked_duration_since(chunk.captured_at)? + .as_secs_f64() + * self.sample_rate as f64; + let index = offset.floor() as usize; + if index >= chunk.samples.len() { + chunk_index += 1; + continue; + } + let left = chunk.samples[index]; + let right = chunk.samples.get(index + 1).copied().unwrap_or(left); + output.push(left + (right - left) * offset.fract() as f32); + break; + } + } + Some(output) + } +} +fn sample_duration(frames: usize, rate: u32) -> Duration { + Duration::from_nanos(frames as u64 * 1_000_000_000 / u64::from(rate)) +} +fn finite_sample(sample: f32) -> f32 { + if sample.is_finite() { + sample.clamp(-1.0, 1.0) + } else { + 0.0 + } +} + +#[derive(Default)] +pub(super) struct EchoProcessor { + microphone: PcmNormalizer, + reference: PcmNormalizer, + canceller: EchoCanceller, + rate: u32, +} +impl EchoProcessor { + pub fn process( + &mut self, + input: &[f32], + channels: usize, + rate: u32, + reference: Option<&[f32]>, + ) -> Vec { + if channels == 0 || rate == 0 || input.len() < channels { + return Vec::new(); + } + if rate != self.rate { + *self = Self::default(); + self.rate = rate; + } + let microphone = input + .chunks_exact(channels) + .map(|frame| frame.iter().map(|&s| finite_sample(s)).sum::() / channels as f32) + .collect::>(); + let available = reference.filter(|samples| samples.len() == microphone.len()); + let render = match available { + Some(samples) => samples + .iter() + .map(|&s| finite_sample(s)) + .collect::>(), + None => vec![0.0; microphone.len()], + }; + let microphone = self + .microphone + .resample(µphone, rate, DICTATION_SAMPLE_RATE); + // Advance both resamplers through a gap, but never train on synthetic silence. + let render = self + .reference + .resample(&render, rate, DICTATION_SAMPLE_RATE); + if available.is_none() { + self.canceller.reset(); + return microphone; + } + self.canceller.process(µphone, &render) + } +} +const ECHO_FILTER_TAPS: usize = 1_024; +const ECHO_ADAPTATION_RATE: f32 = 0.12; +const ECHO_DOUBLE_TALK_RATIO: f32 = 2.5; +const ECHO_MIN_REFERENCE_POWER: f32 = 0.000_001; + +/// Conservative normalized-LMS echo canceller for a mono render reference. +/// +/// The filter is intentionally bounded and stateful. It learns the acoustic path while +/// the render signal is present, but pauses adaptation when near-end energy is clearly +/// stronger than the estimated echo (double talk). It is not a replacement for a +/// platform voice-processing stack; hosts therefore treat it as an optional best-effort +/// processor and bypass it when the render reference is unavailable. +#[derive(Debug, Clone)] +struct EchoCanceller { + coefficients: Vec, + history: Vec, + history_cursor: usize, +} + +impl Default for EchoCanceller { + fn default() -> Self { + Self { + coefficients: vec![0.0; ECHO_FILTER_TAPS], + history: vec![0.0; ECHO_FILTER_TAPS], + history_cursor: 0, + } + } +} + +impl EchoCanceller { + fn reset(&mut self) { + self.coefficients.fill(0.0); + self.history.fill(0.0); + self.history_cursor = 0; + } + + fn process(&mut self, microphone: &[f32], reference: &[f32]) -> Vec { + if microphone.is_empty() || reference.is_empty() { + return microphone.to_vec(); + } + let mut output = Vec::with_capacity(microphone.len()); + for (index, &sample) in microphone.iter().enumerate() { + let Some(&reference_sample) = reference.get(index) else { + // A resampler may produce one fewer sample at a chunk boundary. Do not + // repeat the last render value across the missing tail; preserve the + // microphone sample and let the next reference chunk re-establish alignment. + output.push(sample); + continue; + }; + self.history[self.history_cursor] = reference_sample; + let mut estimate = 0.0; + let mut reference_power = 0.0; + for tap in 0..ECHO_FILTER_TAPS { + let history_index = + (self.history_cursor + ECHO_FILTER_TAPS - tap) % ECHO_FILTER_TAPS; + let value = self.history[history_index]; + estimate += self.coefficients[tap] * value; + reference_power += value * value; + } + if !estimate.is_finite() { + self.reset(); + output.push(sample); + continue; + } + let error = sample - estimate; + let near_power = sample * sample; + let should_adapt = reference_power > ECHO_MIN_REFERENCE_POWER + && (estimate.abs() < 0.01 + || estimate * estimate * ECHO_DOUBLE_TALK_RATIO >= near_power + || near_power < ECHO_MIN_REFERENCE_POWER); + if should_adapt { + let normalization = reference_power + 0.000_01; + for tap in 0..ECHO_FILTER_TAPS { + let history_index = + (self.history_cursor + ECHO_FILTER_TAPS - tap) % ECHO_FILTER_TAPS; + self.coefficients[tap] += + ECHO_ADAPTATION_RATE * error * self.history[history_index] / normalization; + self.coefficients[tap] = self.coefficients[tap].clamp(-2.0, 2.0); + } + } + output.push(if error.is_finite() { error } else { 0.0 }); + self.history_cursor = (self.history_cursor + 1) % ECHO_FILTER_TAPS; + } + output + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audio::{AudioProcessingConfig, AudioProcessor}; + #[test] + fn non_finite_microphone_samples_do_not_poison_future_audio() { + let mut p = AudioProcessor::new(AudioProcessingConfig { + echo_cancellation_enabled: true, + ..Default::default() + }); + let far = vec![0.1; 160]; + let mut mic = vec![0.2; 160]; + mic[0] = f32::NAN; + mic[1] = f32::INFINITY; + p.process_with_reference(&mic, 1, 16_000, Some(&far)) + .unwrap(); + let out = p + .process_with_reference(&vec![0.2; 160], 1, 16_000, Some(&far)) + .unwrap(); + assert!(out + .pcm_i16_le + .as_chunks::<2>() + .0 + .iter() + .any(|b| i16::from_le_bytes([b[0], b[1]]) != 0)); + assert!(out.level.is_finite()); + } + #[test] + fn reference_conversion_has_no_rounding_drift_or_startup_backlog() { + let epoch = Instant::now(); + let b = ReferenceAudioBuffer::new(48_000); + b.push_interleaved_at(&vec![-0.75; 4800], 1, epoch); + let start = epoch + Duration::from_millis(100); + let mut published = 0_u64; + for callback in 0..4000_u64 { + let total = ((callback + 1) * 512 * 48_000).div_ceil(44_100); + b.push_interleaved_at( + &vec![0.25; (total - published) as usize], + 1, + start + sample_duration(published as usize, 48_000), + ); + published = total; + let chunk = b + .take_for_input_at( + 512, + 44_100, + start + sample_duration(callback as usize * 512, 44_100), + ) + .expect("equal-duration streams must stay aligned"); + assert_eq!(chunk.len(), 512); + assert!(chunk.iter().all(|&s| (s - 0.25).abs() < 1e-5)); + } + assert!( + b.state + .lock() + .unwrap() + .chunks + .iter() + .map(|c| c.samples.len()) + .sum::() + <= 96_558 + ); + } + #[test] + fn reference_gap_does_not_reuse_stale_samples() { + let epoch = Instant::now(); + let b = ReferenceAudioBuffer::new(16_000); + b.push_interleaved_at(&[0.25; 80], 1, epoch); + b.push_interleaved_at(&[0.75; 160], 1, epoch + Duration::from_millis(10)); + assert!(b.take_for_input_at(160, 16_000, epoch).is_none()); + assert_eq!( + b.take_for_input_at(160, 16_000, epoch + Duration::from_millis(10)) + .unwrap(), + vec![0.75; 160] + ); + b.invalidate(); + assert!(b + .take_for_input_at(160, 16_000, epoch + Duration::from_millis(10)) + .is_none()); + } + #[test] + fn reference_gaps_preserve_dsp_format_and_tail_duration() { + for rate in [16_000, 44_100, 48_000] { + let mut p = AudioProcessor::new(AudioProcessingConfig { + enhancement_enabled: true, + echo_cancellation_enabled: true, + ..Default::default() + }); + let mut samples = 0; + for available in [true, false, true, false] { + let far = vec![0.0; 317]; + if let Some(out) = p.process_with_reference( + &vec![0.25; 317], + 1, + rate, + available.then_some(far.as_slice()), + ) { + samples += out.pcm_i16_le.len() / 2; + } + } + if let Some(tail) = p.finish() { + samples += tail.pcm_i16_le.len() / 2; + } + assert_eq!( + samples, + (4_u64 * 317 * 16_000).div_ceil(u64::from(rate)) as usize + ); + assert!(p.finish().is_none()); + } + } + #[test] + fn startup_offset_is_aligned_before_the_causal_filter() { + let epoch = Instant::now(); + let b = ReferenceAudioBuffer::new(16_000); + let mut seed = 0x91234567_u32; + let far = (0..40000) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + ((seed >> 8) as f32 / 16777216.0 - 0.5) * 0.4 + }) + .collect::>(); + let mut c = EchoCanceller::default(); + let mut input_power = 0.0; + let mut output_power = 0.0; + for callback in 0..200 { + let time = 2400 + callback * 160; + let start = if callback == 0 { 0 } else { time }; + b.push_interleaved_at( + &far[start..time + 160], + 1, + epoch + sample_duration(start, 16_000), + ); + let r = b + .take_for_input_at(160, 16_000, epoch + sample_duration(time, 16_000)) + .unwrap(); + let mic = far[time - 160..time] + .iter() + .map(|s| s * 0.7) + .collect::>(); + let out = c.process(&mic, &r); + if callback >= 100 { + input_power += mic.iter().map(|s| s * s).sum::(); + output_power += out.iter().map(|s| s * s).sum::(); + } + } + assert!( + output_power < input_power * 0.5, + "{output_power} >= {input_power}" + ); + } + #[test] + fn echo_canceller_reduces_delayed_render_signal() { + let mut canceller = EchoCanceller::default(); + let mut reference = vec![0.0_f32; 16_000]; + for (index, sample) in reference.iter_mut().enumerate() { + *sample = (index as f32 * 0.071).sin() * 0.25; + } + let delay = 96; + let microphone = reference + .iter() + .enumerate() + .map(|(index, _)| { + reference + .get(index.saturating_sub(delay)) + .copied() + .unwrap_or_default() + * 0.7 + }) + .collect::>(); + let mut cancelled = Vec::new(); + for (mic, far) in microphone.chunks(160).zip(reference.chunks(160)) { + cancelled.extend(canceller.process(mic, far)); + } + let input_rms = microphone[2_000..] + .iter() + .map(|sample| sample * sample) + .sum::(); + let output_rms = cancelled[2_000..] + .iter() + .map(|sample| sample * sample) + .sum::(); + assert!(output_rms < input_rms * 0.35, "{output_rms} >= {input_rms}"); + } + + #[test] + fn echo_canceller_preserves_near_end_during_double_talk() { + let mut canceller = EchoCanceller::default(); + let mut reference = vec![0.0_f32; 12_000]; + for (index, sample) in reference.iter_mut().enumerate() { + // Deterministic broadband-ish reference gives the adaptive filter enough + // information to learn the delayed acoustic path. + *sample = (((index as u32) + .wrapping_mul(1_103_515_245) + .wrapping_add(12_345) + >> 16) as f32 + / 32_768.0 + - 1.0) + * 0.2; + } + let delay = 96; + let echo = |index: usize| { + reference + .get(index.saturating_sub(delay)) + .copied() + .unwrap_or_default() + * 0.7 + }; + for (index, far) in reference.chunks(160).enumerate() { + let start = index * 160; + let microphone = (start..start + far.len()).map(echo).collect::>(); + let _ = canceller.process(µphone, far); + } + + let mut output = Vec::new(); + let mut near_end = Vec::new(); + for (index, far) in reference[6_000..].chunks(160).enumerate() { + let start = 6_000 + index * 160; + let near = (start..start + far.len()) + .map(|sample| (sample as f32 * 0.017).sin() * 0.22) + .collect::>(); + let microphone = far + .iter() + .enumerate() + .map(|(offset, _sample)| echo(start + offset) + near[offset]) + .collect::>(); + output.extend(canceller.process(µphone, far)); + near_end.extend(near); + } + let dot = output + .iter() + .zip(&near_end) + .map(|(processed, near)| processed * near) + .sum::(); + let near_power = near_end.iter().map(|sample| sample * sample).sum::(); + assert!( + dot > near_power * 0.35, + "near-end correlation was suppressed" + ); + } +} diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 44e900442..420b9e4c7 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -186,6 +186,13 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Boolean, reason: "local_only_microphone_capture", }, + PreferenceField { + rust_name: "echo_cancellation_enabled", + key: "echoCancellationEnabled", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Boolean, + reason: "local_only_microphone_capture", + }, PreferenceField { rust_name: "audio_enhancement_enabled", key: "audioEnhancementEnabled", diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index b82b76a0e..52992f475 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -335,6 +335,7 @@ impl DictationContext { .then(|| (preferences.silence_auto_stop_seconds * 1_000.0).round() as u64), audio_processing: AudioProcessingConfig { enhancement_enabled: preferences.audio_enhancement_enabled, + echo_cancellation_enabled: preferences.echo_cancellation_enabled, dsp: preferences.audio_dsp, }, }, diff --git a/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs b/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs index 7d8fef64d..a4725742a 100644 --- a/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs +++ b/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs @@ -593,7 +593,12 @@ async fn optional_runtime_files_are_not_download_progress() { let directory = f.store.model_dir(&Model::TextStream.target()).unwrap(); std::fs::create_dir_all(directory.join(".runtime/cache")).unwrap(); let unrelated = std::fs::File::create(directory.join(".runtime/cache/wheels")).unwrap(); - unrelated.set_len(5 * 1024 * 1024 * 1024).unwrap(); + // A modest non-model file is enough to catch accidental runtime-directory + // accounting without relying on sparse-file support from the host filesystem. + unrelated.set_len(5 * 1024 * 1024).unwrap(); + // Release the handle before the download commits a replacement model directory; + // Windows otherwise reports access denied while removing the fixture tree. + drop(unrelated); let status = f.service.status().unwrap(); assert_eq!(status[Model::TextStream.index()].downloaded_bytes, 0); f.service.set_enabled(Model::TextStream, true).unwrap(); diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 1f363893f..6ee3c0331 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -228,7 +228,7 @@ pub use api::{ }; pub use audio::{ encode_dictation_wav, AudioProcessingConfig, AudioProcessor, NormalizedPcmChunk, - PcmNormalizer, DICTATION_SAMPLE_RATE, + PcmNormalizer, ReferenceAudioBuffer, DICTATION_SAMPLE_RATE, }; pub use auxiliary::{ AsrCallLabel, AuxiliaryApi, RepolishRequest, RetranscriptionFailure, RetranscriptionResult, diff --git a/openless-all/app/crates/openless-core/src/preferences.rs b/openless-all/app/crates/openless-core/src/preferences.rs index 817d7663e..49352da57 100644 --- a/openless-all/app/crates/openless-core/src/preferences.rs +++ b/openless-all/app/crates/openless-core/src/preferences.rs @@ -124,6 +124,7 @@ fn preferences_json_preserving_unknown( fn strip_local_only_preferences(value: &mut serde_json::Value) { if let serde_json::Value::Object(object) = value { object.remove("lowLatencyDictationEnabled"); + object.remove("echoCancellationEnabled"); object.remove("audioDsp"); object.remove("lowMemoryBackgroundEnabled"); } @@ -312,6 +313,10 @@ impl PreferencesStore { "lowLatencyDictationEnabled".to_string(), serde_json::Value::Bool(state.low_latency_dictation_enabled), ); + object.insert( + "echoCancellationEnabled".to_string(), + serde_json::Value::Bool(state.echo_cancellation_enabled), + ); } let parsed: UserPreferences = serde_json::from_value(value.clone()) .map_err(|_| persistence_error("validate restored preferences"))?; @@ -480,11 +485,13 @@ mod tests { fn cloud_snapshot_omits_local_microphone_warm_capture_preference() { let mut value = serde_json::json!({ "lowLatencyDictationEnabled": true, + "echoCancellationEnabled": true, "audioDsp": {"noiseReduction": "purevox"}, "microphoneDeviceName": "Built-in Mic", }); strip_local_only_preferences(&mut value); assert!(value.get("lowLatencyDictationEnabled").is_none()); + assert!(value.get("echoCancellationEnabled").is_none()); assert!(value.get("audioDsp").is_none()); assert_eq!(value["microphoneDeviceName"], "Built-in Mic"); } diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index ea06f2ee9..a7f5a9e34 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -427,6 +427,9 @@ pub struct UserPreferences { /// Disabled by default until an audio-quality baseline supports changing it. #[serde(default)] pub audio_enhancement_enabled: bool, + /// Optional local playback-reference cancellation; independent of enhancement. + #[serde(default)] + pub echo_cancellation_enabled: bool, #[serde(default)] pub audio_dsp: crate::audio_dsp::AudioDspConfig, /// Reconnect the current ASR after recording ends and submit the whole PCM. Default off. @@ -1028,6 +1031,8 @@ struct UserPreferencesWire { #[serde(default)] audio_enhancement_enabled: bool, #[serde(default)] + echo_cancellation_enabled: bool, + #[serde(default)] audio_dsp: crate::audio_dsp::AudioDspConfig, #[serde(default)] stable_transcription_enabled: bool, @@ -1314,6 +1319,7 @@ impl Default for UserPreferencesWire { capsule_transcript_font_size: prefs.capsule_transcript_font_size, mute_during_recording: prefs.mute_during_recording, audio_enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, audio_dsp: prefs.audio_dsp.normalized(), stable_transcription_enabled: prefs.stable_transcription_enabled, low_latency_dictation_enabled: prefs.low_latency_dictation_enabled, @@ -1535,6 +1541,7 @@ impl<'de> Deserialize<'de> for UserPreferences { capsule_transcript_font_size: wire.capsule_transcript_font_size.clamp(12, 20), mute_during_recording: wire.mute_during_recording, audio_enhancement_enabled: wire.audio_enhancement_enabled, + echo_cancellation_enabled: wire.echo_cancellation_enabled, audio_dsp: wire.audio_dsp.normalized(), stable_transcription_enabled: wire.stable_transcription_enabled, low_latency_dictation_enabled: wire.low_latency_dictation_enabled, @@ -1934,6 +1941,7 @@ impl Default for UserPreferences { capsule_transcript_font_size: default_capsule_transcript_font_size(), mute_during_recording: false, audio_enhancement_enabled: false, + echo_cancellation_enabled: false, audio_dsp: Default::default(), stable_transcription_enabled: false, low_latency_dictation_enabled: false, diff --git a/openless-all/app/e2e/settings-regressions.spec.ts b/openless-all/app/e2e/settings-regressions.spec.ts index d0bf5de97..3e9c0d46c 100644 --- a/openless-all/app/e2e/settings-regressions.spec.ts +++ b/openless-all/app/e2e/settings-regressions.spec.ts @@ -236,7 +236,9 @@ test('voice activation opens model setup without leaving the multimodal pipeline }, testInfo) => { await page.addInitScript(() => { Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); - Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' }); + Object.defineProperty(navigator, 'userAgent', { + get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + }); }); await page.goto('http://127.0.0.1:5326/?os=win'); await page.getByRole('button', { name: '设置', exact: true }).click(); @@ -244,9 +246,7 @@ test('voice activation opens model setup without leaving the multimodal pipeline const categories = dialog.getByRole('navigation', { name: '设置分类' }); await categories.getByRole('button', { name: 'AI 服务与模型', exact: true }).click(); await dialog.getByRole('button', { name: '多模态模式', exact: true }).click(); - const models = dialog - .locator('.ol-service-views') - .getByRole('button', { name: /^本地模型/ }); + const models = dialog.locator('.ol-service-views').getByRole('button', { name: /^本地模型/ }); await expect(models).toBeDisabled(); await categories.getByRole('button', { name: '录音与输入', exact: true }).click(); await dialog.getByRole('switch', { name: '语音唤醒听写', exact: true }).click(); @@ -288,3 +288,23 @@ test('voice activation opens model setup without leaving the multimodal pipeline await categories.getByRole('button', { name: 'AI 服务与模型', exact: true }).click(); await expect(models).toBeDisabled(); }); + +test('playback echo cancellation is opt-in and does not change output muting', async ({ + page, +}, testInfo) => { + const dialog = await openSettings(page); + const cancellation = dialog.getByRole('switch', { name: '播放回声抑制', exact: true }); + const mute = dialog.getByRole('switch', { name: '录音时静音', exact: true }); + await expect(cancellation).not.toBeChecked(); + const muted = await mute.isChecked(); + await cancellation.click(); + await expect(cancellation).toBeChecked(); + await expect(mute).toHaveAttribute('aria-checked', String(muted)); + await dialog.getByRole('button', { name: '外观与语言', exact: true }).click(); + await dialog.getByRole('button', { name: '录音与输入', exact: true }).click(); + await expect(cancellation).toBeChecked(); + await cancellation.scrollIntoViewIfNeeded(); + await page.screenshot({ path: testInfo.outputPath('playback-echo-cancellation.png') }); + await cancellation.click(); + await expect(cancellation).not.toBeChecked(); +}); diff --git a/openless-all/app/linux-egui/src/audio.rs b/openless-all/app/linux-egui/src/audio.rs index adb75bc7a..86722120b 100644 --- a/openless-all/app/linux-egui/src/audio.rs +++ b/openless-all/app/linux-egui/src/audio.rs @@ -38,6 +38,7 @@ const AUDIO_PROCESSING_QUEUE_CAPACITY: usize = 64; #[cfg(target_os = "linux")] struct RawAudioFrame { + captured_at: std::time::Instant, samples: Vec, channels: usize, input_sample_rate: u32, @@ -76,6 +77,7 @@ impl AudioProcessingWorker { consumer: Arc, progress: Arc, writer: Option>>, + reference_slot: Arc>>>, ) -> Result<(Self, SyncSender), BackendError> { let (sender, receiver) = sync_channel::(AUDIO_PROCESSING_QUEUE_CAPACITY); let started = std::time::Instant::now(); @@ -84,9 +86,23 @@ impl AudioProcessingWorker { .spawn(move || { let mut processor = openless_core::AudioProcessor::new(processing); while let Ok(frame) = receiver.recv() { - if let Some(chunk) = - processor.process(&frame.samples, frame.channels, frame.input_sample_rate) - { + let reference = reference_slot + .lock() + .expect("Linux echo reference lock poisoned") + .clone(); + let reference_samples = reference.as_ref().and_then(|buffer| { + buffer.take_for_input_at( + frame.samples.len() / frame.channels.max(1), + frame.input_sample_rate, + frame.captured_at, + ) + }); + if let Some(chunk) = processor.process_with_reference( + &frame.samples, + frame.channels, + frame.input_sample_rate, + reference_samples.as_deref(), + ) { consumer.consume_pcm_chunk(&chunk.pcm_i16_le); if let Some(writer) = &writer { if let Err(error) = writer @@ -426,14 +442,20 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { runtime_error, startup, } = args; - let (processor, frame_tx) = - match AudioProcessingWorker::start(audio_processing, consumer, progress, writer) { - Ok(worker) => worker, - Err(error) => { - let _ = startup.send(Err(error)); - return; - } - }; + let reference_slot = Arc::new(std::sync::Mutex::new(None)); + let (processor, frame_tx) = match AudioProcessingWorker::start( + audio_processing, + consumer, + progress, + writer, + Arc::clone(&reference_slot), + ) { + Ok(worker) => worker, + Err(error) => { + let _ = startup.send(Err(error)); + return; + } + }; let mut result = Err(BackendError::new( BackendErrorCode::Platform, "no Linux audio backend is available", @@ -463,6 +485,8 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { &frame_tx, &stop, &runtime_error, + &reference_slot, + audio_processing.echo_cancellation_enabled, ) { Ok(stream) => { result = Ok(stream); @@ -479,10 +503,10 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { } } - let stream = match result { - Ok(stream) => { + let streams = match result { + Ok(streams) => { let _ = startup.send(Ok(())); - stream + streams } Err(error) => { log::error!( @@ -499,7 +523,12 @@ fn run_audio_thread(args: LinuxAudioThreadArgs) { while !stop.load(std::sync::atomic::Ordering::Acquire) { std::thread::park_timeout(std::time::Duration::from_millis(25)); } - drop(stream); + drop(streams.microphone); + reference_slot + .lock() + .expect("Linux echo reference lock poisoned") + .take(); + drop(streams.reference); drop(frame_tx); let _ = processor.finish(); } @@ -511,6 +540,12 @@ fn audio_backend_order() -> [&'static str; 3] { ["pipewire", "pulseaudio", "alsa"] } +#[cfg(target_os = "linux")] +struct LinuxAudioStreams { + microphone: cpal::Stream, + reference: Option, +} + #[cfg(target_os = "linux")] #[allow(clippy::too_many_arguments)] fn try_start_audio_stream( @@ -520,18 +555,155 @@ fn try_start_audio_stream( frame_tx: &SyncSender, stop: &Arc, runtime_error: &Arc>>, + reference_slot: &Arc>>>, + echo_cancellation_enabled: bool, +) -> Result { + let device = select_input_device(host, preferred_device_name).map_err(|error| { + BackendError::new(error.code, format!("{backend} backend: {}", error.message)) + })?; + let stream = try_start_microphone_stream(&device, backend, frame_tx, stop, runtime_error)?; + let reference_stream = if echo_cancellation_enabled && backend != "alsa" { + match start_reference_stream(host, backend, reference_slot, stop) { + Ok(stream) => Some(stream), + Err(error) => { + log::warn!( + "{backend} playback monitor unavailable; recording without AEC: {error}" + ); + if backend == "pipewire" { + start_pulse_reference_fallback(reference_slot, stop) + } else { + None + } + } + } + } else { + None + }; + Ok(LinuxAudioStreams { + microphone: stream, + reference: reference_stream, + }) +} + +#[cfg(target_os = "linux")] +fn select_playback_monitor_device( + host: &cpal::Host, + backend: &str, +) -> Result { + use cpal::traits::{DeviceTrait, HostTrait}; + + let default_output_id = host + .default_output_device() + .and_then(|device| device.id().ok()) + .map(|id| id.id().to_owned()); + let devices = host + .devices() + .map_err(|error| classify_audio_error("enumerate playback monitors", error.to_string()))?; + for device in devices { + if !device.supports_input() { + continue; + } + let Ok(id) = device.id() else { + continue; + }; + if is_default_playback_reference(backend, id.id(), default_output_id.as_deref()) { + return Ok(device); + } + } + Err(BackendError::new( + BackendErrorCode::Unsupported, + format!("{backend} monitor of the default playback output was not found"), + )) +} + +#[cfg(any(target_os = "linux", test))] +fn is_default_playback_reference(backend: &str, id: &str, default_output: Option<&str>) -> bool { + match backend { + "pulseaudio" => { + default_output.is_some_and(|sink| id.strip_suffix(".monitor") == Some(sink)) + } + "pipewire" => id == "sink_default" || default_output == Some(id), + _ => false, + } +} + +#[cfg(target_os = "linux")] +fn start_pulse_reference_fallback( + reference_slot: &Arc>>>, + stop: &Arc, +) -> Option { + let host_id = cpal::available_hosts() + .into_iter() + .find(|id| id.name().eq_ignore_ascii_case("pulseaudio"))?; + let host = match cpal::host_from_id(host_id) { + Ok(host) => host, + Err(error) => { + log::debug!("PulseAudio reference fallback host unavailable: {error}"); + return None; + } + }; + match start_reference_stream(&host, "pulseaudio", reference_slot, stop) { + Ok(stream) => { + log::info!("using PulseAudio playback monitor as PipeWire AEC fallback"); + Some(stream) + } + Err(error) => { + log::debug!("PulseAudio playback monitor fallback unavailable: {error}"); + None + } + } +} + +#[cfg(target_os = "linux")] +fn start_reference_stream( + host: &cpal::Host, + backend: &str, + reference_slot: &Arc>>>, + stop: &Arc, ) -> Result { use cpal::traits::{DeviceTrait, StreamTrait}; - let device = select_input_device(host, preferred_device_name).map_err(|error| { - BackendError::new(error.code, format!("{backend} backend: {}", error.message)) + let device = select_playback_monitor_device(host, backend)?; + let supported = device.default_input_config().map_err(|error| { + classify_audio_error("default playback monitor config", error.to_string()) })?; + let config: cpal::StreamConfig = supported.config(); + let sample_rate = config.sample_rate; + let channels = usize::from(config.channels); + let reference = Arc::new(openless_core::ReferenceAudioBuffer::new(sample_rate)); + let stream = build_linux_reference_stream( + &device, + &config, + supported.sample_format(), + Arc::clone(&reference), + channels, + Arc::clone(stop), + )?; + stream + .play() + .map_err(|error| classify_audio_error("start playback monitor", error.to_string()))?; + *reference_slot + .lock() + .expect("Linux echo reference lock poisoned") = Some(reference); + Ok(stream) +} + +#[cfg(target_os = "linux")] +fn try_start_microphone_stream( + device: &cpal::Device, + backend: &str, + frame_tx: &SyncSender, + stop: &Arc, + runtime_error: &Arc>>, +) -> Result { + use cpal::traits::{DeviceTrait, StreamTrait}; + let mut native = None; let mut start = |supported: cpal::SupportedStreamConfig| -> Result { let accepting = Arc::new(std::sync::atomic::AtomicBool::new(false)); let config = supported.config(); let stream = build_input_stream( - &device, + device, &config, supported.sample_format(), supported.sample_rate(), @@ -623,6 +795,60 @@ fn native_input_alternatives( candidates } +#[cfg(target_os = "linux")] +fn build_linux_reference_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + sample_format: cpal::SampleFormat, + reference: Arc, + channels: usize, + stop: Arc, +) -> Result { + use cpal::traits::DeviceTrait; + macro_rules! make_stream { + ($t:ty, $to_f32:expr) => {{ + let reference = Arc::clone(&reference); + let error_reference = Arc::clone(&reference); + let stop_for_callback = Arc::clone(&stop); + device + .build_input_stream( + *config, + move |data: &[$t], info| { + if stop_for_callback.load(std::sync::atomic::Ordering::Acquire) { + return; + } + let captured_at = capture_time(info); + let samples = data.iter().copied().map($to_f32).collect::>(); + reference.push_interleaved_at(&samples, channels, captured_at); + }, + move |error| { + error_reference.invalidate(); + log::warn!("[linux-audio] playback monitor stream error: {error}"); + }, + None, + ) + .map_err(|error| classify_audio_error("build playback monitor", error.to_string())) + }}; + } + match sample_format { + cpal::SampleFormat::F32 => make_stream!(f32, |s: f32| s), + cpal::SampleFormat::F64 => make_stream!(f64, |s: f64| s as f32), + cpal::SampleFormat::I64 => make_stream!(i64, |s: i64| (s as f64 / i64::MAX as f64) as f32), + cpal::SampleFormat::U64 => make_stream!(u64, |s: u64| ((s as f64 / u64::MAX as f64) * 2.0 + - 1.0) as f32), + cpal::SampleFormat::U32 => make_stream!(u32, |s: u32| (s as f64 / u32::MAX as f64 * 2.0 + - 1.0) as f32), + cpal::SampleFormat::I16 => make_stream!(i16, |s: i16| s as f32 / i16::MAX as f32), + cpal::SampleFormat::U16 => make_stream!(u16, |s: u16| (s as f32 - 32768.0) / 32768.0), + cpal::SampleFormat::I32 => make_stream!(i32, |s: i32| s as f32 / i32::MAX as f32), + cpal::SampleFormat::I8 => make_stream!(i8, |s: i8| s as f32 / i8::MAX as f32), + cpal::SampleFormat::U8 => make_stream!(u8, |s: u8| (s as f32 - 128.0) / 128.0), + other => Err(BackendError::new( + BackendErrorCode::Unsupported, + format!("unsupported playback monitor sample format: {other:?}"), + )), + } +} #[cfg(target_os = "linux")] fn select_input_device( host: &cpal::Host, @@ -684,14 +910,16 @@ fn build_input_stream( device .build_input_stream::<$sample, _, _>( *config, - move |data: &[$sample], _| { + move |data: &[$sample], info| { if !accepting_audio.load(std::sync::atomic::Ordering::Acquire) || stop_for_callback.load(std::sync::atomic::Ordering::Acquire) { return; } + let captured_at = capture_time(info); let samples = data.iter().copied().map($to_f32).collect::>(); match frame_tx.try_send(RawAudioFrame { + captured_at, samples, channels, input_sample_rate, @@ -878,6 +1106,14 @@ fn classify_audio_error(context: &str, message: String) -> BackendError { BackendError::new(code, format!("Linux audio {context} failed: {message}")) } +#[cfg(target_os = "linux")] +fn capture_time(info: &cpal::InputCallbackInfo) -> std::time::Instant { + let now = std::time::Instant::now(); + let timestamp = info.timestamp(); + let delay = timestamp.callback.duration_since(timestamp.capture); + now.checked_sub(delay).unwrap_or(now) +} + #[cfg(test)] mod tests { use super::*; @@ -956,4 +1192,36 @@ mod tests { fn audio_backends_are_ordered_from_desktop_server_to_universal_fallback() { assert_eq!(audio_backend_order(), ["pipewire", "pulseaudio", "alsa"]); } + + #[test] + fn monitor_selection_uses_route_ids_not_translated_display_names() { + let sink = "alsa_output.usb-interface.analog-stereo"; + assert!(is_default_playback_reference( + "pulseaudio", + &format!("{sink}.monitor"), + Some(sink) + )); + assert!(!is_default_playback_reference( + "pulseaudio", + "another_output.monitor", + Some(sink) + )); + assert!(!is_default_playback_reference( + "pulseaudio", + sink, + Some(sink) + )); + assert!(!is_default_playback_reference( + "pulseaudio", + "any.monitor", + None + )); + assert!(is_default_playback_reference( + "pipewire", + "sink_default", + None + )); + assert!(is_default_playback_reference("pipewire", sink, Some(sink))); + assert!(!is_default_playback_reference("alsa", sink, Some(sink))); + } } diff --git a/openless-all/app/linux-egui/src/i18n.rs b/openless-all/app/linux-egui/src/i18n.rs index 13ca6fe9e..14558880c 100644 --- a/openless-all/app/linux-egui/src/i18n.rs +++ b/openless-all/app/linux-egui/src/i18n.rs @@ -2606,6 +2606,16 @@ pub const CATALOG: &[Msg] = &[ "음성 향상", ), }, + Msg { + key: "settings.recording.echo_cancellation_label", + text: row( + "播放回声抑制", + "播放回聲抑制", + "Playback echo cancellation", + "再生エコーキャンセル", + "재생 에코 제거", + ), + }, Msg { key: "settings.recording.mute_during_recording_label", text: row( @@ -2616,7 +2626,7 @@ pub const CATALOG: &[Msg] = &[ "녹음 중 음소거", ), }, - Msg { + Msg { key: "settings.recording.paste_shortcut_label", text: row( "模拟粘贴快捷键", @@ -4732,6 +4742,16 @@ pub const CATALOG: &[Msg] = &[ ), }, Msg { + key: "settings.recording.echo_cancellation_desc", + text: row( + "保持系统播放声音，同时使用桌面音频输出作为参考抑制回声。部分设备、蓝牙路径或 Linux ALSA 环境可能不可用。", + "保持系統播放聲音，同時使用桌面音訊輸出作為參考抑制回聲。部分裝置、藍牙路徑或 Linux ALSA 環境可能不可用。", + "Use desktop output as a reference while keeping playback audible; some devices, Bluetooth routes, or Linux ALSA setups may be unavailable.", + "再生音を聞きながらデスクトップ出力を参照してエコーを抑制します。デバイス、Bluetooth、Linux ALSA では利用できない場合があります。", + "재생을 계속 들으면서 데스크톱 출력을 참조해 에코를 줄입니다. 일부 장치, Bluetooth 경로 또는 Linux ALSA에서는 사용할 수 없을 수 있습니다.", + ), + }, + Msg { key: "settings.recording.paste_shortcut_desc", text: row( "插入时模拟按下的粘贴键，部分终端类应用需要 Ctrl+Shift+V（仅 Windows / Linux）。", diff --git a/openless-all/app/linux-egui/src/linux_app/settings_save.rs b/openless-all/app/linux-egui/src/linux_app/settings_save.rs index 6bb556f03..ce0a3cf24 100644 --- a/openless-all/app/linux-egui/src/linux_app/settings_save.rs +++ b/openless-all/app/linux-egui/src/linux_app/settings_save.rs @@ -233,6 +233,9 @@ impl OpenLessEguiApp { frontend::view_model::SettingsField::AudioEnhancement => { preferences.audio_enhancement_enabled = !preferences.audio_enhancement_enabled; } + frontend::view_model::SettingsField::EchoCancellation => { + preferences.echo_cancellation_enabled = !preferences.echo_cancellation_enabled; + } frontend::view_model::SettingsField::RecordAudioForDebug => { preferences.record_audio_for_debug = !preferences.record_audio_for_debug; } diff --git a/openless-all/app/linux-egui/src/main.rs b/openless-all/app/linux-egui/src/main.rs index 63fe56c4c..bfa00c512 100644 --- a/openless-all/app/linux-egui/src/main.rs +++ b/openless-all/app/linux-egui/src/main.rs @@ -5874,6 +5874,7 @@ mod linux_app { }; s.mute_while_recording = prefs.mute_during_recording; s.audio_enhancement = prefs.audio_enhancement_enabled; + s.echo_cancellation = prefs.echo_cancellation_enabled; s.audio_dsp = prefs.audio_dsp; s.audio_cue = prefs.audio_cue_on_record; s.launch_at_login = prefs.launch_at_login; diff --git a/openless-all/app/linux-egui/src/ui/frontend/settings.rs b/openless-all/app/linux-egui/src/ui/frontend/settings.rs index 493a62327..a07b21be3 100644 --- a/openless-all/app/linux-egui/src/ui/frontend/settings.rs +++ b/openless-all/app/linux-egui/src/ui/frontend/settings.rs @@ -745,6 +745,17 @@ fn general(ui: &mut egui::Ui, vm: &mut FrontendViewModel, actions: &mut Vec, pub mute_while_recording: bool, pub audio_enhancement: bool, + pub echo_cancellation: bool, pub audio_dsp: openless_core::AudioDspConfig, pub audio_cue: bool, /// 录音胶囊开关与样式（Tauri `capsuleLabel` / `capsuleStyleLabel`）。 @@ -1265,6 +1268,7 @@ impl Default for SettingsFields { microphone_error: None, mute_while_recording: false, audio_enhancement: false, + echo_cancellation: false, audio_dsp: Default::default(), audio_cue: false, show_capsule: true, diff --git a/openless-all/app/scripts/ci-macos-runner-contract.test.mjs b/openless-all/app/scripts/ci-macos-runner-contract.test.mjs new file mode 100644 index 000000000..809edb81c --- /dev/null +++ b/openless-all/app/scripts/ci-macos-runner-contract.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import yaml from 'js-yaml'; + +const workflow = yaml.load( + await readFile(new URL('../../../.github/workflows/ci.yml', import.meta.url), 'utf8'), +); + +const runnerInput = workflow.on.workflow_dispatch.inputs.macos_runner; +assert.equal(runnerInput.type, 'choice'); +assert.equal(runnerInput.default, 'macos-latest'); +assert.deepEqual(runnerInput.options, ['macos-latest', 'macos-15-intel']); + +const crossPlatform = workflow.jobs['cross-platform']; +assert.equal( + crossPlatform['runs-on'], + "${{ matrix.label == 'macOS' && github.event_name == 'workflow_dispatch' && inputs.macos_runner || matrix.os }}", + 'Only manual macOS checks may override the default matrix runner', +); +assert.equal( + workflow.jobs['macos-msrv']['runs-on'], + "${{ github.event_name == 'workflow_dispatch' && inputs.macos_runner || 'macos-latest' }}", + 'MSRV must share the manual fallback and keep its default for push/PR events', +); +assert.ok(crossPlatform.strategy.matrix.include.includes('"os":"windows-latest"')); +assert.equal(crossPlatform.strategy['fail-fast'], false); +for (const gate of [ + 'Build frontend and run frontend/contract tests', + 'Run Codex writable-roots sandbox contract', + 'Run Rust backend unit tests', + 'Run native recorder module tests (Windows)', +]) { + assert.ok( + crossPlatform.steps.some((step) => step.name === gate), + `${gate} must remain enabled`, + ); +} + +console.log('ci-macos-runner-contract.test.mjs passed'); diff --git a/openless-all/app/scripts/macos-runtime-minimum-version-contract.test.mjs b/openless-all/app/scripts/macos-runtime-minimum-version-contract.test.mjs index 0775995e7..fa61da31d 100644 --- a/openless-all/app/scripts/macos-runtime-minimum-version-contract.test.mjs +++ b/openless-all/app/scripts/macos-runtime-minimum-version-contract.test.mjs @@ -1,4 +1,5 @@ import { readFile } from 'node:fs/promises'; +import assert from 'node:assert/strict'; const raw = await readFile(new URL('../src-tauri/tauri.conf.json', import.meta.url), 'utf-8'); const config = JSON.parse(raw); @@ -16,3 +17,15 @@ if (major < 14 || (major === 14 && minor < 0)) { } console.log(`macOS runtime minimum-system-version contract passed: ${minimumSystemVersion}`); + +const plist = await readFile(new URL('../src-tauri/Info.plist', import.meta.url), 'utf8'); +const tap = await readFile( + new URL('../src-tauri/src/playback_reference_macos.m', import.meta.url), + 'utf8', +); +const manifest = await readFile(new URL('../src-tauri/Cargo.toml', import.meta.url), 'utf8'); +assert.match(plist, /NSAudioCaptureUsageDescription<\/key>\s*[^<]+<\/string>/); +assert.match(tap, /@available\(macOS 14\.6, \*\)/); +assert.match(tap, /initWithProcesses:/); +assert.match(tap, /setMuteBehavior:CATapUnmuted/); +assert.match(manifest, /cpal = \{ version = "=0\.15\.3", path = "vendor\/cpal" \}/); diff --git a/openless-all/app/scripts/test-recorder-module.py b/openless-all/app/scripts/test-recorder-module.py index 0e061e778..44db24747 100644 --- a/openless-all/app/scripts/test-recorder-module.py +++ b/openless-all/app/scripts/test-recorder-module.py @@ -30,7 +30,16 @@ thiserror = "1" log = "0.4" """ + "cpal = { version = \"=0.15.3\", path = " + json.dumps((recorder.parents[1] / "vendor/cpal").as_posix()) + " }\n" - + "micyou-dsp = { path = " + json.dumps((recorder.parents[2] / "crates/micyou-dsp").as_posix()) + " }\n") + + "micyou-dsp = { path = " + json.dumps((recorder.parents[2] / "crates/micyou-dsp").as_posix()) + " }\n" + + "[build-dependencies]\ncc = \"1\"\n[target.'cfg(target_os = \"macos\")'.dependencies]\ncoreaudio-sys = \"0.2\"\n") + (root / "build.rs").write_text( + 'fn main() { if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") {\n' + + 'cc::Build::new().file(' + json.dumps(str(recorder.with_name("playback_reference_macos.m"))) + + ').flag("-fobjc-arc").flag("-mmacosx-version-min=14.0").compile("openless_playback_reference");\n' + + 'let compiler=cc::Build::new().get_compiler(); let resource=std::process::Command::new(compiler.path()).arg("-print-resource-dir").output().expect("find Apple compiler runtime"); assert!(resource.status.success());\n' + + 'let runtime=std::path::PathBuf::from(String::from_utf8(resource.stdout).unwrap().trim()).join("lib/darwin"); println!("cargo:rustc-link-search=native={}",runtime.display()); println!("cargo:rustc-link-lib=static=clang_rt.osx");\n' + + 'println!("cargo:rustc-link-lib=framework=Foundation"); println!("cargo:rustc-link-lib=framework=CoreAudio"); } }\n' + ) (root / "src/lib.rs").write_text( "extern crate self as openless_core;\n" "#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackendErrorCode { InvalidArgument }\n" @@ -39,9 +48,10 @@ + "#[path = " + json.dumps(str(recorder.parents[2] / "crates/openless-core/src/audio_dsp.rs"), ensure_ascii=False) + "]\npub mod audio_dsp;\n" "pub use audio_dsp::{AudioDspConfig, AudioDspStage, NoiseReduction};\n" + "#[path = " + json.dumps(str(recorder.parents[2] / "crates/openless-core/src/audio.rs"), ensure_ascii=False) + "]\npub mod audio;\n" - "pub use audio::{AudioProcessingConfig, AudioProcessor};\n" + "pub use audio::{AudioProcessingConfig, AudioProcessor, ReferenceAudioBuffer};\n" "pub trait AudioConsumer: Send + Sync { fn consume_pcm_chunk(&self, pcm: &[u8]); }\n" + "#[path = " + json.dumps(str(recorder), ensure_ascii=False) + "]\npub mod recorder;\n" + + "#[path = " + json.dumps(str(recorder.with_name("native_audio_cue.rs")), ensure_ascii=False) + "]\npub mod native_audio_cue;\n" ) subprocess.run( ["cargo", "check" if check_only else "test", "--manifest-path", str(root / "Cargo.toml"), diff --git a/openless-all/app/src-tauri/Info.plist b/openless-all/app/src-tauri/Info.plist index e0da7d68f..ac22511e2 100644 --- a/openless-all/app/src-tauri/Info.plist +++ b/openless-all/app/src-tauri/Info.plist @@ -2,6 +2,8 @@ + NSAudioCaptureUsageDescription + OpenLess 使用系统播放声音作为临时参考来抑制听写回声，不会保存或上传播放参考。 NSMicrophoneUsageDescription OpenLess 需要麦克风权限来听写你的语音。 NSSpeechRecognitionUsageDescription diff --git a/openless-all/app/src-tauri/build.rs b/openless-all/app/src-tauri/build.rs index 914dac69e..5948fe13c 100644 --- a/openless-all/app/src-tauri/build.rs +++ b/openless-all/app/src-tauri/build.rs @@ -19,6 +19,14 @@ fn main() { // Apple system recognition is independent of downloadable model runtimes. println!("cargo:rustc-link-lib=framework=Speech"); link_macos_compiler_runtime(); + println!("cargo:rerun-if-changed=src/playback_reference_macos.m"); + cc::Build::new() + .file("src/playback_reference_macos.m") + .flag("-fobjc-arc") + .flag("-mmacosx-version-min=14.0") + .compile("openless_playback_reference"); + println!("cargo:rustc-link-lib=framework=Foundation"); + println!("cargo:rustc-link-lib=framework=CoreAudio"); let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); if target_arch == "aarch64" { build_experimental_gemma(); diff --git a/openless-all/app/src-tauri/src/commands/settings.rs b/openless-all/app/src-tauri/src/commands/settings.rs index 2b869abce..b3288cfb5 100644 --- a/openless-all/app/src-tauri/src/commands/settings.rs +++ b/openless-all/app/src-tauri/src/commands/settings.rs @@ -231,6 +231,7 @@ pub(crate) fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, dsp: prefs.audio_dsp, }, ); diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index d3b383201..e59e41bb0 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -150,6 +150,7 @@ pub(crate) fn backend_dependencies( .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, dsp: prefs.audio_dsp, }, ); @@ -2007,6 +2008,7 @@ impl openless_core::VoiceActivationRuntime for TauriVoiceActivationRuntime { ); let audio_processing = openless_core::AudioProcessingConfig { enhancement_enabled: preferences_snapshot.audio_enhancement_enabled, + echo_cancellation_enabled: preferences_snapshot.echo_cancellation_enabled, dsp: preferences_snapshot.audio_dsp, }; if let Some(wake_asr) = wake_asr.as_ref() { @@ -2272,6 +2274,7 @@ impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + echo_cancellation_enabled: prefs.echo_cancellation_enabled, dsp: prefs.audio_dsp, }, ); diff --git a/openless-all/app/src-tauri/src/playback_reference.rs b/openless-all/app/src-tauri/src/playback_reference.rs new file mode 100644 index 000000000..3ed20a61f --- /dev/null +++ b/openless-all/app/src-tauri/src/playback_reference.rs @@ -0,0 +1,128 @@ +//! Playback-only capture, separate from the primary microphone backend. +use super::{CaptureDiagnostics, RecorderError}; +use openless_core::ReferenceAudioBuffer; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::time::Instant; + +pub(super) fn capture_time(info: &cpal::InputCallbackInfo) -> Instant { + let now = Instant::now(); + let timestamp = info.timestamp(); + match timestamp.callback.duration_since(×tamp.capture) { + Some(delay) => now.checked_sub(delay).unwrap_or(now), + None => timestamp + .capture + .duration_since(×tamp.callback) + .and_then(|ahead| now.checked_add(ahead)) + .unwrap_or(now), + } +} + +#[cfg(target_os = "windows")] +pub(super) struct PlaybackReference(cpal::Stream); +#[cfg(target_os = "windows")] +impl Drop for PlaybackReference { + fn drop(&mut self) { + use cpal::traits::StreamTrait; + if let Err(error) = self.0.pause() { + log::warn!("[recorder] echo reference pause failed: {error}"); + } + } +} + +#[cfg(target_os = "windows")] +pub(super) fn start( + stop: &Arc, + diagnostics: &Arc, +) -> Result<(PlaybackReference, Arc), RecorderError> { + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + let device = cpal::default_host() + .default_output_device() + .ok_or_else(|| RecorderError::EngineFailed("no playback output device".into()))?; + let supported = device + .default_output_config() + .map_err(|e| RecorderError::EngineFailed(format!("echo reference output format: {e}")))?; + let config = supported.config(); + let channels = config.channels as usize; + let reference = Arc::new(ReferenceAudioBuffer::new(config.sample_rate.0)); + macro_rules! stream { + ($t:ty,$convert:expr) => {{ + let reference = Arc::clone(&reference); + let error_reference = Arc::clone(&reference); + let stop = Arc::clone(stop); + let callback_diagnostics = Arc::clone(diagnostics); + let error_diagnostics = Arc::clone(diagnostics); + device.build_input_stream( + &config, + move |data: &[$t], info| { + let captured_at = capture_time(info); + if stop.load(Ordering::Acquire) { + return; + } + callback_diagnostics.data_callback(true, true, data.len() / channels.max(1)); + let samples = data.iter().copied().map($convert).collect::>(); + reference.push_interleaved_at(&samples, channels, captured_at); + }, + move |error| { + error_reference.invalidate(); + let count = error_diagnostics.reference.record_error(); + error_diagnostics.event( + "P3", + "[\"H1\",\"H2\"]", + "reference_stream_error", + "reference_fault", + &format!("{{\"error_count\":{count}}}"), + ); + log::warn!("[recorder] echo reference stream error: {error}"); + }, + None, + ) + }}; + } + // WASAPI render endpoints are distinct from input endpoints; CPAL opens + // these with AUDCLNT_STREAMFLAGS_LOOPBACK, never as a microphone. + let stream = match supported.sample_format() { + cpal::SampleFormat::F32 => stream!(f32, |s: f32| s), + cpal::SampleFormat::F64 => stream!(f64, |s: f64| s as f32), + cpal::SampleFormat::I16 => stream!(i16, |s: i16| s as f32 / i16::MAX as f32), + cpal::SampleFormat::I32 => stream!(i32, |s: i32| s as f32 / i32::MAX as f32), + cpal::SampleFormat::I64 => stream!(i64, |s: i64| (s as f64 / i64::MAX as f64) as f32), + cpal::SampleFormat::U8 => stream!(u8, |s: u8| (s as f32 - 128.0) / 128.0), + cpal::SampleFormat::U16 => stream!(u16, |s: u16| (s as f32 - 32768.0) / 32768.0), + cpal::SampleFormat::U32 => stream!(u32, |s: u32| (s as f64 / u32::MAX as f64 * 2.0 - 1.0) + as f32), + cpal::SampleFormat::U64 => stream!(u64, |s: u64| (s as f64 / u64::MAX as f64 * 2.0 - 1.0) + as f32), + cpal::SampleFormat::I8 => stream!(i8, |s: i8| s as f32 / i8::MAX as f32), + _ => { + return Err(RecorderError::EngineFailed( + "unsupported playback reference sample format".into(), + )) + } + } + .map_err(super::classify_build_stream_err)?; + stream + .play() + .map_err(|e| RecorderError::EngineFailed(format!("start playback reference: {e}")))?; + Ok((PlaybackReference(stream), reference)) +} + +#[cfg(target_os = "macos")] +#[path = "playback_reference_macos.rs"] +mod macos; +#[cfg(target_os = "macos")] +pub(super) use macos::{start, PlaybackReference}; + +#[cfg(not(any(target_os = "windows", target_os = "macos")))] +pub(super) struct PlaybackReference; +#[cfg(not(any(target_os = "windows", target_os = "macos")))] +pub(super) fn start( + _stop: &Arc, + _diagnostics: &Arc, +) -> Result<(PlaybackReference, Arc), RecorderError> { + Err(RecorderError::EngineFailed( + "playback reference is unavailable in this host; Linux uses the egui recorder".into(), + )) +} diff --git a/openless-all/app/src-tauri/src/playback_reference_macos.m b/openless-all/app/src-tauri/src/playback_reference_macos.m new file mode 100644 index 000000000..8ffc32fe9 --- /dev/null +++ b/openless-all/app/src-tauri/src/playback_reference_macos.m @@ -0,0 +1,99 @@ +// Keep new tap APIs behind Clang availability checks; ordinary recording still runs on 14.0. +#import +#import +#import +#import +#include +#include + +typedef void (*PlaybackCallback)(void *, const AudioBufferList *, uint64_t); +typedef struct { + AudioObjectID tap; + AudioObjectID aggregate; + AudioDeviceIOProcID proc; + PlaybackCallback callback; + void *context; + mach_timebase_info_data_t timebase; + bool running; +} PlaybackReference; + +static OSStatus playback_io(AudioObjectID device, const AudioTimeStamp *now, + const AudioBufferList *input, const AudioTimeStamp *input_time, + AudioBufferList *output, const AudioTimeStamp *output_time, void *context) { + PlaybackReference *reference=context; + uint64_t host=mach_absolute_time(); + uint64_t captured=(input_time && (input_time->mFlags & kAudioTimeStampHostTimeValid)) ? input_time->mHostTime : host; + uint64_t age=host>captured ? (uint64_t)((long double)(host-captured)*reference->timebase.numer/reference->timebase.denom) : 0; + reference->callback(reference->context,input,age); + return noErr; +} + +void openless_playback_destroy(void *handle) { + if (!handle) return; + PlaybackReference *reference=handle; + if (reference->running) AudioDeviceStop(reference->aggregate,reference->proc); + if (reference->proc) AudioDeviceDestroyIOProcID(reference->aggregate,reference->proc); + if (@available(macOS 14.6, *)) { + if (reference->aggregate) AudioHardwareDestroyAggregateDevice(reference->aggregate); + if (reference->tap) AudioHardwareDestroyProcessTap(reference->tap); + } + free(reference); +} + +void *openless_playback_create(uint32_t *rate, int32_t *status) { + *status=-1; + if (@available(macOS 14.6, *)) { + @autoreleasepool { + PlaybackReference *reference=calloc(1,sizeof(*reference)); + if (!reference) return NULL; + mach_timebase_info(&reference->timebase); + AudioObjectID output=0;UInt32 size=sizeof(output); + AudioObjectPropertyAddress property={kAudioHardwarePropertyDefaultOutputDevice,kAudioObjectPropertyScopeGlobal,kAudioObjectPropertyElementMain}; + *status=AudioObjectGetPropertyData(kAudioObjectSystemObject,&property,0,NULL,&size,&output); + if (*status!=noErr || output==kAudioObjectUnknown) {openless_playback_destroy(reference);return NULL;} + CFStringRef uid=NULL;size=sizeof(uid); + property.mSelector=kAudioDevicePropertyDeviceUID; + *status=AudioObjectGetPropertyData(output,&property,0,NULL,&size,&uid); + if (*status!=noErr || !uid) {openless_playback_destroy(reference);return NULL;} + NSString *device_uid=CFBridgingRelease(uid); + // Unlike CPAL's generic input path, this always taps output, even if + // the selected audio interface also has microphone input channels. + CATapDescription *tap=[[CATapDescription alloc] initWithProcesses:@[] andDeviceUID:device_uid withStream:0]; + [tap setPrivate:YES];[tap setExclusive:YES];[tap setMuteBehavior:CATapUnmuted]; + *status=AudioHardwareCreateProcessTap(tap,&reference->tap); + if (*status!=noErr) {openless_playback_destroy(reference);return NULL;} + AudioStreamBasicDescription format={0};size=sizeof(format); + property.mSelector=kAudioTapPropertyFormat; + *status=AudioObjectGetPropertyData(reference->tap,&property,0,NULL,&size,&format); + if (*status!=noErr || format.mFormatID!=kAudioFormatLinearPCM || !(format.mFormatFlags & kAudioFormatFlagIsFloat) || format.mBitsPerChannel!=32 || format.mSampleRate<=0 || format.mChannelsPerFrame==0) { + if (*status==noErr) *status=kAudioHardwareUnsupportedOperationError; + openless_playback_destroy(reference);return NULL; + } + NSDictionary *properties=@{ + @kAudioAggregateDeviceNameKey: @"OpenLess playback reference", + @kAudioAggregateDeviceUIDKey: [[NSUUID UUID] UUIDString], + @kAudioAggregateDeviceIsPrivateKey: @YES, + @kAudioAggregateDeviceTapAutoStartKey: @YES, + @kAudioAggregateDeviceTapListKey: @[@{ + @kAudioSubTapUIDKey: [[tap UUID] UUIDString], + @kAudioSubTapDriftCompensationKey: @YES + }] + }; + *status=AudioHardwareCreateAggregateDevice((__bridge CFDictionaryRef)properties,&reference->aggregate); + if (*status!=noErr) {openless_playback_destroy(reference);return NULL;} + *rate=(uint32_t)format.mSampleRate; + return reference; + } + } + return NULL; +} + +int32_t openless_playback_start(void *handle, PlaybackCallback callback, void *context) { + PlaybackReference *reference=handle; + reference->callback=callback;reference->context=context; + OSStatus status=AudioDeviceCreateIOProcID(reference->aggregate,playback_io,reference,&reference->proc); + if (status!=noErr) return status; + status=AudioDeviceStart(reference->aggregate,reference->proc); + reference->running=status==noErr; + return status; +} diff --git a/openless-all/app/src-tauri/src/playback_reference_macos.rs b/openless-all/app/src-tauri/src/playback_reference_macos.rs new file mode 100644 index 000000000..1063c7465 --- /dev/null +++ b/openless-all/app/src-tauri/src/playback_reference_macos.rs @@ -0,0 +1,120 @@ +//! An explicit Core Audio output tap, including on duplex audio interfaces. +use super::*; +use coreaudio_sys::AudioBufferList; +use std::ffi::c_void; +use std::time::Duration; + +unsafe extern "C" { + fn openless_playback_create(rate: *mut u32, status: *mut i32) -> *mut c_void; + fn openless_playback_start( + handle: *mut c_void, + callback: extern "C" fn(*mut c_void, *const AudioBufferList, u64), + context: *mut c_void, + ) -> i32; + fn openless_playback_destroy(handle: *mut c_void); +} + +struct CallbackContext { + reference: Arc, + stop: Arc, + diagnostics: Arc, +} + +pub(in super::super) struct PlaybackReference { + handle: *mut c_void, + _context: Box, +} +impl Drop for PlaybackReference { + fn drop(&mut self) { + // Stop and destroy the IOProc before freeing its Rust callback context. + unsafe { + openless_playback_destroy(self.handle); + } + } +} + +extern "C" fn callback(context: *mut c_void, input: *const AudioBufferList, age_ns: u64) { + let context = unsafe { &*(context as *const CallbackContext) }; + if context.stop.load(Ordering::Acquire) || input.is_null() { + return; + } + let captured_at = Instant::now() + .checked_sub(Duration::from_nanos(age_ns)) + .unwrap_or_else(Instant::now); + let input = unsafe { &*input }; + let buffers = unsafe { + std::slice::from_raw_parts(input.mBuffers.as_ptr(), input.mNumberBuffers as usize) + }; + let Some(first) = buffers.first().filter(|buffer| buffer.mNumberChannels > 0) else { + return; + }; + let frames = first.mDataByteSize as usize / 4 / first.mNumberChannels as usize; + let channels = buffers + .iter() + .map(|buffer| buffer.mNumberChannels as usize) + .sum::(); + if channels == 0 { + return; + } + let mut mono = vec![0.0; frames]; + for buffer in buffers { + let count = buffer.mNumberChannels as usize; + if buffer.mData.is_null() + || count == 0 + || buffer.mDataByteSize as usize != frames * count * 4 + { + context.reference.invalidate(); + return; + } + let samples = + unsafe { std::slice::from_raw_parts(buffer.mData.cast::(), frames * count) }; + for (output, frame) in mono.iter_mut().zip(samples.chunks_exact(count)) { + *output += frame + .iter() + .map(|sample| { + if sample.is_finite() { + *sample / channels as f32 + } else { + 0.0 + } + }) + .sum::(); + } + } + context.diagnostics.data_callback(true, true, frames); + context.reference.push_interleaved_at(&mono, 1, captured_at); +} + +pub(in super::super) fn start( + stop: &Arc, + diagnostics: &Arc, +) -> Result<(PlaybackReference, Arc), RecorderError> { + let mut rate = 0; + let mut status = 0; + let handle = unsafe { openless_playback_create(&mut rate, &mut status) }; + if handle.is_null() { + return Err(RecorderError::EngineFailed(format!("system audio tap needs macOS 14.6+ and System Audio Recording permission (status {status})"))); + } + let reference = Arc::new(ReferenceAudioBuffer::new(rate)); + let mut stream = PlaybackReference { + handle, + _context: Box::new(CallbackContext { + reference: Arc::clone(&reference), + stop: Arc::clone(stop), + diagnostics: Arc::clone(diagnostics), + }), + }; + let status = unsafe { + openless_playback_start( + handle, + callback, + (&mut *stream._context as *mut CallbackContext).cast(), + ) + }; + if status != 0 { + return Err(RecorderError::EngineFailed(format!( + "start system audio tap: {status}" + ))); + } + Ok((stream, reference)) +} diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index 40cf087d1..9b437481a 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -125,6 +125,127 @@ fn voice_capture_key(generation: u64) -> u64 { generation | VOICE_CAPTURE_KEY_PREFIX } +/// [DEBUG-ISSUE1223] Per-capture state shared by the two native stream roles. +/// Only the first callback and callbacks immediately following errors are logged. +struct CaptureDiagnostics { + run_id: u64, + microphone: StreamDiagnostics, + reference: StreamDiagnostics, +} + +#[derive(Default)] +struct StreamDiagnostics { + callbacks: AtomicUsize, + errors: AtomicUsize, + error_pending: AtomicBool, +} + +impl StreamDiagnostics { + fn record_error(&self) -> usize { + let count = self.errors.fetch_add(1, Ordering::Relaxed) + 1; + self.error_pending.store(true, Ordering::Release); + count + } + + fn record_callback(&self) -> (usize, bool) { + let count = self.callbacks.fetch_add(1, Ordering::Relaxed) + 1; + let after_error = self.error_pending.swap(false, Ordering::AcqRel); + (count, after_error) + } +} + +impl CaptureDiagnostics { + fn new() -> Self { + static NEXT_RUN: AtomicU64 = AtomicU64::new(1); + Self { + run_id: NEXT_RUN.fetch_add(1, Ordering::Relaxed), + microphone: StreamDiagnostics::default(), + reference: StreamDiagnostics::default(), + } + } + + /// Only code-controlled enums, booleans and numeric facts may enter the payload. + /// Use the existing logger; do not open files on native callback threads. + fn event( + &self, + probe_id: &str, + hypothesis_ids: &str, + event: &str, + outcome: &str, + safe_json: &str, + ) { + let timestamp_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + log::info!( + "[DEBUG-ISSUE1223] {{\"schema\":\"diagnosing-bugs.v1\",\"session_id\":\"s1223-20261019\",\"run_id\":\"pid-{}-capture-{}\",\"probe_id\":\"{}\",\"hypothesis_ids\":{},\"event\":\"{}\",\"outcome\":\"{}\",\"safe\":{},\"ts\":{}}}", + std::process::id(), + self.run_id, + probe_id, + hypothesis_ids, + event, + outcome, + safe_json, + timestamp_ms, + ); + } + + fn data_callback(&self, reference: bool, accepting_audio: bool, frames: usize) { + let stream = if reference { + &self.reference + } else { + &self.microphone + }; + let (count, after_error) = stream.record_callback(); + if count == 1 || after_error { + self.event( + "P5", + "[\"H1\",\"H2\",\"H3\"]", + "data_callback", + if after_error { "after_error" } else { "first_callback" }, + &format!( + "{{\"stream_role\":\"{}\",\"accepting_audio\":{},\"callback_count\":{},\"error_count\":{},\"frames\":{}}}", + if reference { "reference" } else { "microphone" }, + accepting_audio, + count, + stream.errors.load(Ordering::Relaxed), + frames, + ), + ); + } + } +} + +fn handle_microphone_stream_error( + state: &StreamState, + error: cpal::StreamError, + runtime_error_tx: &Sender, +) { + if !state.accepting_audio.load(Ordering::Acquire) { + return; + } + let error_count = state.diagnostics.microphone.record_error(); + // Vendored 0.15 does not turn CoreAudio overload notices into stream errors. + // Preserve its device-loss and backend-error contract. + let kind = match &error { + cpal::StreamError::DeviceNotAvailable => "device_not_available", + cpal::StreamError::BackendSpecific { .. } => "backend_error", + }; + state.diagnostics.event( + "P2", + "[\"H1\",\"H2\",\"H3\"]", + "microphone_stream_error", + "fatal_candidate", + &format!("{{\"error_kind\":\"{kind}\",\"error_count\":{error_count}}}"), + ); + log::error!("[recorder] stream error: {error}"); + let _ = runtime_error_tx.send(RecorderError::EngineFailed(format!("stream: {error}"))); +} + +#[path = "playback_reference.rs"] +mod playback_reference; + /// Downstream that receives resampled Int16 PCM bytes (little-endian). pub trait AudioConsumer: Send + Sync { /// Each delivery is a little-endian byte sequence of Int16 samples; @@ -531,7 +652,8 @@ impl WarmCaptureSink { } fn set_activation_consumer(&self, consumer: Option>) { - self.activation_consumer_during_capture.store(false, Ordering::Release); + self.activation_consumer_during_capture + .store(false, Ordering::Release); *self.activation_consumer.lock() = consumer; self.activation_vad.lock().reset(); } @@ -592,7 +714,10 @@ impl WarmCaptureSink { ) { let even_len = pcm.len() & !1; let incoming = &pcm[..even_len]; - let excess = buffer.len().saturating_add(incoming.len()).saturating_sub(max_bytes); + let excess = buffer + .len() + .saturating_add(incoming.len()) + .saturating_sub(max_bytes); let evictable_preroll = excess.min(*pre_roll_bytes); for _ in 0..(evictable_preroll / 2) { let _ = buffer.pop_front(); @@ -827,8 +952,8 @@ impl AudioConsumer for WarmCaptureSink { && self .activation_consumer_during_capture .load(Ordering::Acquire))) - .then(|| self.activation_consumer.lock().clone()) - .flatten(); + .then(|| self.activation_consumer.lock().clone()) + .flatten(); for target in targets { CaptureSink::deliver(&target, pcm); } @@ -1068,7 +1193,10 @@ impl WarmCaptureHub { pub(crate) fn clear_activation_fault_handler_if(&self, generation: u64) { let mut handler = self.sink.activation_fault_handler.lock(); - if handler.as_ref().is_some_and(|(current, _)| *current == generation) { + if handler + .as_ref() + .is_some_and(|(current, _)| *current == generation) + { *handler = None; } } @@ -1827,6 +1955,35 @@ fn run_audio_thread( startup_tx: Sender>, runtime_error_tx: Sender, ) { + let (reference_stream, reference) = if processing.echo_cancellation_enabled { + match playback_reference::start(&stop_flag, &state.diagnostics) { + Ok((stream, buffer)) => (Some(stream), Some(buffer)), + Err(error) => { + log::warn!( + "[recorder] playback reference unavailable; recording without AEC: {error}" + ); + (None, None) + } + } + } else { + (None, None) + }; + state.diagnostics.event( + "P1", + "[\"H1\",\"H2\",\"H3\"]", + "reference_start", + if reference.is_some() { + "available" + } else { + "fallback" + }, + &format!( + "{{\"aec_requested\":{},\"reference_available\":{}}}", + processing.echo_cancellation_enabled, + reference.is_some() + ), + ); + *state.reference.lock() = reference; let (processor, frame_tx) = match AudioProcessingWorker::start( processing, consumer, @@ -1934,6 +2091,7 @@ fn run_audio_thread( "[recorder] watchdog: 录音回调已停止 {} 秒，触发错误恢复", elapsed.as_secs() ); + state_for_watchdog.diagnostics.event("P4", "[\"H4\"]", "watchdog_fault", "callback_silent", "{}"); let _ = runtime_error_tx_for_watchdog.send(RecorderError::EngineFailed( format!("录音回调静默停止 {} 秒", elapsed.as_secs()), @@ -1949,6 +2107,7 @@ fn run_audio_thread( "[recorder] watchdog: {} 秒内未收到首次回调，触发错误恢复", elapsed.as_secs() ); + state_for_watchdog.diagnostics.event("P4", "[\"H4\"]", "watchdog_fault", "first_callback_timeout", "{}"); let _ = runtime_error_tx_for_watchdog.send(RecorderError::EngineFailed( format!("录音启动后 {} 秒内未收到回调", elapsed.as_secs()), @@ -1981,6 +2140,7 @@ fn run_audio_thread( log::warn!("[recorder] cpal Stream pause before drop failed: {err}"); } drop(stream); + drop(reference_stream); log::info!("[recorder] cpal Stream dropped (mic released)"); *state.processor_sender.lock() = None; drop(frame_tx); @@ -2246,29 +2406,27 @@ fn build_stream_for_format( let runtime_error_for_error = runtime_error_tx.clone(); let error_state = Arc::clone(&state); let err_cb = move |err| { - if !error_state.accepting_audio.load(Ordering::Acquire) { - return; - } - log::error!("[recorder] stream error: {err}"); - let _ = runtime_error_for_error - .send(RecorderError::EngineFailed(format!("stream: {err}"))); + handle_microphone_stream_error(&error_state, err, &runtime_error_for_error); }; device .build_input_stream::<$t, _, _>( config, - move |data: &[$t], _info| { + move |data: &[$t], info| { if !state.accepting_audio.load(Ordering::Acquire) { return; } if !data.is_empty() { state.note_first_pcm(false); } + let captured_at = playback_reference::capture_time(info); + state.diagnostics.data_callback(false, true, data.len() / channels.max(1)); let mut floats = Vec::with_capacity(data.len()); for s in data { floats.push($to_f32(*s)); } *state.last_callback_time.lock() = Some(std::time::Instant::now()); match frame_tx.try_send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at, samples: floats, channels, input_sample_rate: input_sr, @@ -2283,6 +2441,7 @@ fn build_stream_for_format( stop_flag.store(true, Ordering::Release); } Err(TrySendError::Full(_)) => { + state.diagnostics.event("P4", "[\"H4\"]", "processing_backpressure", "queue_full", "{}"); if state.accepting_audio.swap(false, Ordering::AcqRel) { let _ = runtime_error_for_callback.send( RecorderError::EngineFailed( @@ -2330,6 +2489,7 @@ fn build_stream_for_format( } struct RawAudioFrame { + captured_at: std::time::Instant, samples: Vec, channels: usize, input_sample_rate: u32, @@ -2489,10 +2649,13 @@ impl AudioProcessingWorker { } enhancement_enabled = false; } - if let Some(chunk) = processor.process( + let reference = state.reference.lock().clone(); + let render = reference.and_then(|buffer| buffer.take_for_input_at(frame.samples.len()/frame.channels.max(1), frame.input_sample_rate, frame.captured_at)); + if let Some(chunk) = processor.process_with_reference( &frame.samples, frame.channels, frame.input_sample_rate, + render.as_deref(), ) { deliver(chunk); } @@ -2543,6 +2706,8 @@ impl AudioProcessingWorker { /// State carried across callbacks: resample leftovers, diagnostic counters and peak. struct StreamState { + diagnostics: Arc, + reference: Mutex>>, capture_id: u64, created_at: std::time::Instant, first_raw_pcm_us: AtomicU64, @@ -2566,6 +2731,8 @@ impl StreamState { fn new() -> Self { Self { capture_id: NEXT_CAPTURE_ID.fetch_add(1, Ordering::Relaxed), + diagnostics: Arc::new(CaptureDiagnostics::new()), + reference: Mutex::new(None), created_at: std::time::Instant::now(), first_raw_pcm_us: AtomicU64::new(0), first_output_pcm_us: AtomicU64::new(0), @@ -2923,6 +3090,7 @@ mod tests { }; let frame = || { AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.25; samples], channels: 1, input_sample_rate: 48_000, @@ -2958,6 +3126,7 @@ mod tests { backlog: Arc::new(AudioProcessingBacklog::default()), }; let frame = AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.0; 48_000 * 64], channels: 64, input_sample_rate: 48_000, @@ -2999,6 +3168,7 @@ mod tests { block_next_level.store(true, Ordering::Release); sender .send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.25; 997], channels: 1, input_sample_rate: TARGET_SAMPLE_RATE, @@ -3011,6 +3181,7 @@ mod tests { for _ in 0..100 { assert!(sender .try_send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.5; 160], channels: 1, input_sample_rate: TARGET_SAMPLE_RATE, @@ -3045,6 +3216,45 @@ mod tests { assert_eq!(sender.backlog.bytes.load(Ordering::Acquire), 0); } + #[test] + fn playback_reference_uses_capture_times_after_an_early_start() { + let epoch=std::time::Instant::now(); + let state=Arc::new(StreamState::new()); + let reference=Arc::new(openless_core::ReferenceAudioBuffer::new(16_000)); + reference.push_interleaved_at(&[0.9;160],1,epoch); + reference.push_interleaved_at(&[0.0;1280],1,epoch+std::time::Duration::from_millis(10)); + *state.reference.lock()=Some(reference); + let consumer=Arc::new(RecordingConsumer::default()); + let (worker,sender)=AudioProcessingWorker::start( + openless_core::AudioProcessingConfig {echo_cancellation_enabled:true,..Default::default()}, + Arc::clone(&consumer) as Arc,Arc::new(|_|{}),None,state, + ).unwrap(); + for index in 1..=8 { + sender.send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at:epoch+std::time::Duration::from_millis(index*10),samples:vec![0.25;160],channels:1,input_sample_rate:16_000, + })).unwrap(); + } + drop(sender);worker.finish(); + let chunks=consumer.chunks.lock().unwrap();assert_eq!(chunks.len(),8); + assert!(chunks.iter().all(|chunk|decode_i16_le(chunk)==vec![8191;160])); + } + + #[test] + fn diagnostics_and_microphone_errors_stay_capture_scoped() { + let first=StreamState::new();let second=StreamState::new(); + assert_ne!(first.diagnostics.run_id,second.diagnostics.run_id); + let (sender,receiver)=channel(); + first.accepting_audio.store(false,Ordering::Release); + handle_microphone_stream_error(&first,cpal::StreamError::DeviceNotAvailable,&sender); + assert!(receiver.try_recv().is_err());assert_eq!(first.diagnostics.microphone.errors.load(Ordering::Relaxed),0); + first.accepting_audio.store(true,Ordering::Release); + handle_microphone_stream_error(&first,cpal::StreamError::DeviceNotAvailable,&sender); + assert!(matches!(receiver.try_recv(),Ok(RecorderError::EngineFailed(_)))); + assert_eq!(first.diagnostics.microphone.record_callback(),(1,true)); + assert_eq!(first.diagnostics.reference.record_callback(),(1,false)); + assert_eq!(second.diagnostics.microphone.record_callback(),(1,false)); + } + #[test] fn audio_processing_worker_drains_accepted_frames_before_join() { let consumer = Arc::new(RecordingConsumer::default()); @@ -3062,6 +3272,7 @@ mod tests { .unwrap(); sender .send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.25, -0.25], channels: 1, input_sample_rate: TARGET_SAMPLE_RATE, @@ -3100,6 +3311,7 @@ mod tests { .unwrap(); sender .send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.15; 317], channels: 1, input_sample_rate: TARGET_SAMPLE_RATE, @@ -3122,6 +3334,7 @@ mod tests { ); sender .send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![-0.15; 211], channels: 1, input_sample_rate: TARGET_SAMPLE_RATE, @@ -3160,6 +3373,7 @@ mod tests { for _ in 0..8 { sender .send(AudioProcessingMessage::Frame(RawAudioFrame { + captured_at: std::time::Instant::now(), samples: vec![0.25, -0.25], channels: 1, input_sample_rate: TARGET_SAMPLE_RATE, @@ -3628,8 +3842,14 @@ mod tests { sink.set_activation_consumer(None); sink.consume_pcm_chunk(&[5, 6]); - assert_eq!(consumer.chunks.lock().unwrap().as_slice(), &[vec![1, 2, 3, 4]]); - assert!(matches!(&*sink.state.lock(), WarmCaptureSinkState::Ring { .. })); + assert_eq!( + consumer.chunks.lock().unwrap().as_slice(), + &[vec![1, 2, 3, 4]] + ); + assert!(matches!( + &*sink.state.lock(), + WarmCaptureSinkState::Ring { .. } + )); } #[test] diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index 83ca7fdbd..ad0cd7975 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1602,6 +1602,9 @@ export const de: typeof zhCN = { down: 'Move down', reset: 'Reset enhancement settings', }, + echoCancellationLabel: 'Wiedergabe-Echounterdrückung', + echoCancellationDesc: + 'Verwendet eine lokale Wiedergabereferenz, ohne die Wiedergabe stummzuschalten. macOS benötigt 14.6+ und die Berechtigung zur Systemaudioaufnahme. Nicht unterstützte Geräte, Bluetooth-Routen und Linux ALSA verwenden nur das Mikrofon. Gilt ab der nächsten Aufnahme.', audioEnhancementLabel: 'Sprachverbesserung', audioEnhancementDesc: 'RNNoise- oder PureVox-Rauschunterdrückung, automatische Verstärkung und Begrenzung. Standardmäßig aus; Änderungen gelten ab der nächsten Aufnahme.', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index cabdfbd89..ffb814568 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1572,6 +1572,9 @@ export const en: typeof zhCN = { down: 'Move down', reset: 'Reset enhancement settings', }, + echoCancellationLabel: 'Playback echo cancellation', + echoCancellationDesc: + 'Uses a local playback reference while keeping playback audible. On macOS, requires 14.6+ and System Audio Recording permission. Unsupported devices, Bluetooth routes, or Linux ALSA use microphone-only recording. Applies to the next capture.', audioEnhancementLabel: 'Speech enhancement', audioEnhancementDesc: 'Apply RNNoise or PureVox noise reduction, automatic gain and limiting. Off by default; changes apply to the next recording.', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 0378847ea..ad2b8b5f8 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1593,6 +1593,9 @@ export const es: typeof zhCN = { down: 'Move down', reset: 'Reset enhancement settings', }, + echoCancellationLabel: 'Cancelación del eco de reproducción', + echoCancellationDesc: + 'Usa una referencia local sin silenciar la reproducción. En macOS requiere 14.6+ y permiso para grabar audio del sistema. Los dispositivos, rutas Bluetooth o entornos Linux ALSA no compatibles usan solo el micrófono. Se aplica en la próxima captura.', audioEnhancementLabel: 'Mejora de voz', audioEnhancementDesc: 'Reducción de ruido RNNoise o PureVox, ganancia automática y limitación. Desactivada por defecto; los cambios se aplican a la próxima grabación.', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index b9487f656..89873a641 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1613,6 +1613,9 @@ export const fr: typeof zhCN = { down: 'Move down', reset: 'Reset enhancement settings', }, + echoCancellationLabel: 'Annulation de l’écho de lecture', + echoCancellationDesc: + 'Utilise une référence de lecture locale sans couper le son. Sur macOS, nécessite 14.6+ et une autorisation de capture audio système. Les appareils, routes Bluetooth ou configurations Linux ALSA non pris en charge utilisent uniquement le microphone. Prend effet à la prochaine capture.', audioEnhancementLabel: 'Amélioration de la voix', audioEnhancementDesc: 'Réduction du bruit RNNoise ou PureVox, gain automatique et limitation. Désactivée par défaut ; les changements prennent effet au prochain enregistrement.', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index c1d09e207..ade100dea 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1560,6 +1560,9 @@ export const ja: typeof zhCN = { down: 'Move down', reset: 'Reset enhancement settings', }, + echoCancellationLabel: '再生エコーキャンセル', + echoCancellationDesc: + '再生音を止めずにローカルの再生音を参照してエコーを抑制します。macOS は 14.6 以降とシステム音声の収録許可が必要です。非対応デバイス、Bluetooth 経路、Linux ALSA では通常のマイク録音に戻ります。次の収録開始時に適用されます。', audioEnhancementLabel: '音声強調', audioEnhancementDesc: 'RNNoise または PureVox のノイズ抑制、自動ゲイン、リミッターを適用します。既定ではオフ。変更は次の録音から有効です。', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index dff252b94..4c970c55b 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1547,6 +1547,9 @@ export const ko: typeof zhCN = { down: 'Move down', reset: 'Reset enhancement settings', }, + echoCancellationLabel: '재생 에코 제거', + echoCancellationDesc: + '재생을 유지하면서 로컬 재생음을 참조합니다. macOS는 14.6 이상 및 시스템 오디오 녹음 권한이 필요합니다. 지원하지 않는 장치, Bluetooth 경로 또는 Linux ALSA에서는 일반 마이크 녹음으로 전환합니다. 다음 캡처부터 적용됩니다.', audioEnhancementLabel: '음성 향상', audioEnhancementDesc: 'RNNoise 또는 PureVox 소음 억제, 자동 게인 및 리미터를 적용합니다. 기본값은 꺼짐이며 다음 녹음부터 적용됩니다.', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 6b92f58ea..4666e8162 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1494,6 +1494,9 @@ export const zhCN = { down: '下移', reset: '恢复增强默认设置', }, + echoCancellationLabel: '播放回声抑制', + echoCancellationDesc: + '保持系统声音播放，使用本机播放参考抑制回声。macOS 需 14.6 或更新版本及系统音频录制权限；不支持的设备、蓝牙路径或 Linux ALSA 环境回退为普通麦克风录音。下次开始采集时生效。', audioEnhancementLabel: '语音增强', audioEnhancementDesc: '应用 RNNoise 或 PureVox 降噪、自动增益和限幅。默认关闭，设置在下一次录音生效。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 533136f34..ed0393c74 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1496,6 +1496,9 @@ export const zhTW: typeof zhCN = { down: '下移', reset: '恢復增強預設設定', }, + echoCancellationLabel: '播放回聲抑制', + echoCancellationDesc: + '保持系統聲音播放，使用本機播放參考抑制回聲。macOS 需 14.6 或更新版本及系統音訊錄製權限；不支援的裝置、藍牙路徑或 Linux ALSA 環境改用一般麥克風錄音。下次開始擷取時生效。', audioEnhancementLabel: '語音增強', audioEnhancementDesc: '套用 RNNoise 或 PureVox 降噪、自動增益和限幅。預設關閉，設定於下一次錄音生效。', diff --git a/openless-all/app/src/lib/ipc/mock-data.ts b/openless-all/app/src/lib/ipc/mock-data.ts index 29dbcc579..48a7178bb 100644 --- a/openless-all/app/src/lib/ipc/mock-data.ts +++ b/openless-all/app/src/lib/ipc/mock-data.ts @@ -51,6 +51,7 @@ export let mockSettings: UserPreferences = { lowLatencyDictationEnabled: false, lowMemoryBackgroundEnabled: false, audioEnhancementEnabled: false, + echoCancellationEnabled: false, voiceActivationEnabled: false, voiceActivationWakeEnabled: true, voiceActivationFinishEnabled: true, diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index e3f0f0e66..34042ce47 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -392,6 +392,8 @@ export interface UserPreferences { lowLatencyDictationEnabled: boolean; /** Enable the session-frozen speech enhancement chain. Disabled by default. */ audioEnhancementEnabled: boolean; + /** Use a local playback reference for optional echo cancellation. Off by default. */ + echoCancellationEnabled: boolean; audioDsp?: AudioDspConfig; /** Desktop resource policy, applied after restarting the application. */ lowMemoryBackgroundEnabled: boolean; diff --git a/openless-all/app/src/pages/settings/RecordingInputSection.tsx b/openless-all/app/src/pages/settings/RecordingInputSection.tsx index 23373d44f..21631bfd5 100644 --- a/openless-all/app/src/pages/settings/RecordingInputSection.tsx +++ b/openless-all/app/src/pages/settings/RecordingInputSection.tsx @@ -275,6 +275,8 @@ export function RecordingInputSection({ savePrefs({ ...prefs, lowLatencyDictationEnabled }); const onAudioEnhancementChange = (audioEnhancementEnabled: boolean) => savePrefs({ ...prefs, audioEnhancementEnabled }); + const onEchoCancellationChange = (echoCancellationEnabled: boolean) => + savePrefs({ ...prefs, echoCancellationEnabled }); const onMicrophoneDeviceChange = (microphoneDeviceName: string) => savePrefs({ ...prefs, microphoneDeviceName }); const onRestoreClipboardChange = (restoreClipboardAfterPaste: boolean) => @@ -484,6 +486,17 @@ export function RecordingInputSection({ )} + {platformCaps?.platform === 'desktop' && ( + + + + )} <>