diff --git a/.gitattributes b/.gitattributes index 863404ed5..205d91a81 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,9 @@ *.xml text eol=lf openless-all/app/linux-egui/packaging/AppRun text eol=lf openless-all/app/linux-egui/packaging/openless-desktop-integration text eol=lf +# Keep imported model bytes and unified-diff context markers intact. +*.mcy binary +openless-all/app/scripts/micyou-dsp.patch text eol=lf whitespace=-blank-at-eol +# These two notices retain upstream's exact final blank line. +openless-all/app/crates/micyou-dsp/LICENSE text eol=lf whitespace=-blank-at-eof +openless-all/app/crates/micyou-infer/LICENSE text eol=lf whitespace=-blank-at-eof diff --git a/.github/workflows/sync-micyou.yml b/.github/workflows/sync-micyou.yml new file mode 100644 index 000000000..46fec5f49 --- /dev/null +++ b/.github/workflows/sync-micyou.yml @@ -0,0 +1,105 @@ +name: Sync MicYou DSP + +on: + schedule: + - cron: "17 2 * * *" + workflow_dispatch: + inputs: + revision: + description: Optional immutable MicYou commit SHA (empty checks master) + type: string + +permissions: + contents: read + +concurrency: + group: micyou-upstream-sync + cancel-in-progress: false + +jobs: + candidate: + if: github.event_name == 'workflow_dispatch' || github.repository == 'Open-Less/openless' + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + changed: ${{ steps.sync.outputs.changed }} + revision: ${{ steps.sync.outputs.revision }} + steps: + - uses: actions/checkout@v4 + with: + ref: beta + persist-credentials: false + - name: Verify importer and detect relevant upstream changes + id: sync + env: + REVISION: ${{ inputs.revision }} + run: | + python3 openless-all/app/scripts/test_sync_micyou.py + args=() + if [ -n "$REVISION" ]; then args+=(--revision "$REVISION"); fi + python3 openless-all/app/scripts/sync-micyou.py --sync "${args[@]}" --github-output "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@stable + if: steps.sync.outputs.changed == 'true' + - name: Validate models, callback boundaries and the full shared core + if: steps.sync.outputs.changed == 'true' + run: | + cargo test --locked --manifest-path openless-all/app/Cargo.toml -p micyou-dsp --release + cargo test --locked --manifest-path openless-all/app/Cargo.toml -p openless-core --lib -- --test-threads=1 + # Exact paths only. The publish job never compiles candidate code and + # receives only source/model imports plus their checksum manifest. + - uses: actions/upload-artifact@v4 + if: steps.sync.outputs.changed == 'true' + with: + name: micyou-candidate + retention-days: 7 + path: | + openless-all/app/crates/micyou-dsp/src/lib.rs + openless-all/app/crates/micyou-dsp/src/assets/purevox6.mcy + openless-all/app/crates/micyou-dsp/src/assets/aec7.mcy + openless-all/app/crates/micyou-dsp/LICENSE + openless-all/app/crates/micyou-dsp/LICENSE-PureVox.txt + openless-all/app/crates/micyou-dsp/LICENSE-AEC7.txt + openless-all/app/crates/micyou-infer/src/lib.rs + openless-all/app/crates/micyou-infer/src/ops.rs + openless-all/app/crates/micyou-infer/LICENSE + openless-all/app/scripts/micyou-upstream.json + + publish: + needs: candidate + if: needs.candidate.outputs.changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + ref: beta + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: micyou-candidate + path: openless-all/app + - name: Verify imported file checksums (no candidate code execution) + run: python3 openless-all/app/scripts/sync-micyou.py --verify + - uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: beta + branch: fix/micyou-upstream-sync + title: "chore(audio): sync MicYou DSP fixes" + commit-message: "chore(audio): sync MicYou to ${{ needs.candidate.outputs.revision }}" + draft: true + add-paths: | + openless-all/app/crates/micyou-dsp + openless-all/app/crates/micyou-infer + openless-all/app/scripts/micyou-upstream.json + body: | + Import relevant MicYou source/model changes and reapply the reviewed OpenLess DSP adapter. + + Upstream commit: https://github.com/MicYou-Dev/MicYou/commit/${{ needs.candidate.outputs.revision }} + + Automated validation: model property tests and full shared Core regression. + This is a draft update; native platform CI and real microphone listening still require review. This workflow does not enable auto-merge. + PRs created with GITHUB_TOKEN do not automatically trigger the separate PR workflows; run the normal CI workflow on this branch before merging. diff --git a/.gitignore b/.gitignore index 577a97949..3044965fb 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ CapsWriter # Local Linux toolchains, build logs and packaging workspaces. openless-all/app/.cache/ +__pycache__/ diff --git a/docs/audio-startup-latency.md b/docs/audio-startup-latency.md new file mode 100644 index 000000000..50a03b41b --- /dev/null +++ b/docs/audio-startup-latency.md @@ -0,0 +1,59 @@ +# 录音优先与启动延迟 + +目标：快捷键按下后尽早采集麦克风；识别器或处理线程暂时未就绪时，保存已经采到的 +PCM，而不是等待识别器后再开麦。缓冲按需增长、保留原有上限，不新增常驻录音或 +第二套大队列。 + +## 与现有代码的区别 + +[用户提供的讨论](https://chatgpt.com/share/6ad78b77-04b8-83e8-a106-bb97ea5acb5c) +区分了设备冷启动和 ASR 网络等待。当前 beta 已有按键时的 Windows provisional +capture，以及默认关闭的预热/预录选项；#1150 已在 2026-10-12 关闭。因此不能将 +讨论里的早期流程直接当作当前实现。 + +仍有可移走的前置步骤：`PipelineDictationEngine::start` 在提交 provisional capture +或调用普通 recorder 之前等待 `transcription.prepare()`。改为： + +```text +快捷键 / 正式启动 + → 打开或提交已有的麦克风采集 + → PCM 写入现有 BufferedTranscriptionSession + → 准备识别器 + → 建连、按原顺序交付暂存 PCM + → 直接交付后续 PCM +``` + +`CaptureFirstPreparation` 只缓存一次准备结果，复用原有的 PCM 缓冲。 +稳定模式仍在启动返回成功前完成配置准备，但等停止录音后才连接识别器。 +准备失败或用户取消时，会停止已启动的录音并清理会话。 + +这项调整对 Windows 的意义不只是提前调用 recorder：把 provisional capture 先提交 +到正式缓冲，也能避免慢准备一直占用原来较小的暂存窗口。 + +## 资源边界 + +- 不默认开启 `lowLatencyDictationEnabled`；不扩大 500 ms 的既有预录选项。 +- 没有新增 PCM 队列；复用原有 `BufferedTranscriptionSession` 和处理工作线程。 +- 等待期间的 16 kHz、单声道、16-bit PCM 每秒约 32 KB，按实际采集量保存； + 缓冲不是启动时一次性分配到上限。达到既有上限仍走停止/错误路径。 +- DSP 在音频工作线程中运行。设备回调的工作是格式转换和向既有有界队列交付。 +- 这不恢复硬件开始提供数据之前的声音，也没有宣称消除 Windows 驱动冷启动。 + +## 诊断 + +Tauri 录音日志增加 `[audio-startup]` 阶段，只有时间和录音/按键代号，不包含录音内容： + +- `press` 与 `arm_done_us`：从实际按键到捕获路径建立完成。 +- `capture`：关联同一次原生流的各阶段。 +- `device_selected`：设备选择耗时。 +- `stream_built` / `stream_playing`：建流、play 耗时及距打开入口的时间。 +- `first_raw_pcm`：第一个被接受的原生 PCM 回调，包括静音。 +- `first_output_pcm`：第一个可交付给识别器/归档的处理结果。 + +比较 `first_raw_pcm` 与 `first_output_pcm` 可以区分硬件等待与处理/分帧等待。 +Windows 快捷键日志使用 `press` 和 `capture` 关联；预热路线标记为 `warm_hub`。 + +新增回归测试会故意阻塞识别器准备，验证麦克风已开始、开头 PCM 仍能顺序交付， +并验证准备期间的取消无需等待准备完成即可释放麦克风。另验证准备失败清理。 + +本轮未在原生 Windows 麦克风上测量优化前后的 P50/P95，不报告未经测量的毫秒收益。 diff --git a/docs/issue-1026-micyou-audio.md b/docs/issue-1026-micyou-audio.md new file mode 100644 index 000000000..938993700 --- /dev/null +++ b/docs/issue-1026-micyou-audio.md @@ -0,0 +1,114 @@ +# Issue #1026：MicYou 音频增强接入审查 + +审查日期：2026-10-20。工作分支：`fix/issue-1026-micyou-audio`，基于 `origin/beta` 的 `101f5142`。 + +## 改动前实现 + +Issue：https://github.com/Open-Less/openless/issues/1026 + +关闭该 issue 的 PR：https://github.com/Open-Less/openless/pull/1194 + +`openless-all/app/crates/openless-core/src/audio.rs` 中的 `NoiseSuppressor` +按回调音频块的下混 RMS 估计底噪，通过迟滞、保持时间和增益平滑控制衰减。 +这是噪声门类型的处理；语音打开门后，不会从语音中分离背景噪声。 +`AudioEnhancer` 随后施加固定 1.5 倍增益和软限幅，没有自动增益控制。 +当前路径由 Tauri 与 Linux egui 的本机录音适配器调用，不能据此认定移动端也已覆盖。 + +## 上游核对 + +MicYou：https://github.com/MicYou-Dev/MicYou + +核对版本：`d821da032e2b72330ced09678dd127acd2714bef`。 + +源码：`tauri-app/crates/micyou-audio/src/dsp.rs` 与该 crate 的 `Cargo.toml`。 + +上游支持 RNNoise（`nnnoiseless`）、PureVox 模型降噪、自动增益、VAD、 +均衡器、去混响和需要远端参考音频的 AEC。PureVox/AEC 使用 `micyou-infer` +及嵌入模型，并非仅复制一个函数就能使用。 + +需要区别功能名称与算法：上游 `Speexdsp` 分支调用自己的 `SpeexStyleNS`； +不能仅凭名称推断已接入官方 SpeexDSP。去混响分支标注为延迟线梳状滤波。 + +RNNoise 每帧 480 点，处理链以 48 kHz 为工作采样率；OpenLess 输出契约则为 +16 kHz 单声道 PCM。上游 `process_rnnoise_single_channel` 会将不足一帧的余数 +原样输出并清空。上游主链在调用前已聚齐完整帧；移植时必须保留这层契约， +不能孤立搬走辅助函数。 + +## 建议的实现顺序 + +1. 将处理链明确为：输入下混 → 有状态重采样到 48 kHz → 固定 10 ms 分帧 → + 模型降噪 → 有上限的平滑 AGC → 限幅 → 有状态重采样到 16 kHz。 +2. 先完成 RNNoise 后端、跨回调余数缓存和停止录音时的收尾协议，再接入 PureVox。 + 收尾需要处理模型延迟，记录有效样本数，防止补零增加录音长度或丢失尾音。 +3. 将算法与强度、AGC 设置纳入会话冻结配置，并接通持久化、Tauri 和 egui 设置。 + 默认关闭；关闭时沿用原有音频输出兼容性。 +4. PureVox 单独核对模型许可、资源体积、推理负载与各平台构建；模型不可用时需要 + 明确反馈或明确记录降级结果。复用代码保留上游版权和许可说明。 +5. AEC 需要扬声器参考流与同步。当前本机录音入口只有麦克风流，先完成参考流设计， + 再提供可用的 AEC 开关。移动端需要逐个核实实际录音链后接入。 + +## 验证要求 + +- 相同输入以不同回调长度送入，输出长度、内容和收尾保持一致。 +- 覆盖 16/44.1/48 kHz、单双声道、短于一帧、非有限数值与会话重置。 +- 对同一组干净语音和语音混噪样本比较关闭、旧实现、RNNoise 与 PureVox。 + 记录噪声衰减、语音损伤、识别字错率、首字/尾字保留和处理耗时。 +- 分别检查实时识别、最终录音与历史回听，以及 Tauri、egui、移动端实际覆盖情况。 + +## 已落地的实现 + +- 用固定版本 MicYou DSP 与纯 Rust 推理模块替换旧噪声门路径，保留原版权和许可证。 +- 支持 RNNoise、PureVox 6、基础频谱减法及关闭降噪；可调强度、手动增益、AGC + 目标与速度、语音门限、十段 EQ、梳状滤波、处理阶段顺序及一键恢复默认值。 +- 增强总开关仍默认关闭。开启后的默认方案为 RNNoise 80% + AGC，EQ、梳状滤波和 + 语音门限默认关闭。设置按录音会话冻结。 +- 下混、抗混叠重采样、480 点分帧、干湿信号延迟对齐、模型收尾与有效样本数限制。 + 不复用会增删样本的播放缓冲管理。 +- Tauri、egui 与外部 PCM 录音入口共用处理链；鸿蒙主应用到 IME 的语音设置交换 + 包含新参数。实时识别与归档消费同一份处理后的 PCM。 +- 增加录音停止和暖录音会话边界的收尾，避免丢失不足一帧的尾音。 +- 对启用降噪后的 AGC 增益加入活动约束：RNNoise 使用语音概率，PureVox 使用 + 对齐后的湿/干信号能量比，避免 AGC 重新放大被模型拒绝的底噪。 + +AEC7 模型和带参考信号的 DSP 接口已保留并通过合成回声测试。目前宿主没有同步 +扬声器参考采集，因此应用中未启用 AEC，也不提供无效开关。网络传输、虚拟声卡、 +插件宿主和播放缓冲属于 MicYou 的麦克风转发产品功能，不适合直接并入识别录音链。 + +## 验证与限制 + +- 最终 Core 全量回归：1326 通过、3 忽略，包含音频、归档、配置和采集优先的回归。 +- 音频分块一致性覆盖 8/16/44.1/48/96 kHz，含 RNNoise、PureVox、频谱减法和无降噪。 +- 验证短录音、尾部脉冲、非有限数值、立体声跨回调余数、重置和重复停止。 +- 增加默认 AGC 与降噪组合的底噪回归：修复前合成样本输出/输入功率约 0.875； + 修复后 RNNoise 与 PureVox 均通过输出功率小于输入 25% 的断言。 +- 生产 Tauri 录音模块通过独立宿主测试；egui 检查、鸿蒙宿主交叉检查、鸿蒙配置 + 沙箱测试通过。DSP 引擎 Windows、Android、鸿蒙交叉检查通过。 +- 前端生产构建与完整 `npm test` 的 145 个测试文件通过，浏览器预览验证算法切换与设置面板。 +- 完整 Tauri Windows 检查受现有 C 依赖交叉工具链限制；Linux Tauri 本就被仓库禁止。 + 本轮没有原生 Windows/macOS 或手机上的麦克风实测。 + +发布模式纯 DSP 合成吞吐检查，5 秒输入：RNNoise 约 0.039 秒、PureVox 约 0.57 秒。 +这是本机算法吞吐，不包括音频设备、重采样、网络或识别服务的耗时。 +尚未完成真实语音识别字错率对比，不能据此声称识别率提升。 + +离线 A/B 工具（输入为 16 kHz 单声道 PCM WAV，输出文件必须不存在）： + +```sh +cargo run --release --manifest-path openless-all/app/Cargo.toml -p openless-core \ + --example enhance_audio -- input.wav output.wav rnnoise +``` + +最后一个参数支持 `rnnoise`、`purevox`、`spectral`、`off`。 +`off` 只关闭降噪阶段，仍运行默认 AGC/限幅；原始输入 WAV 是完全关闭增强的对照。 + +## 持续跟进与采集优先 + +- `scripts/micyou-upstream.json` 固定上游 revision 与源文件/模型 SHA-256。 +- `scripts/micyou-dsp.patch` 单独记录 OpenLess 适配；同步时先在临时目录应用补丁， + 冲突时不写入工作区。同步器的 6 项离线测试与固定版本重放检查通过。 +- `.github/workflows/sync-micyou.yml` 每日检查相关上游改动，通过模型与 Core 测试后 + 创建/更新 draft PR。未启用自动合并；接口、依赖或补丁冲突仍需审查。 +- 工作流合入默认分支且仓库允许 Actions 创建 PR 后才会生效；GITHUB_TOKEN 创建的 + PR 不自动触发独立 PR 工作流，合并前需运行正常原生平台 CI。 +- `PipelineDictationEngine` 已调整为先采集，再准备识别器。慢准备期间复用原缓冲， + 不添加第二套 PCM 队列，也不默认启用常驻预热。详见 [启动延迟说明](audio-startup-latency.md)。 diff --git a/openless-all/app/Cargo.lock b/openless-all/app/Cargo.lock index 68d03e392..95fa2698e 100644 --- a/openless-all/app/Cargo.lock +++ b/openless-all/app/Cargo.lock @@ -243,6 +243,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "apple-native-keyring-store" version = "1.0.2" @@ -288,6 +294,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -1495,6 +1507,19 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "easyfft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f" +dependencies = [ + "array-init", + "generic_singleton", + "num-complex", + "realfft", + "rustfft", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -2143,6 +2168,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic_singleton" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6e923c8e978e57cf63e2e200ca967d1d20f0ea2662b28f6d4e11c44aa6ab16" +dependencies = [ + "anymap3", + "parking_lot", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -3136,6 +3171,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "micyou-dsp" +version = "0.1.0" +dependencies = [ + "log", + "micyou-infer", + "nnnoiseless", + "rustfft", + "serde", +] + +[[package]] +name = "micyou-infer" +version = "0.1.0" + [[package]] name = "mime" version = "0.3.17" @@ -3329,6 +3379,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nnnoiseless" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c" +dependencies = [ + "easyfft", + "once_cell", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -3401,6 +3461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", + "serde", ] [[package]] @@ -3897,6 +3958,7 @@ dependencies = [ "jsonwebtoken", "log", "md-5", + "micyou-dsp", "once_cell", "parking_lot", "rand 0.8.8", @@ -4472,6 +4534,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -4777,6 +4848,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -5048,6 +5128,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -5587,6 +5681,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -6079,6 +6179,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tree_magic_mini" version = "3.2.2" diff --git a/openless-all/app/Cargo.toml b/openless-all/app/Cargo.toml index 2b500e206..058053e3b 100644 --- a/openless-all/app/Cargo.toml +++ b/openless-all/app/Cargo.toml @@ -2,6 +2,8 @@ resolver = "2" members = [ "crates/openless-core", + "crates/micyou-dsp", + "crates/micyou-infer", "linux-egui", "crates/openless-capsule", "native-capsule", diff --git a/openless-all/app/crates/micyou-dsp/Cargo.toml b/openless-all/app/crates/micyou-dsp/Cargo.toml new file mode 100644 index 000000000..060828ccc --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "micyou-dsp" +version = "0.1.0" +edition = "2021" +license-file = "LICENSE" +publish = false + +[features] +default = ["dsp", "noise-suppression"] +dsp = [] +noise-suppression = ["dsp"] + +[dependencies] +serde = { version = "1", features = ["derive"] } +log = "0.4" +nnnoiseless = { version = "0.5.2", default-features = false } +rustfft = "6.4.1" +micyou-infer = { path = "../micyou-infer" } diff --git a/openless-all/app/crates/micyou-dsp/LICENSE b/openless-all/app/crates/micyou-dsp/LICENSE new file mode 100644 index 000000000..01288814f --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/LICENSE @@ -0,0 +1,725 @@ +MicYou is licensed under the GNU General Public License version 3.0 (GPLv3) +with the MicYou Plugin Exception (Version 1.0). + +Copyright (C) 2026 LanRhyme + +============================================================================== + MicYou Plugin Exception (Version 1.0) +============================================================================== + +As a special exception to Sections 5 and 6 of the GNU General Public License +version 3 ("GPLv3"), under Section 7 (Additional Permissions), the copyright +holders of MicYou grant permission to link, load, or combine independently +created software modules ("Plugins") with MicYou through its defined Plugin +Interfaces, and to convey the resulting combined work, provided that all of +the following conditions are met: + +1. Defined Plugin Interfaces: + This exception applies solely to Plugins that interact with MicYou through + its official, public Plugin Application Programming Interfaces (APIs), + including but not limited to the C ABI (mpl_host_api_t / micyou_plugin_abi.h), + WASM guest-host interfaces, and official Plugin SDKs. This exception does + not grant permission to extract, repackage, or link internal components of + MicYou outside the defined Plugin Interfaces into proprietary works. + +2. Non-Commercial Plugins: + Plugins that are developed, distributed, or used solely for non-commercial + purposes are NOT required to be licensed under the GNU GPLv3. Authors of + such non-commercial Plugins may choose any open-source or proprietary + license of their choice. + +3. Commercial Plugins: + Any commercial distribution, sale, paid subscription, or proprietary + commercial monetization of Plugins that link with or run inside MicYou + requires prior written authorization or a separate commercial license from + the copyright holders of MicYou. + +4. Core Program Protection: + Nothing in this exception modifies the license terms of MicYou itself or + works directly based on MicYou (other than the Plugin combination permitted + herein). Any modification to MicYou's core source code remains strictly + subject to the full terms and conditions of the GNU General Public License + version 3. + +5. Marketplace Distribution: + Distribution via the official MicYou Plugin Marketplace is subject to the + MicYou Plugin Marketplace Policy, which may require GPL-3.0-compatible open + source licensing or official review and authorization. + +============================================================================== + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 +============================================================================== + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/openless-all/app/crates/micyou-dsp/LICENSE-AEC7.txt b/openless-all/app/crates/micyou-dsp/LICENSE-AEC7.txt new file mode 100644 index 000000000..ab6dc217a --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/LICENSE-AEC7.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 a2heng + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/openless-all/app/crates/micyou-dsp/LICENSE-PureVox.txt b/openless-all/app/crates/micyou-dsp/LICENSE-PureVox.txt new file mode 100644 index 000000000..bfa7788c0 --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/LICENSE-PureVox.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 a2heng + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/openless-all/app/crates/micyou-dsp/LICENSE-RNNoise.txt b/openless-all/app/crates/micyou-dsp/LICENSE-RNNoise.txt new file mode 100644 index 000000000..2caa6a9cc --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/LICENSE-RNNoise.txt @@ -0,0 +1,32 @@ +Copyright (c) 2020, Joe Neeman +Copyright (c) 2017, Mozilla +Copyright (c) 2007-2017, Jean-Marc Valin +Copyright (c) 2005-2017, Xiph.Org Foundation +Copyright (c) 2003-2004, Mark Borgerding + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of the Xiph.Org Foundation nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/openless-all/app/crates/micyou-dsp/UPSTREAM.md b/openless-all/app/crates/micyou-dsp/UPSTREAM.md new file mode 100644 index 000000000..4be8b707e --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/UPSTREAM.md @@ -0,0 +1,83 @@ +# MicYou DSP adaptation + +Source: [MicYou-Dev/MicYou](https://github.com/MicYou-Dev/MicYou), commit +`d821da032e2b72330ced09678dd127acd2714bef` (2026-10-20 import). + +`src/lib.rs` comes from `tauri-app/crates/micyou-audio/src/dsp.rs`. +`../micyou-infer/src/{lib,ops}.rs` and the two `src/assets/*.mcy` blobs +come from the corresponding upstream paths. The inference runtime and model +blobs are imported without algorithm changes. Original copyright notices are +retained in all source files. See `LICENSE` for MicYou's complete license and +plugin exception; `LICENSE-PureVox.txt` and `LICENSE-AEC7.txt` are the upstream +MIT model notices. The inference crate also carries MicYou's license. +`LICENSE-RNNoise.txt` is `nnnoiseless` 0.5.2's original BSD notice, including +the RNNoise/Xiph/Mozilla contributors. + +OpenLess changes: + +- Standalone DSP-only manifests, local inference dependency, no audio device + or ONNX Runtime dependency. +- Removed playback buffer management: ASR audio must not duplicate or drop + samples to regulate playback latency. +- Align dry/wet mixing with the neural denoisers' one-hop delay. +- Limit AGC boost on rejected noise, using RNNoise speech probability or + PureVox wet/dry energy ratio. This prevents AGC from undoing suppression + when a conservative amount of dry signal is mixed back in. +- Add serde defaults for DSP settings. +- Define the small AEC failure enum locally instead of depending on MicYou's + audio host. + +OpenLess supplies bounded session settings, mono downmix, anti-aliasing +resampling, fixed 480-sample hops, model-delay compensation, end-of-stream +flush and exact output-duration accounting in `openless-core/src/audio.rs` +and `audio_dsp.rs`. The main microphone path does not install plugin hooks. + +AEC7 and its aligned-reference API are retained and tested here. OpenLess's +microphone hosts do not yet provide synchronized speaker-reference capture, +so the application does not expose or claim active acoustic echo cancellation. +The optional comb filter follows upstream's simple delay subtraction; it is +not a learned room dereverberation model. + +Synthetic throughput check (release build): + +```sh +cargo run --release --manifest-path openless-all/app/Cargo.toml -p micyou-dsp --example benchmark +``` + +These measurements do not establish speech quality or recognition accuracy. + +## Following upstream fixes + +The authoritative current revision and source/model SHA-256 values live in +`../../scripts/micyou-upstream.json`. The version above records the original +import. OpenLess-specific edits live in `../../scripts/micyou-dsp.patch`; +the inference VM retains upstream algorithms with explicit Clippy exceptions +for its numerical kernels and operation signatures. These exceptions are +recorded in the patch; OpenLess application code keeps strict lint checks. + +From the app directory: + +```sh +python3 scripts/sync-micyou.py --verify # offline integrity check +python3 scripts/sync-micyou.py --check # stage latest upstream, no writes +python3 scripts/sync-micyou.py --sync # apply relevant upstream fixes +python3 scripts/sync-micyou.py --refresh-patch # record reviewed adapter edits +``` + +`--check` and `--sync` also accept `--revision <40-character SHA>`. +The importer downloads immutable source/model files, applies the adapter in a +temporary directory, and rejects context conflicts before writing anything. +Local edits require an explicit patch refresh; models and notices cannot be +silently forked by that command. Unrelated upstream commits create no update. + +`.github/workflows/sync-micyou.yml` checks daily, tests relevant changes, then +creates/updates a draft PR against `beta`. It becomes active after the workflow +is present on the repository's default branch and Actions is allowed to create +PRs. Candidate compilation has read-only permissions; the write-permission +job does not execute candidate code. No automatic merge is enabled. Run normal +native platform CI on the draft branch before merging; GITHUB_TOKEN-created PRs +do not automatically trigger separate PR workflows. + +Most upstream implementation/model fixes can follow this route without manual +copying. Upstream API/dependency/format changes that break the adapter or tests +still require review and a deliberate patch/dependency update. diff --git a/openless-all/app/crates/micyou-dsp/examples/benchmark.rs b/openless-all/app/crates/micyou-dsp/examples/benchmark.rs new file mode 100644 index 000000000..13f91c924 --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/examples/benchmark.rs @@ -0,0 +1,34 @@ +//! Synthetic throughput check; does not measure speech recognition accuracy. +use micyou_dsp::{AudioDspSettings, DspProcessor}; +use std::sync::{Arc, RwLock}; + +fn main() { + for algorithm in ["RNNoise", "PureVox", "Speexdsp"] { + let settings = AudioDspSettings { + ns_enabled: true, + ns_type: algorithm.into(), + ns_intensity: 80.0, + agc_enabled: true, + ..Default::default() + }; + let mut dsp = DspProcessor::new(Arc::new(RwLock::new(settings)), None); + let mut state = 0x12345678u32; + let started = std::time::Instant::now(); + for _ in 0..500 { + let mut frame: Vec = (0..480) + .map(|_| { + state = state.wrapping_mul(1664525).wrapping_add(1013904223); + ((state >> 8) as f32 / 16_777_216.0 - 0.5) * 0.1 + }) + .collect(); + dsp.process(&mut frame, 1, 0.0); + assert_eq!(frame.len(), 480); + assert!(frame.iter().all(|s| s.is_finite())); + } + let time = started.elapsed().as_secs_f64(); + println!( + "{algorithm}: 5.00s audio, {time:.3}s processing, real-time factor {:.3}", + time / 5.0 + ); + } +} diff --git a/openless-all/app/crates/micyou-dsp/src/assets/aec7.mcy b/openless-all/app/crates/micyou-dsp/src/assets/aec7.mcy new file mode 100644 index 000000000..8cfc50447 Binary files /dev/null and b/openless-all/app/crates/micyou-dsp/src/assets/aec7.mcy differ diff --git a/openless-all/app/crates/micyou-dsp/src/assets/purevox6.mcy b/openless-all/app/crates/micyou-dsp/src/assets/purevox6.mcy new file mode 100644 index 000000000..798c0a6c7 Binary files /dev/null and b/openless-all/app/crates/micyou-dsp/src/assets/purevox6.mcy differ diff --git a/openless-all/app/crates/micyou-dsp/src/lib.rs b/openless-all/app/crates/micyou-dsp/src/lib.rs new file mode 100644 index 000000000..7db9773db --- /dev/null +++ b/openless-all/app/crates/micyou-dsp/src/lib.rs @@ -0,0 +1,2128 @@ +/* + * MicYou — Turns your Android device into a high-quality PC microphone. + * Copyright (C) 2026 LanRhyme + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version, with the MicYou Plugin Exception. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +// Keep upstream FFT indexing and error conversion intact in the import patch. +#![allow( + clippy::needless_range_loop, + clippy::needless_return_with_question_mark +)] + +use std::collections::VecDeque; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +#[cfg(feature = "noise-suppression")] +use micyou_infer::{Graph, Session}; +#[cfg(feature = "dsp")] +use nnnoiseless::DenoiseState; +#[cfg(feature = "noise-suppression")] +use rustfft::num_complex::Complex; + +// OpenLess adaptation: DSP-only crate; no device, playback or plugin host. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AecFailure { + ModelLoadFailed, + InferenceFailed, +} + +/// Host-provided DSP stage invoked at plugin chain nodes. The first argument +/// is the chain node name: either the legacy synthetic [`PLUGIN_CHAIN_NODE`] +/// (runs every registered plugin, in registry order) or a per-plugin node +/// ([`PLUGIN_NODE_PREFIX`] + plugin id, runs just that plugin). +pub type ExternalDspHook = Box, usize, f64) + Send>; + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[serde(default, rename_all = "camelCase")] +pub struct EqualizerConfig { + pub enabled: bool, + pub pre_amp: f32, + pub gains: Vec, // 10 bands +} + +impl Default for EqualizerConfig { + fn default() -> Self { + Self { + enabled: false, + pre_amp: 0.0, + gains: vec![0.0; 10], + } + } +} + +/// Audio DSP settings, synced from the frontend. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[serde(default, rename_all = "camelCase")] +pub struct AudioDspSettings { + pub gain: f32, // dB, -50 to +50 + pub ns_enabled: bool, + pub ns_type: String, // "PureVox", "RNNoise", "Speexdsp" + pub ns_intensity: f32, // 0..100 + pub dereverb_enabled: bool, + pub dereverb_level: f32, // 0..100 + pub agc_enabled: bool, + pub agc_target: f32, // 0..32767 + pub agc_attack: f32, // raw slider value 1..100, maps to 0.001..0.1 + pub agc_decay: f32, // raw slider value 1..100, maps to 0.0001..0.01 + pub vad_enabled: bool, + pub vad_threshold: f32, // dB, -100..0 + pub aec_enabled: bool, + + /// Output playback ring buffer headroom in milliseconds (100..1200). + #[serde(default = "default_output_buffer_ms")] + pub output_buffer_ms: u32, + + #[serde(default)] + pub processing_chain: Vec, + #[serde(default)] + pub equalizer: EqualizerConfig, +} + +impl Default for AudioDspSettings { + fn default() -> Self { + Self { + gain: 0.0, + ns_enabled: false, + ns_type: "PureVox".to_string(), + ns_intensity: 50.0, + dereverb_enabled: false, + dereverb_level: 50.0, + agc_enabled: false, + agc_target: 16000.0, + agc_attack: 50.0, + agc_decay: 50.0, + vad_enabled: false, + vad_threshold: -40.0, + aec_enabled: false, + output_buffer_ms: 300, + processing_chain: vec![ + "AEC".to_string(), + "NoiseReduction".to_string(), + "Dereverb".to_string(), + "Equalizer".to_string(), + "Amplifier".to_string(), + "AGC".to_string(), + "VAD".to_string(), + ], + equalizer: EqualizerConfig::default(), + } + } +} + +impl AudioDspSettings { + pub fn normalize(&mut self) { + if !matches!(self.ns_type.as_str(), "PureVox" | "RNNoise" | "Speexdsp") { + self.ns_type = Self::default().ns_type; + } + } +} + +/// Default output playback buffer headroom in milliseconds. +fn default_output_buffer_ms() -> u32 { + 300 +} + +// ─── Shared STFT window helpers ─────────────────────────────────────────── + +#[cfg(feature = "noise-suppression")] +fn sqrt_hann_window(size: usize) -> Vec { + (0..size) + .map(|i| { + let value = + 0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / (size - 1) as f64).cos(); + value.sqrt() as f32 + }) + .collect() +} + +#[cfg(feature = "noise-suppression")] +fn overlap_add_gain(window: &[f32], hop_length: usize) -> f32 { + let average = (0..hop_length) + .map(|i| window[i] * window[i] + window[i + hop_length] * window[i + hop_length]) + .sum::() + / hop_length as f32; + if average > 0.001 { + 1.0 / average.sqrt() + } else { + 1.0 + } +} + +// ─── Compiled native models (pure Rust, no ONNX Runtime) ────────────────── + +/// Statically compiled MCYI model blobs and their shared parsed graphs. +/// +/// The blobs were compiled from the ONNX exports (purevox6.onnx / +/// aec7_ep0185.onnx) and validated against onnxruntime — the compiler and +/// the full verification matrix (numpy reference interpreter, per-op and +/// streaming goldens, time-domain DSP goldens) live in the experimental +/// `libmicyou` backend repository; see `docs/pure-rust-inference.md`. +/// Embedding the blobs removes both the ONNX Runtime shared library and +/// runtime model discovery from this codebase. +#[cfg(feature = "noise-suppression")] +mod native_models { + use super::Graph; + use std::sync::OnceLock; + + pub static PUREVOX6_BLOB: &[u8] = include_bytes!("assets/purevox6.mcy"); + pub static AEC7_BLOB: &[u8] = include_bytes!("assets/aec7.mcy"); + + pub fn purevox_graph() -> &'static Graph { + static GRAPH: OnceLock = OnceLock::new(); + GRAPH.get_or_init(|| { + Graph::parse(PUREVOX6_BLOB) + .expect("embedded purevox6.mcy must parse (blob is CI-verified)") + }) + } + + pub fn aec_graph() -> &'static Graph { + static GRAPH: OnceLock = OnceLock::new(); + GRAPH.get_or_init(|| { + Graph::parse(AEC7_BLOB).expect("embedded aec7.mcy must parse (blob is CI-verified)") + }) + } +} + +// ─── PureVox6 native noise suppression ──────────────────────────────────── + +#[cfg(feature = "noise-suppression")] +struct PureVoxProcessor { + sess: Session<'static>, + frame_size: usize, // 960 + hop_length: usize, // 480 + window: Vec, + ola_gain: f32, + previous: Vec, + ola_accumulator: Vec, + /// Model input staging: interleaved (re, im) pairs per bin, [1,481,1,2]. + spec_flat: Vec, + // PureVox6 has 4 independent cache states (flat 1D), fed back each frame. + enc_c: Vec, // [7368] + dec_c: Vec, // [1440] + tfa_c: Vec, // [800] + inter_c: Vec, // [4608] + fft_forward: std::sync::Arc>, + fft_inverse: std::sync::Arc>, +} + +#[cfg(feature = "noise-suppression")] +impl PureVoxProcessor { + fn new() -> Result> { + let frame_size = 960; + let hop_length = 480; + let spec_size = frame_size / 2 + 1; + + let graph = native_models::purevox_graph(); + // Sanity-check the embedded contract once per processor. + if graph.num_inputs() != 5 || graph.num_outputs() != 5 { + return Err(format!( + "purevox6 model contract mismatch: {} in / {} out", + graph.num_inputs(), + graph.num_outputs() + ))?; + } + let sess = Session::new(graph); + + let window = sqrt_hann_window(frame_size); + let ola_gain = overlap_add_gain(&window, hop_length); + + use rustfft::FftPlanner; + let mut planner = FftPlanner::new(); + let fft_forward = planner.plan_fft_forward(frame_size); + let fft_inverse = planner.plan_fft_inverse(frame_size); + + Ok(Self { + sess, + frame_size, + hop_length, + window, + ola_gain, + previous: vec![0.0; hop_length], + ola_accumulator: vec![0.0; frame_size], + spec_flat: vec![0.0f32; spec_size * 2], + enc_c: vec![0.0f32; 7368], + dec_c: vec![0.0f32; 1440], + tfa_c: vec![0.0f32; 800], + inter_c: vec![0.0f32; 4608], + fft_forward, + fft_inverse, + }) + } + + fn process(&mut self, input: &[f32]) -> Vec { + if input.len() != self.hop_length { + return input.to_vec(); + } + + let frame_size = self.frame_size; + let hop_length = self.hop_length; + let spec_size = frame_size / 2 + 1; + + // Build frame: previous + current + let mut fft_buffer = vec![0.0_f32; frame_size]; + fft_buffer[..hop_length].copy_from_slice(&self.previous); + fft_buffer[hop_length..].copy_from_slice(input); + self.previous.copy_from_slice(input); + + // Apply window + for (sample, window) in fft_buffer.iter_mut().zip(&self.window) { + *sample *= window; + } + + // FFT + let mut complex_buf: Vec> = + fft_buffer.iter().map(|&v| Complex::new(v, 0.0)).collect(); + self.fft_forward.process(&mut complex_buf); + + // Model input format: interleaved (re, im) for [1, 481, 1, 2] + for i in 0..spec_size { + self.spec_flat[i * 2] = complex_buf[i].re; + self.spec_flat[i * 2 + 1] = complex_buf[i].im; + } + + // Run inference (input order: spec, enc_c, dec_c, tfa_c, inter_c) + let inputs: [&[f32]; 5] = [ + &self.spec_flat, + &self.enc_c, + &self.dec_c, + &self.tfa_c, + &self.inter_c, + ]; + if let Err(e) = self.sess.run(&inputs) { + log::error!("PureVox inference failed: {e}"); + return input.to_vec(); + } + + // Extract enhanced spectrum (output 0) + match self.sess.output(0) { + Ok(enh) => { + for i in 0..spec_size { + complex_buf[i] = Complex::new(enh[i * 2], enh[i * 2 + 1]); + } + } + Err(e) => { + log::error!("PureVox output read failed: {e}"); + return input.to_vec(); + } + } + // Feed the updated cache states back for the next frame + for (i, cache) in [ + &mut self.enc_c, + &mut self.dec_c, + &mut self.tfa_c, + &mut self.inter_c, + ] + .iter_mut() + .enumerate() + { + if let Err(e) = self.sess.copy_output(i + 1, cache) { + log::error!("PureVox cache readback failed: {e}"); + } + } + for i in spec_size..frame_size { + complex_buf[i] = complex_buf[frame_size - i].conj(); + } + + // IFFT + self.fft_inverse.process(&mut complex_buf); + let scale = 1.0 / frame_size as f32; + for ((sample, complex), window) in fft_buffer.iter_mut().zip(&complex_buf).zip(&self.window) + { + *sample = complex.re * scale * window; + } + + // OLA + for (accumulated, sample) in self.ola_accumulator.iter_mut().zip(&fft_buffer) { + *accumulated += sample; + } + + let mut output = vec![0.0_f32; hop_length]; + for (sample, accumulated) in output.iter_mut().zip(&self.ola_accumulator) { + *sample = accumulated * self.ola_gain; + } + + // Shift accumulator + for i in 0..frame_size - hop_length { + self.ola_accumulator[i] = self.ola_accumulator[i + hop_length]; + } + for i in frame_size - hop_length..frame_size { + self.ola_accumulator[i] = 0.0; + } + + output + } +} + +// ─── AEC7 native acoustic echo cancellation ────────────────────────────── + +#[cfg(feature = "noise-suppression")] +const AEC_WIN_LEN: usize = 960; +#[cfg(feature = "noise-suppression")] +const AEC_HOP_LEN: usize = 480; +#[cfg(feature = "noise-suppression")] +const AEC_NFFT: usize = 960; +#[cfg(feature = "noise-suppression")] +const AEC_N_BINS: usize = AEC_NFFT / 2 + 1; // 481 + +/// AEC7 cache state configuration, in compiled-model graph order. +/// +/// Cache `j` is model input `j + 2` (inputs 0/1 are the mic/far STFT +/// frames) and is refreshed from model output `output_idx` after every +/// frame. `deep_enc_conv` (shape [1,0]) is excluded — the ONNX optimizer +/// folded it away, so it is neither an input nor a meaningful output; its +/// placeholder `deep_enc_conv_o` still appears at output index 5 and is +/// skipped. This matches the original C++ reference implementation. +#[cfg(feature = "noise-suppression")] +const AEC_CACHE_CONFIGS: &[(&str, &[usize], usize)] = &[ + ("res_enc_conv", &[1, 135680], 1), + ("res_enc_tfa", &[1, 248], 2), + ("mic_enc_conv", &[1, 135680], 3), + ("mic_enc_tfa", &[1, 248], 4), + // output[5] = deep_enc_conv_o → skipped + ("deep_enc_tfa", &[1, 336], 6), + ("dec_conv", &[1, 13440], 7), + ("dec_tfa", &[1, 496], 8), + ("inter", &[1, 7680], 9), + ("res_prev1", &[1, 1, 1, 320], 10), + ("res_prev2", &[1, 1, 1, 320], 11), + ("mic_prev1", &[1, 1, 1, 320], 12), + ("mic_prev2", &[1, 1, 1, 320], 13), +]; + +/// Native AEC7 acoustic echo cancellation (pure-Rust VM, no ONNX Runtime). +/// Takes both mic (near-end) and far-end (speaker) audio and cancels echo. +/// Compiled from the aec7_ep0185.onnx model. +#[cfg(feature = "noise-suppression")] +struct AecProcessor { + sess: Session<'static>, + /// STFT frame staging for the two model inputs ([1,2,481] planar: + /// 481 real parts followed by 481 imaginary parts). + mic_frame: Vec, + far_frame: Vec, + /// Cache state tensors in graph order (see AEC_CACHE_CONFIGS). + caches: Vec>, + // STFT / iSTFT / OLA + window: Vec, + /// Running per-sample accumulation of window² for correct COLA normalization. + window_sum: Vec, + mic_previous: Vec, // last 480 mic samples for overlapping frames + far_previous: Vec, // last 480 far-end samples for overlapping frames + ola_accumulator: Vec, // 960 samples + fft_forward: std::sync::Arc>, + fft_inverse: std::sync::Arc>, + + // Pre-allocated scratch buffers — reused every process() call to avoid + // heap allocations on the real-time audio thread. + scratch_mic_frame: Vec, + scratch_far_frame: Vec, + scratch_time_frame: Vec, + scratch_complex: Vec>, +} + +#[cfg(feature = "noise-suppression")] +impl AecProcessor { + fn new() -> Result> { + use rustfft::FftPlanner; + + let graph = native_models::aec_graph(); + if graph.num_inputs() != 14 || graph.num_outputs() != 14 { + return Err(format!( + "aec7 model contract mismatch: {} in / {} out", + graph.num_inputs(), + graph.num_outputs() + ))?; + } + let sess = Session::new(graph); + + // Sine window (sqrt of Hann). Applied once during analysis + // (forward_stft) and once during synthesis (iSTFT in process), + // giving total window effect = Hann (sin^2). + let window = sqrt_hann_window(AEC_WIN_LEN); + + let mut planner = FftPlanner::new(); + let fft_forward = planner.plan_fft_forward(AEC_NFFT); + let fft_inverse = planner.plan_fft_inverse(AEC_NFFT); + + let caches = AEC_CACHE_CONFIGS + .iter() + .map(|(_, shape, _)| vec![0.0f32; shape.iter().product()]) + .collect(); + + Ok(Self { + sess, + mic_frame: vec![0.0f32; 2 * AEC_N_BINS], + far_frame: vec![0.0f32; 2 * AEC_N_BINS], + caches, + window, + window_sum: vec![0.0; AEC_WIN_LEN], + mic_previous: vec![0.0; AEC_HOP_LEN], + far_previous: vec![0.0; AEC_HOP_LEN], + ola_accumulator: vec![0.0; AEC_WIN_LEN], + fft_forward, + fft_inverse, + scratch_mic_frame: vec![0.0f32; AEC_WIN_LEN], + scratch_far_frame: vec![0.0f32; AEC_WIN_LEN], + scratch_time_frame: vec![0.0f32; AEC_WIN_LEN], + scratch_complex: vec![Complex::default(); AEC_NFFT], + }) + } + + fn reset(&mut self) { + for tensor in &mut self.caches { + tensor.fill(0.0); + } + self.window_sum.fill(0.0); + self.mic_previous.fill(0.0); + self.far_previous.fill(0.0); + self.ola_accumulator.fill(0.0); + } + + /// Process one frame (480 samples each of mic and far-end). + /// Returns 480 samples of echo-cancelled audio. + fn process(&mut self, mic_chunk: &[f32], far_chunk: &[f32]) -> Result, String> { + if mic_chunk.len() != AEC_HOP_LEN || far_chunk.len() != AEC_HOP_LEN { + return Err(format!( + "invalid AEC frame lengths: mic={}, far={}, expected={AEC_HOP_LEN}", + mic_chunk.len(), + far_chunk.len() + )); + } + + // ── Build overlapping frames into scratch buffers ── + { + let mic = &mut self.scratch_mic_frame; + mic[..AEC_HOP_LEN].copy_from_slice(&self.mic_previous); + mic[AEC_HOP_LEN..].copy_from_slice(mic_chunk); + } + self.mic_previous.copy_from_slice(mic_chunk); + + { + let far = &mut self.scratch_far_frame; + far[..AEC_HOP_LEN].copy_from_slice(&self.far_previous); + far[AEC_HOP_LEN..].copy_from_slice(far_chunk); + } + self.far_previous.copy_from_slice(far_chunk); + + // ── Apply window + FFT → STFT frames (fill pre-allocated arrays) ── + forward_stft_free( + &self.scratch_mic_frame, + &mut self.mic_frame, + &mut self.scratch_complex, + &self.window, + &self.fft_forward, + ); + forward_stft_free( + &self.scratch_far_frame, + &mut self.far_frame, + &mut self.scratch_complex, + &self.window, + &self.fft_forward, + ); + + // ── Native inference ── + let mut inputs: Vec<&[f32]> = Vec::with_capacity(14); + inputs.push(&self.mic_frame); + inputs.push(&self.far_frame); + for cache in self.caches.iter() { + inputs.push(cache.as_slice()); + } + self.sess.run(&inputs).map_err(|e| e.to_string())?; + + // ── iSTFT + OLA → time domain (reuse scratch buffers) ── + { + let enhanced = self.sess.output(0).map_err(|e| e.to_string())?; + let complex_buf = &mut self.scratch_complex; + for bin in 0..AEC_NFFT { + if bin < AEC_N_BINS { + complex_buf[bin] = Complex::new(enhanced[bin], enhanced[AEC_N_BINS + bin]); + } else { + let mirror = AEC_NFFT - bin; + complex_buf[bin] = + Complex::new(enhanced[mirror], -enhanced[AEC_N_BINS + mirror]); + } + } + self.fft_inverse.process(complex_buf); + let scale = 1.0 / AEC_NFFT as f32; + for ((sample, complex), window) in self + .scratch_time_frame + .iter_mut() + .zip(complex_buf.iter()) + .zip(&self.window) + { + *sample = complex.re * scale * window; + } + } + + // Update cache states from outputs (indices per AEC_CACHE_CONFIGS). + for (cache, &(_, _, output_idx)) in self.caches.iter_mut().zip(AEC_CACHE_CONFIGS.iter()) { + self.sess + .copy_output(output_idx, cache) + .map_err(|e| e.to_string())?; + } + + // ── OLA with per-sample window-sum normalization ── + for i in 0..AEC_WIN_LEN { + self.ola_accumulator[i] += self.scratch_time_frame[i]; + self.window_sum[i] += self.window[i] * self.window[i]; + } + + let mut output = vec![0.0f32; AEC_HOP_LEN]; + for ((sample, norm), accumulated) in output + .iter_mut() + .zip(&self.window_sum) + .zip(&self.ola_accumulator) + { + *sample = if *norm > 1e-6 { + accumulated / norm + } else { + *accumulated + }; + } + + // Shift OLA accumulators + for i in 0..AEC_WIN_LEN - AEC_HOP_LEN { + self.ola_accumulator[i] = self.ola_accumulator[i + AEC_HOP_LEN]; + self.window_sum[i] = self.window_sum[i + AEC_HOP_LEN]; + } + for i in AEC_WIN_LEN - AEC_HOP_LEN..AEC_WIN_LEN { + self.ola_accumulator[i] = 0.0; + self.window_sum[i] = 0.0; + } + + Ok(output) + } +} + +// ─── AEC Delay Estimation and Alignment ─────────────────────────────── + +/// FIFO for far-end audio fed to AEC. The producer feeds the same number of +/// mono reference samples as the near-end frames entering the DSP pipeline, so +/// each 480-sample AEC hop consumes a distinct reference hop. +#[cfg(feature = "noise-suppression")] +struct FarEndBuffer { + buffer: VecDeque, +} + +#[cfg(feature = "noise-suppression")] +impl FarEndBuffer { + fn new() -> Self { + Self { + buffer: VecDeque::with_capacity(AEC_HOP_LEN * 4), + } + } + + fn feed(&mut self, data: &[f32]) { + const MAX_REFERENCE_SAMPLES: usize = 48_000 * 2; + + if data.len() >= MAX_REFERENCE_SAMPLES { + self.buffer.clear(); + self.buffer + .extend(data[data.len() - MAX_REFERENCE_SAMPLES..].iter().copied()); + return; + } + + let overflow = self + .buffer + .len() + .saturating_add(data.len()) + .saturating_sub(MAX_REFERENCE_SAMPLES); + if overflow > 0 { + self.buffer.drain(..overflow); + } + self.buffer.extend(data.iter().copied()); + } + + fn take_hop(&mut self) -> Option> { + if self.buffer.len() < AEC_HOP_LEN { + return None; + } + Some(self.buffer.drain(..AEC_HOP_LEN).collect()) + } + + fn clear(&mut self) { + self.buffer.clear(); + } +} + +/// Free function: compute STFT of a windowed frame, filling `out` in place. +/// Avoids borrowing `AecProcessor` as a whole so that callers can pass in +/// individual field references from the struct. +#[cfg(feature = "noise-suppression")] +fn forward_stft_free( + windowed_frame: &[f32], + out: &mut [f32], + complex_buf: &mut [Complex], + window: &[f32], + fft_forward: &std::sync::Arc>, +) { + for i in 0..AEC_WIN_LEN { + complex_buf[i] = Complex::new(windowed_frame[i] * window[i], 0.0); + } + fft_forward.process(complex_buf); + + for bin in 0..AEC_N_BINS { + out[bin] = complex_buf[bin].re; + out[AEC_N_BINS + bin] = complex_buf[bin].im; + } +} + +// ─── Speexdsp-style spectral subtraction noise suppression ────────────────── + +/// A simple spectral subtraction noise suppressor inspired by Speex's approach. +/// Uses FFT to estimate noise floor and subtract it from the signal. +#[cfg(feature = "dsp")] +struct SpeexStyleNS { + frame_size: usize, + noise_estimate: Vec, // Running noise floor estimate per frequency bin + adaptation_rate: f32, + fft_forward: std::sync::Arc>, + fft_inverse: std::sync::Arc>, +} + +#[cfg(feature = "dsp")] +impl SpeexStyleNS { + fn new() -> Self { + use rustfft::FftPlanner; + let frame_size = 480; + let mut planner = FftPlanner::new(); + let fft_forward = planner.plan_fft_forward(frame_size); + let fft_inverse = planner.plan_fft_inverse(frame_size); + Self { + frame_size, + noise_estimate: vec![0.0; frame_size / 2 + 1], + adaptation_rate: 0.02, + fft_forward, + fft_inverse, + } + } + + fn process(&mut self, data: &mut [f32], intensity: f32) { + use rustfft::num_complex::Complex; + + let len = data.len(); + if len < self.frame_size { + return; + } + + let num_frames = len / self.frame_size; + let mix = (intensity / 100.0).clamp(0.0, 1.0); + let spec_size = self.frame_size / 2 + 1; + + for frame_idx in 0..num_frames { + let offset = frame_idx * self.frame_size; + let frame = &data[offset..offset + self.frame_size]; + + let mut complex: Vec> = + frame.iter().map(|&s| Complex::new(s, 0.0)).collect(); + + self.fft_forward.process(&mut complex); + + // Compute magnitude and update noise estimate + for (bin, noise_estimate) in complex + .iter() + .zip(self.noise_estimate.iter_mut()) + .take(spec_size) + { + let mag = bin.norm(); + *noise_estimate = + *noise_estimate * (1.0 - self.adaptation_rate) + mag * self.adaptation_rate; + } + + // Spectral subtraction + for i in 0..spec_size { + let mag = complex[i].norm(); + let phase = complex[i].arg(); + let noise = self.noise_estimate[i] * mix * 2.0; + let clean_mag = (mag - noise).max(mag * 0.05); + complex[i] = Complex::from_polar(clean_mag, phase); + if i > 0 && i < self.frame_size - i { + complex[self.frame_size - i] = complex[i].conj(); + } + } + + self.fft_inverse.process(&mut complex); + + let scale = 1.0 / self.frame_size as f32; + for i in 0..self.frame_size { + data[offset + i] = complex[i].re * scale; + } + } + } +} + +// ─── Equalizer (10-band Biquad Peaking EQ) ────────────────────────────────── + +struct BiquadFilter { + a0: f64, + a1: f64, + a2: f64, + b1: f64, + b2: f64, + x1: f64, + x2: f64, + y1: f64, + y2: f64, +} + +impl BiquadFilter { + fn new() -> Self { + Self { + a0: 1.0, + a1: 0.0, + a2: 0.0, + b1: 0.0, + b2: 0.0, + x1: 0.0, + x2: 0.0, + y1: 0.0, + y2: 0.0, + } + } + + fn set_peaking_eq(&mut self, sample_rate: f64, center_freq: f64, q: f64, db_gain: f64) { + let w0 = 2.0 * std::f64::consts::PI * center_freq / sample_rate; + let alpha = w0.sin() / (2.0 * q); + let a = 10.0_f64.powf(db_gain / 40.0); + + let b0_raw = 1.0 + alpha * a; + let b1_raw = -2.0 * w0.cos(); + let b2_raw = 1.0 - alpha * a; + let a0_raw = 1.0 + alpha / a; + let a1_raw = -2.0 * w0.cos(); + let a2_raw = 1.0 - alpha / a; + + self.a0 = b0_raw / a0_raw; + self.a1 = b1_raw / a0_raw; + self.a2 = b2_raw / a0_raw; + self.b1 = a1_raw / a0_raw; + self.b2 = a2_raw / a0_raw; + } + + fn process(&mut self, x: f64) -> f64 { + let y = self.a0 * x + self.a1 * self.x1 + self.a2 * self.x2 + - self.b1 * self.y1 + - self.b2 * self.y2; + self.x2 = self.x1; + self.x1 = x; + self.y2 = self.y1; + self.y1 = y; + y + } +} + +struct EqualizerEffect { + filters_ch1: Vec, + filters_ch2: Vec, + pre_amp_gain: f32, + frequencies: [f64; 10], +} + +impl EqualizerEffect { + fn new() -> Self { + let mut eq = Self { + filters_ch1: (0..10).map(|_| BiquadFilter::new()).collect(), + filters_ch2: (0..10).map(|_| BiquadFilter::new()).collect(), + pre_amp_gain: 1.0, + frequencies: [ + 31.25, 62.5, 125.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0, + ], + }; + eq.update_filters(&EqualizerConfig::default()); + eq + } + + fn update_filters(&mut self, config: &EqualizerConfig) { + self.pre_amp_gain = 10.0_f32.powf(config.pre_amp / 20.0); + let sample_rate = 48000.0; + for i in 0..10 { + let gain = if i < config.gains.len() { + config.gains[i] as f64 + } else { + 0.0 + }; + self.filters_ch1[i].set_peaking_eq(sample_rate, self.frequencies[i], 1.0, gain); + self.filters_ch2[i].set_peaking_eq(sample_rate, self.frequencies[i], 1.0, gain); + } + } + + fn process(&mut self, data: &mut [f32], channels: usize) { + if channels == 1 { + for sample in data.iter_mut() { + let mut s = (*sample * self.pre_amp_gain) as f64; + for i in 0..10 { + s = self.filters_ch1[i].process(s); + } + *sample = s as f32; + } + } else if channels == 2 { + for (i, sample) in data.iter_mut().enumerate() { + let mut s = (*sample * self.pre_amp_gain) as f64; + if i % 2 == 0 { + for j in 0..10 { + s = self.filters_ch1[j].process(s); + } + } else { + for j in 0..10 { + s = self.filters_ch2[j].process(s); + } + } + *sample = s as f32; + } + } + } +} + +// ─── Main DSP Processor ───────────────────────────────────────────────────── + +/// The main DSP processor. Operates on f32 PCM samples at 48kHz. +pub struct DspProcessor { + settings: Arc>, + dry_delay: VecDeque, + noise_speech_probability: f32, + // RNNoise states - separate per channel to avoid RNN state cross-contamination + #[cfg(feature = "dsp")] + denoiser_left: Box>, + #[cfg(feature = "dsp")] + denoiser_right: Box>, + #[cfg(feature = "dsp")] + ns_buffer_left: Vec, + #[cfg(feature = "dsp")] + ns_buffer_right: Vec, + // PureVox native processor - separate per channel + #[cfg(feature = "noise-suppression")] + purevox_left: Option, + #[cfg(feature = "noise-suppression")] + purevox_right: Option, + + /// Once processor construction fails, stop retrying to avoid expensive + /// rebuilds every frame on the real-time audio thread. + #[cfg(feature = "noise-suppression")] + purevox_load_failed: bool, + + // AEC7 native acoustic echo cancellation. + // Stereo is downmixed to mono before AEC, then upmixed back (reference pattern). + #[cfg(feature = "noise-suppression")] + aec: Option, + #[cfg(feature = "noise-suppression")] + /// Latches model loading or inference failures until the next transport + /// session, preventing expensive retries on the real-time audio thread. + aec_session_failed: bool, + #[cfg(feature = "noise-suppression")] + aec_failure: Option, + /// Far-end buffer for AEC. + #[cfg(feature = "noise-suppression")] + far_end: FarEndBuffer, + + // Speexdsp-style NS + #[cfg(feature = "dsp")] + speex_ns: SpeexStyleNS, + // Equalizer + equalizer: EqualizerEffect, + // Dereverb state + dereverb_buffer_left: Vec, + dereverb_buffer_right: Vec, + dereverb_index: usize, + // AGC envelope follower + agc_envelope: f32, + // AGC smoothed gain (avoids sudden gain jumps causing pops) + agc_smoothed_gain: f32, + // VAD fade state (0.0 = muted, 1.0 = full) + vad_fade: f32, + // Spectrum snapshots + raw_spectrum: Vec, + processed_spectrum: Vec, + // Frame accumulation buffer (align to 480-sample frames for noise reduction) + accum_buffer: Vec, + output_buffer: Vec, + to_process_buf: Vec, + /// Optional external DSP stage injected by the host (plugin system). + /// Invoked when the processing chain reaches a plugin node: the legacy + /// synthetic `"Plugins"` node or a per-plugin `"Plugin:"` node. + /// Kept as a closure so `micyou-audio` stays independent of the + /// plugin crate (no dependency cycle). + external_hook: Option, +} + +/// Legacy synthetic processing-chain node that invokes the external plugin +/// DSP stage for **all** registered plugins at once. Since issue #347 every +/// plugin normally owns a dedicated [`PLUGIN_NODE_PREFIX`]-prefixed node so +/// it can be reordered independently; this node is kept for backward +/// compatibility with persisted settings and still runs every registered +/// plugin (registry order) when reached. +pub const PLUGIN_CHAIN_NODE: &str = "Plugins"; + +/// Prefix of per-plugin processing-chain nodes: `Plugin:`. +/// Each registered DSP plugin occupies exactly one such node in +/// `AudioDspSettings.processing_chain`, at whatever position the user +/// dragged it to. +pub const PLUGIN_NODE_PREFIX: &str = "Plugin:"; + +/// Build the processing-chain node name for a plugin id. +pub fn plugin_chain_node(plugin_id: &str) -> String { + format!("{PLUGIN_NODE_PREFIX}{plugin_id}") +} + +/// Extract the plugin id from a per-plugin chain node name. +/// Returns `None` for built-in stages and the legacy [`PLUGIN_CHAIN_NODE`]. +pub fn parse_plugin_chain_node(node: &str) -> Option<&str> { + node.strip_prefix(PLUGIN_NODE_PREFIX) +} + +const RNNOISE_FRAME_SIZE: usize = 480; + +impl DspProcessor { + pub fn new(settings: Arc>, _model_dir: Option) -> Self { + Self { + settings: settings.clone(), + dry_delay: VecDeque::new(), + noise_speech_probability: 0.0, + #[cfg(feature = "dsp")] + denoiser_left: DenoiseState::new(), + #[cfg(feature = "dsp")] + denoiser_right: DenoiseState::new(), + #[cfg(feature = "dsp")] + ns_buffer_left: Vec::with_capacity(RNNOISE_FRAME_SIZE * 2), + #[cfg(feature = "dsp")] + ns_buffer_right: Vec::with_capacity(RNNOISE_FRAME_SIZE * 2), + #[cfg(feature = "noise-suppression")] + purevox_left: None, + #[cfg(feature = "noise-suppression")] + purevox_right: None, + #[cfg(feature = "noise-suppression")] + purevox_load_failed: false, + + #[cfg(feature = "noise-suppression")] + aec: None, + #[cfg(feature = "noise-suppression")] + aec_session_failed: false, + #[cfg(feature = "noise-suppression")] + aec_failure: None, + #[cfg(feature = "noise-suppression")] + far_end: FarEndBuffer::new(), + + #[cfg(feature = "dsp")] + speex_ns: SpeexStyleNS::new(), + equalizer: EqualizerEffect::new(), + dereverb_buffer_left: vec![0.0; 480], + dereverb_buffer_right: vec![0.0; 480], + dereverb_index: 0, + agc_envelope: 0.0, + agc_smoothed_gain: 1.0, + vad_fade: 1.0, + raw_spectrum: vec![0.0; 64], + processed_spectrum: vec![0.0; 64], + accum_buffer: Vec::new(), + output_buffer: vec![0.0; 960], + to_process_buf: Vec::new(), + external_hook: None, + } + } + + /// Attach the external plugin DSP stage (see [`PLUGIN_CHAIN_NODE`] and + /// [`PLUGIN_NODE_PREFIX`] nodes). + pub fn set_external_hook(&mut self, hook: Option) { + self.external_hook = hook; + } + + /// Process a chunk of f32 PCM audio in-place. + /// Returns (raw_rms, processed_rms) for level metering. + /// Internally accumulates to 480-sample aligned frames before noise reduction, + /// matching the KMP AudioProcessorPipeline behavior. + pub fn process(&mut self, data: &mut Vec, channels: usize, queued_ms: f64) -> (f32, f32) { + if data.is_empty() { + return (0.0, 0.0); + } + + let raw_rms = compute_rms(data); + self.compute_spectrum(data, true); + + // Frame accumulation: align to 480*channels samples before processing, + // matching KMP's AudioProcessorPipeline behavior. + // Noise reduction requires exactly 480-sample frames. + // Processing variable-size chunks through AGC/EQ causes artifacts. + self.accum_buffer.extend_from_slice(data); + let samples_per_frame = RNNOISE_FRAME_SIZE * channels.max(1); + let frame_count = self.accum_buffer.len() / samples_per_frame; + + if frame_count == 0 { + data.clear(); + return (raw_rms, 0.0); + } + + let process_count = frame_count * samples_per_frame; + self.to_process_buf.clear(); + self.to_process_buf + .extend_from_slice(&self.accum_buffer[..process_count]); + self.accum_buffer.drain(..process_count); + + let mut to_process = std::mem::take(&mut self.to_process_buf); + + let settings = self + .settings + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + self.equalizer.update_filters(&settings.equalizer); + + for effect in &settings.processing_chain { + match effect.as_str() { + "AEC" => + { + #[cfg(feature = "noise-suppression")] + if settings.aec_enabled { + self.apply_aec(&mut to_process, channels.max(1)); + } + } + "NoiseReduction" => { + if settings.ns_enabled { + self.apply_noise_reduction(&mut to_process, channels.max(1), &settings); + } + } + "Dereverb" => { + if settings.dereverb_enabled { + self.apply_dereverb( + &mut to_process, + channels.max(1), + settings.dereverb_level, + ); + } + } + "Equalizer" => { + if settings.equalizer.enabled { + self.equalizer.process(&mut to_process, channels); + } + } + "Amplifier" => { + if settings.gain.abs() > 0.01 { + let gain_linear = 10.0_f32.powf(settings.gain / 20.0); + for sample in to_process.iter_mut() { + *sample *= gain_linear; + } + } + } + "AGC" => { + if settings.agc_enabled { + let attack_rate = settings.agc_attack / 1000.0; + let decay_rate = settings.agc_decay / 10000.0; + self.apply_agc( + &mut to_process, + settings.agc_target, + attack_rate, + decay_rate, + if !settings.ns_enabled + || settings.ns_type == "Speexdsp" + || self.noise_speech_probability >= 0.5 + { + 5.0 + } else { + 1.0 + }, + ); + } + } + "VAD" if settings.vad_enabled => { + self.apply_vad(&mut to_process, settings.vad_threshold); + } + PLUGIN_CHAIN_NODE => { + // Legacy synthetic plugin stage (may be absent): runs + // every registered plugin at this chain position. + if let Some(hook) = &mut self.external_hook { + hook( + PLUGIN_CHAIN_NODE, + &mut to_process, + channels.max(1), + queued_ms, + ); + } + } + node if node.starts_with(PLUGIN_NODE_PREFIX) => { + // Per-plugin stage (issue #347): runs just the plugin + // named by this node, at its own position in the chain. + if let Some(hook) = &mut self.external_hook { + hook(node, &mut to_process, channels.max(1), queued_ms); + } + } + _ => {} + } + } + + self.output_buffer.clear(); + self.output_buffer.extend_from_slice(&to_process); + + // Soft clip — avoids harsh hard-clipping artifacts (crackling/pops) + for sample in self.output_buffer.iter_mut() { + *sample = soft_clip(*sample); + } + + self.to_process_buf = to_process; + + data.clear(); + data.extend_from_slice(&self.output_buffer); + + let processed_rms = compute_rms(data); + self.compute_spectrum(data, false); + + (raw_rms, processed_rms) + } + + pub fn get_spectrums(&self) -> (Vec, Vec) { + (self.raw_spectrum.clone(), self.processed_spectrum.clone()) + } + + /// Feed far-end (speaker) audio for AEC. + #[cfg(feature = "noise-suppression")] + pub fn set_far_end_audio(&mut self, far_end: &[f32]) { + self.far_end.feed(far_end); + } + + /// No-op when the optional DSP feature is disabled. This keeps the public + /// audio API usable by callers that build the crate without ONNX support. + #[cfg(not(feature = "noise-suppression"))] + pub fn set_far_end_audio(&mut self, _far_end: &[f32]) {} + + #[cfg(feature = "noise-suppression")] + pub fn take_aec_failure(&mut self) -> Option { + self.aec_failure.take() + } + + #[cfg(not(feature = "noise-suppression"))] + pub fn take_aec_failure(&mut self) -> Option { + None + } + + /// Reset stream-local AEC state when a new transport session begins. + #[cfg(feature = "noise-suppression")] + pub fn reset_aec_session(&mut self) { + self.far_end.clear(); + self.aec_failure = None; + // Model loading and inference failures are scoped to one transport + // session. A failed inference discards the VM session, so clearing the + // latch here causes the next session to build a fresh one rather than + // reusing potentially corrupted runtime state. + self.aec_session_failed = false; + if let Some(aec) = &mut self.aec { + aec.reset(); + } + } + + #[cfg(not(feature = "noise-suppression"))] + pub fn reset_aec_session(&mut self) {} + + // ── AEC Acoustic Echo Cancellation ──────────────────────────────────── + + #[cfg(feature = "noise-suppression")] + fn ensure_aec_loaded(&mut self) -> bool { + if self.aec.is_some() { + return true; + } + if self.aec_session_failed { + return false; + } + + match AecProcessor::new() { + Ok(processor) => { + log::info!("[DSP] AEC7 native model ready (embedded, no ONNX Runtime)"); + self.aec = Some(processor); + true + } + Err(error) => { + log::error!("[DSP] Failed to initialize AEC7 model: {error}"); + self.fail_aec_load(AecFailure::ModelLoadFailed); + false + } + } + } + + #[cfg(feature = "noise-suppression")] + fn fail_aec_load(&mut self, failure: AecFailure) { + self.aec_session_failed = true; + self.aec_failure = Some(failure); + } + + #[cfg(feature = "noise-suppression")] + fn fail_aec_inference(&mut self) { + // Do not retry a failing VM session for every audio packet. Keep the + // failure latched for this transport session and rebuild on the next + // reset_aec_session(). + self.aec = None; + self.aec_session_failed = true; + self.aec_failure = Some(AecFailure::InferenceFailed); + } + + #[cfg(feature = "noise-suppression")] + fn process_aec_mono(&mut self, input: &[f32]) -> Result, String> { + if !input.len().is_multiple_of(AEC_HOP_LEN) { + return Err(format!( + "invalid AEC input length: {}, expected a multiple of {AEC_HOP_LEN}", + input.len() + )); + } + + let processor = self + .aec + .as_mut() + .expect("AEC model must be loaded before processing"); + let mut output = Vec::with_capacity(input.len()); + + for chunk in input.as_chunks::().0 { + match self.far_end.take_hop() { + Some(far_chunk) => output.extend(processor.process(chunk, &far_chunk)?), + None => output.extend_from_slice(chunk), + } + } + + Ok(output) + } + + #[cfg(feature = "noise-suppression")] + fn apply_aec(&mut self, data: &mut Vec, channels: usize) { + if !self.ensure_aec_loaded() { + return; + } + + let mono = (channels > 1).then(|| { + data.chunks_exact(channels) + .map(|frame| frame.iter().sum::() / channels as f32) + .collect::>() + }); + let input = mono.as_deref().unwrap_or(data); + let clean = match self.process_aec_mono(input) { + Ok(clean) => clean, + Err(error) => { + log::error!("[DSP] AEC inference failed: {error}"); + self.fail_aec_inference(); + return; + } + }; + + if channels == 1 { + *data = clean; + } else { + data.clear(); + data.reserve(clean.len() * channels); + for sample in clean { + data.extend(std::iter::repeat_n(sample, channels)); + } + } + } + + // ── Noise Reduction Dispatcher ────────────────────────────────────────── + + fn apply_noise_reduction( + &mut self, + data: &mut Vec, + channels: usize, + settings: &AudioDspSettings, + ) { + let original = data.clone(); + match settings.ns_type.as_str() { + #[cfg(feature = "dsp")] + "RNNoise" => self.apply_rnnoise(data, channels, 100.0), + #[cfg(feature = "noise-suppression")] + "PureVox" => self.apply_purevox(data, channels, 100.0), + #[cfg(feature = "dsp")] + "Speexdsp" => { + self.apply_speex(data, channels, settings.ns_intensity); + return; + } + _ => return, + } + // OpenLess: mix against the previous hop, matching model latency. + if self.dry_delay.is_empty() { + self.dry_delay.resize(RNNOISE_FRAME_SIZE * channels, 0.0); + } + self.dry_delay.extend(original); + if settings.ns_type == "PureVox" { + // PureVox has no VAD output. Avoid boosting strongly rejected + // stationary noise by comparing wet energy with aligned dry audio. + let dry_power = self + .dry_delay + .iter() + .take(data.len()) + .map(|s| s * s) + .sum::() + / data.len().max(1) as f32; + let wet_rms = compute_rms(data); + self.noise_speech_probability = if wet_rms > dry_power.sqrt() * 0.2 { + 1.0 + } else { + 0.0 + }; + } + let mix = (settings.ns_intensity / 100.0).clamp(0.0, 1.0); + for sample in data { + let dry = self.dry_delay.pop_front().unwrap_or(0.0); + *sample = dry * (1.0 - mix) + *sample * mix; + } + } + + // ── RNNoise (nnnoiseless) ─────────────────────────────────────────────── + + #[cfg(feature = "dsp")] + fn apply_rnnoise(&mut self, data: &mut Vec, channels: usize, intensity: f32) { + if data.is_empty() || channels == 0 { + return; + } + + if channels >= 2 { + let frames = data.len() / 2; + let mut left: Vec = Vec::with_capacity(frames); + let mut right: Vec = Vec::with_capacity(frames); + for i in 0..frames { + left.push(data[i * 2]); + right.push(data[i * 2 + 1]); + } + + // Process each channel with its own denoiser (no RNN state cross-contamination) + let left_probability = Self::process_rnnoise_single_channel( + &mut left, + &mut self.ns_buffer_left, + &mut self.denoiser_left, + intensity, + ); + let right_probability = Self::process_rnnoise_single_channel( + &mut right, + &mut self.ns_buffer_right, + &mut self.denoiser_right, + intensity, + ); + self.noise_speech_probability = left_probability.max(right_probability); + + data.clear(); + for i in 0..frames { + data.push(left[i]); + data.push(right[i]); + } + } else { + self.noise_speech_probability = Self::process_rnnoise_single_channel( + data, + &mut self.ns_buffer_left, + &mut self.denoiser_left, + intensity, + ); + } + } + + #[cfg(feature = "dsp")] + fn process_rnnoise_single_channel( + data: &mut [f32], + ns_buffer: &mut Vec, + denoiser: &mut DenoiseState<'static>, + intensity: f32, + ) -> f32 { + let mix = (intensity / 100.0).clamp(0.0, 1.0); + let input_len = data.len(); + ns_buffer.extend_from_slice(data); + + let mut output = Vec::with_capacity(input_len); + let mut speech_probability = 0.0_f32; + + while ns_buffer.len() >= RNNOISE_FRAME_SIZE { + let frame: Vec = ns_buffer.drain(..RNNOISE_FRAME_SIZE).collect(); + + let input_frame: Vec = frame.iter().map(|s| s * 32767.0).collect(); + let mut output_frame = vec![0.0f32; RNNOISE_FRAME_SIZE]; + + let vad_prob = denoiser.process_frame(&mut output_frame, &input_frame); + speech_probability = speech_probability.max(vad_prob); + + for i in 0..RNNOISE_FRAME_SIZE { + let clean = output_frame[i] / 32767.0; + let original = frame[i]; + output.push(original * (1.0 - mix) + clean * mix); + } + } + + for sample in ns_buffer.drain(..) { + output.push(sample); + } + + output.truncate(input_len); + while output.len() < input_len { + output.push(0.0); + } + + data.copy_from_slice(&output); + speech_probability + } + + #[cfg(feature = "noise-suppression")] + fn apply_purevox(&mut self, data: &mut Vec, channels: usize, intensity: f32) { + // Lazy init for both channels — only attempt once; if construction + // fails, mark it so we don't retry on every audio frame. + if self.purevox_left.is_none() && !self.purevox_load_failed { + match PureVoxProcessor::new() { + Ok(proc) => { + log::info!("[DSP] PureVox native model ready (L, embedded)"); + self.purevox_left = Some(proc); + } + Err(e) => { + log::error!("[DSP] Failed to initialize PureVox model: {e}"); + self.purevox_load_failed = true; + self.apply_rnnoise(data, channels, intensity); + return; + } + } + } + if channels >= 2 && self.purevox_right.is_none() && !self.purevox_load_failed { + match PureVoxProcessor::new() { + Ok(proc) => { + log::info!("[DSP] PureVox native model ready (R, embedded)"); + self.purevox_right = Some(proc); + } + Err(e) => { + log::error!("[DSP] Failed to initialize PureVox model for R channel: {e}"); + } + } + } + + if channels >= 2 { + let frames = data.len() / 2; + let mut left: Vec = Vec::with_capacity(frames); + let mut right: Vec = Vec::with_capacity(frames); + for i in 0..frames { + left.push(data[i * 2]); + right.push(data[i * 2 + 1]); + } + + Self::process_purevox_single_channel(&mut left, &mut self.purevox_left, intensity); + Self::process_purevox_single_channel(&mut right, &mut self.purevox_right, intensity); + + data.clear(); + for i in 0..frames { + data.push(left[i]); + data.push(right[i]); + } + } else { + Self::process_purevox_single_channel(data, &mut self.purevox_left, intensity); + } + } + + #[cfg(feature = "noise-suppression")] + fn process_purevox_single_channel( + data: &mut [f32], + purevox: &mut Option, + intensity: f32, + ) { + let mix = (intensity / 100.0).clamp(0.0, 1.0); + + if let Some(proc) = purevox { + let mut output = Vec::with_capacity(data.len()); + for chunk in data.chunks(480) { + let clean = proc.process(chunk); + for i in 0..chunk.len() { + let clean_sample = if i < clean.len() { clean[i] } else { chunk[i] }; + output.push(chunk[i] * (1.0 - mix) + clean_sample * mix); + } + } + data.copy_from_slice(&output); + } + } + + #[cfg(feature = "dsp")] + fn apply_speex(&mut self, data: &mut Vec, channels: usize, intensity: f32) { + let input_len = data.len(); + + // Stereo: separate channels, process each independently, re-interleave + if channels >= 2 && input_len >= 2 { + let frames = input_len / 2; + let mut left: Vec = Vec::with_capacity(frames); + let mut right: Vec = Vec::with_capacity(frames); + for i in 0..frames { + left.push(data[i * 2]); + right.push(data[i * 2 + 1]); + } + + self.speex_ns.process(&mut left, intensity); + self.speex_ns.process(&mut right, intensity); + + // Re-interleave + data.clear(); + for i in 0..frames { + data.push(left[i]); + data.push(right[i]); + } + } else { + // Mono + self.speex_ns.process(data, intensity); + } + } + + // ── Dereverb (delay-line comb filter, matching KMP DereverbEffect) ───── + + fn apply_dereverb(&mut self, data: &mut [f32], channels: usize, level: f32) { + let mix = (level / 100.0).clamp(0.0, 1.0); + if mix <= 0.0 || channels == 0 { + return; + } + + let delay = 480usize; + + // Ensure buffers are sized + if self.dereverb_buffer_left.len() != delay { + self.dereverb_buffer_left = vec![0.0; delay]; + } + if self.dereverb_buffer_right.len() != delay { + self.dereverb_buffer_right = vec![0.0; delay]; + } + + if channels == 1 { + let buf = &mut self.dereverb_buffer_left; + for sample in data.iter_mut() { + let delayed = buf[self.dereverb_index]; + buf[self.dereverb_index] = *sample; + *sample = (*sample - delayed * mix).clamp(-1.0, 1.0); + self.dereverb_index += 1; + if self.dereverb_index >= delay { + self.dereverb_index = 0; + } + } + } else { + let buf_l = &mut self.dereverb_buffer_left; + let buf_r = &mut self.dereverb_buffer_right; + let mut i = 0; + while i + 1 < data.len() { + let delayed_l = buf_l[self.dereverb_index]; + let delayed_r = buf_r[self.dereverb_index]; + buf_l[self.dereverb_index] = data[i]; + buf_r[self.dereverb_index] = data[i + 1]; + data[i] = (data[i] - delayed_l * mix).clamp(-1.0, 1.0); + data[i + 1] = (data[i + 1] - delayed_r * mix).clamp(-1.0, 1.0); + self.dereverb_index += 1; + if self.dereverb_index >= delay { + self.dereverb_index = 0; + } + i += 2; + } + } + } + + // ── AGC ───────────────────────────────────────────────────────────────── + + fn apply_agc( + &mut self, + data: &mut [f32], + target: f32, + attack: f32, + decay: f32, + maximum_gain: f32, + ) { + let target_linear = target / 32767.0; + // Noise gate threshold: below this level, don't apply AGC gain + // (prevents amplifying hiss/noise floor during speech pauses) + let gate_threshold = 0.005_f32; // ~ -46dB + + for sample in data.iter_mut() { + let abs_sample = sample.abs(); + if abs_sample > self.agc_envelope { + self.agc_envelope += attack * (abs_sample - self.agc_envelope); + } else { + self.agc_envelope += decay * (abs_sample - self.agc_envelope); + } + if self.agc_envelope > gate_threshold { + let desired_gain = target_linear / self.agc_envelope; + let clamped_gain = desired_gain.clamp(0.1, maximum_gain); + // Smooth gain transition to avoid pops (exponential moving average) + let smooth_factor = 0.005_f32; + self.agc_smoothed_gain += smooth_factor * (clamped_gain - self.agc_smoothed_gain); + *sample *= self.agc_smoothed_gain; + } else { + // Below noise gate: smoothly reduce gain toward unity + let smooth_factor = 0.002_f32; + self.agc_smoothed_gain += smooth_factor * (1.0 - self.agc_smoothed_gain); + *sample *= self.agc_smoothed_gain; + } + } + } + + // ── VAD ───────────────────────────────────────────────────────────────── + + fn apply_vad(&mut self, data: &mut [f32], threshold_db: f32) { + let rms = compute_rms(data); + let rms_db = if rms > 1e-10 { + 20.0 * rms.log10() + } else { + -100.0 + }; + let target_fade = if rms_db >= threshold_db { 1.0 } else { 0.0 }; + let fade_speed = if target_fade > self.vad_fade { + 0.1 + } else { + 0.02 + }; + self.vad_fade += fade_speed * (target_fade - self.vad_fade); + self.vad_fade = self.vad_fade.clamp(0.0, 1.0); + for sample in data.iter_mut() { + *sample *= self.vad_fade; + } + } + + // ── Spectrum ───────────────────────────────────────────────────────────── + + fn compute_spectrum(&mut self, data: &[f32], is_raw: bool) { + let bands = 64; + let target = if is_raw { + &mut self.raw_spectrum + } else { + &mut self.processed_spectrum + }; + if target.len() != bands { + target.resize(bands, 0.0); + } + if data.is_empty() { + for v in target.iter_mut() { + *v = 0.0; + } + return; + } + + const BANDS: usize = 64; + static BAND_LIMITS: std::sync::OnceLock<[f32; BANDS + 1]> = std::sync::OnceLock::new(); + let limits = BAND_LIMITS.get_or_init(|| { + let mut array = [0.0; BANDS + 1]; + for (i, limit) in array.iter_mut().enumerate() { + *limit = (i as f32 / BANDS as f32).powf(1.5); + } + array + }); + + for (band_idx, band_val) in target.iter_mut().enumerate() { + let start = limits[band_idx] * data.len() as f32; + let end = limits[band_idx + 1] * data.len() as f32; + let start = start as usize; + let end = (end as usize).min(data.len()); + if start >= end { + *band_val *= 0.85; + continue; + } + let mut sum = 0.0_f32; + for sample in &data[start..end] { + sum += sample * sample; + } + let rms = (sum / (end - start) as f32).sqrt(); + let db = if rms > 1e-10 { + 20.0 * rms.log10() + } else { + -100.0 + }; + let normalized = ((db + 60.0) / 60.0).clamp(0.0, 1.0); + if normalized > *band_val { + *band_val = normalized; + } else { + *band_val = *band_val * 0.85 + normalized * 0.15; + } + } + } +} + +fn compute_rms(data: &[f32]) -> f32 { + if data.is_empty() { + return 0.0; + } + let sum: f32 = data.iter().map(|s| s * s).sum(); + (sum / data.len() as f32).sqrt() +} + +/// Soft clip — smooth polynomial knee to avoid harsh hard-clipping artifacts. +/// Identity below 0.95, smooth Hermite compression to ±1.0 at ±2.0. +fn soft_clip(sample: f32) -> f32 { + let a = sample.abs(); + if a <= 0.95 { + sample + } else if a <= 2.0 { + let sign = sample.signum(); + let t = (a - 0.95) / 1.05; // 0..1 over [0.95, 2.0] + // Hermite smoothstep: C1 continuous, f(0)=0, f(1)=1, f'(0)=f'(1)=0 + let s = t * t * (3.0 - 2.0 * t); + sign * (0.95 + 0.05 * s) + } else { + sample.signum() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plugin_hook_runs_when_chain_contains_plugin_node() { + let settings = Arc::new(RwLock::new(AudioDspSettings { + processing_chain: vec![ + "AEC".to_string(), + PLUGIN_CHAIN_NODE.to_string(), + "Amplifier".to_string(), + ], + ..Default::default() + })); + let mut processor = DspProcessor::new(settings, None); + let calls = std::sync::Arc::new(std::sync::Mutex::new(0usize)); + let calls_clone = calls.clone(); + processor.set_external_hook(Some(Box::new( + move |node: &str, data: &mut Vec, channels: usize, _queued_ms: f64| { + assert_eq!(node, PLUGIN_CHAIN_NODE); + assert_eq!(channels, 1); + for sample in data.iter_mut() { + *sample += 0.5; // visible marker: plugin output + } + *calls_clone.lock().unwrap() += 1; + }, + ))); + + let mut data = vec![0.1f32; 960]; + processor.process(&mut data, 1, 10.0); + assert_eq!( + *calls.lock().unwrap(), + 1, + "hook must run once per chain pass" + ); + // Amplifier(0dB) + soft clip: plugin marker survives in output + assert!( + data.iter().any(|s| (*s - 0.6).abs() < 0.001), + "plugin output must reach the final buffer" + ); + } + + #[test] + fn plugin_hook_skipped_when_chain_has_no_plugin_node() { + let settings = Arc::new(RwLock::new(AudioDspSettings::default())); + let mut processor = DspProcessor::new(settings, None); + let calls = std::sync::Arc::new(std::sync::Mutex::new(0usize)); + let calls_clone = calls.clone(); + processor.set_external_hook(Some(Box::new( + move |_node: &str, _data: &mut Vec, _ch: usize, _q: f64| { + *calls_clone.lock().unwrap() += 1; + }, + ))); + + let mut data = vec![0.1f32; 960]; + processor.process(&mut data, 1, 10.0); + assert_eq!( + *calls.lock().unwrap(), + 0, + "hook must not run without the Plugins node" + ); + } + + #[test] + fn per_plugin_nodes_invoke_hook_in_chain_order() { + let settings = Arc::new(RwLock::new(AudioDspSettings { + processing_chain: vec![ + "AEC".to_string(), + plugin_chain_node("eq"), + "Amplifier".to_string(), + plugin_chain_node("comp"), + ], + ..Default::default() + })); + let mut processor = DspProcessor::new(settings, None); + let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let seen_clone = seen.clone(); + processor.set_external_hook(Some(Box::new( + move |node: &str, _data: &mut Vec, _channels: usize, _queued_ms: f64| { + seen_clone.lock().unwrap().push(node.to_string()); + }, + ))); + + let mut data = vec![0.1f32; 960]; + processor.process(&mut data, 1, 10.0); + assert_eq!( + *seen.lock().unwrap(), + vec![plugin_chain_node("eq"), plugin_chain_node("comp")], + "each per-plugin node must invoke the hook exactly once, in chain order" + ); + } + + #[test] + fn plugin_node_helpers_round_trip() { + assert_eq!(plugin_chain_node("my.eq"), "Plugin:my.eq"); + assert_eq!(parse_plugin_chain_node("Plugin:my.eq"), Some("my.eq")); + assert_eq!(parse_plugin_chain_node(PLUGIN_CHAIN_NODE), None); + assert_eq!(parse_plugin_chain_node("Equalizer"), None); + assert_eq!(parse_plugin_chain_node("Plugin:"), Some("")); + } + + #[test] + fn unsupported_noise_reduction_type_uses_default() { + let mut settings = AudioDspSettings { + ns_type: "RemovedAlgorithm".to_string(), + ..Default::default() + }; + + settings.normalize(); + + assert_eq!(settings.ns_type, AudioDspSettings::default().ns_type); + } + + #[cfg(feature = "noise-suppression")] + #[test] + fn far_end_buffer_consumes_distinct_hops_in_order() { + let mut buffer = FarEndBuffer::new(); + let samples: Vec = (0..AEC_HOP_LEN * 2 + 60) + .map(|sample| sample as f32) + .collect(); + buffer.feed(&samples); + + let first = buffer.take_hop().unwrap(); + let second = buffer.take_hop().unwrap(); + assert_eq!(first[0], 0.0); + assert_eq!(first[AEC_HOP_LEN - 1], 479.0); + assert_eq!(second[0], 480.0); + assert_eq!(second[AEC_HOP_LEN - 1], 959.0); + assert!(buffer.take_hop().is_none()); + + buffer.clear(); + assert!(buffer.take_hop().is_none()); + } + + #[cfg(feature = "noise-suppression")] + #[test] + fn new_session_retries_aec_model_after_load_failure() { + let settings = Arc::new(RwLock::new(AudioDspSettings::default())); + let mut processor = DspProcessor::new(settings, None); + processor.aec_session_failed = true; + processor.aec_failure = Some(AecFailure::ModelLoadFailed); + processor.far_end.feed(&vec![1.0; AEC_HOP_LEN]); + + processor.reset_aec_session(); + + assert!(!processor.aec_session_failed); + assert_eq!(processor.aec_failure, None); + assert!(processor.far_end.take_hop().is_none()); + } + + #[cfg(feature = "noise-suppression")] + #[test] + fn inference_failure_is_latched_until_next_session() { + let settings = Arc::new(RwLock::new(AudioDspSettings::default())); + let mut processor = DspProcessor::new(settings, None); + + processor.fail_aec_inference(); + + assert!(processor.aec.is_none()); + assert!(processor.aec_session_failed); + assert_eq!( + processor.take_aec_failure(), + Some(AecFailure::InferenceFailed) + ); + assert_eq!(processor.take_aec_failure(), None); + assert!(!processor.ensure_aec_loaded()); + + processor.reset_aec_session(); + + assert!(!processor.aec_session_failed); + assert_eq!(processor.aec_failure, None); + } + + #[cfg(feature = "noise-suppression")] + #[test] + fn invalid_aec_input_length_is_rejected_before_processing() { + let settings = Arc::new(RwLock::new(AudioDspSettings::default())); + let mut processor = DspProcessor::new(settings, None); + let input = vec![0.0; AEC_HOP_LEN - 1]; + + let error = processor.process_aec_mono(&input).unwrap_err(); + + assert!(error.contains("invalid AEC input length")); + assert_eq!(input.len(), AEC_HOP_LEN - 1); + } + + #[test] + fn test_gain_positive() { + let settings = Arc::new(RwLock::new(AudioDspSettings { + gain: 20.0, + ..Default::default() + })); + let mut processor = DspProcessor::new(settings, None); + let mut data = vec![0.1; 480]; + processor.process(&mut data, 1, 80.0); + assert!(data[0] > 0.9, "Expected amplified sample, got {}", data[0]); + } + + #[test] + fn test_gain_negative() { + let settings = Arc::new(RwLock::new(AudioDspSettings { + gain: -20.0, + ..Default::default() + })); + let mut processor = DspProcessor::new(settings, None); + let mut data = vec![0.5; 480]; + processor.process(&mut data, 1, 80.0); + assert!(data[0] < 0.1, "Expected attenuated sample, got {}", data[0]); + } + + #[test] + fn test_vad_mutes_quiet() { + let settings = Arc::new(RwLock::new(AudioDspSettings { + vad_enabled: true, + vad_threshold: -10.0, + ..Default::default() + })); + let mut processor = DspProcessor::new(settings, None); + let mut data = vec![0.001; 960]; + for _ in 0..20 { + processor.process(&mut data, 1, 80.0); + } + assert!( + data[data.len() - 1].abs() < 0.01, + "Expected muted, got {}", + data[data.len() - 1] + ); + } + + #[test] + fn test_agc_boosts_quiet() { + let settings = Arc::new(RwLock::new(AudioDspSettings { + agc_enabled: true, + agc_target: 16000.0, + agc_attack: 90.0, + agc_decay: 10.0, + ..Default::default() + })); + let mut processor = DspProcessor::new(settings, None); + let mut data: Vec = vec![0.01; 4800]; + for _ in 0..10 { + processor.process(&mut data, 1, 80.0); + } + assert!( + data[data.len() - 1].abs() > 0.01, + "AGC should have amplified the signal" + ); + } +} + +// ─── Native model property tests ────────────────────────────────────────── +// Exact golden/parity verification of the embedded blobs lives in the +// libmicyou experimental repository (see docs/pure-rust-inference.md); the +// tests here are fixture-free behavioural properties of the integration. + +#[cfg(all(test, feature = "noise-suppression"))] +mod native_tests { + use super::*; + + /// Deterministic LCG so property tests need no rand dependency. + struct Lcg(u64); + impl Lcg { + fn next_f32(&mut self) -> f32 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + ((self.0 >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } + } + + fn rms(x: &[f32]) -> f32 { + (x.iter().map(|v| v * v).sum::() / x.len().max(1) as f32).sqrt() + } + + #[test] + fn purevox_suppresses_stationary_noise() { + let mut proc = PureVoxProcessor::new().expect("PureVoxProcessor must build"); + let mut rng = Lcg(0x5eed_2026_0927_0001); + let hop = 480usize; + let warmup = 100usize; + let measure = 100usize; + let mut out_tail: Vec = Vec::with_capacity(measure * hop); + let mut in_tail: Vec = Vec::with_capacity(measure * hop); + for f in 0..warmup + measure { + let frame: Vec = (0..hop).map(|_| rng.next_f32() * 0.05).collect(); + let out = proc.process(&frame); + if f >= warmup { + out_tail.extend_from_slice(&out); + in_tail.extend_from_slice(&frame); + } + } + let (r_out, r_in) = (rms(&out_tail), rms(&in_tail)); + println!( + "purevox noise: in_rms={r_in:.5} out_rms={r_out:.5} ratio={}", + r_out / r_in + ); + assert!(out_tail.iter().all(|v| v.is_finite())); + assert!( + r_out < 0.8 * r_in, + "stationary noise must be attenuated (ratio {})", + r_out / r_in + ); + } + + #[test] + fn aec_attenuates_noise_echo() { + // Aligned noise-like far-end echo is the model's strong suit (tonal + // signals are deliberately treated as possible near-end speech and + // only mildly attenuated). Noise echo gets crushed (>20 dB in the + // numpy reference), so a loose 0.5 ratio threshold is a robust + // wiring+function signal without being flaky. + let mut proc = AecProcessor::new().expect("AecProcessor must build"); + let mut rng = Lcg(0x5eed_2026_0927_0003); + let hop = 480usize; + let warmup = 150usize; + let measure = 50usize; + let mut out_tail: Vec = Vec::with_capacity(measure * hop); + let mut mic_tail: Vec = Vec::with_capacity(measure * hop); + for f in 0..warmup + measure { + let far: Vec = (0..hop).map(|_| rng.next_f32() * 0.3).collect(); + let mic: Vec = far.iter().map(|v| 0.7 * v).collect(); + let out = proc.process(&mic, &far).expect("aec frame must succeed"); + if f >= warmup { + out_tail.extend_from_slice(&out); + mic_tail.extend_from_slice(&mic); + } + } + let (r_out, r_mic) = (rms(&out_tail), rms(&mic_tail)); + println!( + "aec noise echo: mic_rms={r_mic:.5} out_rms={r_out:.5} ratio={}", + r_out / r_mic + ); + assert!(out_tail.iter().all(|v| v.is_finite())); + assert!( + r_out < 0.5 * r_mic, + "aligned noise echo must be attenuated (ratio {})", + r_out / r_mic + ); + } + + #[test] + fn dsp_processor_chain_smoke() { + // Full chain: AEC → PureVox NS through the public DspProcessor API. + let mut settings = AudioDspSettings::default(); + settings.ns_enabled = true; + settings.ns_type = "PureVox".to_string(); + settings.ns_intensity = 100.0; + settings.aec_enabled = true; + let settings = Arc::new(RwLock::new(settings)); + let mut dsp = DspProcessor::new(settings, None); + + let mut rng = Lcg(0x5eed_2026_0927_0002); + let hop = 480usize; + for f in 0..60 { + let far: Vec = (0..hop).map(|_| rng.next_f32() * 0.2).collect(); + dsp.set_far_end_audio(&far); + let mut data: Vec = (0..hop) + .map(|i| far[i] * 0.5 + rng.next_f32() * 0.03) + .collect(); + let (raw, processed) = dsp.process(&mut data, 1, 200.0); + assert!(raw.is_finite() && processed.is_finite(), "frame {f}"); + // accumulation may defer output for partial frames; once flowing, + // length must be preserved for 480-aligned inputs + if !data.is_empty() { + assert_eq!(data.len() % hop, 0, "frame {f}: ragged output"); + assert!(data.iter().all(|v| v.is_finite()), "frame {f}: NaN/inf"); + } + } + assert!( + dsp.take_aec_failure().is_none(), + "AEC must not fail with embedded model" + ); + } +} diff --git a/openless-all/app/crates/micyou-infer/Cargo.toml b/openless-all/app/crates/micyou-infer/Cargo.toml new file mode 100644 index 000000000..9e88a89c3 --- /dev/null +++ b/openless-all/app/crates/micyou-infer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "micyou-infer" +version = "0.1.0" +edition = "2021" +license-file = "LICENSE" +publish = false + +[dependencies] diff --git a/openless-all/app/crates/micyou-infer/LICENSE b/openless-all/app/crates/micyou-infer/LICENSE new file mode 100644 index 000000000..01288814f --- /dev/null +++ b/openless-all/app/crates/micyou-infer/LICENSE @@ -0,0 +1,725 @@ +MicYou is licensed under the GNU General Public License version 3.0 (GPLv3) +with the MicYou Plugin Exception (Version 1.0). + +Copyright (C) 2026 LanRhyme + +============================================================================== + MicYou Plugin Exception (Version 1.0) +============================================================================== + +As a special exception to Sections 5 and 6 of the GNU General Public License +version 3 ("GPLv3"), under Section 7 (Additional Permissions), the copyright +holders of MicYou grant permission to link, load, or combine independently +created software modules ("Plugins") with MicYou through its defined Plugin +Interfaces, and to convey the resulting combined work, provided that all of +the following conditions are met: + +1. Defined Plugin Interfaces: + This exception applies solely to Plugins that interact with MicYou through + its official, public Plugin Application Programming Interfaces (APIs), + including but not limited to the C ABI (mpl_host_api_t / micyou_plugin_abi.h), + WASM guest-host interfaces, and official Plugin SDKs. This exception does + not grant permission to extract, repackage, or link internal components of + MicYou outside the defined Plugin Interfaces into proprietary works. + +2. Non-Commercial Plugins: + Plugins that are developed, distributed, or used solely for non-commercial + purposes are NOT required to be licensed under the GNU GPLv3. Authors of + such non-commercial Plugins may choose any open-source or proprietary + license of their choice. + +3. Commercial Plugins: + Any commercial distribution, sale, paid subscription, or proprietary + commercial monetization of Plugins that link with or run inside MicYou + requires prior written authorization or a separate commercial license from + the copyright holders of MicYou. + +4. Core Program Protection: + Nothing in this exception modifies the license terms of MicYou itself or + works directly based on MicYou (other than the Plugin combination permitted + herein). Any modification to MicYou's core source code remains strictly + subject to the full terms and conditions of the GNU General Public License + version 3. + +5. Marketplace Distribution: + Distribution via the official MicYou Plugin Marketplace is subject to the + MicYou Plugin Marketplace Policy, which may require GPL-3.0-compatible open + source licensing or official review and authorization. + +============================================================================== + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 +============================================================================== + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/openless-all/app/crates/micyou-infer/src/lib.rs b/openless-all/app/crates/micyou-infer/src/lib.rs new file mode 100644 index 000000000..061a117a0 --- /dev/null +++ b/openless-all/app/crates/micyou-infer/src/lib.rs @@ -0,0 +1,998 @@ +/* + * MicYou — Turns your Android device into a high-quality PC microphone. + * Copyright (C) 2026 LanRhyme + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version, with the MicYou Plugin Exception. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +//! `micyou-infer` — a tiny, dependency-free inference VM for **statically +//! compiled** MCYI model blobs. +//! +//! This crate replaces `ort`/ONNX Runtime for the PureVox6 noise suppressor +//! and the AEC7 echo canceller. The models are compiled ahead of time by the +//! `tools/onnx-port` compiler in the experimental +//! [libmicyou](https://github.com/OrientCOMPASS/libmicyou) repository (numpy +//! reference interpreter → constant-folded static instruction stream + flat +//! f32 weight blob), following the same methodology as the silero-v4 +//! pure-Rust port. Compilation folds every shape-structural node away, so at +//! runtime the VM only executes numeric kernels over a pre-packed f32 arena — +//! no graph parsing, no dynamic shapes, no allocations per frame. +//! +//! # Semantics contract +//! +//! The executable specification of the blob format and every kernel is +//! `onnxref/replay.py` (numpy) in the libmicyou repository. The Rust kernels +//! are a mechanical transcription, verified three ways there: +//! +//! 1. `parity_ort.py` — numpy interpreter ↔ onnxruntime, streaming frames, +//! bit-approximate (≤ ~2e-5 f32 noise); +//! 2. `replay_check.py` — compiled blob ↔ interpreter (numpy replay); +//! 3. golden fixtures (`gen_golden.py`) — per-op kernels and full end-to-end +//! streaming sequences. +//! +//! This repository ships only the runtime artifacts (VM + compiled blobs); +//! `docs/pure-rust-inference.md` records the verification matrix and results. +//! +//! # Safety model +//! +//! Kernels operate on raw `*const f32`/`*mut f32` into a single arena. +//! Soundness rests on one invariant, enforced by the compiler's live-range +//! packing and re-checked in debug builds ([`Graph::validate_layout`]): +//! at every op, the output region never overlaps any input region. + +// Preserve upstream numerical kernels and operation signatures for reproducible imports. +#![allow( + clippy::needless_range_loop, + clippy::excessive_precision, + clippy::approx_constant, + clippy::manual_checked_ops, + clippy::manual_range_contains, + clippy::unnecessary_fold, + clippy::too_many_arguments, + clippy::chunks_exact_to_as_chunks, + clippy::type_complexity +)] + +use std::fmt; + +mod ops; + +// ─── blob format constants (keep in sync with onnxref/compiler.py) ─────── + +/// Blob magic: `MCYI` (MiCYou Inference). +pub const MAGIC: [u8; 4] = *b"MCYI"; +/// Blob format version produced by the onnx-port compiler and expected here. +pub const VERSION: u32 = 1; + +/// Sentinel input position meaning "absent optional input". +pub const NULL_SLOT: u32 = u32::MAX; + +// Slot kinds +const SLOT_INPUT: u8 = 0; +const SLOT_CONST: u8 = 1; +const SLOT_ARENA: u8 = 2; +const SLOT_ALIAS: u8 = 3; + +// Opcodes (mirror onnxref/compiler.py OPS) +const OP_ADD: u16 = 1; +const OP_SUB: u16 = 2; +const OP_MUL: u16 = 3; +const OP_DIV: u16 = 4; +const OP_POW: u16 = 5; +const OP_SIGMOID: u16 = 6; +const OP_SQRT: u16 = 7; +const OP_LOG: u16 = 8; +const OP_CLIP: u16 = 9; +const OP_MATMUL: u16 = 10; +const OP_CONV: u16 = 11; +const OP_CONV_T: u16 = 12; +const OP_GRU: u16 = 13; +const OP_BATCH_NORM: u16 = 14; +const OP_LAYER_NORM: u16 = 15; +const OP_TRANSPOSE: u16 = 16; +const OP_CONCAT: u16 = 17; +const OP_SLICE: u16 = 18; +const OP_PAD: u16 = 19; +const OP_EXPAND: u16 = 20; +const OP_GATHER: u16 = 21; +const OP_REDUCE_MEAN: u16 = 22; +const OP_REDUCE_L2: u16 = 23; +const OP_RESIZE: u16 = 24; +const OP_COPY: u16 = 25; + +/// Errors surfaced by parsing or execution. +#[derive(Debug)] +pub enum InferError { + BadMagic, + BadVersion(u32), + Truncated, + BadSlot(u32), + UnknownOpcode(u16), + BadOperand(&'static str), + BadInputCount { + expected: usize, + got: usize, + }, + BadInputLen { + input: usize, + expected: usize, + got: usize, + }, + LayoutOverlap { + op: usize, + }, + BadOutput(usize), +} + +impl fmt::Display for InferError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BadMagic => write!(f, "not an MCYI blob (bad magic)"), + Self::BadVersion(v) => write!(f, "unsupported MCYI version {v}"), + Self::Truncated => write!(f, "MCYI blob truncated"), + Self::BadSlot(s) => write!(f, "slot id {s} out of range"), + Self::UnknownOpcode(c) => write!(f, "unknown opcode {c}"), + Self::BadOperand(what) => write!(f, "malformed operands for {what}"), + Self::BadInputCount { expected, got } => { + write!(f, "model expects {expected} inputs, got {got}") + } + Self::BadInputLen { + input, + expected, + got, + } => { + write!(f, "input {input} expects {expected} f32 values, got {got}") + } + Self::LayoutOverlap { op } => write!(f, "arena layout overlap at op {op}"), + Self::BadOutput(i) => write!(f, "output {i} unavailable"), + } + } +} + +impl std::error::Error for InferError {} + +type Result = std::result::Result; + +// ─── little-endian reader ──────────────────────────────────────────────── + +struct Reader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> Reader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + if self.pos + n > self.data.len() { + return Err(InferError::Truncated); + } + let s = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + fn u16(&mut self) -> Result { + let b = self.take(2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) + } + fn u32(&mut self) -> Result { + let b = self.take(4)?; + Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]])) + } + fn i32(&mut self) -> Result { + let b = self.take(4)?; + Ok(i32::from_le_bytes([b[0], b[1], b[2], b[3]])) + } + fn i64(&mut self) -> Result { + let b = self.take(8)?; + Ok(i64::from_le_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ])) + } + fn f32(&mut self) -> Result { + let b = self.take(4)?; + Ok(f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + } + fn skip(&mut self, n: usize) -> Result<()> { + self.take(n)?; + Ok(()) + } +} + +// ─── graph ─────────────────────────────────────────────────────────────── + +#[derive(Debug)] +struct Slot { + kind: u8, + /// Own shape of this slot (aliases carry the reshaped view). + shape: Box<[u32]>, + numel: usize, + /// CONST: element offset into `const_data`. ARENA: element offset into + /// the session arena. ALIAS: index of the aliased slot. INPUT: index + /// into the bound-input table (filled at parse). + off: u32, + /// Storage-owning slot after alias resolution (== own id unless alias). + root: u32, +} + +#[derive(Debug)] +struct Op { + code: u16, + ins: Box<[u32]>, + outs: Box<[u32]>, + i64: Box<[i64]>, + f32: Box<[f32]>, +} + +/// A parsed, immutable MCYI graph. Cheap to share between sessions +/// (`Sync`); parsing copies the weight blob into aligned storage once. +pub struct Graph { + slots: Vec, + ops: Vec, + /// Slot ids of the model inputs, in blob order. + inputs: Vec, + /// Slot ids of the model outputs, in blob order. + outputs: Vec, + const_data: Box<[f32]>, + arena_len: usize, +} + +impl Graph { + /// Parse an MCYI blob. `data` is typically an `include_bytes!` asset. + pub fn parse(data: &[u8]) -> Result { + let mut r = Reader::new(data); + let magic = r.take(4)?; + if magic != MAGIC { + return Err(InferError::BadMagic); + } + let version = r.u32()?; + if version != VERSION { + return Err(InferError::BadVersion(version)); + } + let n_slots = r.u32()? as usize; + let n_ops = r.u32()? as usize; + let n_in = r.u32()? as usize; + let n_out = r.u32()? as usize; + let arena_len = r.u32()? as usize; + let const_len = r.u32()? as usize; + + let mut inputs = Vec::with_capacity(n_in); + for _ in 0..n_in { + inputs.push(r.u32()?); + } + let mut outputs = Vec::with_capacity(n_out); + for _ in 0..n_out { + outputs.push(r.u32()?); + } + + // slot table (byte-length prefixed) + let slot_bytes = r.u32()? as usize; + let slot_end = r.pos + slot_bytes; + let mut slots: Vec = Vec::with_capacity(n_slots); + for _ in 0..n_slots { + let kind = r.u8()?; + let ndim = r.u8()? as usize; + r.skip(2)?; // pad u16 + let off = r.i32()? as u32; + let mut shape = Vec::with_capacity(ndim); + for _ in 0..ndim { + let d = r.i32()?; + if d < 0 { + return Err(InferError::Truncated); + } + shape.push(d as u32); + } + let numel = shape.iter().fold(1usize, |a, &d| a * d as usize); + slots.push(Slot { + kind, + shape: shape.into_boxed_slice(), + numel, + off, + root: u32::MAX, + }); + } + if r.pos != slot_end { + return Err(InferError::Truncated); + } + + // op table (byte-length prefixed) + let op_bytes = r.u32()? as usize; + let op_end = r.pos + op_bytes; + let mut op_vec: Vec = Vec::with_capacity(n_ops); + for _ in 0..n_ops { + let code = r.u16()?; + let n_i64 = r.u16()? as usize; + let n_ins = r.u16()? as usize; + let n_outs = r.u16()? as usize; + let n_f32 = r.u16()? as usize; + let mut ins = Vec::with_capacity(n_ins); + for _ in 0..n_ins { + ins.push(r.u32()?); + } + let mut outs = Vec::with_capacity(n_outs); + for _ in 0..n_outs { + outs.push(r.u32()?); + } + let mut i64s = Vec::with_capacity(n_i64); + for _ in 0..n_i64 { + i64s.push(r.i64()?); + } + let mut f32s = Vec::with_capacity(n_f32); + for _ in 0..n_f32 { + f32s.push(r.f32()?); + } + op_vec.push(Op { + code, + ins: ins.into_boxed_slice(), + outs: outs.into_boxed_slice(), + i64: i64s.into_boxed_slice(), + f32: f32s.into_boxed_slice(), + }); + } + if r.pos != op_end { + return Err(InferError::Truncated); + } + + // const data (f32 LE) + let bytes = r.take(const_len * 4)?; + let mut const_data = Vec::with_capacity(const_len); + for chunk in bytes.chunks_exact(4) { + const_data.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + + let mut g = Graph { + slots, + ops: op_vec, + inputs, + outputs, + const_data: const_data.into_boxed_slice(), + arena_len, + }; + g.finalize()?; + Ok(g) + } + + /// Resolve alias chains, map input slots to bound-input indices and + /// sanity-check references. + fn finalize(&mut self) -> Result<()> { + let n = self.slots.len() as u32; + // alias roots (chains are acyclic by construction; guard anyway) + for sid in 0..self.slots.len() { + let mut root = sid as u32; + let mut depth = 0u32; + while self.slots[root as usize].kind == SLOT_ALIAS { + root = self.slots[root as usize].off; + if root >= n || depth > n { + return Err(InferError::BadSlot(root)); + } + depth += 1; + } + self.slots[sid].root = root; + } + // input slots get their bound-table index in `off` + for (i, &sid) in self.inputs.iter().enumerate() { + if sid as usize >= self.slots.len() { + return Err(InferError::BadSlot(sid)); + } + let s = &mut self.slots[sid as usize]; + if s.kind != SLOT_INPUT { + return Err(InferError::BadOperand("graph input is not an INPUT slot")); + } + s.off = i as u32; + } + for &sid in self.outputs.iter() { + if sid as usize >= self.slots.len() { + return Err(InferError::BadSlot(sid)); + } + let root = self.slots[sid as usize].root; + if self.slots[root as usize].kind == SLOT_INPUT { + return Err(InferError::BadOperand( + "graph output aliases an input (compiler must emit COPY)", + )); + } + } + for (oi, op) in self.ops.iter().enumerate() { + for &sid in op.ins.iter().chain(op.outs.iter()) { + if sid != NULL_SLOT && sid as usize >= self.slots.len() { + return Err(InferError::BadSlot(sid)); + } + } + for &sid in op.outs.iter() { + if sid == NULL_SLOT { + continue; + } + let root = self.slots[sid as usize].root as usize; + if self.slots[root].kind != SLOT_ARENA { + return Err(InferError::BadOperand("op output is not an ARENA slot")); + } + let end = self.slots[root].off as usize + self.slots[sid as usize].numel; + if end > self.arena_len { + return Err(InferError::LayoutOverlap { op: oi }); + } + } + } + #[cfg(debug_assertions)] + self.validate_layout()?; + Ok(()) + } + + /// Debug-build check of the core soundness invariant: per op, output + /// regions never overlap input regions of the same op. + #[cfg(debug_assertions)] + fn validate_layout(&self) -> Result<()> { + for (oi, op) in self.ops.iter().enumerate() { + let mut regions: Vec<(usize, usize, bool)> = Vec::new(); // (start, end, is_out) + for &sid in op.ins.iter() { + if sid == NULL_SLOT { + continue; + } + let s = &self.slots[sid as usize]; + if s.kind != SLOT_ARENA { + continue; + } + let root = &self.slots[s.root as usize]; + regions.push((root.off as usize, root.off as usize + s.numel, false)); + } + for &sid in op.outs.iter() { + if sid == NULL_SLOT { + continue; + } + let s = &self.slots[sid as usize]; + let root = &self.slots[s.root as usize]; + regions.push((root.off as usize, root.off as usize + s.numel, true)); + } + for i in 0..regions.len() { + if !regions[i].2 { + continue; + } + for j in 0..regions.len() { + if i == j { + continue; + } + // inputs may overlap each other (read-read is fine) + if !regions[j].2 && regions[i].0 < regions[j].1 && regions[j].0 < regions[i].1 { + return Err(InferError::LayoutOverlap { op: oi }); + } + } + } + } + Ok(()) + } + + /// Number of model inputs. + pub fn num_inputs(&self) -> usize { + self.inputs.len() + } + /// Number of model outputs. + pub fn num_outputs(&self) -> usize { + self.outputs.len() + } + /// Shape (f32 element counts per axis) of model input `i`. + pub fn input_shape(&self, i: usize) -> Option<&[u32]> { + self.inputs + .get(i) + .map(|&sid| &*self.slots[sid as usize].shape) + } + /// Shape of model output `i`. + pub fn output_shape(&self, i: usize) -> Option<&[u32]> { + self.outputs + .get(i) + .map(|&sid| &*self.slots[sid as usize].shape) + } + /// Element count expected for model input `i`. + pub fn input_len(&self, i: usize) -> Option { + self.inputs + .get(i) + .map(|&sid| self.slots[sid as usize].numel) + } + /// Element count of model output `i`. + pub fn output_len(&self, i: usize) -> Option { + self.outputs + .get(i) + .map(|&sid| self.slots[sid as usize].numel) + } + /// Total f32 elements in the embedded weight/constant blob. + pub fn const_len(&self) -> usize { + self.const_data.len() + } + /// Total f32 elements in the runtime arena. + pub fn arena_len(&self) -> usize { + self.arena_len + } + /// Number of emitted ops. + pub fn num_ops(&self) -> usize { + self.ops.len() + } +} + +// ─── session ───────────────────────────────────────────────────────────── + +struct Bound { + ptr: *const f32, +} + +/// Mutable execution context over a [`Graph`]: owns the f32 arena and the +/// per-run scratch buffer. Not `Clone`; create one per audio channel. +pub struct Session<'g> { + g: &'g Graph, + arena: Vec, + bound: Vec, + scratch: Vec, + scratch2: Vec, + /// Per-op pre-transposed GRU weight tables `(Wt, Rt)` for GRU ops whose + /// W/R inputs are constant (the shipped models). Layout per direction: + /// `Wt[dir][k][j] = W[dir][j][k]`, `Rt[dir][k][j] = R[dir][j][k]`. + /// Built once here so the audio thread never re-transposes (~0.6 ms per + /// AEC7 frame otherwise). Parallel to `g.ops`; `None` for other opcodes. + gru_pre: Vec, Box<[f32]>)>>, +} + +// SAFETY: `bound` holds caller pointers that are only dereferenced during +// `run()` and are cleared (set to dangling-empty) before `run` returns, so +// a `Session` never carries live references to another thread's memory. +// Everything else (arena, scratch, graph ref) is plain owned/shared data. +unsafe impl Send for Session<'_> {} + +impl<'g> Session<'g> { + pub fn new(g: &'g Graph) -> Self { + let gru_pre = g + .ops + .iter() + .map(|op| { + if op.code == OP_GRU { + build_gru_pre(g, op) + } else { + None + } + }) + .collect(); + Self { + g, + arena: vec![0.0f32; g.arena_len], + bound: Vec::new(), + scratch: Vec::new(), + scratch2: Vec::new(), + gru_pre, + } + } + + /// Execute one frame. `inputs` must match [`Graph::num_inputs`] in + /// order, each with exactly [`Graph::input_len`] elements. Outputs are + /// read afterwards via [`Session::output`]; they stay valid until the + /// next `run`. + pub fn run(&mut self, inputs: &[&[f32]]) -> Result<()> { + if inputs.len() != self.g.inputs.len() { + return Err(InferError::BadInputCount { + expected: self.g.inputs.len(), + got: inputs.len(), + }); + } + for (i, (&sid, inp)) in self.g.inputs.iter().zip(inputs.iter()).enumerate() { + let expect = self.g.slots[sid as usize].numel; + if inp.len() != expect { + return Err(InferError::BadInputLen { + input: i, + expected: expect, + got: inp.len(), + }); + } + } + self.bound.clear(); + for inp in inputs { + self.bound.push(Bound { ptr: inp.as_ptr() }); + } + let n_ops = self.g.ops.len(); + for oi in 0..n_ops { + self.exec(oi)?; + } + // Do not let caller pointers outlive the call. + self.bound.clear(); + Ok(()) + } + + /// Output `i` of the last [`Session::run`]. Valid until the next run. + pub fn output(&self, i: usize) -> Result<&[f32]> { + let &sid = self.g.outputs.get(i).ok_or(InferError::BadOutput(i))?; + let s = &self.g.slots[sid as usize]; + let root = &self.g.slots[s.root as usize]; + match root.kind { + SLOT_ARENA => Ok(&self.arena[root.off as usize..root.off as usize + s.numel]), + SLOT_CONST => Ok(&self.g.const_data[root.off as usize..root.off as usize + s.numel]), + _ => Err(InferError::BadOutput(i)), + } + } + + /// Copy output `i` into `dst` (must be exactly `output_len(i)` long). + pub fn copy_output(&self, i: usize, dst: &mut [f32]) -> Result<()> { + let src = self.output(i)?; + if dst.len() != src.len() { + return Err(InferError::BadOutput(i)); + } + dst.copy_from_slice(src); + Ok(()) + } + + // ── internals ── + + /// Resolve a slot to (storage pointer, element count). + /// + /// SAFETY: caller must hold the borrow of self for the pointer's + /// lifetime; INPUT pointers are only valid while `bound` is populated + /// (i.e. during `run`). + fn slot_ptr(&self, sid: u32) -> Result<(*const f32, usize)> { + let s = &self.g.slots[sid as usize]; + let root = &self.g.slots[s.root as usize]; + match root.kind { + SLOT_INPUT => { + let b = &self.bound[root.off as usize]; + Ok((b.ptr, s.numel)) + } + SLOT_CONST => Ok(( + unsafe { self.g.const_data.as_ptr().add(root.off as usize) }, + s.numel, + )), + SLOT_ARENA => Ok(( + unsafe { self.arena.as_ptr().add(root.off as usize) }, + s.numel, + )), + _ => Err(InferError::BadSlot(sid)), + } + } + + fn exec(&mut self, oi: usize) -> Result<()> { + let op = &self.g.ops[oi]; + let g = self.g; + + // Resolve inputs (≤16) and outputs (≤2) to raw pointers + shapes. + let mut ip: [*const f32; 16] = [std::ptr::null(); 16]; + let mut ilen: [usize; 16] = [0; 16]; + if op.ins.len() > 16 { + return Err(InferError::BadOperand("too many inputs")); + } + for (k, &sid) in op.ins.iter().enumerate() { + if sid != NULL_SLOT { + let (p, l) = self.slot_ptr(sid)?; + ip[k] = p; + ilen[k] = l; + } + } + if op.outs.len() > 2 { + return Err(InferError::BadOperand("too many outputs")); + } + + let arena = self.arena.as_mut_ptr(); + let scratch = &mut self.scratch; + let scratch2 = &mut self.scratch2; + + // shape borrows (from the immutable graph) + let sh = |sid: u32| -> &[u32] { &g.slots[sid as usize].shape }; + + match op.code { + OP_ADD | OP_SUB | OP_MUL | OP_DIV | OP_POW => { + let bin = match op.code { + OP_ADD => ops::BinOp::Add, + OP_SUB => ops::BinOp::Sub, + OP_MUL => ops::BinOp::Mul, + OP_DIV => ops::BinOp::Div, + _ => ops::BinOp::Pow, + }; + require2(ip, ilen, 2)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::binary(bin, ip[0], sh(op.ins[0]), ip[1], sh(op.ins[1]), dst, sh(o)); + } + } + OP_SIGMOID | OP_SQRT | OP_LOG => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::unary(op.code, ip[0], dst, ilen[0]); + } + } + OP_CLIP => { + require2(ip, ilen, 1)?; + let has_lo = op.i64[0] != 0; + let has_hi = op.i64[1] != 0; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::clip( + ip[0], + dst, + ilen[0], + has_lo.then_some(op.f32[0]), + has_hi.then_some(op.f32[1]), + ); + } + } + OP_MATMUL => { + require2(ip, ilen, 2)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::matmul(ip[0], sh(op.ins[0]), ip[1], sh(op.ins[1]), dst, sh(o)); + } + } + OP_CONV | OP_CONV_T => { + let transpose = op.code == OP_CONV_T; + let a = &op.i64; + let need = if transpose { 14 } else { 12 }; + if a.len() < need { + return Err(InferError::BadOperand("conv attrs")); + } + let attrs = ops::ConvAttrs { + kh: a[0] as usize, + kw: a[1] as usize, + sh: a[2] as usize, + sw: a[3] as usize, + dh: a[4] as usize, + dw: a[5] as usize, + pt: a[6] as i32, + pl: a[7] as i32, + pb: a[8] as i32, + pr: a[9] as i32, + group: a[10] as usize, + opt_h: if transpose { a[12] as usize } else { 0 }, + opt_w: if transpose { a[13] as usize } else { 0 }, + }; + let bias = if a[11] != 0 { + require2(ip, ilen, 3)?; + Some(ip[2]) + } else { + None + }; + require2(ip, ilen, 2)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::conv( + transpose, + ip[0], + sh(op.ins[0]), + ip[1], + sh(op.ins[1]), + bias, + dst, + sh(o), + &attrs, + scratch, + ); + } + } + OP_GRU => { + if op.ins.len() != 5 || op.outs.len() != 2 { + return Err(InferError::BadOperand("gru inputs/outputs")); + } + require2(ip, ilen, 3)?; + let hidden = op.i64[0] as usize; + let direction = op.i64[1]; + let lbr = op.i64[2] != 0; + let b = if ip[3].is_null() { None } else { Some(ip[3]) }; + let h0 = if ip[4].is_null() { None } else { Some(ip[4]) }; + // outputs: [Y, Y_h]; either may be NULL_SLOT + let y = if op.outs[0] != NULL_SLOT { + Some(unsafe { arena.add(g.slots[op.outs[0] as usize].off as usize) }) + } else { + None + }; + let yh = if op.outs.len() > 1 && op.outs[1] != NULL_SLOT { + Some(unsafe { arena.add(g.slots[op.outs[1] as usize].off as usize) }) + } else { + None + }; + let pre = self + .gru_pre + .get(oi) + .and_then(|v| v.as_ref()) + .map(|(pw, pr)| (&pw[..], &pr[..])); + unsafe { + ops::gru( + ip[0], + sh(op.ins[0]), + ip[1], + ip[2], + b, + h0, + y, + yh, + hidden, + direction, + lbr, + pre, + scratch, + ); + } + } + OP_BATCH_NORM => { + require2(ip, ilen, 5)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::batch_norm( + ip[0], + sh(op.ins[0]), + ip[1], + ip[2], + ip[3], + ip[4], + dst, + op.f32[0], + ); + } + } + OP_LAYER_NORM => { + let bias = if op.ins[2] != NULL_SLOT { + require2(ip, ilen, 3)?; + Some(ip[2]) + } else { + require2(ip, ilen, 2)?; + None + }; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::layer_norm( + ip[0], + sh(op.ins[0]), + ip[1], + bias, + dst, + op.i64[0] as usize, + op.f32[0], + ); + } + } + OP_TRANSPOSE => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::transpose(ip[0], sh(op.ins[0]), &op.i64, dst); + } + } + OP_CONCAT => { + let axis = op.i64[0] as usize; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + let mut ins: Vec<(*const f32, &[u32])> = Vec::with_capacity(op.ins.len()); + for (k, &sid) in op.ins.iter().enumerate() { + if ip[k].is_null() { + return Err(InferError::BadOperand("concat null input")); + } + ins.push((ip[k], sh(sid))); + } + unsafe { + ops::concat(&ins, axis, dst, sh(o)); + } + } + OP_SLICE => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::slice(ip[0], sh(op.ins[0]), &op.i64, dst, sh(o)); + } + } + OP_PAD => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::pad(ip[0], sh(op.ins[0]), &op.i64, op.f32[0], dst, sh(o)); + } + } + OP_EXPAND => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::expand(ip[0], sh(op.ins[0]), dst, sh(o)); + } + } + OP_GATHER => { + require2(ip, ilen, 1)?; + let axis = op.i64[0]; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::gather(ip[0], sh(op.ins[0]), axis, &op.i64[1..], dst, sh(o)); + } + } + OP_REDUCE_MEAN | OP_REDUCE_L2 => { + require2(ip, ilen, 1)?; + let keepdims = op.i64[0] != 0; + let n_ax = op.i64[1] as usize; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::reduce( + op.code == OP_REDUCE_L2, + ip[0], + sh(op.ins[0]), + &op.i64[2..2 + n_ax], + keepdims, + dst, + sh(o), + ); + } + } + OP_RESIZE => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + ops::resize(ip[0], sh(op.ins[0]), dst, sh(o), scratch, scratch2); + } + } + OP_COPY => { + require2(ip, ilen, 1)?; + let o = op.outs[0]; + let dst = unsafe { arena.add(g.slots[o as usize].off as usize) }; + unsafe { + std::ptr::copy_nonoverlapping(ip[0], dst, ilen[0]); + } + } + other => return Err(InferError::UnknownOpcode(other)), + } + Ok(()) + } +} + +/// Pre-transpose constant GRU W/R into projection-friendly layouts. +fn build_gru_pre(g: &Graph, op: &Op) -> Option<(Box<[f32]>, Box<[f32]>)> { + let w_slot = &g.slots[op.ins[1] as usize]; + let r_slot = &g.slots[op.ins[2] as usize]; + let w_root = &g.slots[w_slot.root as usize]; + let r_root = &g.slots[r_slot.root as usize]; + if w_root.kind != SLOT_CONST || r_root.kind != SLOT_CONST { + return None; + } + if w_slot.shape.len() != 3 || r_slot.shape.len() != 3 { + return None; + } + let dir = w_slot.shape[0] as usize; + let h3 = w_slot.shape[1] as usize; + let in_dim = w_slot.shape[2] as usize; + let h = r_slot.shape[2] as usize; + if h3 != 3 * h || r_slot.shape[0] as usize != dir || r_slot.shape[1] as usize != h3 { + return None; + } + let w = &g.const_data[w_root.off as usize..w_root.off as usize + w_slot.numel]; + let r = &g.const_data[r_root.off as usize..r_root.off as usize + r_slot.numel]; + let mut wt = vec![0.0f32; dir * in_dim * h3]; + let mut rt = vec![0.0f32; dir * h * h3]; + for d in 0..dir { + for j in 0..h3 { + for k in 0..in_dim { + wt[(d * in_dim + k) * h3 + j] = w[d * h3 * in_dim + j * in_dim + k]; + } + for k in 0..h { + rt[(d * h + k) * h3 + j] = r[d * h3 * h + j * h + k]; + } + } + } + Some((wt.into_boxed_slice(), rt.into_boxed_slice())) +} + +fn require2(ip: [*const f32; 16], _ilen: [usize; 16], n: usize) -> Result<()> { + for p in ip.iter().take(n) { + if p.is_null() { + return Err(InferError::BadOperand("missing required input")); + } + } + Ok(()) +} diff --git a/openless-all/app/crates/micyou-infer/src/ops.rs b/openless-all/app/crates/micyou-infer/src/ops.rs new file mode 100644 index 000000000..23bdda258 --- /dev/null +++ b/openless-all/app/crates/micyou-infer/src/ops.rs @@ -0,0 +1,1757 @@ +/* + * MicYou — Turns your Android device into a high-quality PC microphone. + * Copyright (C) 2026 LanRhyme + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version, with the MicYou Plugin Exception. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +//! Numeric kernels for the MCYI VM. +//! +//! Every kernel is a mechanical transcription of the corresponding numpy +//! reference in `onnxref/replay.py` (libmicyou repository, `tools/onnx-port`; +//! itself validated bit-approximately against onnxruntime). All arithmetic is +//! f32; rounding points mirror the reference so streaming goldens stay +//! within ~1e-5. +//! +//! # Safety & performance contract +//! +//! Kernels receive raw pointers (the VM cannot express disjoint arena +//! borrows through `&mut Vec`). Each kernel immediately re-materializes +//! them as `&[f32]`/`&mut [f32]` slices. This is sound **and** essential +//! for speed: the compiler's live-range packing guarantees output regions +//! never overlap input regions of the same op (re-validated in debug +//! builds), and slice references give LLVM the `noalias` facts it needs — +//! the same loops over raw pointers derived from one arena base do **not** +//! auto-vectorize (~2.5× slower in measurements). + +/// Element-wise binary operations. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum BinOp { + Add, + Sub, + Mul, + Div, + Pow, +} + +const MAX_NDIM: usize = 6; + +#[inline] +fn numel(shape: &[u32]) -> usize { + shape.iter().fold(1usize, |a, &d| a * d as usize) +} + +#[inline] +fn row_major_strides(shape: &[u32]) -> [usize; MAX_NDIM] { + let mut st = [0usize; MAX_NDIM]; + let n = shape.len(); + if n == 0 { + return st; + } + st[n - 1] = 1; + for k in (0..n - 1).rev() { + st[k] = st[k + 1] * shape[k + 1] as usize; + } + st +} + +/// Map own-shape strides into the output axis space (right-aligned), using +/// stride 0 for broadcast (size-1 or missing) axes. +fn broadcast_strides(ish: &[u32], osh: &[u32]) -> [usize; MAX_NDIM] { + let own = row_major_strides(ish); + let mut out = [0usize; MAX_NDIM]; + let n = osh.len(); + debug_assert!(ish.len() <= n); + let pad = n - ish.len(); + for k in 0..n { + if k < pad { + out[k] = 0; // axis absent in input → broadcast + } else { + let ik = k - pad; + out[k] = if ish[ik] == 1 { 0 } else { own[ik] }; + } + } + out +} + +/// Cephes-style f32 `exp`: degree-6 Taylor on the reduced argument, +/// relative error ≈ 1.2e-7 — indistinguishable from libm `expf` at the +/// tolerances this VM is validated against (2e-4), and several times +/// faster with straight-line vectorizable code (sigmoid/tanh dominate the +/// GRU/gate paths: >100k calls per AEC7 frame). +/// +/// Range behavior (matches expf within 1e-37 absolute — far below any +/// tolerance-relevant threshold): +/// * `x > 88.0` → +inf +/// * `x < -87.0` → 0.0 +#[inline] +pub fn fast_exp(x: f32) -> f32 { + const LN2_HI: f32 = 0.6931457519531250; // trailing zero bits → k*LN2_HI exact + const LN2_LO: f32 = 1.4286068203094172e-6; + const LOG2E: f32 = 1.4426950408889634; + // Fully branchless (clamp + copysign) so callers vectorize: outside the + // f32-normal range the result saturates to ~1.7e38 / ~2e-38 — within + // 1e-37 absolute of the true expf value, i.e. far below any tolerance + // that matters here (sigmoid/tanh saturate identically). + let xc = x.clamp(-87.3, 88.0); + let t = xc * LOG2E; + let k = (t + 0.5f32.copysign(t)) as i32; // round-to-nearest, branchless + let kf = k as f32; + let r = (xc - kf * LN2_HI) - kf * LN2_LO; + let p = 1.0 + + r * (1.0 + + r * (0.5 + + r * (0.16666667 + r * (0.041666668 + r * (0.008333334 + r * 0.0013888889))))); + let bits = ((127 + k) as u32) << 23; + p * f32::from_bits(bits) +} + +/// `tanh` via `fast_exp(2z)`: absolute error ≈ 6e-8 near zero, ~1.2e-7 +/// elsewhere — within golden tolerances, avoids libm call overhead. +#[inline] +pub fn fast_tanh(z: f32) -> f32 { + // branchless: fast_exp saturates at ~1.7e38, so (e-1)/(e+1) → 1.0 in f32 + let e = fast_exp(2.0 * z); + (e - 1.0) / (e + 1.0) +} + +#[inline(always)] +fn sigmoid(x: f32) -> f32 { + 1.0 / (1.0 + fast_exp(-x)) +} + +/// NumPy-style broadcasting binary op. `osh` is the (statically known) +/// broadcast result shape. +/// +/// Per-op dispatch happens ONCE per call (never per element) so every inner +/// loop monomorphizes into branch-free, auto-vectorizable code. The Pow +/// scalar-square case (the only Pow in both models) ignores `y` and becomes +/// a bare x*x — bit-identical to correctly-rounded powf(x, 2). +/// +/// # Safety +/// See module docs. +#[inline(always)] +pub(crate) unsafe fn binary_impl( + op: BinOp, + ap: *const f32, + ash: &[u32], + bp: *const f32, + bsh: &[u32], + op_: *mut f32, + osh: &[u32], +) { + let a = std::slice::from_raw_parts(ap, numel(ash)); + let b = std::slice::from_raw_parts(bp, numel(bsh)); + let o = std::slice::from_raw_parts_mut(op_, numel(osh)); + match op { + BinOp::Add => binary_core(a, ash, b, bsh, o, osh, |x, y| x + y), + BinOp::Sub => binary_core(a, ash, b, bsh, o, osh, |x, y| x - y), + BinOp::Mul => binary_core(a, ash, b, bsh, o, osh, |x, y| x * y), + BinOp::Div => binary_core(a, ash, b, bsh, o, osh, |x, y| x / y), + BinOp::Pow => { + if b.len() == 1 && b[0] == 2.0 { + binary_core(a, ash, b, bsh, o, osh, |x, _y| x * x) + } else { + binary_core(a, ash, b, bsh, o, osh, |x, y| x.powf(y)) + } + } + } +} + +#[inline(always)] +fn binary_core f32>( + a: &[f32], + ash: &[u32], + b: &[f32], + bsh: &[u32], + o: &mut [f32], + osh: &[u32], + f: F, +) { + debug_assert!(osh.len() <= MAX_NDIM); + let on = o.len(); + let n = osh.len(); + + // fast path: identical shapes → flat zip + if a.len() == on && b.len() == on && ash == osh && bsh == osh { + for i in 0..on { + o[i] = f(a[i], b[i]); + } + return; + } + // scalar fast paths + if b.len() == 1 && a.len() == on && ash == osh { + let bv = b[0]; + for i in 0..on { + o[i] = f(a[i], bv); + } + return; + } + if a.len() == 1 && b.len() == on && bsh == osh { + let av = a[0]; + for i in 0..on { + o[i] = f(av, b[i]); + } + return; + } + + if n == 0 { + o[0] = f(a[0], b[0]); + return; + } + + let astr = broadcast_strides(ash, osh); + let bstr = broadcast_strides(bsh, osh); + let inner = osh[n - 1] as usize; + let outer = if inner == 0 { 0 } else { on / inner }; + let alast = astr[n - 1]; + let blast = bstr[n - 1]; + debug_assert!(inner == 0 || alast <= 1); + debug_assert!(inner == 0 || blast <= 1); + + let mut coords = [0usize; MAX_NDIM]; + for oi in 0..outer { + let mut aoff = 0usize; + let mut boff = 0usize; + for k in 0..n - 1 { + aoff += coords[k] * astr[k]; + boff += coords[k] * bstr[k]; + } + let dst = &mut o[oi * inner..(oi + 1) * inner]; + match (alast, blast) { + (1, 1) => { + let sa = &a[aoff..aoff + inner]; + let sb = &b[boff..boff + inner]; + for j in 0..inner { + dst[j] = f(sa[j], sb[j]); + } + } + (1, _) => { + let sa = &a[aoff..aoff + inner]; + let bv = b[boff]; + for j in 0..inner { + dst[j] = f(sa[j], bv); + } + } + (_, 1) => { + let av = a[aoff]; + let sb = &b[boff..boff + inner]; + for j in 0..inner { + dst[j] = f(av, sb[j]); + } + } + _ => { + let av = a[aoff]; + let bv = b[boff]; + for j in 0..inner { + dst[j] = f(av, bv); + } + } + } + // odometer over outer axes 0..n-2 + if n >= 2 { + let mut k = n - 2; + loop { + coords[k] += 1; + if coords[k] < osh[k] as usize { + break; + } + coords[k] = 0; + if k == 0 { + break; + } + k -= 1; + } + } + } +} + +/// SIGMOID (6) / SQRT (7) / LOG (8). +/// +/// # Safety +/// See module docs. +#[inline(always)] +pub(crate) unsafe fn unary_impl(opcode: u16, xp: *const f32, op_: *mut f32, n: usize) { + let x = std::slice::from_raw_parts(xp, n); + let o = std::slice::from_raw_parts_mut(op_, n); + match opcode { + 6 => { + for i in 0..n { + o[i] = sigmoid(x[i]); + } + } + 7 => { + for i in 0..n { + o[i] = x[i].sqrt(); + } + } + _ => { + for i in 0..n { + o[i] = x[i].ln(); + } + } + } +} + +/// # Safety +/// See module docs. +pub unsafe fn clip(xp: *const f32, op_: *mut f32, n: usize, lo: Option, hi: Option) { + let x = std::slice::from_raw_parts(xp, n); + let o = std::slice::from_raw_parts_mut(op_, n); + match (lo, hi) { + (Some(l), Some(h)) => { + for i in 0..n { + o[i] = if x[i] < l { + l + } else if x[i] > h { + h + } else { + x[i] + }; + } + } + (Some(l), None) => { + for i in 0..n { + o[i] = if x[i] < l { l } else { x[i] }; + } + } + (None, Some(h)) => { + for i in 0..n { + o[i] = if x[i] > h { h } else { x[i] }; + } + } + (None, None) => o.copy_from_slice(x), + } +} + +/// Batched MatMul: `a [..., M, K] × b [..., K, N] → o [..., M, N]` with +/// numpy batch broadcasting (b is 2-D in both shipped models). +/// +/// # Safety +/// See module docs. +#[inline(always)] +pub(crate) unsafe fn matmul_impl( + ap: *const f32, + ash: &[u32], + bp: *const f32, + bsh: &[u32], + op_: *mut f32, + osh: &[u32], +) { + debug_assert!(ash.len() >= 2 && bsh.len() >= 2 && osh.len() >= 2); + debug_assert!(bsh.len() == 2 || bsh.len() == osh.len()); + let a = std::slice::from_raw_parts(ap, numel(ash)); + let b = std::slice::from_raw_parts(bp, numel(bsh)); + let o = std::slice::from_raw_parts_mut(op_, numel(osh)); + + let nb = osh.len() - 2; + let m = osh[nb] as usize; + let n = osh[nb + 1] as usize; + let k = ash[ash.len() - 1] as usize; + debug_assert_eq!(bsh[bsh.len() - 2] as usize, k); + + // Batch-axis strides come from the FULL tensor layout (a batch step + // skips the whole trailing [M,K] / [K,N] block), right-aligned with + // stride-0 broadcasting for size-1 batch dims. + fn batch_strides(ish: &[u32], nb: usize) -> [usize; MAX_NDIM] { + let full = row_major_strides(ish); + let mut out = [0usize; MAX_NDIM]; + let inb = ish.len().saturating_sub(2); + let pad = nb.saturating_sub(inb); + for kk in 0..nb { + if kk >= pad { + let ik = kk - pad; + out[kk] = if ish[ik] == 1 { 0 } else { full[ik] }; + } + } + out + } + let astr = batch_strides(ash, nb); + let bstr = batch_strides(bsh, nb); + + let n_batches = numel(&osh[..nb]).max(1); + let mut coords = [0usize; MAX_NDIM]; + for batch in 0..n_batches { + let mut aoff = 0usize; + let mut boff = 0usize; + for d in 0..nb { + aoff += coords[d] * astr[d]; + boff += coords[d] * bstr[d]; + } + let abase = &a[aoff..]; // [M, K] + let bbase = &b[boff..]; // [K, N] + let obase = &mut o[batch * m * n..(batch + 1) * m * n]; + for i in 0..m { + let orow = &mut obase[i * n..(i + 1) * n]; + orow.fill(0.0); + let arow = &abase[i * k..(i + 1) * k]; + for kk in 0..k { + let av = arow[kk]; + let brow = &bbase[kk * n..(kk + 1) * n]; + for j in 0..n { + orow[j] += av * brow[j]; + } + } + } + if nb > 0 { + let mut d = nb - 1; + loop { + coords[d] += 1; + if coords[d] < osh[d] as usize { + break; + } + coords[d] = 0; + if d == 0 { + break; + } + d -= 1; + } + } + } +} + +/// Conv2d / ConvTranspose2d attribute block (decoded by the VM from the +/// op's i64 attribute table). `pb`/`pr`/`opt_*` are carried for +/// completeness — output sizes come from the compiled output slot shapes. +#[allow(dead_code)] +pub struct ConvAttrs { + pub kh: usize, + pub kw: usize, + pub sh: usize, + pub sw: usize, + pub dh: usize, + pub dw: usize, + pub pt: i32, + pub pl: i32, + pub pb: i32, + pub pr: i32, + pub group: usize, + pub opt_h: usize, + pub opt_w: usize, +} + +/// NCHW Conv2d (`transpose == false`, weight `[M, C/g, kh, kw]`) or +/// ConvTranspose2d (`transpose == true`, weight `[C, M/g, kh, kw]`). +/// `scratch` is used for im2col staging (general path only). +/// +/// # Safety +/// See module docs. +#[allow(clippy::too_many_arguments)] +#[inline(always)] +pub(crate) unsafe fn conv_impl( + transpose: bool, + xp: *const f32, + xsh: &[u32], + wp: *const f32, + wsh: &[u32], + bias: Option<*const f32>, + op_: *mut f32, + osh: &[u32], + at: &ConvAttrs, + scratch: &mut Vec, +) { + debug_assert_eq!(xsh.len(), 4); + let x = std::slice::from_raw_parts(xp, numel(xsh)); + let w = std::slice::from_raw_parts(wp, numel(wsh)); + let o = std::slice::from_raw_parts_mut(op_, numel(osh)); + let bias = bias.map(|bp| { + let m = if transpose { + (wsh[1] as usize) * at.group + } else { + wsh[0] as usize + }; + std::slice::from_raw_parts(bp, m) + }); + + let nbatch = xsh[0] as usize; + let c_in = xsh[1] as usize; + let h_in = xsh[2] as usize; + let w_in = xsh[3] as usize; + let (kh, kw, sh, sw, dh, dw, group) = (at.kh, at.kw, at.sh, at.sw, at.dh, at.dw, at.group); + let (pt, pl) = (at.pt as i64, at.pl as i64); + let h_out = osh[2] as usize; + let w_out = osh[3] as usize; + let cpg = c_in / group; + let hw_in = h_in * w_in; + let hw_out = h_out * w_out; + + if !transpose { + let m_out = wsh[0] as usize; + let mpg = m_out / group; + + if kh == 1 && kw == 1 && sh == 1 && sw == 1 && dh == 1 && dw == 1 { + // pointwise == GEMM per group: [mpg, cpg] × [cpg, HW] + for nb in 0..nbatch { + for gi in 0..group { + let xg = &x[nb * c_in * hw_in + gi * cpg * hw_in..]; + let wg = &w[gi * mpg * cpg..]; + let og = &mut o[nb * m_out * hw_out + gi * mpg * hw_out..]; + for mi in 0..mpg { + let orow = &mut og[mi * hw_out..(mi + 1) * hw_out]; + let bv = bias.map_or(0.0, |b| b[gi * mpg + mi]); + orow.fill(bv); + for ci in 0..cpg { + let wv = wg[mi * cpg + ci]; + let xrow = &xg[ci * hw_in..(ci + 1) * hw_in]; + for p in 0..hw_out { + orow[p] += wv * xrow[p]; + } + } + } + } + } + return; + } + + if group == c_in && m_out == c_in { + // depthwise: w [C, 1, kh, kw] — axpy passes so the output row is + // contiguous in the inner loop. + for nb in 0..nbatch { + for c in 0..c_in { + let xp_ = &x[(nb * c_in + c) * hw_in..]; + let wp_ = &w[c * kh * kw..]; + let op2 = &mut o[(nb * c_in + c) * hw_out..]; + let bv = bias.map_or(0.0, |b| b[c]); + op2[..hw_out].fill(bv); + for oh in 0..h_out { + let orow = &mut op2[oh * w_out..(oh + 1) * w_out]; + for i in 0..kh { + let ih = (oh * sh + i * dh) as i64 - pt; + if ih < 0 || ih >= h_in as i64 { + continue; + } + let xrow = &xp_[ih as usize * w_in..]; + for j in 0..kw { + let wv = wp_[i * kw + j]; + let base = j as i64 * dw as i64 - pl; + // valid ow: 0 ≤ ow*sw + base < w_in + let lo = if base >= 0 { + 0 + } else { + ((-base + sw as i64 - 1) / sw as i64) as usize + }; + let max_ow = (w_in as i64 - 1 - base) / sw as i64; + let hi_excl = (max_ow + 1).clamp(0, w_out as i64) as usize; + for ow in lo..hi_excl { + let iw = (ow as i64 * sw as i64 + base) as usize; + orow[ow] += wv * xrow[iw]; + } + } + } + } + } + } + return; + } + + // general grouped conv: im2col + GEMM + let col_k = cpg * kh * kw; + scratch.clear(); + scratch.resize(col_k * hw_out, 0.0); + for nb in 0..nbatch { + for gi in 0..group { + let xg = &x[nb * c_in * hw_in + gi * cpg * hw_in..]; + { + let cols: &mut [f32] = scratch; + for ci in 0..cpg { + for i in 0..kh { + for j in 0..kw { + let col_row = ci * kh * kw + i * kw + j; + for oh in 0..h_out { + let ih = (oh * sh + i * dh) as i64 - pt; + let row_ok = ih >= 0 && ih < h_in as i64; + let crow = &mut cols[col_row * hw_out + oh * w_out..][..w_out]; + for ow in 0..w_out { + let iw = (ow * sw + j * dw) as i64 - pl; + crow[ow] = if row_ok && iw >= 0 && iw < w_in as i64 { + xg[ci * hw_in + ih as usize * w_in + iw as usize] + } else { + 0.0 + }; + } + } + } + } + } + } + let wg = &w[gi * mpg * col_k..]; + let og = &mut o[nb * m_out * hw_out + gi * mpg * hw_out..]; + let cols: &[f32] = scratch; + for mi in 0..mpg { + let orow = &mut og[mi * hw_out..(mi + 1) * hw_out]; + let bv = bias.map_or(0.0, |b| b[gi * mpg + mi]); + orow.fill(bv); + for kk in 0..col_k { + let wv = wg[mi * col_k + kk]; + let crow = &cols[kk * hw_out..(kk + 1) * hw_out]; + for p in 0..hw_out { + orow[p] += wv * crow[p]; + } + } + } + } + } + return; + } + + // ── ConvTranspose: scatter-add ── + let mpg = wsh[1] as usize; + let m_out = mpg * group; + debug_assert_eq!(osh[1] as usize, m_out); + o.fill(0.0); + for nb in 0..nbatch { + for gi in 0..group { + for ci in 0..cpg { + let c = gi * cpg + ci; + let xpc = &x[nb * c_in * hw_in + c * hw_in..]; + let wpc = &w[c * mpg * kh * kw..]; + for a in 0..h_in { + let oh_base = a as i64 * sh as i64 - pt; + for bidx in 0..w_in { + let xv = xpc[a * w_in + bidx]; + let ow_base = bidx as i64 * sw as i64 - pl; + for i in 0..kh { + let oh = oh_base + i as i64 * dh as i64; + if oh < 0 || oh >= h_out as i64 { + continue; + } + for j in 0..kw { + let ow = ow_base + j as i64 * dw as i64; + if ow < 0 || ow >= w_out as i64 { + continue; + } + let pos = (oh as usize) * w_out + ow as usize; + for mi in 0..mpg { + o[nb * m_out * hw_out + (gi * mpg + mi) * hw_out + pos] += + wpc[mi * kh * kw + i * kw + j] * xv; + } + } + } + } + } + } + } + } + if let Some(b) = bias { + for nb in 0..nbatch { + for m in 0..m_out { + let bv = b[m]; + let p0 = nb * m_out * hw_out + m * hw_out; + for p in 0..hw_out { + o[p0 + p] += bv; + } + } + } + } +} + +/// ONNX GRU (gate order z,r,h; optional biases; forward/backward/ +/// bidirectional; `linear_before_reset` 0/1). +/// +/// X `[seq, batch, in]`, W `[dir, 3h, in]`, R `[dir, 3h, h]`, +/// B `[dir, 6h]` (W-biases ‖ R-biases), H0 `[dir, batch, h]`. +/// Y `[seq, dir, batch, h]` (optional), Y_h `[dir, batch, h]` (optional). +/// +/// # Safety +/// See module docs. +#[allow(clippy::too_many_arguments)] +#[inline(always)] +pub(crate) unsafe fn gru_impl( + xp: *const f32, + xsh: &[u32], + wp: *const f32, + rp: *const f32, + b: Option<*const f32>, + h0: Option<*const f32>, + y: Option<*mut f32>, + yh: Option<*mut f32>, + hidden: usize, + direction: i64, + lbr: bool, + pre: Option<(&[f32], &[f32])>, + scratch: &mut Vec, +) { + debug_assert_eq!(xsh.len(), 3); + let seq = xsh[0] as usize; + let batch = xsh[1] as usize; + let in_dim = xsh[2] as usize; + let h = hidden; + let h3 = 3 * h; + let num_dir = if direction == 2 { 2usize } else { 1usize }; + + let x = std::slice::from_raw_parts(xp, seq * batch * in_dim); + let w = std::slice::from_raw_parts(wp, num_dir * h3 * in_dim); + let r = std::slice::from_raw_parts(rp, num_dir * h3 * h); + let b = b.map(|bp| std::slice::from_raw_parts(bp, num_dir * 6 * h)); + let h0 = h0.map(|hp| std::slice::from_raw_parts(hp, num_dir * batch * h)); + let mut y = y.map(|yp| std::slice::from_raw_parts_mut(yp, seq * num_dir * batch * h)); + let mut yh = yh.map(|yp| std::slice::from_raw_parts_mut(yp, num_dir * batch * h)); + + // scratch layout (split_at_mut → mutually noalias slices → vectorizes): + // wt [in*h3] | rt [h*h3] | xproj [seq*batch*h3] | state [batch*h] + // | rproj [batch*h3] | bz [h3] | zt [batch*h] | rtg [batch*h] + let wt_n = in_dim * h3; + let rt_n = h * h3; + let xp_n = seq * batch * h3; + let total = wt_n + rt_n + xp_n + batch * h + batch * h3 + h3 + 2 * batch * h; + if scratch.len() < total { + scratch.resize(total, 0.0); + } + let (wt_s, rest) = scratch.split_at_mut(wt_n); + let (rt_s, rest) = rest.split_at_mut(rt_n); + let (xproj_s, rest) = rest.split_at_mut(xp_n); + let (state_s, rest) = rest.split_at_mut(batch * h); + let (rproj_s, rest) = rest.split_at_mut(batch * h3); + let (bz_s, rest) = rest.split_at_mut(h3); + let (zt_s, rtg_s) = rest.split_at_mut(batch * h); + + for w_idx in 0..num_dir { + let reverse = direction == 1 || (direction == 2 && w_idx == 1); + + // Projections need W/R transposed to [in][3h] / [h][3h] for + // contiguous inner loops. For constant weights (the shipped models) + // the Session pre-transposes once; otherwise fall back to a + // per-call transpose into scratch. + let (wt_use, rt_use): (&[f32], &[f32]) = match pre { + Some((pw, pr)) => ( + &pw[w_idx * wt_n..(w_idx + 1) * wt_n], + &pr[w_idx * rt_n..(w_idx + 1) * rt_n], + ), + None => { + let wd = &w[w_idx * h3 * in_dim..(w_idx + 1) * h3 * in_dim]; + let rd = &r[w_idx * h3 * h..(w_idx + 1) * h3 * h]; + for j in 0..h3 { + for kk in 0..in_dim { + wt_s[kk * h3 + j] = wd[j * in_dim + kk]; + } + for kk in 0..h { + rt_s[kk * h3 + j] = rd[j * h + kk]; + } + } + (&*wt_s, &*rt_s) + } + }; + // xproj[t] = X[t] @ Wdᵀ : [batch, in] × [in, h3] + // Loop order kk-outer keeps each transposed weight row L1-hot while + // it feeds every batch row (batch-major would re-stream the whole + // weight matrix per row). Per-element accumulation order over kk is + // unchanged → bit-identical results. + for t in 0..seq { + let xb = &x[t * batch * in_dim..(t + 1) * batch * in_dim]; + let pb = &mut xproj_s[t * batch * h3..(t + 1) * batch * h3]; + pb.fill(0.0); + for kk in 0..in_dim { + let wrow = &wt_use[kk * h3..(kk + 1) * h3]; + for i in 0..batch { + let xv = xb[i * in_dim + kk]; + let prow = &mut pb[i * h3..(i + 1) * h3]; + for j in 0..h3 { + prow[j] += xv * wrow[j]; + } + } + } + } + // combined gate biases bz[j] = Wb[j] + Rb[j] + bz_s[..h3].fill(0.0); + if let Some(bb) = b { + let bd = &bb[w_idx * 6 * h..(w_idx + 1) * 6 * h]; + for j in 0..h3 { + bz_s[j] = bd[j] + bd[h3 + j]; + } + } + // initial state + state_s[..batch * h].fill(0.0); + if let Some(hp) = h0 { + state_s[..batch * h].copy_from_slice(&hp[w_idx * batch * h..(w_idx + 1) * batch * h]); + } + + for si in 0..seq { + let t = if reverse { seq - 1 - si } else { si }; + let pb = &xproj_s[t * batch * h3..(t + 1) * batch * h3]; + // rproj = state @ Rᵀ : [batch, h] × [h, h3] — kk-outer for + // weight reuse (same accumulation order → bit-identical) + rproj_s.fill(0.0); + for kk in 0..h { + let wrow = &rt_use[kk * h3..(kk + 1) * h3]; + for i in 0..batch { + let sv = state_s[i * h + kk]; + let orow = &mut rproj_s[i * h3..(i + 1) * h3]; + for j in 0..h3 { + orow[j] += sv * wrow[j]; + } + } + } + // gates (elementwise passes — vectorize; rounding order matches + // replay.py exactly) + for i in 0..batch { + let prow = &pb[i * h3..(i + 1) * h3]; + let rprow = &rproj_s[i * h3..(i + 1) * h3]; + let zt = &mut zt_s[i * h..(i + 1) * h]; + let rtg = &mut rtg_s[i * h..(i + 1) * h]; + let st = &mut state_s[i * h..(i + 1) * h]; + let (wb_h, rb_h): (&[f32], Option<&[f32]>) = match b { + Some(bb) => ( + &bb[w_idx * 6 * h + 2 * h..w_idx * 6 * h + 3 * h], + Some(&bb[w_idx * 6 * h + 5 * h..w_idx * 6 * h + 6 * h]), + ), + None => (&[], None), + }; + for j in 0..h { + zt[j] = sigmoid((prow[j] + rprow[j]) + bz_s[j]); + } + for j in 0..h { + rtg[j] = sigmoid((prow[h + j] + rprow[h + j]) + bz_s[h + j]); + } + if lbr { + match rb_h { + Some(rbh) => { + for j in 0..h { + let ht = fast_tanh( + (prow[2 * h + j] + wb_h[j]) + + rtg[j] * (rprow[2 * h + j] + rbh[j]), + ); + st[j] = (1.0 - zt[j]) * ht + zt[j] * st[j]; + } + } + None => { + for j in 0..h { + let ht = fast_tanh(prow[2 * h + j] + rtg[j] * rprow[2 * h + j]); + st[j] = (1.0 - zt[j]) * ht + zt[j] * st[j]; + } + } + } + } else { + match rb_h { + Some(rbh) => { + for j in 0..h { + let ht = fast_tanh( + (prow[2 * h + j] + rtg[j] * rprow[2 * h + j]) + + (wb_h[j] + rbh[j]), + ); + st[j] = (1.0 - zt[j]) * ht + zt[j] * st[j]; + } + } + None => { + for j in 0..h { + let ht = fast_tanh(prow[2 * h + j] + rtg[j] * rprow[2 * h + j]); + st[j] = (1.0 - zt[j]) * ht + zt[j] * st[j]; + } + } + } + } + } + if let Some(ys) = y.as_deref_mut() { + // Y layout [seq, dir, batch, h] + let dst = &mut ys[(t * num_dir + w_idx) * batch * h..][..batch * h]; + dst.copy_from_slice(&state_s[..batch * h]); + } + } + if let Some(yhs) = yh.as_deref_mut() { + let dst = &mut yhs[w_idx * batch * h..(w_idx + 1) * batch * h]; + dst.copy_from_slice(&state_s[..batch * h]); + } + } +} + +/// BatchNormalization (inference), NCHW, channels on axis 1. +/// +/// # Safety +/// See module docs. +#[allow(clippy::too_many_arguments)] +pub unsafe fn batch_norm( + xp: *const f32, + xsh: &[u32], + scale: *const f32, + bias: *const f32, + mean: *const f32, + var: *const f32, + op_: *mut f32, + eps: f32, +) { + let c = xsh[1] as usize; + let spatial = numel(xsh) / c; + let x = std::slice::from_raw_parts(xp, numel(xsh)); + let o = std::slice::from_raw_parts_mut(op_, numel(xsh)); + let scale = std::slice::from_raw_parts(scale, c); + let bias = std::slice::from_raw_parts(bias, c); + let mean = std::slice::from_raw_parts(mean, c); + let var = std::slice::from_raw_parts(var, c); + for ch in 0..c { + let inv = 1.0 / (var[ch] + eps).sqrt(); + let sc = scale[ch]; + let bi = bias[ch]; + let mu = mean[ch]; + let xs = &x[ch * spatial..(ch + 1) * spatial]; + let os = &mut o[ch * spatial..(ch + 1) * spatial]; + for p in 0..spatial { + os[p] = ((xs[p] - mu) * inv) * sc + bi; + } + } +} + +/// LayerNormalization (opset-17 semantics): normalize over dims `axis..`; +/// scale/bias have the shape of the normalized tail. +/// +/// # Safety +/// See module docs. +#[inline(always)] +pub(crate) unsafe fn layer_norm_impl( + xp: *const f32, + xsh: &[u32], + scale: *const f32, + bias: Option<*const f32>, + op_: *mut f32, + axis: usize, + eps: f32, +) { + let block: usize = xsh[axis..].iter().fold(1usize, |a, &d| a * d as usize); + let total = numel(xsh); + let leading = total / block; + let x = std::slice::from_raw_parts(xp, total); + let o = std::slice::from_raw_parts_mut(op_, total); + let scale = std::slice::from_raw_parts(scale, block); + let bias = bias.map(|bp| std::slice::from_raw_parts(bp, block)); + for l in 0..leading { + let xs = &x[l * block..(l + 1) * block]; + let os = &mut o[l * block..(l + 1) * block]; + let mut sum = 0.0f32; + for i in 0..block { + sum += xs[i]; + } + let mean = sum / block as f32; + let mut vs = 0.0f32; + for i in 0..block { + let c = xs[i] - mean; + vs += c * c; + } + let inv = 1.0 / (vs / block as f32 + eps).sqrt(); + match bias { + Some(bp) => { + for i in 0..block { + let c = xs[i] - mean; + os[i] = ((c * inv) * scale[i]) + bp[i]; + } + } + None => { + for i in 0..block { + let c = xs[i] - mean; + os[i] = (c * inv) * scale[i]; + } + } + } + } +} + +/// General N-D transpose. +/// +/// # Safety +/// See module docs. +pub unsafe fn transpose(xp: *const f32, xsh: &[u32], perm: &[i64], op_: *mut f32) { + let n = xsh.len(); + debug_assert!(n <= MAX_NDIM && n == perm.len()); + let total = numel(xsh); + let x = std::slice::from_raw_parts(xp, total); + let o = std::slice::from_raw_parts_mut(op_, total); + let mut osh = [0u32; MAX_NDIM]; + for k in 0..n { + osh[k] = xsh[perm[k] as usize]; + } + let xstr = row_major_strides(xsh); + // source-index delta when the output odometer advances on axis k + let mut dsrc = [0usize; MAX_NDIM]; + for k in 0..n { + dsrc[k] = xstr[perm[k] as usize]; + } + let mut src = 0usize; + let mut coords = [0usize; MAX_NDIM]; + for oi in 0..total { + o[oi] = x[src]; + if oi + 1 == total { + break; + } + let mut k = n - 1; + loop { + coords[k] += 1; + src += dsrc[k]; + if coords[k] < osh[k] as usize { + break; + } + coords[k] = 0; + src -= dsrc[k] * osh[k] as usize; + if k == 0 { + break; + } + k -= 1; + } + } +} + +/// Concat along `axis`. +/// +/// # Safety +/// See module docs. +pub unsafe fn concat(ins: &[(*const f32, &[u32])], axis: usize, op_: *mut f32, osh: &[u32]) { + let inner: usize = osh[axis + 1..].iter().fold(1usize, |a, &d| a * d as usize); + let outer: usize = osh[..axis].iter().fold(1usize, |a, &d| a * d as usize); + let out_axis = osh[axis] as usize; + let o = std::slice::from_raw_parts_mut(op_, numel(osh)); + for oi in 0..outer { + let mut cum = 0usize; + for &(ptr, sh) in ins { + let rows = sh[axis] as usize; + let src = std::slice::from_raw_parts(ptr, numel(sh)); + let from = oi * rows * inner; + let to = oi * out_axis * inner + cum * inner; + o[to..to + rows * inner].copy_from_slice(&src[from..from + rows * inner]); + cum += rows; + } + debug_assert_eq!(cum, out_axis); + } +} + +/// Slice with compile-time-normalized ranges. +/// i64 attrs: `[n, (axis, start, end, step) × n]`; for negative steps, +/// `end == -1` means "through index 0" (ONNX semantics) and the compiler +/// encodes an empty selection as `(start=0, end=0)`. +/// +/// # Safety +/// See module docs. +pub unsafe fn slice(xp: *const f32, xsh: &[u32], attrs: &[i64], op_: *mut f32, osh: &[u32]) { + let n = xsh.len(); + debug_assert!(n <= MAX_NDIM && n >= 1); + let total_in = numel(xsh); + let total = numel(osh); + let x = std::slice::from_raw_parts(xp, total_in); + if total == 0 { + return; + } + let o = std::slice::from_raw_parts_mut(op_, total); + let cnt = attrs[0] as usize; + let mut start = [0i64; MAX_NDIM]; + let mut step = [1i64; MAX_NDIM]; + let mut counts = [1usize; MAX_NDIM]; + for d in 0..n { + counts[d] = xsh[d] as usize; + } + for k in 0..cnt { + let axis = attrs[1 + 4 * k] as usize; + let s = attrs[2 + 4 * k]; + let e = attrs[3 + 4 * k]; + let st = attrs[4 + 4 * k]; + start[axis] = s; + step[axis] = st; + let c = if st > 0 { + ((e - s) + st - 1) / st + } else if s == 0 && e == 0 { + 0 // empty-selection sentinel from the compiler + } else { + ((s - e) + (-st) - 1) / (-st) + }; + counts[axis] = c.max(0) as usize; + } + debug_assert_eq!(counts[..n].iter().fold(1usize, |a, &d| a * d), total); + let xstr = row_major_strides(xsh); + + // fast path: contiguous run along the last axis + if n == 1 || step[n - 1] == 1 { + let inner_run = counts[n - 1]; + let outer_n = total / inner_run.max(1); + let mut coords = [0usize; MAX_NDIM]; + let mut dst = 0usize; + for oi in 0..outer_n { + if oi > 0 && n >= 2 { + let mut k = n - 2; + loop { + coords[k] += 1; + if coords[k] < counts[k] { + break; + } + coords[k] = 0; + if k == 0 { + break; + } + k -= 1; + } + } + let mut src = start[n - 1] as usize * xstr[n - 1]; + for k in 0..n - 1 { + src += (start[k] + coords[k] as i64 * step[k]) as usize * xstr[k]; + } + o[dst..dst + inner_run].copy_from_slice(&x[src..src + inner_run]); + dst += inner_run; + } + return; + } + + // general path: per-element odometer + let mut coords = [0usize; MAX_NDIM]; + for oi in 0..total { + if oi > 0 { + let mut k = n - 1; + loop { + coords[k] += 1; + if coords[k] < counts[k] { + break; + } + coords[k] = 0; + if k == 0 { + break; + } + k -= 1; + } + } + let mut src = 0usize; + for k in 0..n { + src += (start[k] + coords[k] as i64 * step[k]) as usize * xstr[k]; + } + o[oi] = x[src]; + } +} + +/// Constant padding. i64 attrs: `[begin_0..begin_{n-1}, end_0..end_{n-1}]`. +/// +/// # Safety +/// See module docs. +pub unsafe fn pad(xp: *const f32, xsh: &[u32], pads: &[i64], cv: f32, op_: *mut f32, osh: &[u32]) { + let n = xsh.len(); + debug_assert!(n <= MAX_NDIM && pads.len() == 2 * n); + let o = std::slice::from_raw_parts_mut(op_, numel(osh)); + o.fill(cv); + let in_total = numel(xsh); + if in_total == 0 { + return; + } + let x = std::slice::from_raw_parts(xp, in_total); + let ostr = row_major_strides(osh); + let inner = xsh[n - 1] as usize; // contiguous in both src and dst + let outer = in_total / inner; + let mut coords = [0usize; MAX_NDIM]; + for oi in 0..outer { + if oi > 0 && n >= 2 { + let mut k = n - 2; + loop { + coords[k] += 1; + if coords[k] < xsh[k] as usize { + break; + } + coords[k] = 0; + if k == 0 { + break; + } + k -= 1; + } + } + let mut dst = pads[n - 1] as usize * ostr[n - 1]; + for k in 0..n - 1 { + dst += (coords[k] + pads[k] as usize) * ostr[k]; + } + o[dst..dst + inner].copy_from_slice(&x[oi * inner..(oi + 1) * inner]); + } +} + +/// Broadcast copy (Expand): every input dim is 1 or equal to the out dim. +/// +/// # Safety +/// See module docs. +pub unsafe fn expand(xp: *const f32, xsh: &[u32], op_: *mut f32, osh: &[u32]) { + let n = osh.len(); + let on = numel(osh); + let o = std::slice::from_raw_parts_mut(op_, on); + let a = std::slice::from_raw_parts(xp, numel(xsh)); + if n == 0 { + o[0] = a[0]; + return; + } + if a.len() == on { + o.copy_from_slice(a); + return; + } + let astr = broadcast_strides(xsh, osh); + let inner = osh[n - 1] as usize; + let outer = if inner == 0 { 0 } else { on / inner }; + let alast = astr[n - 1]; + let mut coords = [0usize; MAX_NDIM]; + for oi in 0..outer { + let mut aoff = 0usize; + for k in 0..n - 1 { + aoff += coords[k] * astr[k]; + } + let dst = &mut o[oi * inner..(oi + 1) * inner]; + if alast == 1 { + dst.copy_from_slice(&a[aoff..aoff + inner]); + } else { + dst.fill(a[aoff]); + } + if n >= 2 { + let mut k = n - 2; + loop { + coords[k] += 1; + if coords[k] < osh[k] as usize { + break; + } + coords[k] = 0; + if k == 0 { + break; + } + k -= 1; + } + } + } +} + +/// Gather with constant indices (`idx` = attribute values after the axis). +/// +/// # Safety +/// See module docs. +pub unsafe fn gather( + xp: *const f32, + xsh: &[u32], + axis: i64, + idx: &[i64], + op_: *mut f32, + osh: &[u32], +) { + let n = xsh.len(); + let axis = if axis < 0 { + (axis + n as i64) as usize + } else { + axis as usize + }; + let inner: usize = xsh[axis + 1..].iter().fold(1usize, |a, &d| a * d as usize); + let outer: usize = xsh[..axis].iter().fold(1usize, |a, &d| a * d as usize); + let dim = xsh[axis] as i64; + debug_assert_eq!(numel(osh), outer * idx.len() * inner); + let x = std::slice::from_raw_parts(xp, numel(xsh)); + let o = std::slice::from_raw_parts_mut(op_, numel(osh)); + for oi in 0..outer { + for (t, &ix) in idx.iter().enumerate() { + let ixd = if ix < 0 { ix + dim } else { ix } as usize; + let from = (oi * dim as usize + ixd) * inner; + let to = (oi * idx.len() + t) * inner; + o[to..to + inner].copy_from_slice(&x[from..from + inner]); + } + } +} + +/// ReduceMean / ReduceL2 over the given axes (l2 = sqrt of summed squares). +/// +/// # Safety +/// See module docs. +#[inline(always)] +pub(crate) unsafe fn reduce_impl( + l2: bool, + xp: *const f32, + xsh: &[u32], + axes: &[i64], + _keepdims: bool, + op_: *mut f32, + osh: &[u32], +) { + let n = xsh.len(); + debug_assert!(n <= MAX_NDIM); + let total_in = numel(xsh); + let total_out = numel(osh); + let x = std::slice::from_raw_parts(xp, total_in); + let o = std::slice::from_raw_parts_mut(op_, total_out); + o.fill(0.0); + let mut reduced = [false; MAX_NDIM]; + for &a in axes { + let a = if a < 0 { + (a + n as i64) as usize + } else { + a as usize + }; + reduced[a] = true; + } + + // Vectorized special cases (cover every reduction in both models): + if axes.len() == 1 { + let ax = { + let a = axes[0]; + if a < 0 { + (a + n as i64) as usize + } else { + a as usize + } + }; + if ax == n - 1 && n >= 1 { + // contiguous trailing axis → row sums + let run = xsh[ax] as usize; + let rows = total_out; + for row in 0..rows { + let xs = &x[row * run..(row + 1) * run]; + if l2 { + let mut acc = 0.0f32; + for i in 0..run { + acc += xs[i] * xs[i]; + } + o[row] = acc.sqrt(); + } else { + let mut acc = 0.0f32; + for i in 0..run { + acc += xs[i]; + } + o[row] = acc / run as f32; + } + } + return; + } + if ax == 1 && n == 4 { + // NCHW channel axis → axpy passes over the contiguous HW plane + let c = xsh[1] as usize; + let hw = (xsh[2] as usize) * (xsh[3] as usize); + let nb = xsh[0] as usize; + for nn in 0..nb { + for ch in 0..c { + let xs = &x[(nn * c + ch) * hw..(nn * c + ch + 1) * hw]; + let os = &mut o[nn * hw..(nn + 1) * hw]; + if l2 { + for p in 0..hw { + os[p] += xs[p] * xs[p]; + } + } else { + for p in 0..hw { + os[p] += xs[p]; + } + } + } + } + if l2 { + for i in 0..total_out { + o[i] = o[i].sqrt(); + } + } else { + let inv = 1.0f32 / c as f32; + for i in 0..total_out { + o[i] *= inv; + } + } + return; + } + } + // output flat index as a linear form over input coordinates + let mut ostr = [0usize; MAX_NDIM]; + { + let mut acc = 1usize; + for k in (0..n).rev() { + if !reduced[k] { + ostr[k] = acc; + acc *= xsh[k] as usize; + } + } + debug_assert_eq!(acc, total_out); + } + let mut count = 1usize; + for k in 0..n { + if reduced[k] { + count *= xsh[k] as usize; + } + } + let mut dst = 0usize; + let mut coords = [0usize; MAX_NDIM]; + for xi in 0..total_in { + if xi > 0 { + // advance the input odometer, maintaining dst incrementally + let mut k = n - 1; + loop { + coords[k] += 1; + dst += ostr[k]; + if coords[k] < xsh[k] as usize { + break; + } + coords[k] = 0; + dst -= ostr[k] * xsh[k] as usize; + if k == 0 { + break; + } + k -= 1; + } + } + let v = x[xi]; + o[dst] += if l2 { v * v } else { v }; + } + if l2 { + for i in 0..total_out { + o[i] = o[i].sqrt(); + } + } else { + let inv = 1.0f32 / count as f32; + for i in 0..total_out { + o[i] *= inv; + } + } +} + +/// Resize: linear mode, half_pixel transform, target sizes from `osh`. +/// Separable per-dimension passes ping-pong between the scratch buffers; +/// the final pass writes to `o`. +/// +/// # Safety +/// See module docs. +pub unsafe fn resize( + xp: *const f32, + xsh: &[u32], + op_: *mut f32, + osh: &[u32], + scratch: &mut Vec, + scratch2: &mut Vec, +) { + let n = xsh.len(); + debug_assert_eq!(n, osh.len()); + debug_assert!(n <= MAX_NDIM); + let out_total = numel(osh); + let x = std::slice::from_raw_parts(xp, numel(xsh)); + let mut max_total = 1usize; + let mut dims: [usize; MAX_NDIM] = [0; MAX_NDIM]; + let mut m = 0usize; + for d in 0..n { + let mx = xsh[d].max(osh[d]) as usize; + max_total *= mx; + if xsh[d] != osh[d] { + dims[m] = d; + m += 1; + } + } + if m == 0 { + let o = std::slice::from_raw_parts_mut(op_, out_total); + o.copy_from_slice(x); + return; + } + scratch.resize(max_total, 0.0); + scratch2.resize(max_total, 0.0); + let s1 = scratch.as_mut_ptr(); + let s2 = scratch2.as_mut_ptr(); + + // pass p: src = x (p=0) / s1 (odd) / s2 (even, >0); dst = o (last) / + // s1 (even) / s2 (odd) — src and dst are always distinct buffers. + let mut cur_shape = [0u32; MAX_NDIM]; + cur_shape[..n].copy_from_slice(xsh); + for pass in 0..m { + let d = dims[pass]; + let last = pass + 1 == m; + let in_d = cur_shape[d] as usize; + let out_d = osh[d] as usize; + let ratio = in_d as f32 / out_d as f32; + let outer: usize = cur_shape[..d].iter().fold(1usize, |a, &v| a * v as usize); + let inner: usize = cur_shape[d + 1..n] + .iter() + .fold(1usize, |a, &v| a * v as usize); + let cur_total = outer * in_d * inner; + let out_pass_total = outer * out_d * inner; + + // per-dim coordinate tables (tiny; recomputed per pass) + let mut lohi: Vec<(i64, i64)> = Vec::with_capacity(out_d); + let mut frac_tab: Vec = Vec::with_capacity(out_d); + for p in 0..out_d { + let coord = (p as f32 + 0.5) * ratio - 0.5; + let lo = coord.floor() as i64; + let lo_c = lo.clamp(0, in_d as i64 - 1); + let hi_c = (lo + 1).clamp(0, in_d as i64 - 1); + lohi.push((lo_c, hi_c)); + frac_tab.push(coord - lo as f32); + } + + let src_ptr: *const f32 = if pass == 0 { + x.as_ptr() + } else if pass % 2 == 1 { + s1 as *const f32 + } else { + s2 as *const f32 + }; + let dst_ptr: *mut f32 = if last { + op_ + } else if pass % 2 == 0 { + s1 + } else { + s2 + }; + let src = std::slice::from_raw_parts(src_ptr, cur_total); + let dst = std::slice::from_raw_parts_mut(dst_ptr, out_pass_total); + resize_pass(src, dst, outer, in_d, out_d, inner, &lohi, &frac_tab); + cur_shape[d] = osh[d]; + } +} + +#[allow(clippy::too_many_arguments)] +fn resize_pass( + src: &[f32], + dst: &mut [f32], + outer: usize, + in_d: usize, + out_d: usize, + inner: usize, + lohi: &[(i64, i64)], + frac_tab: &[f32], +) { + for oo in 0..outer { + for p in 0..out_d { + let (lo, hi) = lohi[p]; + let frac = frac_tab[p]; + let slow = &src[(oo * in_d + lo as usize) * inner..][..inner]; + let shigh = &src[(oo * in_d + hi as usize) * inner..][..inner]; + let drow = &mut dst[(oo * out_d + p) * inner..][..inner]; + for i in 0..inner { + let lv = slow[i]; + drow[i] = lv + (shigh[i] - lv) * frac; + } + } + } +} + +// ─── AVX2+FMA runtime dispatch ───────────────────────────────────────────── +// +// The workspace builds for baseline x86-64 (SSE2) so the shipped binaries +// run everywhere; ONNX Runtime's MLAS gets its speed from runtime CPUID +// dispatch to AVX kernels. We mirror that: each hot kernel body is compiled +// twice from the SAME source (an `#[inline(always)]` impl inlined into a +// `#[target_feature(enable = "avx2,fma")]` wrapper and into the baseline +// entry point), selected once per process via `is_x86_feature_detected`. +// Non-x86 targets (e.g. macOS arm64) use the baseline path, where LLVM +// already auto-vectorizes with NEON. + +#[cfg(target_arch = "x86_64")] +fn have_avx2() -> bool { + use std::sync::OnceLock; + static DETECTED: OnceLock = OnceLock::new(); + *DETECTED.get_or_init(|| { + std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("fma") + }) +} + +pub unsafe fn binary( + op: BinOp, + ap: *const f32, + ash: &[u32], + bp: *const f32, + bsh: &[u32], + op_: *mut f32, + osh: &[u32], +) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return binary_avx2(op, ap, ash, bp, bsh, op_, osh); + } + } + binary_impl(op, ap, ash, bp, bsh, op_, osh) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn binary_avx2( + op: BinOp, + ap: *const f32, + ash: &[u32], + bp: *const f32, + bsh: &[u32], + op_: *mut f32, + osh: &[u32], +) { + binary_impl(op, ap, ash, bp, bsh, op_, osh) +} + +pub unsafe fn unary(opcode: u16, xp: *const f32, op_: *mut f32, n: usize) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return unary_avx2(opcode, xp, op_, n); + } + } + unary_impl(opcode, xp, op_, n) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn unary_avx2(opcode: u16, xp: *const f32, op_: *mut f32, n: usize) { + unary_impl(opcode, xp, op_, n) +} + +pub unsafe fn matmul( + ap: *const f32, + ash: &[u32], + bp: *const f32, + bsh: &[u32], + op_: *mut f32, + osh: &[u32], +) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return matmul_avx2(ap, ash, bp, bsh, op_, osh); + } + } + matmul_impl(ap, ash, bp, bsh, op_, osh) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn matmul_avx2( + ap: *const f32, + ash: &[u32], + bp: *const f32, + bsh: &[u32], + op_: *mut f32, + osh: &[u32], +) { + matmul_impl(ap, ash, bp, bsh, op_, osh) +} + +pub unsafe fn conv( + transpose: bool, + xp: *const f32, + xsh: &[u32], + wp: *const f32, + wsh: &[u32], + bias: Option<*const f32>, + op_: *mut f32, + osh: &[u32], + at: &ConvAttrs, + scratch: &mut Vec, +) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return conv_avx2(transpose, xp, xsh, wp, wsh, bias, op_, osh, at, scratch); + } + } + conv_impl(transpose, xp, xsh, wp, wsh, bias, op_, osh, at, scratch) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn conv_avx2( + transpose: bool, + xp: *const f32, + xsh: &[u32], + wp: *const f32, + wsh: &[u32], + bias: Option<*const f32>, + op_: *mut f32, + osh: &[u32], + at: &ConvAttrs, + scratch: &mut Vec, +) { + conv_impl(transpose, xp, xsh, wp, wsh, bias, op_, osh, at, scratch) +} + +pub unsafe fn gru( + xp: *const f32, + xsh: &[u32], + wp: *const f32, + rp: *const f32, + b: Option<*const f32>, + h0: Option<*const f32>, + y: Option<*mut f32>, + yh: Option<*mut f32>, + hidden: usize, + direction: i64, + lbr: bool, + pre: Option<(&[f32], &[f32])>, + scratch: &mut Vec, +) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return gru_avx2( + xp, xsh, wp, rp, b, h0, y, yh, hidden, direction, lbr, pre, scratch, + ); + } + } + gru_impl( + xp, xsh, wp, rp, b, h0, y, yh, hidden, direction, lbr, pre, scratch, + ) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn gru_avx2( + xp: *const f32, + xsh: &[u32], + wp: *const f32, + rp: *const f32, + b: Option<*const f32>, + h0: Option<*const f32>, + y: Option<*mut f32>, + yh: Option<*mut f32>, + hidden: usize, + direction: i64, + lbr: bool, + pre: Option<(&[f32], &[f32])>, + scratch: &mut Vec, +) { + gru_impl( + xp, xsh, wp, rp, b, h0, y, yh, hidden, direction, lbr, pre, scratch, + ) +} + +pub unsafe fn reduce( + l2: bool, + xp: *const f32, + xsh: &[u32], + axes: &[i64], + _keepdims: bool, + op_: *mut f32, + osh: &[u32], +) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return reduce_avx2(l2, xp, xsh, axes, _keepdims, op_, osh); + } + } + reduce_impl(l2, xp, xsh, axes, _keepdims, op_, osh) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn reduce_avx2( + l2: bool, + xp: *const f32, + xsh: &[u32], + axes: &[i64], + _keepdims: bool, + op_: *mut f32, + osh: &[u32], +) { + reduce_impl(l2, xp, xsh, axes, _keepdims, op_, osh) +} + +pub unsafe fn layer_norm( + xp: *const f32, + xsh: &[u32], + scale: *const f32, + bias: Option<*const f32>, + op_: *mut f32, + axis: usize, + eps: f32, +) { + #[cfg(target_arch = "x86_64")] + { + if have_avx2() { + return layer_norm_avx2(xp, xsh, scale, bias, op_, axis, eps); + } + } + layer_norm_impl(xp, xsh, scale, bias, op_, axis, eps) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn layer_norm_avx2( + xp: *const f32, + xsh: &[u32], + scale: *const f32, + bias: Option<*const f32>, + op_: *mut f32, + axis: usize, + eps: f32, +) { + layer_norm_impl(xp, xsh, scale, bias, op_, axis, eps) +} diff --git a/openless-all/app/crates/openless-capsule/src/supervisor.rs b/openless-all/app/crates/openless-capsule/src/supervisor.rs index 213695f07..9d36b6131 100644 --- a/openless-all/app/crates/openless-capsule/src/supervisor.rs +++ b/openless-all/app/crates/openless-capsule/src/supervisor.rs @@ -106,6 +106,23 @@ async fn run_instance( frames: &mut watch::Receiver>, on_action: &Arc, on_health: &HealthCallback, +) -> io::Result<()> { + run_instance_with_handshake_timeout( + command, + frames, + on_action, + on_health, + Duration::from_secs(5), + ) + .await +} + +async fn run_instance_with_handshake_timeout( + command: &mut Command, + frames: &mut watch::Receiver>, + on_action: &Arc, + on_health: &HealthCallback, + handshake_timeout: Duration, ) -> io::Result<()> { command .stdin(std::process::Stdio::piped()) @@ -127,7 +144,7 @@ async fn run_instance( ); let mut ready = false; let mut message_bytes = Vec::new(); - let deadline = tokio::time::sleep(Duration::from_secs(5)); + let deadline = tokio::time::sleep(handshake_timeout); tokio::pin!(deadline); let result = async { loop { @@ -434,13 +451,24 @@ mod health_tests { }); let action: Arc = Arc::new(|_, _| panic!("unexpected action")); - let task = tokio::spawn(async move { - run_instance(&mut helper(true), &mut receiver, &action, ¬ify).await - }); - tokio::time::timeout(Duration::from_secs(5), health.changed()) + let mut task = tokio::spawn(async move { + // The Windows test fixture cold-starts PowerShell, unlike the + // production native helper. Do not make its startup speed part + // of the protocol/lifecycle contract. Production retains 5 s. + run_instance_with_handshake_timeout( + &mut helper(true), + &mut receiver, + &action, + ¬ify, + Duration::from_secs(15), + ) .await - .unwrap() - .unwrap(); + }); + tokio::select! { + changed = health.changed() => changed.expect("helper health channel closed before Ready"), + result = &mut task => panic!("helper exited before Ready: {result:?}"), + _ = tokio::time::sleep(Duration::from_secs(20)) => panic!("test helper did not report Ready within its startup budget"), + } assert!(health.borrow().is_none()); frames.send_replace(None); assert!(tokio::time::timeout(Duration::from_secs(2), task) diff --git a/openless-all/app/crates/openless-core/Cargo.toml b/openless-all/app/crates/openless-core/Cargo.toml index 80b76bc93..88b071bbb 100644 --- a/openless-all/app/crates/openless-core/Cargo.toml +++ b/openless-all/app/crates/openless-core/Cargo.toml @@ -8,6 +8,7 @@ rust-version = "1.99" publish = false [dependencies] +micyou-dsp = { path = "../micyou-dsp" } anyhow = "1" argon2 = { version = "0.6.0", features = ["zeroize"] } base64 = "0.23.1" diff --git a/openless-all/app/crates/openless-core/examples/enhance_audio.rs b/openless-all/app/crates/openless-core/examples/enhance_audio.rs new file mode 100644 index 000000000..7acf34d98 --- /dev/null +++ b/openless-all/app/crates/openless-core/examples/enhance_audio.rs @@ -0,0 +1,74 @@ +//! Offline A/B tool. Input must be canonical 16 kHz mono PCM WAV. +//! cargo run --release -p openless-core --example enhance_audio -- input.wav output.wav rnnoise +use openless_core::{AudioDspConfig, AudioProcessingConfig, AudioProcessor, NoiseReduction}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().collect(); + if !(3..=4).contains(&args.len()) { + return Err( + "usage: enhance_audio INPUT.wav OUTPUT.wav [rnnoise|purevox|spectral|off]".into(), + ); + } + if std::path::Path::new(&args[1]) == std::path::Path::new(&args[2]) { + return Err("input and output must be different files".into()); + } + let algorithm = match args.get(3).map(String::as_str).unwrap_or("rnnoise") { + "rnnoise" => NoiseReduction::Rnnoise, + "purevox" => NoiseReduction::Purevox, + "spectral" => NoiseReduction::Spectral, + "off" => NoiseReduction::Off, + _ => return Err("unknown noise reduction algorithm".into()), + }; + let wav = std::fs::read(&args[1])?; + let pcm = openless_core::audio::decode_dictation_wav(&wav)?; + let samples: Vec = pcm + .as_chunks::<2>() + .0 + .iter() + .map(|p| i16::from_le_bytes([p[0], p[1]]) as f32 / 32768.0) + .collect(); + let mut processor = AudioProcessor::new(AudioProcessingConfig { + enhancement_enabled: true, + dsp: AudioDspConfig { + noise_reduction: algorithm, + ..Default::default() + }, + }); + let started = std::time::Instant::now(); + let mut output = Vec::new(); + for input in samples.chunks(320) { + if let Some(chunk) = processor.process(input, 1, 16_000) { + output.extend(chunk.pcm_i16_le); + } + } + if let Some(chunk) = processor.finish() { + output.extend(chunk.pcm_i16_le); + } + let elapsed = started.elapsed().as_secs_f64(); + assert_eq!( + output.len(), + pcm.len(), + "enhancement changed recording duration" + ); + let result = openless_core::encode_dictation_wav(&output)?; + // Create-new also rejects an existing output path and an alias/symlink to + // the input, preserving recordings used for comparison. + use std::io::Write; + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[2])? + .write_all(&result)?; + fn rms(pcm: &[u8]) -> f64 { + let power = pcm + .as_chunks::<2>() + .0 + .iter() + .map(|p| (i16::from_le_bytes([p[0], p[1]]) as f64 / 32768.0).powi(2)) + .sum::(); + (power / (pcm.len() / 2).max(1) as f64).sqrt() + } + let duration = samples.len() as f64 / 16_000.0; + println!("algorithm={algorithm:?} duration={duration:.3}s processing={elapsed:.3}s real_time_factor={:.3} input_rms={:.6} output_rms={:.6}", elapsed / duration.max(1e-9), rms(pcm), rms(&output)); + Ok(()) +} diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 4256e0983..f5b809476 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2320,6 +2320,11 @@ struct BackendEngineProgress { impl EngineProgressSink for BackendEngineProgress { fn publish(&self, session_id: SessionId, progress: EngineProgress) -> Result<(), BackendError> { match progress { + EngineProgress::CaptureReady => { + let state = self.state.read().expect("backend state lock poisoned"); + ensure_active_session(&state, session_id)?; + self.phase_changed.notify_waiters(); + } EngineProgress::RecordingLevel { elapsed_ms, level } => { let mut state = self.state.write().expect("backend state lock poisoned"); ensure_active_session(&state, session_id)?; @@ -5290,6 +5295,7 @@ impl OpenLessBackend { pre_roll_ms: 500, audio_processing: crate::audio::AudioProcessingConfig { enhancement_enabled: preferences.audio_enhancement_enabled, + dsp: preferences.audio_dsp, }, }; match self.deps.dictation_engine.arm_audio_capture(request) { @@ -8003,6 +8009,38 @@ impl OpenLessBackend { if let Some(session) = ready { break session; } + let session_id = expected_session_id.expect("stop is bound to an active session"); + let resources = self.voice_sessions.hold_resources(session_id)?; + let engine = Arc::clone(&self.deps.dictation_engine); + let state = Arc::clone(&self.state); + let phase_changed = Arc::clone(&self.phase_changed); + own_voice_effect( + &self.deps.task_spawner, + Box::pin(async move { + let _resources = resources; + loop { + let changed = phase_changed.notified(); + match engine.stop_recording(session_id).await { + Ok(false) => {} + // finish() reports cached stop errors without discarding recovery audio. + Ok(true) | Err(_) => return Ok(()), + } + { + let state = state.read().expect("backend state lock poisoned"); + if state.dictation.session_id != Some(session_id) + || state.dictation.phase != DictationPhase::Starting + { + return Ok(()); + } + } + tokio::select! { + _ = changed => {} + _ = _resources.cancel.cancelled() => return Ok(()), + } + } + }), + ) + .await?; changed.await; }; @@ -17896,6 +17934,227 @@ mod tests { assert_eq!(backend.snapshot().dictation.phase, DictationPhase::Idle); } + #[tokio::test] + async fn stop_during_provider_preparation_releases_capture_before_finalization() { + use crate::ports::{ + ActiveRecording, AudioConsumer, AudioRecorder, PreparedTranscription, RecordingArchive, + RecordingProgressSink, TextStreamSink, TranscriptionEngine, TranscriptionSession, + }; + use crate::testing::{ + FixtureAudioRecorder, FixtureTextPolisher, FixtureTranscriptionEngine, + }; + + struct BlockedPreparation { + entered: Arc, + release: Arc, + transcriber: FixtureTranscriptionEngine, + } + impl TranscriptionEngine for BlockedPreparation { + fn prepare( + self: Arc, + session_id: SessionId, + context: Arc, + ) -> BoxFuture<'static, Result, BackendError>> + { + Box::pin(async move { + self.entered.notify_one(); + self.release.notified().await; + Arc::new(self.transcriber.clone()) + .prepare(session_id, context) + .await + }) + } + fn start( + &self, + session_id: SessionId, + context: Arc, + partials: Arc, + ) -> BoxFuture<'static, Result, BackendError>> + { + self.transcriber.start(session_id, context, partials) + } + } + + struct Recorder { + inner: FixtureAudioRecorder, + stop_fails: bool, + archive: Arc>>>, + } + struct Recording { + inner: Box, + stop_fails: bool, + } + impl ActiveRecording for Recording { + fn archive(&self) -> Option> { + self.inner.archive() + } + fn stop(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { + let Recording { inner, stop_fails } = *self; + let stopping = inner.stop(); + Box::pin(async move { + stopping.await?; + if stop_fails { + Err(BackendError::new( + BackendErrorCode::Platform, + "fixture stop failure", + )) + } else { + Ok(()) + } + }) + } + } + impl AudioRecorder for Recorder { + fn start( + &self, + session_id: SessionId, + context: Arc, + consumer: Arc, + progress: Arc, + ) -> BoxFuture<'static, Result, BackendError>> { + let starting = self.inner.start(session_id, context, consumer, progress); + let archive = self.archive.clone(); + let stop_fails = self.stop_fails; + Box::pin(async move { + let recording = starting.await?; + *archive.lock().unwrap() = recording.archive(); + Ok(Box::new(Recording { + inner: recording, + stop_fails, + }) as Box) + }) + } + } + + for stable in [false, true] { + for publish_level in [false, true] { + for (stop_before_start, drop_stop, stop_fails) in [ + (false, false, false), + (true, false, false), + (true, true, false), + (false, false, true), + (true, false, true), + ] { + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let recorder = FixtureAudioRecorder::new( + vec![vec![1, 0, 2, 0]], + if publish_level { + vec![(1, 0.1)] + } else { + vec![] + }, + ) + .with_archived_recording(true); + let archive = Arc::new(Mutex::new(None)); + let transcriber = FixtureTranscriptionEngine::successful("captured prefix", 1); + let mut deps = BackendDependencies::unsupported(); + deps.credential_store = + Arc::new(crate::credentials::InMemoryCredentialStore::default()); + deps.dictation_engine = Arc::new(crate::DictationEngineRouter::new(Arc::new( + crate::PipelineDictationEngine::new( + Arc::new(Recorder { + inner: recorder.clone(), + stop_fails, + archive: archive.clone(), + }), + Arc::new(BlockedPreparation { + entered: entered.clone(), + release: release.clone(), + transcriber: transcriber.clone(), + }), + Arc::new(FixtureTextPolisher::successful("unused")), + ), + ))); + let data_dir = TestDataDir::new("stop-during-provider-preparation"); + let backend = Arc::new( + OpenLessBackend::new( + BackendConfig { + data_dir: data_dir.path().to_path_buf(), + ..Default::default() + }, + deps, + ) + .unwrap(), + ); + backend.start().await.unwrap(); + let mut prefs = backend.get_preferences(); + prefs.stable_transcription_enabled = stable; + prefs.record_audio_for_debug = true; + backend.set_preferences(prefs).unwrap(); + let options = DictationStartOptions { + insert_text: false, + ..Default::default() + }; + let reservation = backend + .reserve_dictation_session(false, options.output_target, None) + .unwrap(); + let session_id = reservation.session_id; + let mut stopping = Some(Box::pin(backend.stop_dictation())); + if stop_before_start { + assert!( + futures_util::poll!(stopping.as_mut().unwrap().as_mut()).is_pending() + ); + tokio::task::yield_now().await; + assert!( + futures_util::poll!(stopping.as_mut().unwrap().as_mut()).is_pending() + ); + } + if drop_stop { + drop(stopping.take()); + } + let starting = tokio::spawn({ + let backend = backend.clone(); + async move { backend.start_reserved_dictation(reservation, options).await } + }); + entered.notified().await; + let wait_for_stop = async { + while recorder.stop_count() == 0 { + if let Some(stopping) = stopping.as_mut() { + tokio::select! { + result = stopping => panic!("finalized before provider preparation: {result:?}"), + _ = tokio::task::yield_now() => {} + } + } else { + tokio::task::yield_now().await; + } + } + }; + let stopped_before_prepare = + tokio::time::timeout(std::time::Duration::from_secs(2), wait_for_stop) + .await; + assert!(transcriber.pcm().is_empty()); + release.notify_one(); + assert_eq!(starting.await.unwrap().unwrap(), session_id); + let result = match stopping { + Some(stopping) => stopping.await, + None => backend.stop_dictation().await, + }; + if stop_fails { + assert_eq!(result.unwrap_err().code, BackendErrorCode::Platform); + assert_eq!( + backend.list_history().unwrap()[0].error_code.as_deref(), + Some("transcribeFailed") + ); + } else { + let result = result.unwrap(); + assert_eq!(result.session_id, session_id); + assert_eq!(result.polished_text, "captured prefix"); + assert_eq!(transcriber.pcm(), vec![1, 0, 2, 0]); + } + assert_eq!(recorder.stop_count(), 1); + assert!(archive.lock().unwrap().as_ref().unwrap().is_available()); + assert_eq!( + backend.list_history().unwrap()[0].has_audio_recording, + Some(true) + ); + assert_eq!(backend.snapshot().dictation.phase, DictationPhase::Idle); + assert!(stopped_before_prepare.is_ok(), "Stop kept capture open: stable={stable}, level={publish_level}, early={stop_before_start}, dropped={drop_stop}, failed={stop_fails}"); + } + } + } + } + async fn assert_pending_insertion_cleanup(drop_start: bool) { struct DelayedSession { restoring: Arc, diff --git a/openless-all/app/crates/openless-core/src/audio.rs b/openless-all/app/crates/openless-core/src/audio.rs index acfc652e4..254ec2684 100644 --- a/openless-all/app/crates/openless-core/src/audio.rs +++ b/openless-all/app/crates/openless-core/src/audio.rs @@ -13,240 +13,160 @@ pub struct NormalizedPcmChunk { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct AudioProcessingConfig { pub enhancement_enabled: bool, + pub dsp: crate::audio_dsp::AudioDspConfig, } -/// Small, deterministic microphone enhancement that does not require a model -/// or a platform-specific audio dependency. -#[derive(Debug, Clone, Copy)] -pub struct AudioEnhancer { - enabled: bool, - noise_suppressor: NoiseSuppressor, +/// Shared processor for native and external microphone hosts. +/// Enhancement is session-frozen. Hosts must deliver `finish()` before closing +/// their ASR consumer or recording archive. +pub struct AudioProcessor { + config: AudioProcessingConfig, + normalizer: PcmNormalizer, + enhanced: Option, + finished: bool, } -#[derive(Debug, Clone, Copy)] -struct NoiseSuppressor { - enabled: bool, - noise_floor: f32, - gain: f32, - gate_open: bool, - hold_remaining_secs: f32, +impl std::fmt::Debug for AudioProcessor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AudioProcessor") + .field("config", &self.config) + .finish_non_exhaustive() + } } -impl NoiseSuppressor { - const INITIAL_NOISE_FLOOR: f32 = 0.008; - const MIN_THRESHOLD: f32 = 0.012; - const CLOSE_THRESHOLD_MULTIPLIER: f32 = 2.0; - const OPEN_THRESHOLD_MULTIPLIER: f32 = 2.5; - const ATTENUATED_GAIN: f32 = 0.25; - const NOISE_FLOOR_DECAY_SECS: f32 = 0.25; - const NOISE_FLOOR_RISE_SECS: f32 = 8.0; - const HOLD_SECS: f32 = 0.050; - const GAIN_ATTACK_SECS: f32 = 0.010; - const GAIN_RELEASE_SECS: f32 = 0.080; - - fn new(enabled: bool) -> Self { +impl AudioProcessor { + pub fn new(config: AudioProcessingConfig) -> Self { Self { - enabled, - noise_floor: Self::INITIAL_NOISE_FLOOR, - gain: 1.0, - gate_open: false, - hold_remaining_secs: 0.0, + config, + normalizer: Default::default(), + enhanced: None, + finished: false, } } - fn reset(&mut self) { - self.noise_floor = Self::INITIAL_NOISE_FLOOR; - self.gain = 1.0; - self.gate_open = false; - self.hold_remaining_secs = 0.0; + pub fn reset(&mut self) { + *self = Self::new(self.config); } - fn process_in_place(&mut self, samples: &mut [f32], channels: usize, sample_rate: u32) { - if !self.enabled || samples.is_empty() || channels == 0 || sample_rate == 0 { - return; - } - let frame_count = samples.len() / channels; - if frame_count == 0 { - return; + pub fn process( + &mut self, + input: &[f32], + channels: usize, + rate: u32, + ) -> Option { + if self.finished || input.is_empty() || channels == 0 || rate == 0 { + return None; } - let processed_samples = frame_count * channels; - let rms = downmixed_rms(&samples[..processed_samples], channels); - let duration_secs = frame_count as f32 / sample_rate as f32; - let close_threshold = - (self.noise_floor * Self::CLOSE_THRESHOLD_MULTIPLIER).max(Self::MIN_THRESHOLD); - let open_threshold = (self.noise_floor * Self::OPEN_THRESHOLD_MULTIPLIER) - .max(close_threshold * 1.5) - .max(Self::MIN_THRESHOLD); - - if rms <= close_threshold { - let adaptation_secs = if rms < self.noise_floor { - Self::NOISE_FLOOR_DECAY_SECS - } else { - Self::NOISE_FLOOR_RISE_SECS - }; - let coefficient = 1.0 - (-duration_secs / adaptation_secs).exp(); - self.noise_floor += (rms - self.noise_floor) * coefficient.clamp(0.0, 1.0); - self.noise_floor = self.noise_floor.max(0.0); + if !self.config.enhancement_enabled { + return self.normalizer.process(input, channels, rate); } - - if self.gate_open { - if rms <= close_threshold { - self.hold_remaining_secs -= duration_secs; - if self.hold_remaining_secs <= 0.0 { - self.gate_open = false; - self.hold_remaining_secs = 0.0; - } - } else { - self.hold_remaining_secs = Self::HOLD_SECS; - } - } else if rms >= open_threshold { - self.gate_open = true; - self.hold_remaining_secs = Self::HOLD_SECS; + let stream = self + .enhanced + .get_or_insert_with(|| EnhancedStream::new(self.config.dsp, channels, rate)); + if stream.channels != channels || stream.rate != rate { + log::error!("audio format changed during an enhancement session; reset required"); + return None; } - - let target_gain = if self.gate_open { - 1.0 - } else if rms <= close_threshold { - Self::ATTENUATED_GAIN - } else { - let knee = - (rms - close_threshold) / (open_threshold - close_threshold).max(f32::EPSILON); - Self::ATTENUATED_GAIN + (1.0 - Self::ATTENUATED_GAIN) * knee.clamp(0.0, 1.0) - }; - let smoothing_secs = if target_gain < self.gain { - Self::GAIN_RELEASE_SECS - } else { - Self::GAIN_ATTACK_SECS - }; - let coefficient = 1.0 - (-duration_secs / smoothing_secs).exp(); - self.gain += (target_gain - self.gain) * coefficient.clamp(0.0, 1.0); - - for sample in &mut samples[..processed_samples] { - if sample.is_finite() { - *sample *= self.gain; + stream.interleaved.extend(input.iter().map(|s| { + if s.is_finite() { + s.clamp(-1.0, 1.0) } else { - *sample = 0.0; + 0.0 } + })); + let complete = stream.interleaved.len() / channels * channels; + let mono = downmix_to_mono(&stream.interleaved[..complete], channels); + stream.interleaved.drain(..complete); + stream.input_frames += mono.len() as u64; + let working = stream.input_resampler.push(&mono, false); + let processed = stream.frames(&working, false); + pcm_chunk(&processed) + } + + /// Idempotent end-of-stream barrier. Flushes fractional resampling state, + /// a partial 10 ms frame and model lookahead without adding padding to PCM. + pub fn finish(&mut self) -> Option { + if self.finished { + return None; } + self.finished = true; + let stream = self.enhanced.as_mut()?; + let working = stream.input_resampler.push(&[], true); + let processed = stream.frames(&working, true); + pcm_chunk(&processed) } } -impl Default for NoiseSuppressor { - fn default() -> Self { - Self::new(false) - } -} - -fn downmixed_rms(samples: &[f32], channels: usize) -> f32 { - let frame_count = samples.len() / channels; - if frame_count == 0 { - return 0.0; - } - let square_sum = samples - .chunks_exact(channels) - .map(|frame| { - let mono = frame - .iter() - .map(|sample| if sample.is_finite() { *sample } else { 0.0 }) - .sum::() - / channels as f32; - f64::from(mono) * f64::from(mono) - }) - .sum::(); - (square_sum / frame_count as f64).sqrt() as f32 -} - -impl Default for AudioEnhancer { - fn default() -> Self { - Self::new(AudioProcessingConfig::default()) - } +struct EnhancedStream { + channels: usize, + rate: u32, + input_resampler: crate::audio_dsp::SpeechResampler, + output_resampler: crate::audio_dsp::SpeechResampler, + dsp: micyou_dsp::DspProcessor, + interleaved: Vec, + pending: std::collections::VecDeque, + skip: usize, + model_delay: usize, + input_frames: u64, + emitted_frames: u64, } -impl AudioEnhancer { - const INPUT_GAIN: f32 = 1.5; - const SOFT_CLIP_KNEE: f32 = 0.75; - - pub fn new(config: AudioProcessingConfig) -> Self { +impl EnhancedStream { + fn new(config: crate::audio_dsp::AudioDspConfig, channels: usize, rate: u32) -> Self { + let config = config.normalized(); Self { - enabled: config.enhancement_enabled, - noise_suppressor: NoiseSuppressor::new(config.enhancement_enabled), + channels, + rate, + input_resampler: crate::audio_dsp::SpeechResampler::new(rate, 48_000), + output_resampler: crate::audio_dsp::SpeechResampler::new(48_000, DICTATION_SAMPLE_RATE), + dsp: micyou_dsp::DspProcessor::new( + std::sync::Arc::new(std::sync::RwLock::new(config.settings())), + None, + ), + interleaved: Vec::new(), + pending: Default::default(), + skip: config.model_delay(), + model_delay: config.model_delay(), + input_frames: 0, + emitted_frames: 0, } } - pub fn reset(&mut self) { - self.noise_suppressor.reset(); - } - - pub fn process_in_place(&mut self, samples: &mut [f32]) { - self.process_in_place_with_format(samples, 1, DICTATION_SAMPLE_RATE); - } - - fn process_in_place_with_format( - &mut self, - samples: &mut [f32], - channels: usize, - sample_rate: u32, - ) { - if !self.enabled { - return; + fn frames(&mut self, input: &[f32], finish: bool) -> Vec { + self.pending.extend(input); + if finish { + let tail = self.pending.len().div_ceil(480) * 480 + self.model_delay; + self.pending.resize(tail, 0.0); } - self.noise_suppressor - .process_in_place(samples, channels, sample_rate); - for sample in samples { - let boosted = *sample * Self::INPUT_GAIN; - *sample = Self::soft_clip(boosted); + let mut output = Vec::new(); + while self.pending.len() >= 480 { + let mut frame: Vec = self.pending.drain(..480).collect(); + self.dsp.process(&mut frame, 1, 0.0); + let skip = self.skip.min(frame.len()); + self.skip -= skip; + output.extend(self.output_resampler.push(&frame[skip..], false)); } - } - - fn soft_clip(sample: f32) -> f32 { - let sample = sample.clamp(-1.0, 1.0); - let magnitude = sample.abs(); - if magnitude <= Self::SOFT_CLIP_KNEE { - return sample; + if finish { + output.extend(self.output_resampler.push(&[], true)); } - let normalized = (magnitude - Self::SOFT_CLIP_KNEE) / (1.0 - Self::SOFT_CLIP_KNEE); - let eased = normalized * normalized * (3.0 - 2.0 * normalized); - sample.signum() * (Self::SOFT_CLIP_KNEE + (1.0 - Self::SOFT_CLIP_KNEE) * eased) + // The last padded model frame may produce additional samples. Bound + // output by real capture duration, independently of callback sizes. + let valid = (self.input_frames * DICTATION_SAMPLE_RATE as u64).div_ceil(self.rate as u64); + output.truncate(valid.saturating_sub(self.emitted_frames) as usize); + self.emitted_frames += output.len() as u64; + output } } -/// Shared stateful processor used by native microphone hosts. -#[derive(Debug)] -pub struct AudioProcessor { - normalizer: PcmNormalizer, - enhancer: AudioEnhancer, -} - -impl AudioProcessor { - pub fn new(config: AudioProcessingConfig) -> Self { - Self { - normalizer: PcmNormalizer::default(), - enhancer: AudioEnhancer::new(config), - } - } - - pub fn reset(&mut self) { - self.normalizer = PcmNormalizer::default(); - self.enhancer.reset(); - } - - pub fn process( - &mut self, - interleaved: &[f32], - channels: usize, - input_sample_rate: u32, - ) -> Option { - if !self.enhancer.enabled { - return self - .normalizer - .process(interleaved, channels, input_sample_rate); - } - let mut enhanced = interleaved.to_vec(); - self.enhancer - .process_in_place_with_format(&mut enhanced, channels, input_sample_rate); - self.normalizer - .process(&enhanced, channels, input_sample_rate) +fn pcm_chunk(samples: &[f32]) -> Option { + if samples.is_empty() { + return None; } + let (pcm_i16_le, rms) = quantize_to_i16_le(samples); + Some(NormalizedPcmChunk { + pcm_i16_le, + level: (rms * LEVEL_RMS_GAIN).clamp(0.0, 1.0), + }) } /// Stateful interleaved-audio normalizer. @@ -433,11 +353,15 @@ fn quantize_to_i16_le(samples: &[f32]) -> (Vec, f32) { (bytes, rms) } +#[cfg(test)] +#[path = "audio/dsp_tests.rs"] +mod dsp_tests; + #[cfg(test)] mod tests { use super::*; - fn decode(bytes: &[u8]) -> Vec { + pub(super) fn decode(bytes: &[u8]) -> Vec { bytes .as_chunks::<2>() .0 @@ -473,104 +397,6 @@ mod tests { assert!(normalizer.process(&[0.0], 1, 0).is_none()); } - #[test] - fn enhancement_boosts_quiet_samples_and_soft_clips_loud_samples() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let mut samples = [0.5, -0.5, 1.0, -1.0]; - enhancer.process_in_place(&mut samples); - assert_eq!(samples[0], 0.75); - assert_eq!(samples[1], -0.75); - assert_eq!(samples[2], 1.0); - assert_eq!(samples[3], -1.0); - } - - #[test] - fn enhancement_replaces_non_finite_samples_and_stays_in_range() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let mut samples = [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 2.0, -2.0]; - enhancer.process_in_place(&mut samples); - assert!(samples.iter().all(|sample| sample.is_finite())); - assert!(samples.iter().all(|sample| (-1.0..=1.0).contains(sample))); - assert_eq!(samples[0], 0.0); - } - - #[test] - fn noise_suppression_attenuates_quiet_frames_and_recovers_for_speech() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let quiet = [0.004_f32; 160]; - let mut processed_quiet = quiet; - for _ in 0..24 { - processed_quiet = quiet; - enhancer.process_in_place(&mut processed_quiet); - } - assert!(processed_quiet[0].abs() < quiet[0].abs() * 0.5); - - let mut speech = [0.2_f32; 160]; - enhancer.process_in_place(&mut speech); - assert!(speech[0] > 0.1); - } - - #[test] - fn noise_suppression_reset_restores_initial_gain_state() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let quiet = [0.004_f32; 160]; - let mut settled = quiet; - for _ in 0..24 { - enhancer.process_in_place(&mut settled); - } - enhancer.reset(); - let mut after_reset = quiet; - enhancer.process_in_place(&mut after_reset); - assert!(after_reset[0] > settled[0]); - } - - #[test] - fn noise_suppression_keeps_stereo_frame_processing_consistent() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let mut samples = [0.004_f32; 320]; - enhancer.process_in_place_with_format(&mut samples, 2, DICTATION_SAMPLE_RATE); - assert_eq!(samples[0], samples[1]); - assert!(samples.iter().all(|sample| sample.is_finite())); - } - - #[test] - fn noise_suppression_does_not_learn_sustained_speech_as_noise() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - for _ in 0..1_000 { - let mut speech = [0.2_f32; 160]; - enhancer.process_in_place(&mut speech); - assert!(speech[0] > 0.25); - } - } - - #[test] - fn noise_suppression_uses_downmixed_rms_for_stereo_input() { - let mut enhancer = AudioEnhancer::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let mut out_of_phase = [0.2_f32; 320]; - for frame in out_of_phase.as_chunks_mut::<2>().0 { - frame[1] = -0.2; - } - for _ in 0..24 { - enhancer.process_in_place_with_format(&mut out_of_phase, 2, DICTATION_SAMPLE_RATE); - } - assert!(out_of_phase[0].abs() < 0.15); - assert_eq!(out_of_phase[0], -out_of_phase[1]); - } - #[test] fn processor_disabled_output_matches_normalizer() { let input = [0.0, 1.0, 0.5, -0.5]; @@ -582,51 +408,6 @@ mod tests { ); } - #[test] - fn processor_applies_noise_suppression_to_worker_facing_output() { - let quiet = [0.004_f32; 160]; - let mut processor = AudioProcessor::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let mut output = None; - for _ in 0..24 { - output = processor.process(&quiet, 1, DICTATION_SAMPLE_RATE); - } - let output = output.unwrap(); - let mut normalizer = PcmNormalizer::default(); - let baseline = normalizer - .process(&quiet, 1, DICTATION_SAMPLE_RATE) - .unwrap(); - let output_peak = decode(&output.pcm_i16_le) - .into_iter() - .map(|sample| i32::from(sample).abs()) - .max() - .unwrap_or(0); - let baseline_peak = decode(&baseline.pcm_i16_le) - .into_iter() - .map(|sample| i32::from(sample).abs()) - .max() - .unwrap_or(0); - assert!(output_peak < baseline_peak); - } - - #[test] - fn processor_enhancement_preserves_canonical_output_contract() { - let mut processor = AudioProcessor::new(AudioProcessingConfig { - enhancement_enabled: true, - }); - let output = processor - .process(&[0.25, -0.25, 0.5, -0.5], 1, DICTATION_SAMPLE_RATE) - .unwrap(); - assert!(output.pcm_i16_le.len().is_multiple_of(2)); - assert!(output.level.is_finite()); - assert!((0.0..=1.0).contains(&output.level)); - assert_eq!( - decode(&output.pcm_i16_le), - vec![12287, -12287, 24575, -24575] - ); - } - #[test] fn processor_reset_discards_resampling_state() { let mut processor = AudioProcessor::new(AudioProcessingConfig::default()); diff --git a/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs b/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs new file mode 100644 index 000000000..bd04ee7be --- /dev/null +++ b/openless-all/app/crates/openless-core/src/audio/dsp_tests.rs @@ -0,0 +1,197 @@ +use super::tests::decode; +use super::*; +use crate::{AudioDspConfig, NoiseReduction}; + +fn config(algorithm: NoiseReduction) -> AudioProcessingConfig { + AudioProcessingConfig { + enhancement_enabled: true, + dsp: AudioDspConfig { + noise_reduction: algorithm, + agc_enabled: false, + ..Default::default() + }, + } +} + +fn render( + input: &[f32], + rate: u32, + channels: usize, + block: usize, + config: AudioProcessingConfig, +) -> Vec { + let mut processor = AudioProcessor::new(config); + let mut bytes = Vec::new(); + for chunk in input.chunks(block) { + if let Some(output) = processor.process(chunk, channels, rate) { + assert!(output.level.is_finite() && (0.0..=1.0).contains(&output.level)); + bytes.extend(output.pcm_i16_le); + } + } + if let Some(output) = processor.finish() { + bytes.extend(output.pcm_i16_le); + } + assert!(processor.finish().is_none()); + assert!(processor.process(&[0.5; 480], 1, rate).is_none()); + bytes +} + +fn signal(size: usize) -> Vec { + (0..size).map(|i| (i as f32 * 0.073).sin() * 0.2).collect() +} + +#[test] +fn enhanced_output_is_independent_of_callback_boundaries() { + for algorithm in [ + NoiseReduction::Off, + NoiseReduction::Rnnoise, + NoiseReduction::Spectral, + NoiseReduction::Purevox, + ] { + for rate in [8_000, 16_000, 44_100, 48_000, 96_000] { + let input = signal(rate as usize / 12 + 37); + let whole = render(&input, rate, 1, input.len(), config(algorithm)); + let partitioned = render(&input, rate, 1, 137, config(algorithm)); + assert_eq!( + whole.len(), + (input.len() as u64 * 16_000).div_ceil(rate as u64) as usize * 2 + ); + assert_eq!(whole, partitioned, "algorithm={algorithm:?}, rate={rate}"); + } + } +} + +#[test] +fn short_recordings_flush_without_padding_or_losing_samples() { + for algorithm in [ + NoiseReduction::Off, + NoiseReduction::Rnnoise, + NoiseReduction::Purevox, + ] { + for len in [1, 7, 159, 160, 161, 479, 480, 481] { + let pcm = render(&signal(len), 16_000, 1, 13, config(algorithm)); + assert_eq!(pcm.len(), len * 2, "algorithm={algorithm:?}, len={len}"); + } + } +} + +#[test] +fn dry_mix_is_aligned_and_preserves_the_last_impulse() { + // At 1% wet, the dry component should keep its original time position. + let mut input = vec![0.0; 1600]; + input[160] = 0.5; + input[1590] = -0.5; + let mut c = config(NoiseReduction::Rnnoise); + c.dsp.noise_strength = 1; + let pcm = decode(&render(&input, 16_000, 1, 73, c)); + assert!(pcm[160] > 10_000, "first impulse moved or lost"); + assert!(pcm[1590] < -10_000, "last impulse moved or lost"); +} + +#[test] +fn stereo_downmix_and_partial_channel_frames_are_consistent() { + let mono = signal(1733); + let stereo: Vec = mono.iter().flat_map(|s| [*s, *s]).collect(); + let c = config(NoiseReduction::Rnnoise); + assert_eq!( + render(&mono, 44_100, 1, 51, c), + render(&stereo, 44_100, 2, 79, c) + ); +} + +#[test] +fn invalid_samples_are_sanitized_and_reset_restarts_the_session() { + let mut p = AudioProcessor::new(config(NoiseReduction::Rnnoise)); + assert!(p.process(&[], 1, 16_000).is_none()); + assert!(p.process(&[1.0], 0, 16_000).is_none()); + assert!(p.process(&[1.0], 1, 0).is_none()); + let _ = p.process(&[f32::NAN, f32::INFINITY, f32::NEG_INFINITY], 1, 16_000); + let output = p.finish().unwrap(); + assert!(output.level.is_finite()); + assert_eq!(output.pcm_i16_le.len(), 6); + p.reset(); + let input = signal(1733); + let mut output = p.process(&input, 1, 16_000).unwrap().pcm_i16_le; + output.extend(p.finish().unwrap().pcm_i16_le); + assert_eq!( + output, + render(&input, 16_000, 1, 99, config(NoiseReduction::Rnnoise)) + ); +} + +#[test] +fn resampling_suppresses_out_of_band_tones() { + fn tone(frequency: f32) -> Vec { + (0..4800) + .map(|i| (std::f32::consts::TAU * frequency * i as f32 / 48_000.0).sin() * 0.3) + .collect() + } + fn rms(pcm: Vec) -> f64 { + let samples = decode(&pcm); + (samples[160..samples.len() - 160] + .iter() + .map(|s| f64::from(*s).powi(2)) + .sum::() + / (samples.len() - 320) as f64) + .sqrt() + } + let c = config(NoiseReduction::Off); + let speech_band = rms(render(&tone(1000.0), 48_000, 1, 197, c)); + let alias_band = rms(render(&tone(12_000.0), 48_000, 1, 197, c)); + assert!( + alias_band < speech_band * 0.02, + "alias RMS {alias_band}, passband RMS {speech_band}" + ); +} + +#[test] +fn partial_preferences_migrate_and_invalid_controls_are_bounded() { + let config: AudioDspConfig = serde_json::from_str(r#"{"noiseReduction":"purevox"}"#).unwrap(); + assert_eq!(config.noise_reduction, NoiseReduction::Purevox); + assert!(config.agc_enabled); + let mut config = config; + config.gain_db = 127; + config.noise_strength = 255; + config.processing_order.fill(crate::AudioDspStage::Agc); + let normalized = config.normalized(); + assert_eq!(normalized.gain_db, 30); + assert_eq!(normalized.noise_strength, 100); + assert_eq!( + normalized.processing_order, + AudioDspConfig::default().processing_order + ); +} + +#[test] +fn default_agc_does_not_amplify_denoised_room_noise_back_to_original_level() { + let mut seed = 0x12345678u32; + let noise: Vec = (0..48_000) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + ((seed >> 8) as f32 / 16_777_216.0 - 0.5) * 0.1 + }) + .collect(); + for algorithm in [NoiseReduction::Rnnoise, NoiseReduction::Purevox] { + let config = AudioProcessingConfig { + enhancement_enabled: true, + dsp: AudioDspConfig { + noise_reduction: algorithm, + ..Default::default() + }, + }; + let output = decode(&render(&noise, 16_000, 1, 320, config)); + let input_power = noise[32_000..] + .iter() + .map(|s| f64::from(*s).powi(2)) + .sum::(); + let output_power = output[32_000..] + .iter() + .map(|s| (f64::from(*s) / 32768.0).powi(2)) + .sum::(); + assert!( + output_power < input_power * 0.25, + "{algorithm:?}: output/input power {}", + output_power / input_power + ); + } +} diff --git a/openless-all/app/crates/openless-core/src/audio_dsp.rs b/openless-all/app/crates/openless-core/src/audio_dsp.rs new file mode 100644 index 000000000..3e639acaf --- /dev/null +++ b/openless-all/app/crates/openless-core/src/audio_dsp.rs @@ -0,0 +1,215 @@ +//! Session-frozen speech enhancement, adapted from MicYou's 48 kHz DSP chain. +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum NoiseReduction { + Off, + #[default] + Rnnoise, + Purevox, + Spectral, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum AudioDspStage { + NoiseReduction, + Dereverb, + Equalizer, + Amplifier, + Agc, + Vad, +} + +impl AudioDspStage { + fn name(self) -> &'static str { + match self { + Self::NoiseReduction => "NoiseReduction", + Self::Dereverb => "Dereverb", + Self::Equalizer => "Equalizer", + Self::Amplifier => "Amplifier", + Self::Agc => "AGC", + Self::Vad => "VAD", + } + } +} + +/// Integer controls make persisted/network settings finite and bounded. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct AudioDspConfig { + pub noise_reduction: NoiseReduction, + pub noise_strength: u8, + pub gain_db: i8, + pub agc_enabled: bool, + /// Target peak envelope as a percentage of full scale. + pub agc_target: u8, + pub agc_attack: u8, + pub agc_decay: u8, + pub vad_enabled: bool, + pub vad_threshold_db: i8, + /// Optional comb filter, not a learned room dereverberation model. + pub dereverb_level: u8, + pub equalizer_enabled: bool, + pub equalizer_preamp_db: i8, + pub equalizer_bands_db: [i8; 10], + pub processing_order: [AudioDspStage; 6], +} + +impl Default for AudioDspConfig { + fn default() -> Self { + use AudioDspStage::*; + Self { + noise_reduction: crate::audio_dsp::NoiseReduction::Rnnoise, + noise_strength: 80, + gain_db: 0, + agc_enabled: true, + agc_target: 35, + agc_attack: 50, + agc_decay: 50, + vad_enabled: false, + vad_threshold_db: -50, + dereverb_level: 0, + equalizer_enabled: false, + equalizer_preamp_db: 0, + equalizer_bands_db: [0; 10], + processing_order: [NoiseReduction, Dereverb, Equalizer, Amplifier, Agc, Vad], + } + } +} + +impl AudioDspConfig { + pub fn normalized(mut self) -> Self { + self.noise_strength = self.noise_strength.min(100); + self.gain_db = self.gain_db.clamp(-30, 30); + self.agc_target = self.agc_target.clamp(5, 90); + self.agc_attack = self.agc_attack.clamp(1, 100); + self.agc_decay = self.agc_decay.clamp(1, 100); + self.vad_threshold_db = self.vad_threshold_db.clamp(-80, -10); + self.dereverb_level = self.dereverb_level.min(100); + self.equalizer_preamp_db = self.equalizer_preamp_db.clamp(-20, 20); + self.equalizer_bands_db = self.equalizer_bands_db.map(|gain| gain.clamp(-12, 12)); + if self + .processing_order + .iter() + .enumerate() + .any(|(i, stage)| self.processing_order[..i].contains(stage)) + { + self.processing_order = Self::default().processing_order; + } + self + } + + pub(crate) fn model_delay(self) -> usize { + if self.noise_strength > 0 + && matches!( + self.noise_reduction, + NoiseReduction::Rnnoise | NoiseReduction::Purevox + ) + { + 480 + } else { + 0 + } + } + + pub(crate) fn settings(self) -> micyou_dsp::AudioDspSettings { + let c = self.normalized(); + micyou_dsp::AudioDspSettings { + gain: c.gain_db as f32, + ns_enabled: c.noise_reduction != NoiseReduction::Off && c.noise_strength > 0, + ns_type: match c.noise_reduction { + NoiseReduction::Purevox => "PureVox", + NoiseReduction::Spectral => "Speexdsp", + _ => "RNNoise", + } + .into(), + ns_intensity: c.noise_strength as f32, + dereverb_enabled: c.dereverb_level > 0, + dereverb_level: c.dereverb_level as f32, + agc_enabled: c.agc_enabled, + agc_target: c.agc_target as f32 / 100.0 * 32767.0, + agc_attack: c.agc_attack as f32, + agc_decay: c.agc_decay as f32, + vad_enabled: c.vad_enabled, + vad_threshold: c.vad_threshold_db as f32, + equalizer: micyou_dsp::EqualizerConfig { + enabled: c.equalizer_enabled, + pre_amp: c.equalizer_preamp_db as f32, + gains: c.equalizer_bands_db.map(f32::from).to_vec(), + }, + processing_chain: c.processing_order.map(|stage| stage.name().into()).to_vec(), + ..Default::default() + } + } +} + +/// A streaming windowed-sinc resampler. Integer positions avoid callback-size +/// dependent phase drift; lookahead is flushed once at the end of a session. +pub(crate) struct SpeechResampler { + source_rate: u32, + target_rate: u32, + samples: std::collections::VecDeque, + base: i64, + total: u64, + emitted: u64, +} + +impl SpeechResampler { + const HALF: i64 = 24; + + pub fn new(source_rate: u32, target_rate: u32) -> Self { + Self { + source_rate, + target_rate, + samples: Default::default(), + base: 0, + total: 0, + emitted: 0, + } + } + + pub fn push(&mut self, samples: &[f32], finish: bool) -> Vec { + self.samples + .extend(samples.iter().map(|s| if s.is_finite() { *s } else { 0.0 })); + self.total += samples.len() as u64; + let limit = (self.total * self.target_rate as u64).div_ceil(self.source_rate as u64); + let mut output = Vec::new(); + while self.emitted < limit { + let numerator = self.emitted * self.source_rate as u64; + let center = (numerator / self.target_rate as u64) as i64; + if !finish && center + Self::HALF >= self.total as i64 { + break; + } + let fraction = (numerator % self.target_rate as u64) as f64 / self.target_rate as f64; + let cutoff = (self.target_rate as f64 / self.source_rate as f64).min(1.0) * 0.94; + let mut value = 0.0; + let mut weight_sum = 0.0; + for index in center - Self::HALF + 1..=center + Self::HALF { + let distance = (index - center) as f64 - fraction; + let x = std::f64::consts::PI * distance * cutoff; + let sinc = if x.abs() < 1e-12 { 1.0 } else { x.sin() / x }; + let window = + 0.5 + 0.5 * (std::f64::consts::PI * distance / Self::HALF as f64).cos(); + let weight = sinc * window * cutoff; + let sample = if index >= self.base && index < self.total as i64 { + self.samples[(index - self.base) as usize] as f64 + } else { + 0.0 + }; + value += sample * weight; + weight_sum += weight; + } + output.push((value / weight_sum) as f32); + self.emitted += 1; + } + let next_center = (self.emitted * self.source_rate as u64 / self.target_rate as u64) as i64; + let retain_from = (next_center - Self::HALF).max(0); + let discard = (retain_from - self.base).max(0) as usize; + let discard = discard.min(self.samples.len()); + self.samples.drain(..discard); + self.base += discard as i64; + output + } +} diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 67cc380e5..6163cd56b 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -193,6 +193,13 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Boolean, reason: "local_only_microphone_capture", }, + PreferenceField { + rust_name: "audio_dsp", + key: "audioDsp", + class: PreferenceClass::Excluded, + shape: PreferenceShape::Object, + reason: "local_only_microphone_capture", + }, PreferenceField { rust_name: "voice_activation_enabled", key: "voiceActivationEnabled", diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index 9a9cb977a..b82b76a0e 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -335,6 +335,7 @@ impl DictationContext { .then(|| (preferences.silence_auto_stop_seconds * 1_000.0).round() as u64), audio_processing: AudioProcessingConfig { enhancement_enabled: preferences.audio_enhancement_enabled, + dsp: preferences.audio_dsp, }, }, pipeline_mode: crate::shared_types::effective_pipeline_mode( @@ -703,7 +704,8 @@ pub(crate) fn local_asr_fallback( ) -> Option { if !preferences.local_asr_fallback_enabled || !local_asr_supported - || !crate::cloud_providers::SHARED_CLOUD_ASR_PROVIDER_TYPES.contains(&asr.provider_type.as_str()) + || !crate::cloud_providers::SHARED_CLOUD_ASR_PROVIDER_TYPES + .contains(&asr.provider_type.as_str()) { return None; } @@ -854,6 +856,11 @@ mod tests { local_asr_active_model: "qwen3-asr-1.7b".to_string(), stable_transcription_enabled: true, audio_enhancement_enabled: true, + audio_dsp: crate::AudioDspConfig { + noise_reduction: crate::NoiseReduction::Purevox, + noise_strength: 33, + ..Default::default() + }, history_max_entries: Some(100), audio_recording_max_entries: Some(7), working_languages: vec!["简体中文".to_string(), "English".to_string()], @@ -881,6 +888,7 @@ mod tests { preferences.microphone_device_name = "changed".to_string(); preferences.active_asr_provider = "changed".to_string(); preferences.stable_transcription_enabled = false; + preferences.audio_dsp.noise_strength = 99; assert_eq!( context.recording.microphone_device_name.as_deref(), Some("USB microphone") @@ -888,6 +896,11 @@ mod tests { assert_eq!(context.asr.provider_id, "local-qwen3"); assert!(context.recording.transcribe_after_stop); assert!(context.recording.audio_processing.enhancement_enabled); + assert_eq!( + context.recording.audio_processing.dsp.noise_reduction, + crate::NoiseReduction::Purevox + ); + assert_eq!(context.recording.audio_processing.dsp.noise_strength, 33); assert_eq!(context.asr.model.as_deref(), Some("qwen3-asr-1.7b")); // Audio archives have their own user-visible limit. History retention // may be much larger and must not silently override the recording cap. diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 01c777f60..08d46eafe 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -8,7 +8,8 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, RwLock}; -use futures_util::future::BoxFuture; +use futures_util::future::{BoxFuture, Shared}; +use futures_util::FutureExt; use crate::dictation_context::{DictationContext, DictationOutputTarget}; use crate::errors::{BackendError, BackendErrorCode}; @@ -44,6 +45,41 @@ pub struct PipelineDictationEngine { pending_captures: Arc>>>, } +/// Reuses the existing PCM buffer while capture starts ahead of provider +/// preparation. Preparation still completes before start() reports success, +/// including stable mode; concurrent finish/attach shares the same result. +struct CaptureFirstPreparation { + engine: Arc, + session_id: SessionId, + context: Arc, + result: Arc, BackendError>>>, +} + +impl CaptureFirstPreparation { + fn prepare(&self) -> BoxFuture<'static, Result, BackendError>> { + let engine = Arc::clone(&self.engine); + let session_id = self.session_id; + let context = Arc::clone(&self.context); + let result = Arc::clone(&self.result); + Box::pin(async move { + result + .get_or_init(|| engine.prepare(session_id, context)) + .await + .clone() + }) + } +} + +impl PreparedTranscription for CaptureFirstPreparation { + fn start( + &self, + partials: Arc, + ) -> BoxFuture<'static, Result, BackendError>> { + let ready = self.prepare(); + Box::pin(async move { ready.await?.start(partials).await }) + } +} + struct PipelineSession { context: RwLock>, cancelled: AtomicBool, @@ -56,9 +92,49 @@ struct PipelineSession { resources: Mutex, } +type RecordingStop = Shared>>; + +// Early stop, finalization and cancellation must join the same native close. +struct PipelineRecording { + recording: Option>, + archive: Option>, + stopped: Option, +} + +impl PipelineRecording { + fn new(recording: Box) -> Self { + Self { + archive: recording.archive(), + recording: Some(recording), + stopped: None, + } + } + + fn archive(&self) -> Option> { + self.archive.clone() + } + + fn stop_recording(&mut self) -> RecordingStop { + self.stopped + .get_or_insert_with(|| { + self.recording + .take() + .expect("pipeline recording is available until stop begins") + .stop() + .shared() + }) + .clone() + } + + fn stop(mut self) -> RecordingStop { + self.stop_recording() + } +} + #[derive(Default)] struct PipelineResources { - recording: Option>, + recording: Option, + stop_recording_requested: bool, transcription: Option>, buffered: Option>, prepared: Option>, @@ -211,20 +287,13 @@ impl DictationEngine for PipelineDictationEngine { } } - let prepared = match transcription_engine - .prepare(session_id, Arc::clone(&context)) - .await - { - Ok(prepared) => prepared, - Err(error) => { - remove_session(&sessions, session_id, &session); - return Err(if session.cancelled.load(Ordering::Acquire) { - cancelled_error("dictation cancelled while preparing the provider") - } else { - error - }); - } - }; + let preparation = Arc::new(CaptureFirstPreparation { + engine: transcription_engine, + session_id, + context: Arc::clone(&context), + result: Arc::new(tokio::sync::OnceCell::new()), + }); + let prepared: Arc = preparation.clone(); if session.cancelled.load(Ordering::Acquire) { remove_session(&sessions, session_id, &session); return Err(cancelled_error( @@ -318,8 +387,8 @@ impl DictationEngine for PipelineDictationEngine { } }; - let mut recording = Some(recording); - { + let mut recording = Some(PipelineRecording::new(recording)); + let stopping = { let mut resources = session .resources .lock() @@ -327,7 +396,15 @@ impl DictationEngine for PipelineDictationEngine { if !session.cancelled.load(Ordering::Acquire) { resources.recording = recording.take(); } - } + if resources.stop_recording_requested { + resources + .recording + .as_mut() + .map(PipelineRecording::stop_recording) + } else { + None + } + }; if let Some(recording) = recording { let stop_result = recording.stop().await; let cancel_result = @@ -339,6 +416,36 @@ impl DictationEngine for PipelineDictationEngine { "dictation cancelled while the recorder was starting", )); } + // No provider preparation, configuration I/O or network work may + // delay opening/promoting the microphone. The existing bounded + // transcription buffer retains the captured prefix during prepare. + let preparation_result = async { + progress.publish(session_id, EngineProgress::CaptureReady)?; + if let Some(stopping) = stopping { + // finish() retains the archive and reports any native stop failure. + let _ = stopping.await; + } + preparation.prepare().await + } + .await; + if preparation_result.is_err() || session.cancelled.load(Ordering::Acquire) { + let recording = session + .resources + .lock() + .expect("pipeline resource lock poisoned") + .recording + .take(); + if let Some(recording) = recording { + let _ = recording.stop().await; + } + let _ = cancel_transcription_once(&session, Arc::clone(&buffered)).await; + remove_session(&sessions, session_id, &session); + return Err(if session.cancelled.load(Ordering::Acquire) { + cancelled_error("dictation cancelled while preparing the provider") + } else { + preparation_result.err().expect("preparation failed") + }); + } if let Some(spawner) = local_startup_spawner.filter(|_| !stable) { let attaching = buffered.attach(); let progress = Arc::clone(&buffered.inner.progress); @@ -393,6 +500,35 @@ impl DictationEngine for PipelineDictationEngine { }) } + fn stop_recording(&self, session_id: SessionId) -> BoxFuture<'static, Result> { + let session = self + .sessions + .lock() + .expect("pipeline session lock poisoned") + .get(&session_id) + .cloned(); + let Some(session) = session else { + return Box::pin(async { Ok(false) }); + }; + let stopping = { + let mut resources = session + .resources + .lock() + .expect("pipeline resource lock poisoned"); + resources.stop_recording_requested = true; + resources + .recording + .as_mut() + .map(PipelineRecording::stop_recording) + }; + Box::pin(async move { + if let Some(stopping) = stopping { + stopping.await?; + } + Ok(true) + }) + } + fn start_transcription( &self, session_id: SessionId, @@ -2098,6 +2234,55 @@ mod tests { } } + #[tokio::test] + async fn early_recording_stop_shares_cleanup_failure_and_preserves_the_archive() { + struct BlockingStop { + stops: Arc, + release: Arc, + archive: Arc, + } + impl ActiveRecording for BlockingStop { + fn archive(&self) -> Option> { + Some(self.archive.clone()) + } + fn stop(self: Box) -> BoxFuture<'static, Result<(), BackendError>> { + self.stops.fetch_add(1, Ordering::AcqRel); + Box::pin(async move { + self.release.notified().await; + Err(BackendError::new( + BackendErrorCode::Platform, + "fixture stop failure", + )) + }) + } + } + let stops = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let archive = Arc::new(FixtureArchive { + available: AtomicBool::new(true), + discards: Arc::new(AtomicUsize::new(0)), + pcm: vec![1, 0], + }); + let mut recording = PipelineRecording::new(Box::new(BlockingStop { + stops: stops.clone(), + release: release.clone(), + archive: archive.clone(), + })); + let retained = recording.archive().unwrap(); + let mut early_stop = Box::pin(recording.stop_recording()); + let mut terminal_stop = Box::pin(recording.stop()); + assert!(futures_util::poll!(early_stop.as_mut()).is_pending()); + assert!(futures_util::poll!(terminal_stop.as_mut()).is_pending()); + assert_eq!(stops.load(Ordering::Acquire), 1); + release.notify_one(); + let (early, terminal) = tokio::join!(early_stop, terminal_stop); + assert_eq!(early.unwrap_err().message, "fixture stop failure"); + assert_eq!(terminal.unwrap_err().message, "fixture stop failure"); + assert!(retained.is_available()); + assert_eq!(retained.read_pcm().await.unwrap(), vec![1, 0]); + assert_eq!(archive.discards.load(Ordering::Acquire), 0); + } + #[derive(Default)] struct LimitRecordingProgress { limits: AtomicUsize, @@ -3209,7 +3394,7 @@ mod tests { } #[tokio::test] - async fn provider_preparation_failure_does_not_start_the_recorder() { + async fn provider_preparation_failure_stops_capture_and_removes_the_session() { let stops = Arc::new(AtomicUsize::new(0)); let engine = PipelineDictationEngine::new( Arc::new(ExposedRecorder { @@ -3229,10 +3414,129 @@ mod tests { .unwrap_err(); assert_eq!(error.code, BackendErrorCode::Provider); - assert_eq!(stops.load(Ordering::Acquire), 0); + assert_eq!(stops.load(Ordering::Acquire), 1); assert!(engine.sessions.lock().unwrap().is_empty()); } + struct ReadyFixturePreparation(Arc); + + impl PreparedTranscription for ReadyFixturePreparation { + fn start( + &self, + _partials: Arc, + ) -> BoxFuture<'static, Result, BackendError>> { + let session = self.0.clone(); + Box::pin(async move { Ok(session as Arc) }) + } + } + + struct BlockedPreparationTranscriber { + entered: Arc, + release: Arc, + calls: Arc, + session: Arc, + } + + impl TranscriptionEngine for BlockedPreparationTranscriber { + fn prepare( + self: Arc, + _id: SessionId, + _context: Arc, + ) -> BoxFuture<'static, Result, BackendError>> { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::AcqRel); + self.entered.notify_one(); + self.release.notified().await; + Ok(Arc::new(ReadyFixturePreparation(self.session.clone())) + as Arc) + }) + } + + fn start( + &self, + _id: SessionId, + _context: Arc, + _partials: Arc, + ) -> BoxFuture<'static, Result, BackendError>> { + Box::pin(async { unreachable!("must use the prepared provider") }) + } + } + + #[tokio::test] + async fn capture_starts_before_slow_preparation_and_cancellation_releases_it_promptly() { + for cancel in [false, true] { + let consumer = Arc::new(Mutex::new(None)); + let stops = Arc::new(AtomicUsize::new(0)); + let pcm = Arc::new(Mutex::new(Vec::new())); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let calls = Arc::new(AtomicUsize::new(0)); + let engine = Arc::new(PipelineDictationEngine::new( + Arc::new(ExposedRecorder { + consumer: consumer.clone(), + stops: stops.clone(), + }), + Arc::new(BlockedPreparationTranscriber { + entered: entered.clone(), + release: release.clone(), + calls: calls.clone(), + session: Arc::new(FixtureTranscriptionSession { + pcm: pcm.clone(), + cancels: Arc::new(AtomicUsize::new(0)), + finish_entered: None, + finish_release: None, + }), + }), + noop_polisher(), + )); + let id = SessionId::new(); + let progress = Arc::new(RecordingProgress::default()); + let starting = tokio::spawn({ + let engine = engine.clone(); + let progress = progress.clone(); + async move { engine.start(id, raw_dictation_context(), progress).await } + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + assert!( + consumer.lock().unwrap().is_some(), + "microphone must precede prepare" + ); + assert!(pcm.lock().unwrap().is_empty()); + consumer + .lock() + .unwrap() + .as_ref() + .unwrap() + .consume_pcm_chunk(&[2, 0]); + if cancel { + tokio::time::timeout(std::time::Duration::from_secs(2), engine.cancel(id)) + .await + .unwrap() + .unwrap(); + assert_eq!( + stops.load(Ordering::Acquire), + 1, + "cancel waited for provider preparation" + ); + } + release.notify_one(); + let result = starting.await.unwrap(); + if cancel { + assert_eq!(result.unwrap_err().code, BackendErrorCode::Cancelled); + assert!(pcm.lock().unwrap().is_empty()); + } else { + result.unwrap(); + engine.finish(id, progress).await.unwrap(); + assert_eq!(&*pcm.lock().unwrap(), &[1, 0, 2, 0]); + } + assert_eq!(calls.load(Ordering::Acquire), 1); + assert_eq!(stops.load(Ordering::Acquire), 1); + assert!(engine.sessions.lock().unwrap().is_empty()); + } + } + #[tokio::test] async fn cancel_while_realtime_asr_starts_stops_recorder_and_cancels_late_session() { let consumer = Arc::new(Mutex::new(None)); diff --git a/openless-all/app/crates/openless-core/src/external_audio.rs b/openless-all/app/crates/openless-core/src/external_audio.rs index 24e3e66d8..7e42b1ab7 100644 --- a/openless-all/app/crates/openless-core/src/external_audio.rs +++ b/openless-all/app/crates/openless-core/src/external_audio.rs @@ -37,6 +37,7 @@ struct ExternalRecordingState { bytes_received: u64, consumer: Arc, progress: Arc, + processor: Option, } struct ExternalActiveRecording { @@ -72,11 +73,22 @@ impl ExternalAudioRecorder { "external audio session was replaced", )); } - expected + let mut state = expected .state .lock() - .expect("external audio state lock poisoned") - .active = false; + .expect("external audio state lock poisoned"); + if let Some(chunk) = state + .processor + .as_mut() + .and_then(crate::AudioProcessor::finish) + { + state.consumer.consume_pcm_chunk(&chunk.pcm_i16_le); + if let Some(archive) = &expected.archive { + archive.append(&chunk.pcm_i16_le); + } + } + state.active = false; + drop(state); if let Some(archive) = &expected.archive { archive.finish(); } @@ -131,6 +143,11 @@ impl AudioRecorder for ExternalAudioRecorder { bytes_received: 0, consumer, progress, + processor: context + .recording + .audio_processing + .enhancement_enabled + .then(|| crate::AudioProcessor::new(context.recording.audio_processing)), }), archive, }); @@ -173,12 +190,29 @@ impl AudioRecorder for ExternalAudioRecorder { "external audio session is not active", )); } + state.bytes_received = state.bytes_received.saturating_add(pcm.len() as u64); + let elapsed_ms = state.bytes_received.saturating_mul(1_000) / 32_000; + if let Some(processor) = &mut state.processor { + let samples: Vec = pcm + .as_chunks::<2>() + .0 + .iter() + .map(|pair| i16::from_le_bytes([pair[0], pair[1]]) as f32 / 32768.0) + .collect(); + if let Some(chunk) = processor.process(&samples, 1, crate::audio::DICTATION_SAMPLE_RATE) + { + state.consumer.consume_pcm_chunk(&chunk.pcm_i16_le); + if let Some(archive) = &session.archive { + archive.append(&chunk.pcm_i16_le); + } + return state.progress.publish_level(elapsed_ms, chunk.level); + } + return Ok(()); + } if let Some(archive) = &session.archive { archive.append(pcm); } state.consumer.consume_pcm_chunk(pcm); - state.bytes_received = state.bytes_received.saturating_add(pcm.len() as u64); - let elapsed_ms = state.bytes_received.saturating_mul(1_000) / 32_000; state .progress .publish_level(elapsed_ms, pcm_i16_le_rms(pcm)) @@ -267,6 +301,52 @@ mod tests { use super::*; + #[tokio::test] + async fn enhanced_external_input_flushes_the_same_complete_pcm_to_asr_and_archive() { + let directory = std::env::temp_dir().join(format!( + "openless-enhanced-external-{}", + uuid::Uuid::new_v4() + )); + let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); + let id = SessionId::new(); + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; + context.recording.archive_enabled = true; + context.recording.audio_processing.enhancement_enabled = true; + let consumer = Arc::new(RecordingConsumer::default()); + let recording = recorder + .start( + id, + Arc::new(context), + consumer.clone(), + Arc::new(RecordingProgress::default()), + ) + .await + .unwrap(); + let archive = recording.archive().unwrap(); + let input: Vec = (0..1777) + .flat_map(|i| (((i as f32 * 0.13).sin() * 8000.0) as i16).to_le_bytes()) + .collect(); + for frame in input.chunks(74) { + recorder.feed_pcm(id, frame).unwrap(); + } + let before_stop = consumer.0.lock().unwrap().len(); + recording.stop().await.unwrap(); + let output = consumer.0.lock().unwrap().clone(); + assert!( + before_stop < output.len(), + "end-of-stream tail was not delivered" + ); + assert_eq!(output.len(), input.len()); + assert_ne!(output, input); + assert_eq!(archive.read_pcm().await.unwrap(), output); + assert!(recorder.feed_pcm(id, &[0, 0]).is_err()); + archive.discard().await.unwrap(); + let _ = std::fs::remove_dir(directory); + } + #[derive(Default)] struct RecordingConsumer(Mutex>); diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 827099da0..7a4368e0a 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -9,6 +9,8 @@ pub mod android_types; pub mod api; pub mod asr; pub mod audio; +pub mod audio_dsp; +pub use audio_dsp::{AudioDspConfig, AudioDspStage, NoiseReduction}; pub mod auxiliary; pub mod cli; mod cloud_providers; @@ -224,7 +226,7 @@ pub use api::{ QaVoiceCaptureSession, StartupSnapshot, VoiceTranscriptionSession, }; pub use audio::{ - encode_dictation_wav, AudioEnhancer, AudioProcessingConfig, AudioProcessor, NormalizedPcmChunk, + encode_dictation_wav, AudioProcessingConfig, AudioProcessor, NormalizedPcmChunk, PcmNormalizer, DICTATION_SAMPLE_RATE, }; pub use auxiliary::{ diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 3ab20b2a4..87ab414f2 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -282,6 +282,8 @@ pub enum EngineStage { #[derive(Debug, Clone, PartialEq)] pub enum EngineProgress { + /// The recorder is registered; provider preparation may still be pending. + CaptureReady, RecordingLevel { elapsed_ms: u64, level: f32 }, RecordingLimitReached, RecordingFault(BackendError), @@ -342,6 +344,12 @@ pub trait DictationEngine: Send + Sync + 'static { progress: std::sync::Arc, ) -> BoxFuture<'static, Result>; + /// Retain an early stop request, preserving PCM and stop errors for finish(). + /// Returns false until the session is registered, or for legacy engines. + fn stop_recording(&self, _session_id: SessionId) -> BoxFuture<'static, Result> { + Box::pin(async { Ok(false) }) + } + /// Replace the immutable session snapshot before finalization when a host /// action (currently Android's finish-and-translate gesture) is only known /// at stop time. Implementations that retain the context must override this diff --git a/openless-all/app/crates/openless-core/src/preferences.rs b/openless-all/app/crates/openless-core/src/preferences.rs index 3df4a7aa1..817d7663e 100644 --- a/openless-all/app/crates/openless-core/src/preferences.rs +++ b/openless-all/app/crates/openless-core/src/preferences.rs @@ -124,6 +124,7 @@ fn preferences_json_preserving_unknown( fn strip_local_only_preferences(value: &mut serde_json::Value) { if let serde_json::Value::Object(object) = value { object.remove("lowLatencyDictationEnabled"); + object.remove("audioDsp"); object.remove("lowMemoryBackgroundEnabled"); } } @@ -479,10 +480,12 @@ mod tests { fn cloud_snapshot_omits_local_microphone_warm_capture_preference() { let mut value = serde_json::json!({ "lowLatencyDictationEnabled": true, + "audioDsp": {"noiseReduction": "purevox"}, "microphoneDeviceName": "Built-in Mic", }); strip_local_only_preferences(&mut value); assert!(value.get("lowLatencyDictationEnabled").is_none()); + assert!(value.get("audioDsp").is_none()); assert_eq!(value["microphoneDeviceName"], "Built-in Mic"); } diff --git a/openless-all/app/crates/openless-core/src/provider_registry.rs b/openless-all/app/crates/openless-core/src/provider_registry.rs index 9a6cb792d..3f5c973b6 100644 --- a/openless-all/app/crates/openless-core/src/provider_registry.rs +++ b/openless-all/app/crates/openless-core/src/provider_registry.rs @@ -530,6 +530,21 @@ impl DictationEngine for DictationEngineRouter { }) } + fn stop_recording(&self, session_id: SessionId) -> BoxFuture<'static, Result> { + let routed = self + .active + .read() + .expect("active dictation provider lock poisoned") + .get(&session_id) + .cloned(); + Box::pin(async move { + match routed { + Some(routed) => routed.engine.stop_recording(session_id).await, + None => Ok(false), + } + }) + } + fn feed_audio(&self, session_id: SessionId, pcm: &[u8]) -> Result<(), BackendError> { let routed = self .active diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index a133a6ff4..09786dcfa 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -423,10 +423,12 @@ pub struct UserPreferences { /// Temporarily mute system output during recording; restore the original mute state on stop/cancel/error. #[serde(default)] pub mute_during_recording: bool, - /// Apply conservative noise suppression, lightweight gain, and a soft limiter to native microphone input. + /// Apply the session-frozen speech enhancement chain to microphone input. /// Disabled by default until an audio-quality baseline supports changing it. #[serde(default)] pub audio_enhancement_enabled: bool, + #[serde(default)] + pub audio_dsp: crate::audio_dsp::AudioDspConfig, /// Reconnect the current ASR after recording ends and submit the whole PCM. Default off. #[serde(default)] pub stable_transcription_enabled: bool, @@ -1023,6 +1025,8 @@ struct UserPreferencesWire { #[serde(default)] audio_enhancement_enabled: bool, #[serde(default)] + audio_dsp: crate::audio_dsp::AudioDspConfig, + #[serde(default)] stable_transcription_enabled: bool, #[serde(default)] low_latency_dictation_enabled: bool, @@ -1305,6 +1309,7 @@ impl Default for UserPreferencesWire { capsule_transcript_font_size: prefs.capsule_transcript_font_size, mute_during_recording: prefs.mute_during_recording, audio_enhancement_enabled: prefs.audio_enhancement_enabled, + audio_dsp: prefs.audio_dsp.normalized(), stable_transcription_enabled: prefs.stable_transcription_enabled, low_latency_dictation_enabled: prefs.low_latency_dictation_enabled, low_memory_background_enabled: prefs.low_memory_background_enabled, @@ -1524,6 +1529,7 @@ impl<'de> Deserialize<'de> for UserPreferences { capsule_transcript_font_size: wire.capsule_transcript_font_size.clamp(12, 20), mute_during_recording: wire.mute_during_recording, audio_enhancement_enabled: wire.audio_enhancement_enabled, + audio_dsp: wire.audio_dsp.normalized(), stable_transcription_enabled: wire.stable_transcription_enabled, low_latency_dictation_enabled: wire.low_latency_dictation_enabled, low_memory_background_enabled: wire.low_memory_background_enabled, @@ -1919,6 +1925,7 @@ impl Default for UserPreferences { capsule_transcript_font_size: default_capsule_transcript_font_size(), mute_during_recording: false, audio_enhancement_enabled: false, + audio_dsp: Default::default(), stable_transcription_enabled: false, low_latency_dictation_enabled: false, low_memory_background_enabled: false, diff --git a/openless-all/app/harmony/rust/src/voice_configuration.rs b/openless-all/app/harmony/rust/src/voice_configuration.rs index 8e9dbafc5..c10f763d5 100644 --- a/openless-all/app/harmony/rust/src/voice_configuration.rs +++ b/openless-all/app/harmony/rust/src/voice_configuration.rs @@ -19,6 +19,7 @@ const FIELDS: &[&str] = &[ "polishSkipMaxChars", "stableTranscriptionEnabled", "audioEnhancementEnabled", + "audioDsp", "recordAudioForDebug", ]; diff --git a/openless-all/app/linux-egui/src/audio.rs b/openless-all/app/linux-egui/src/audio.rs index 3c51ae00d..19d10b1dc 100644 --- a/openless-all/app/linux-egui/src/audio.rs +++ b/openless-all/app/linux-egui/src/audio.rs @@ -101,6 +101,20 @@ impl AudioProcessingWorker { .publish_level(started.elapsed().as_millis() as u64, chunk.level); } } + if let Some(chunk) = processor.finish() { + consumer.consume_pcm_chunk(&chunk.pcm_i16_le); + if let Some(writer) = &writer { + if let Err(error) = writer + .lock() + .expect("Linux WAV writer lock poisoned") + .append(&chunk.pcm_i16_le) + { + log::warn!("Linux recording archive write failed: {error}"); + } + } + let _ = + progress.publish_level(started.elapsed().as_millis() as u64, chunk.level); + } }) .map_err(|error| { BackendError::new( diff --git a/openless-all/app/linux-egui/src/i18n.rs b/openless-all/app/linux-egui/src/i18n.rs index 66b044a7d..29bc7c752 100644 --- a/openless-all/app/linux-egui/src/i18n.rs +++ b/openless-all/app/linux-egui/src/i18n.rs @@ -149,6 +149,35 @@ const fn row( // that the zh-CN column is fully populated and that every key actually // referenced resolves. pub const CATALOG: &[Msg] = &[ + Msg { key: "settings.audioDsp.disabledHint", text: row("开启上方语音增强开关后可调整，当前配置已保留。", "開啟上方語音增強開關後可調整，目前設定已保留。", "Enable speech enhancement above to adjust these settings. Your configuration is retained.", "上の音声強調を有効にすると調整できます。設定は保持されています。", "위에서 음성 향상을 켜면 조정할 수 있습니다. 설정은 유지됩니다.") }, + Msg { key: "settings.audioDsp.moveUp", text: row("上移处理阶段", "上移處理階段", "Move stage up", "処理を上へ移動", "처리 단계 위로 이동") }, + Msg { key: "settings.audioDsp.moveDown", text: row("下移处理阶段", "下移處理階段", "Move stage down", "処理を下へ移動", "처리 단계 아래로 이동") }, + Msg { key: "settings.audioDsp.noiseStage", text: row("降噪", "降噪", "Noise reduction", "ノイズ低減", "잡음 감소") }, + Msg { key: "settings.audioDsp.combStage", text: row("梳状滤波", "梳狀濾波", "Comb filter", "コムフィルター", "콤 필터") }, + Msg { key: "settings.audioDsp.Speech_enhancement_settings", text: row("语音增强设置", "語音增強設置", "Speech enhancement settings", "音声強調設定", "음성 향상 설정") }, + Msg { key: "settings.audioDsp.RNNoise_is_lightweight_PureVox_uses_more_CPU_Changes_apply_to_the_next_recording", text: row("RNNoise 更轻量，PureVox 模型降噪耗时较高。设置在下一次录音生效。", "RNNoise 更輕量，PureVox 模型降噪耗時較高。設置在下一次錄音生效。", "RNNoise is lightweight; PureVox uses more CPU. Changes apply to the next recording.", "RNNoise は軽量、PureVox は CPU 使用量が多めです。設定は次の録音に適用されます。", "RNNoise는 가볍고 PureVox는 CPU를 더 사용합니다. 설정은 다음 녹음에 적용됩니다.") }, + Msg { key: "settings.audioDsp.Noise_reduction_off", text: row("关闭降噪", "關闭降噪", "Noise reduction off", "ノイズ低減オフ", "잡음 감소 끄기") }, + Msg { key: "settings.audioDsp.Spectral_subtraction", text: row("频谱减法", "頻譜減法", "Spectral subtraction", "スペクトル減算法", "스펙트럼 차감") }, + Msg { key: "settings.audioDsp.Off", text: row("关闭降噪", "關闭降噪", "Off", "オフ", "끄기") }, + Msg { key: "settings.audioDsp.Noise_reduction", text: row("降噪强度 %", "降噪強度 %", "Noise reduction %", "ノイズ低減 %", "잡음 감소 %") }, + Msg { key: "settings.audioDsp.Input_gain_dB", text: row("输入增益 dB", "輸入增益 dB", "Input gain dB", "入力ゲイン dB", "입력 게인 dB") }, + Msg { key: "settings.audioDsp.Automatic_gain", text: row("自动增益", "自動增益", "Automatic gain", "自動ゲイン", "자동 게인") }, + Msg { key: "settings.audioDsp.Target_level", text: row("目标电平 %", "目標電平 %", "Target level %", "目標レベル %", "목표 레벨 %") }, + Msg { key: "settings.audioDsp.Attack_speed", text: row("启动速度", "啟動速度", "Attack speed", "アタック速度", "어택 속도") }, + Msg { key: "settings.audioDsp.Release_speed", text: row("释放速度", "釋放速度", "Release speed", "リリース速度", "릴리스 속도") }, + Msg { key: "settings.audioDsp.Speech_gate_does_not_stop_recording", text: row("语音门限（不自动停止录音）", "語音門限（不自動停止錄音）", "Speech gate (does not stop recording)", "音声ゲート（録音は停止しません）", "음성 게이트 (녹음을 중지하지 않음)") }, + Msg { key: "settings.audioDsp.Threshold_dB", text: row("门限 dB", "門限 dB", "Threshold dB", "しきい値 dB", "임계값 dB") }, + Msg { key: "settings.audioDsp.A_high_threshold_can_remove_quiet_speech", text: row("门限过高可能丢失轻声。", "門限過高可能丟失輕聲。", "A high threshold can remove quiet speech.", "しきい値が高いと小さな声が失われる場合があります。", "임계값이 높으면 작은 목소리가 사라질 수 있습니다.") }, + Msg { key: "settings.audioDsp.Tone_and_room_adjustments", text: row("音色与房间调整", "音色與房間調整", "Tone and room adjustments", "音色と室内音響の調整", "음색 및 실내 음향 조정") }, + Msg { key: "settings.audioDsp.The_MicYou_comb_filter_can_color_speech_Start_at_0", text: row("梳状滤波沿用 MicYou 的延迟相减方式，可能改变音色，建议从 0% 开始。", "梳狀濾波沿用 MicYou 的延遲相減方式，可能改變音色，建議從 0% 开始。", "The MicYou comb filter can color speech. Start at 0%.", "MicYou のコムフィルターは音色を変える場合があります。0% から始めてください。", "MicYou 콤 필터는 음색을 바꿀 수 있습니다. 0%부터 시작하세요.") }, + Msg { key: "settings.audioDsp.Comb_filter", text: row("梳状滤波 %", "梳狀濾波 %", "Comb filter %", "コムフィルター %", "콤 필터 %") }, + Msg { key: "settings.audioDsp.10_band_equalizer", text: row("十段均衡器", "十段均衡器", "10-band equalizer", "10 バンドイコライザー", "10밴드 이퀄라이저") }, + Msg { key: "settings.audioDsp.Preamp_dB", text: row("前置增益 dB", "前置增益 dB", "Preamp dB", "プリアンプ dB", "프리앰프 dB") }, + Msg { key: "settings.audioDsp.Processing_order", text: row("处理顺序", "處理順序", "Processing order", "処理順序", "처리 순서") }, + Msg { key: "settings.audioDsp.Equalizer", text: row("均衡器", "均衡器", "Equalizer", "イコライザー", "이퀄라이저") }, + Msg { key: "settings.audioDsp.Input_gain", text: row("输入增益", "輸入增益", "Input gain", "入力ゲイン", "입력 게인") }, + Msg { key: "settings.audioDsp.Speech_gate", text: row("语音门限", "語音門限", "Speech gate", "音声ゲート", "음성 게이트") }, + Msg { key: "settings.audioDsp.Reset_enhancement_settings", text: row("恢复增强默认设置", "恢復增強默認設置", "Reset enhancement settings", "強調設定を初期化", "향상 설정 초기화") }, Msg { key: "settings.providers.presets.chatgptSiwc", text: row("ChatGPT Plan", "ChatGPT Plan", "ChatGPT Plan", "ChatGPT Plan", "ChatGPT Plan") }, Msg { key: "siwc.description", text: row("符合条件的文本润色和翻译使用你的 ChatGPT 计划或积分余额。ASR 仍使用原渠道；额度不足时不会自动切换到 API Key。", "符合條件的文字潤飾和翻譯使用你的 ChatGPT 方案或點數。ASR 仍使用原管道；額度不足時不會自動切換至 API Key。", "Eligible text polishing and translation use your ChatGPT plan or credits. ASR keeps its existing provider. Usage limits never switch you to an API key automatically.", "対象の文章校正と翻訳には ChatGPT プランまたはクレジットを使用します。音声認識は従来のプロバイダーを使い、上限到達時に API キーへ自動切り替えしません。", "지원되는 텍스트 다듬기와 번역에 ChatGPT 요금제 또는 크레딧을 사용합니다. 음성 인식은 기존 서비스를 사용하며 한도에 도달해도 API 키로 자동 전환하지 않습니다.") }, Msg { key: "siwc.account", text: row("ChatGPT 账号", "ChatGPT 帳號", "ChatGPT account", "ChatGPT アカウント", "ChatGPT 계정") }, @@ -2569,11 +2598,11 @@ pub const CATALOG: &[Msg] = &[ Msg { key: "settings.recording.audio_enhancement_label", text: row( - "轻量麦克风增强", - "輕量麥克風增強", - "Lightweight microphone enhancement", - "軽量マイク拡張", - "가벼운 마이크 향상", + "语音增强", + "語音增強", + "Speech enhancement", + "音声強調", + "음성 향상", ), }, Msg { @@ -4680,11 +4709,11 @@ pub const CATALOG: &[Msg] = &[ Msg { key: "settings.recording.audio_enhancement_desc", text: row( - "对本机麦克风应用轻量降噪、增益与软限幅，不包含回声消除。默认关闭。", - "對本機麥克風套用輕量降噪、增益與軟限幅，不包含回聲消除。預設關閉。", - "Applies light gain, soft limiting, and conservative noise suppression to the local microphone. It does not include echo cancellation. Off by default.", - "ローカルマイクに軽いノイズ抑制、ゲイン、ソフトリミッターを適用します。エコーキャンセルは含みません。既定ではオフです。", - "로컬 마이크에 가벼운 소음 억제, 게인과 소프트 리미터를 적용합니다. 에코 제거는 포함하지 않으며 기본값은 꺼짐입니다.", + "应用 RNNoise 或 PureVox 降噪、自动增益和限幅。默认关闭，设置在下一次录音生效。", + "套用 RNNoise 或 PureVox 降噪、自動增益和限幅。預設關閉，設定於下一次錄音生效。", + "Apply RNNoise or PureVox noise reduction, automatic gain and limiting. Off by default; changes apply to the next recording.", + "RNNoise または PureVox のノイズ抑制、自動ゲイン、リミッターを適用します。既定ではオフ。変更は次の録音から有効です。", + "RNNoise 또는 PureVox 소음 억제, 자동 게인 및 리미터를 적용합니다. 기본값은 꺼짐이며 다음 녹음부터 적용됩니다.", ), }, Msg { diff --git a/openless-all/app/linux-egui/src/main.rs b/openless-all/app/linux-egui/src/main.rs index 8b691ab6b..6daf7ac10 100644 --- a/openless-all/app/linux-egui/src/main.rs +++ b/openless-all/app/linux-egui/src/main.rs @@ -5835,6 +5835,7 @@ mod linux_app { }; s.mute_while_recording = prefs.mute_during_recording; s.audio_enhancement = prefs.audio_enhancement_enabled; + s.audio_dsp = prefs.audio_dsp; s.audio_cue = prefs.audio_cue_on_record; s.launch_at_login = prefs.launch_at_login; s.streaming_save_clipboard = prefs.streaming_insert_save_clipboard; @@ -7481,6 +7482,12 @@ mod linux_app { frontend::view_model::FrontendAction::SettingsToggle(field) => { self.apply_settings_toggle(field); } + frontend::view_model::FrontendAction::SettingsAudioDsp(config) => { + if let Some(preferences) = self.preferences.as_mut() { + preferences.audio_dsp = config.normalized(); + } + self.save_settings_if_dirty(); + } frontend::view_model::FrontendAction::SettingsCombo(field, index) => { if matches!( field, diff --git a/openless-all/app/linux-egui/src/ui/frontend/audio_dsp.rs b/openless-all/app/linux-egui/src/ui/frontend/audio_dsp.rs new file mode 100644 index 000000000..0430a5424 --- /dev/null +++ b/openless-all/app/linux-egui/src/ui/frontend/audio_dsp.rs @@ -0,0 +1,123 @@ +use eframe::egui; +use openless_core::{AudioDspConfig, AudioDspStage, NoiseReduction}; +use openless_linux_egui::{tr_l10n, Lang}; + +pub fn settings(ui: &mut egui::Ui, config: &mut AudioDspConfig, lang: Lang, enabled: bool) -> bool { + let before = *config; + let label = |key| tr_l10n(lang, key); + egui::CollapsingHeader::new(label("settings.audioDsp.Speech_enhancement_settings")) + .id_salt("audio-dsp-settings").default_open(false).show(ui, |ui| { + if !enabled { ui.small(label("settings.audioDsp.disabledHint")); } + ui.add_enabled_ui(enabled, |ui| { + ui.small(label("settings.audioDsp.RNNoise_is_lightweight_PureVox_uses_more_CPU_Changes_apply_to_the_next_recording")); + ui.add_space(8.0); + ui.strong(label("settings.audioDsp.noiseStage")); + egui::ComboBox::from_id_salt("audio-dsp-algorithm") + .width(ui.available_width().min(280.0)) + .selected_text(match config.noise_reduction { + NoiseReduction::Off => label("settings.audioDsp.Noise_reduction_off"), + NoiseReduction::Rnnoise => "RNNoise", + NoiseReduction::Purevox => "PureVox 6", + NoiseReduction::Spectral => label("settings.audioDsp.Spectral_subtraction"), + }) + .show_ui(ui, |ui| { + for (value, name) in [(NoiseReduction::Off, label("settings.audioDsp.Off")), (NoiseReduction::Rnnoise, "RNNoise"), (NoiseReduction::Purevox, "PureVox 6"), (NoiseReduction::Spectral, label("settings.audioDsp.Spectral_subtraction"))] { + ui.selectable_value(&mut config.noise_reduction, value, name); + } + }); + if config.noise_reduction != NoiseReduction::Off { + slider(ui, label("settings.audioDsp.Noise_reduction"), &mut config.noise_strength, 0..=100, "%"); + } + ui.separator(); + ui.strong(label("settings.audioDsp.Input_gain")); + slider(ui, label("settings.audioDsp.Input_gain_dB"), &mut config.gain_db, -30..=30, " dB"); + ui.checkbox(&mut config.agc_enabled, label("settings.audioDsp.Automatic_gain")); + if config.agc_enabled { + slider(ui, label("settings.audioDsp.Target_level"), &mut config.agc_target, 5..=90, "%"); + slider(ui, label("settings.audioDsp.Attack_speed"), &mut config.agc_attack, 1..=100, ""); + slider(ui, label("settings.audioDsp.Release_speed"), &mut config.agc_decay, 1..=100, ""); + } + ui.separator(); + ui.strong(label("settings.audioDsp.Speech_gate")); + ui.checkbox(&mut config.vad_enabled, label("settings.audioDsp.Speech_gate_does_not_stop_recording")); + if config.vad_enabled { + slider(ui, label("settings.audioDsp.Threshold_dB"), &mut config.vad_threshold_db, -80..=-10, " dB"); + ui.small(label("settings.audioDsp.A_high_threshold_can_remove_quiet_speech")); + } + ui.add_space(8.0); + ui.collapsing(label("settings.audioDsp.Tone_and_room_adjustments"), |ui| { + ui.small(label("settings.audioDsp.The_MicYou_comb_filter_can_color_speech_Start_at_0")); + slider(ui, label("settings.audioDsp.Comb_filter"), &mut config.dereverb_level, 0..=100, "%"); + ui.checkbox(&mut config.equalizer_enabled, label("settings.audioDsp.10_band_equalizer")); + if config.equalizer_enabled { + slider(ui, label("settings.audioDsp.Preamp_dB"), &mut config.equalizer_preamp_db, -20..=20, " dB"); + let columns = if ui.available_width() >= 540.0 { 2 } else { 1 }; + egui::Grid::new("audio-dsp-eq").num_columns(columns).spacing([16.0, 8.0]).show(ui, |ui| { + let width = ((ui.available_width() - 16.0 * (columns - 1) as f32) / columns as f32).max(100.0); + for (index, (band, name)) in config.equalizer_bands_db.iter_mut().zip(["31 Hz", "63 Hz", "125 Hz", "250 Hz", "500 Hz", "1 kHz", "2 kHz", "4 kHz", "8 kHz", "16 kHz"]).enumerate() { + ui.push_id(index, |ui| ui.vertical(|ui| { + ui.set_width(width); + ui.label(name); + ui.spacing_mut().slider_width = (width - 80.0).clamp(60.0, 220.0); + ui.add(egui::Slider::new(band, -12..=12).suffix(" dB")); + })); + if (index + 1) % columns == 0 { ui.end_row(); } + } + }); + } + }); + ui.collapsing(label("settings.audioDsp.Processing_order"), |ui| { + let mut swap = None; + for (i, stage) in config.processing_order.iter().enumerate() { + let name = match stage { + AudioDspStage::NoiseReduction => label("settings.audioDsp.noiseStage"), + AudioDspStage::Dereverb => label("settings.audioDsp.combStage"), + AudioDspStage::Equalizer => label("settings.audioDsp.Equalizer"), + AudioDspStage::Amplifier => label("settings.audioDsp.Input_gain"), + AudioDspStage::Agc => label("settings.audioDsp.Automatic_gain"), + AudioDspStage::Vad => label("settings.audioDsp.Speech_gate"), + }; + ui.push_id(i, |ui| ui.horizontal(|ui| { + ui.monospace(format!("{}", i + 1)); + let active = match stage { + AudioDspStage::NoiseReduction => config.noise_reduction != NoiseReduction::Off && config.noise_strength > 0, + AudioDspStage::Dereverb => config.dereverb_level > 0, + AudioDspStage::Equalizer => config.equalizer_enabled, + AudioDspStage::Amplifier => config.gain_db != 0, + AudioDspStage::Agc => config.agc_enabled, + AudioDspStage::Vad => config.vad_enabled, + }; + ui.add_enabled(active, egui::Label::new(name)); + if ui.add_enabled(i > 0, egui::Button::new("↑")).on_hover_text(label("settings.audioDsp.moveUp")).clicked() { swap = Some((i, i - 1)); } + if ui.add_enabled(i + 1 < config.processing_order.len(), egui::Button::new("↓")).on_hover_text(label("settings.audioDsp.moveDown")).clicked() { swap = Some((i, i + 1)); } + })); + } + if let Some((a, b)) = swap { config.processing_order.swap(a, b); } + }); + ui.add_space(10.0); + if ui.button(label("settings.audioDsp.Reset_enhancement_settings")).clicked() { *config = Default::default(); } + }); + }); + before != *config +} + +fn slider( + ui: &mut egui::Ui, + title: &str, + value: &mut T, + range: std::ops::RangeInclusive, + suffix: &str, +) { + let narrow = ui.available_width() < 420.0; + if narrow { + ui.label(title); + } + ui.horizontal(|ui| { + if !narrow { + ui.add_sized([160.0, 24.0], egui::Label::new(title).wrap()); + } + ui.spacing_mut().slider_width = (ui.available_width() - 85.0).clamp(60.0, 240.0); + ui.add(egui::Slider::new(value, range).suffix(suffix)); + }); + ui.add_space(4.0); +} diff --git a/openless-all/app/linux-egui/src/ui/frontend/mod.rs b/openless-all/app/linux-egui/src/ui/frontend/mod.rs index 132271c89..873cae560 100644 --- a/openless-all/app/linux-egui/src/ui/frontend/mod.rs +++ b/openless-all/app/linux-egui/src/ui/frontend/mod.rs @@ -1,3 +1,4 @@ +mod audio_dsp; pub mod capsule_motion; pub mod cloud_sync; pub mod corrections; diff --git a/openless-all/app/linux-egui/src/ui/frontend/settings.rs b/openless-all/app/linux-egui/src/ui/frontend/settings.rs index 8a8599511..5dcd97f23 100644 --- a/openless-all/app/linux-egui/src/ui/frontend/settings.rs +++ b/openless-all/app/linux-egui/src/ui/frontend/settings.rs @@ -631,6 +631,14 @@ fn general(ui: &mut egui::Ui, vm: &mut FrontendViewModel, actions: &mut Vec, pub mute_while_recording: bool, pub audio_enhancement: bool, + pub audio_dsp: openless_core::AudioDspConfig, pub audio_cue: bool, /// 录音胶囊开关与样式（Tauri `capsuleLabel` / `capsuleStyleLabel`）。 pub show_capsule: bool, @@ -1258,6 +1260,7 @@ impl Default for SettingsFields { microphone_error: None, mute_while_recording: false, audio_enhancement: false, + audio_dsp: Default::default(), audio_cue: false, show_capsule: true, capsule_style: 0, diff --git a/openless-all/app/scripts/micyou-dsp.patch b/openless-all/app/scripts/micyou-dsp.patch new file mode 100644 index 000000000..12d2d8819 --- /dev/null +++ b/openless-all/app/scripts/micyou-dsp.patch @@ -0,0 +1,356 @@ +--- a/crates/micyou-dsp/src/lib.rs ++++ b/crates/micyou-dsp/src/lib.rs +@@ -13,7 +13,12 @@ + * GNU General Public License for more details. + */ + +-#[cfg(feature = "noise-suppression")] ++// Keep upstream FFT indexing and error conversion intact in the import patch. ++#![allow( ++ clippy::needless_range_loop, ++ clippy::needless_return_with_question_mark ++)] ++ + use std::collections::VecDeque; + use std::path::PathBuf; + use std::sync::{Arc, RwLock}; +@@ -25,8 +30,12 @@ + #[cfg(feature = "noise-suppression")] + use rustfft::num_complex::Complex; + +-#[cfg(feature = "noise-suppression")] +-use crate::AecFailure; ++// OpenLess adaptation: DSP-only crate; no device, playback or plugin host. ++#[derive(Debug, Clone, Copy, PartialEq, Eq)] ++pub enum AecFailure { ++ ModelLoadFailed, ++ InferenceFailed, ++} + + /// Host-provided DSP stage invoked at plugin chain nodes. The first argument + /// is the chain node name: either the legacy synthetic [`PLUGIN_CHAIN_NODE`] +@@ -35,7 +44,7 @@ + pub type ExternalDspHook = Box, usize, f64) + Send>; + + #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +-#[serde(rename_all = "camelCase")] ++#[serde(default, rename_all = "camelCase")] + pub struct EqualizerConfig { + pub enabled: bool, + pub pre_amp: f32, +@@ -54,7 +63,7 @@ + + /// Audio DSP settings, synced from the frontend. + #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +-#[serde(rename_all = "camelCase")] ++#[serde(default, rename_all = "camelCase")] + pub struct AudioDspSettings { + pub gain: f32, // dB, -50 to +50 + pub ns_enabled: bool, +@@ -874,76 +883,13 @@ + } + } + +-// ─── Buffer Level Manager (replaces adaptive resampler) ───────────────────── +- +-/// Manages playback timing by monitoring the output buffer level. +-/// Instead of continuous resampling (which introduces interpolation artifacts), +-/// this uses simple sample duplication/dropping only when the buffer drifts +-/// significantly from the target level. +-struct BufferLevelManager { +- underrun_count: u32, +- overrun_count: u32, +-} +- +-impl BufferLevelManager { +- fn new() -> Self { +- Self { +- underrun_count: 0, +- overrun_count: 0, +- } +- } +- +- fn process(&mut self, input: &[f32], channels: usize, queued_ms: f64, output: &mut Vec) { +- output.clear(); +- if channels == 0 || input.is_empty() { +- output.extend_from_slice(input); +- return; +- } +- +- let frames = input.len() / channels; +- +- // Buffer critically low (< 15ms): duplicate last frame to prevent underrun +- if queued_ms < 15.0 { +- self.underrun_count += 1; +- self.overrun_count = 0; +- // Duplicate the last frame +- if frames >= 1 && self.underrun_count <= 3 { +- output.reserve(input.len() + channels); +- output.extend_from_slice(input); +- let last_frame_start = (frames - 1) * channels; +- for c in 0..channels { +- output.push(input[last_frame_start + c]); +- } +- return; +- } +- output.extend_from_slice(input); +- return; +- } +- +- // Buffer critically high (> 300ms): drop first frame to prevent overflow +- if queued_ms > 300.0 { +- self.overrun_count += 1; +- self.underrun_count = 0; +- if frames > 2 && self.overrun_count <= 3 { +- output.extend_from_slice(&input[channels..]); +- return; +- } +- output.extend_from_slice(input); +- return; +- } +- +- // Normal range: pass through unchanged +- self.underrun_count = 0; +- self.overrun_count = 0; +- output.extend_from_slice(input); +- } +-} +- + // ─── Main DSP Processor ───────────────────────────────────────────────────── + + /// The main DSP processor. Operates on f32 PCM samples at 48kHz. + pub struct DspProcessor { + settings: Arc>, ++ dry_delay: VecDeque, ++ noise_speech_probability: f32, + // RNNoise states - separate per channel to avoid RNN state cross-contamination + #[cfg(feature = "dsp")] + denoiser_left: Box>, +@@ -983,9 +929,6 @@ + speex_ns: SpeexStyleNS, + // Equalizer + equalizer: EqualizerEffect, +- // Adaptive Resampler +- // Buffer level manager (replaces adaptive resampler) +- buffer_manager: BufferLevelManager, + // Dereverb state + dereverb_buffer_left: Vec, + dereverb_buffer_right: Vec, +@@ -1042,6 +985,8 @@ + pub fn new(settings: Arc>, _model_dir: Option) -> Self { + Self { + settings: settings.clone(), ++ dry_delay: VecDeque::new(), ++ noise_speech_probability: 0.0, + #[cfg(feature = "dsp")] + denoiser_left: DenoiseState::new(), + #[cfg(feature = "dsp")] +@@ -1069,7 +1014,6 @@ + #[cfg(feature = "dsp")] + speex_ns: SpeexStyleNS::new(), + equalizer: EqualizerEffect::new(), +- buffer_manager: BufferLevelManager::new(), + dereverb_buffer_left: vec![0.0; 480], + dereverb_buffer_right: vec![0.0; 480], + dereverb_index: 0, +@@ -1176,6 +1120,14 @@ + settings.agc_target, + attack_rate, + decay_rate, ++ if !settings.ns_enabled ++ || settings.ns_type == "Speexdsp" ++ || self.noise_speech_probability >= 0.5 ++ { ++ 5.0 ++ } else { ++ 1.0 ++ }, + ); + } + } +@@ -1186,7 +1138,12 @@ + // Legacy synthetic plugin stage (may be absent): runs + // every registered plugin at this chain position. + if let Some(hook) = &mut self.external_hook { +- hook(PLUGIN_CHAIN_NODE, &mut to_process, channels.max(1), queued_ms); ++ hook( ++ PLUGIN_CHAIN_NODE, ++ &mut to_process, ++ channels.max(1), ++ queued_ms, ++ ); + } + } + node if node.starts_with(PLUGIN_NODE_PREFIX) => { +@@ -1200,8 +1157,8 @@ + } + } + +- self.buffer_manager +- .process(&to_process, channels, queued_ms, &mut self.output_buffer); ++ self.output_buffer.clear(); ++ self.output_buffer.extend_from_slice(&to_process); + + // Soft clip — avoids harsh hard-clipping artifacts (crackling/pops) + for sample in self.output_buffer.iter_mut() { +@@ -1368,14 +1325,45 @@ + channels: usize, + settings: &AudioDspSettings, + ) { ++ let original = data.clone(); + match settings.ns_type.as_str() { + #[cfg(feature = "dsp")] +- "RNNoise" => self.apply_rnnoise(data, channels, settings.ns_intensity), ++ "RNNoise" => self.apply_rnnoise(data, channels, 100.0), + #[cfg(feature = "noise-suppression")] +- "PureVox" => self.apply_purevox(data, channels, settings.ns_intensity), ++ "PureVox" => self.apply_purevox(data, channels, 100.0), + #[cfg(feature = "dsp")] +- "Speexdsp" => self.apply_speex(data, channels, settings.ns_intensity), +- _ => {} ++ "Speexdsp" => { ++ self.apply_speex(data, channels, settings.ns_intensity); ++ return; ++ } ++ _ => return, ++ } ++ // OpenLess: mix against the previous hop, matching model latency. ++ if self.dry_delay.is_empty() { ++ self.dry_delay.resize(RNNOISE_FRAME_SIZE * channels, 0.0); ++ } ++ self.dry_delay.extend(original); ++ if settings.ns_type == "PureVox" { ++ // PureVox has no VAD output. Avoid boosting strongly rejected ++ // stationary noise by comparing wet energy with aligned dry audio. ++ let dry_power = self ++ .dry_delay ++ .iter() ++ .take(data.len()) ++ .map(|s| s * s) ++ .sum::() ++ / data.len().max(1) as f32; ++ let wet_rms = compute_rms(data); ++ self.noise_speech_probability = if wet_rms > dry_power.sqrt() * 0.2 { ++ 1.0 ++ } else { ++ 0.0 ++ }; ++ } ++ let mix = (settings.ns_intensity / 100.0).clamp(0.0, 1.0); ++ for sample in data { ++ let dry = self.dry_delay.pop_front().unwrap_or(0.0); ++ *sample = dry * (1.0 - mix) + *sample * mix; + } + } + +@@ -1397,18 +1385,19 @@ + } + + // Process each channel with its own denoiser (no RNN state cross-contamination) +- Self::process_rnnoise_single_channel( ++ let left_probability = Self::process_rnnoise_single_channel( + &mut left, + &mut self.ns_buffer_left, + &mut self.denoiser_left, + intensity, + ); +- Self::process_rnnoise_single_channel( ++ let right_probability = Self::process_rnnoise_single_channel( + &mut right, + &mut self.ns_buffer_right, + &mut self.denoiser_right, + intensity, + ); ++ self.noise_speech_probability = left_probability.max(right_probability); + + data.clear(); + for i in 0..frames { +@@ -1416,7 +1405,7 @@ + data.push(right[i]); + } + } else { +- Self::process_rnnoise_single_channel( ++ self.noise_speech_probability = Self::process_rnnoise_single_channel( + data, + &mut self.ns_buffer_left, + &mut self.denoiser_left, +@@ -1431,12 +1420,13 @@ + ns_buffer: &mut Vec, + denoiser: &mut DenoiseState<'static>, + intensity: f32, +- ) { ++ ) -> f32 { + let mix = (intensity / 100.0).clamp(0.0, 1.0); + let input_len = data.len(); + ns_buffer.extend_from_slice(data); + + let mut output = Vec::with_capacity(input_len); ++ let mut speech_probability = 0.0_f32; + + while ns_buffer.len() >= RNNOISE_FRAME_SIZE { + let frame: Vec = ns_buffer.drain(..RNNOISE_FRAME_SIZE).collect(); +@@ -1444,7 +1434,8 @@ + let input_frame: Vec = frame.iter().map(|s| s * 32767.0).collect(); + let mut output_frame = vec![0.0f32; RNNOISE_FRAME_SIZE]; + +- let _vad_prob = denoiser.process_frame(&mut output_frame, &input_frame); ++ let vad_prob = denoiser.process_frame(&mut output_frame, &input_frame); ++ speech_probability = speech_probability.max(vad_prob); + + for i in 0..RNNOISE_FRAME_SIZE { + let clean = output_frame[i] / 32767.0; +@@ -1463,6 +1454,7 @@ + } + + data.copy_from_slice(&output); ++ speech_probability + } + + #[cfg(feature = "noise-suppression")] +@@ -1618,7 +1610,14 @@ + + // ── AGC ───────────────────────────────────────────────────────────────── + +- fn apply_agc(&mut self, data: &mut [f32], target: f32, attack: f32, decay: f32) { ++ fn apply_agc( ++ &mut self, ++ data: &mut [f32], ++ target: f32, ++ attack: f32, ++ decay: f32, ++ maximum_gain: f32, ++ ) { + let target_linear = target / 32767.0; + // Noise gate threshold: below this level, don't apply AGC gain + // (prevents amplifying hiss/noise floor during speech pauses) +@@ -1633,7 +1632,7 @@ + } + if self.agc_envelope > gate_threshold { + let desired_gain = target_linear / self.agc_envelope; +- let clamped_gain = desired_gain.clamp(0.1, 5.0); ++ let clamped_gain = desired_gain.clamp(0.1, maximum_gain); + // Smooth gain transition to avoid pops (exponential moving average) + let smooth_factor = 0.005_f32; + self.agc_smoothed_gain += smooth_factor * (clamped_gain - self.agc_smoothed_gain); +--- a/crates/micyou-infer/src/lib.rs ++++ b/crates/micyou-infer/src/lib.rs +@@ -48,6 +48,19 @@ + //! packing and re-checked in debug builds ([`Graph::validate_layout`]): + //! at every op, the output region never overlaps any input region. + ++// Preserve upstream numerical kernels and operation signatures for reproducible imports. ++#![allow( ++ clippy::needless_range_loop, ++ clippy::excessive_precision, ++ clippy::approx_constant, ++ clippy::manual_checked_ops, ++ clippy::manual_range_contains, ++ clippy::unnecessary_fold, ++ clippy::too_many_arguments, ++ clippy::chunks_exact_to_as_chunks, ++ clippy::type_complexity ++)] ++ + use std::fmt; + + mod ops; diff --git a/openless-all/app/scripts/micyou-upstream.json b/openless-all/app/scripts/micyou-upstream.json new file mode 100644 index 000000000..611eefcf8 --- /dev/null +++ b/openless-all/app/scripts/micyou-upstream.json @@ -0,0 +1,43 @@ +{ + "repository": "MicYou-Dev/MicYou", + "revision": "d821da032e2b72330ced09678dd127acd2714bef", + "patchSha256": "7f204189f5ce7bb9f0250093bf6f5823e81e9c97777936d5f9bed54c210dd5ee", + "files": { + "crates/micyou-dsp/src/lib.rs": { + "upstreamSha256": "9e3cb90b48c843554d75704188d4490d197f0134f57cf2fbc2f4a2aad1753d24", + "localSha256": "33174dfb93c42d4fd45dd36b721def47393bdce780324e8c58694f66f9585fa4" + }, + "crates/micyou-infer/src/lib.rs": { + "upstreamSha256": "a2d412f007ee15148e9a3684db8f59559e1057f62109f6ed5a1ca5fe951f5883", + "localSha256": "34385b6d707059937f8cfd6f4366b94fc3e38adedf4682435282c99d5f521264" + }, + "crates/micyou-infer/src/ops.rs": { + "upstreamSha256": "718c52a97d87a18043822f55f876ee03231429d5989361938f7e5b8d6dd459e9", + "localSha256": "718c52a97d87a18043822f55f876ee03231429d5989361938f7e5b8d6dd459e9" + }, + "crates/micyou-dsp/src/assets/purevox6.mcy": { + "upstreamSha256": "88fe0a1f44b93942770c97c576eab84ff193370b96cb311671900c07ef213147", + "localSha256": "88fe0a1f44b93942770c97c576eab84ff193370b96cb311671900c07ef213147" + }, + "crates/micyou-dsp/src/assets/aec7.mcy": { + "upstreamSha256": "a199b0210227775a16ce092c3c66046b8a0d2a663a6a02d8cb7d60f9e56e66a5", + "localSha256": "a199b0210227775a16ce092c3c66046b8a0d2a663a6a02d8cb7d60f9e56e66a5" + }, + "crates/micyou-dsp/LICENSE": { + "upstreamSha256": "9638e134977de4ee0c9745cf94e3a2c64c2252f88057fd5092731b9d74e75816", + "localSha256": "9638e134977de4ee0c9745cf94e3a2c64c2252f88057fd5092731b9d74e75816" + }, + "crates/micyou-infer/LICENSE": { + "upstreamSha256": "9638e134977de4ee0c9745cf94e3a2c64c2252f88057fd5092731b9d74e75816", + "localSha256": "9638e134977de4ee0c9745cf94e3a2c64c2252f88057fd5092731b9d74e75816" + }, + "crates/micyou-dsp/LICENSE-PureVox.txt": { + "upstreamSha256": "06c669821b66a2d5cb87a0eb24794e21950534bc2b1fc87f0d4b81d72999a09d", + "localSha256": "06c669821b66a2d5cb87a0eb24794e21950534bc2b1fc87f0d4b81d72999a09d" + }, + "crates/micyou-dsp/LICENSE-AEC7.txt": { + "upstreamSha256": "c41ed2d1fe4ff8660f2b7250343f078316aad743a4871ce205482236ca5f8df6", + "localSha256": "c41ed2d1fe4ff8660f2b7250343f078316aad743a4871ce205482236ca5f8df6" + } + } +} diff --git a/openless-all/app/scripts/micyou-upstream.test.mjs b/openless-all/app/scripts/micyou-upstream.test.mjs new file mode 100644 index 000000000..7993ca3eb --- /dev/null +++ b/openless-all/app/scripts/micyou-upstream.test.mjs @@ -0,0 +1,15 @@ +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const appRoot = fileURLToPath(new URL('..', import.meta.url)); +const python = process.platform === 'win32' ? 'python' : 'python3'; +for (const args of [['scripts/test_sync_micyou.py'], ['scripts/sync-micyou.py', '--verify']]) { + const result = spawnSync(python, args, { + cwd: appRoot, + env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' }, + stdio: 'inherit', + }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); +} +console.log('MicYou sync importer and pinned source checks passed'); diff --git a/openless-all/app/scripts/package-linux-egui.sh b/openless-all/app/scripts/package-linux-egui.sh index 3b700c8c2..f7990a3cd 100644 --- a/openless-all/app/scripts/package-linux-egui.sh +++ b/openless-all/app/scripts/package-linux-egui.sh @@ -90,6 +90,10 @@ stage_common() { "$root/usr/share/applications/openless.desktop" install -Dm644 "$PACKAGING/top.openless.OpenLess.metainfo.xml" \ "$root/usr/share/metainfo/top.openless.OpenLess.metainfo.xml" + for notice in LICENSE LICENSE-RNNoise.txt LICENSE-PureVox.txt LICENSE-AEC7.txt UPSTREAM.md; do + install -Dm644 "$APP_ROOT/crates/micyou-dsp/$notice" \ + "$root/usr/share/doc/openless/micyou/$notice" + done # 多档 hicolor 尺寸：桌面环境按需选档（任务栏 32/48、菜单 64/128、大图标 256/512）。 # 旧的单档安装把 512×512 的图放进了 256x256 目录（那档根本不是 256px）。 for spec in "${HICOLOR_ICONS[@]}"; do diff --git a/openless-all/app/scripts/sync-micyou.py b/openless-all/app/scripts/sync-micyou.py new file mode 100644 index 000000000..2a3ad9146 --- /dev/null +++ b/openless-all/app/scripts/sync-micyou.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Reproducible MicYou imports. Updates are staged and patched before any writes.""" +import argparse +import difflib +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import subprocess +import tempfile +import urllib.request + +APP_ROOT = Path(__file__).resolve().parents[1] +MANIFEST = "scripts/micyou-upstream.json" +PATCH = "scripts/micyou-dsp.patch" +REPOSITORY = "MicYou-Dev/MicYou" +PATCHABLE = { + "crates/micyou-dsp/src/lib.rs", + "crates/micyou-infer/src/lib.rs", + "crates/micyou-infer/src/ops.rs", +} +# Deliberately fixed destinations: never let upstream metadata choose files to +# overwrite, run downloaded scripts, or introduce workflow/config changes. +IMPORTS = { + "crates/micyou-dsp/src/lib.rs": "tauri-app/crates/micyou-audio/src/dsp.rs", + "crates/micyou-infer/src/lib.rs": "tauri-app/crates/micyou-infer/src/lib.rs", + "crates/micyou-infer/src/ops.rs": "tauri-app/crates/micyou-infer/src/ops.rs", + "crates/micyou-dsp/src/assets/purevox6.mcy": "tauri-app/crates/micyou-audio/src/assets/purevox6.mcy", + "crates/micyou-dsp/src/assets/aec7.mcy": "tauri-app/crates/micyou-audio/src/assets/aec7.mcy", + "crates/micyou-dsp/LICENSE": "LICENSE", + "crates/micyou-infer/LICENSE": "LICENSE", + "crates/micyou-dsp/LICENSE-PureVox.txt": "tauri-app/src-tauri/resources/LICENSE-PureVox.txt", + "crates/micyou-dsp/LICENSE-AEC7.txt": "tauri-app/src-tauri/resources/LICENSE-AEC7.txt", +} + + +def canonical(data, path): + return data if path.endswith(".mcy") else data.replace(b"\r\n", b"\n") + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def revision(value): + if not re.fullmatch(r"[0-9a-f]{40}", value): + raise ValueError("an immutable, lowercase 40-character commit SHA is required") + return value + + +def download(url): + request = urllib.request.Request(url, headers={"User-Agent": "OpenLess-MicYou-Sync"}) + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read(16 * 1024 * 1024 + 1) + if len(data) > 16 * 1024 * 1024: + raise ValueError("upstream file exceeds import size limit") + return data + + +def fetch_sources(sha): + cache = {} + result = {} + for local, remote in IMPORTS.items(): + if remote not in cache: + cache[remote] = download(f"https://raw.githubusercontent.com/{REPOSITORY}/{sha}/{remote}") + result[local] = canonical(cache[remote], local) + return result + + +def load_manifest(root): + manifest = json.loads((root / MANIFEST).read_text()) + revision(manifest["revision"]) + if manifest.get("repository") != REPOSITORY or set(manifest["files"]) != set(IMPORTS): + raise ValueError("unexpected import repository or file list") + return manifest + + +def verify_local(root, manifest): + if digest((root / PATCH).read_bytes()) != manifest["patchSha256"]: + raise ValueError("local adapter patch changed; run --refresh-patch after reviewing it") + for local, hashes in manifest["files"].items(): + if digest(canonical((root / local).read_bytes(), local)) != hashes["localSha256"]: + raise ValueError(f"local imported file changed: {local}; run --refresh-patch to record reviewed adaptations") + + +def apply_patch(sources, patch): + # Reject file additions, deletions, renames and arbitrary destinations even + # if somebody accidentally edits the adapter patch incorrectly. + text = patch.decode("utf-8") + old_paths = re.findall(r"^--- a/(.+)$", text, re.MULTILINE) + new_paths = re.findall(r"^\+\+\+ b/(.+)$", text, re.MULTILINE) + if not old_paths or old_paths != new_paths or any(p not in PATCHABLE for p in new_paths): + raise ValueError("adapter patch must modify only the three imported Rust source files") + if any(line.startswith(("diff --git", "rename ", "new file mode", "deleted file mode")) for line in text.splitlines()): + raise ValueError("adapter patch must be a plain unified diff") + with tempfile.TemporaryDirectory(prefix="openless-micyou-stage-") as directory: + stage = Path(directory) + for local, data in sources.items(): + path = stage / PurePosixPath(local) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + patch_file = stage / "adapter.patch" + patch_file.write_bytes(patch) + for args in [["--check"], []]: + result = subprocess.run(["git", "-c", "core.autocrlf=false", "-c", "core.eol=lf", "apply", *args, "--whitespace=error-all", str(patch_file)], cwd=stage, capture_output=True, text=True) + if result.returncode: + raise ValueError("MicYou adapter no longer applies cleanly; review upstream changes and refresh the patch:\n" + result.stderr) + return {local: (stage / local).read_bytes() for local in IMPORTS} + + +def write_import(root, sources, manifest): + previous = {local: (root / local).read_bytes() for local in sources} + manifest_path = root / MANIFEST + old_manifest = manifest_path.read_bytes() if manifest_path.exists() else None + try: + for local, data in sources.items(): + (root / local).write_bytes(data) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + except BaseException: + for local, data in previous.items(): + (root / local).write_bytes(data) + if old_manifest is not None: + manifest_path.write_bytes(old_manifest) + raise + + +def refresh_patch(root, manifest): + sources = fetch_sources(manifest["revision"]) + patch = "" + current = {} + for local, source in sources.items(): + local_data = canonical((root / local).read_bytes(), local) + current[local] = local_data + if local in PATCHABLE: + patch += "".join(difflib.unified_diff(source.decode().splitlines(keepends=True), local_data.decode().splitlines(keepends=True), fromfile="a/" + local, tofile="b/" + local)) + elif source != local_data: + raise ValueError(f"do not silently fork an upstream model or license: {local}") + patch_bytes = patch.encode() + if apply_patch(sources, patch_bytes) != current: + raise ValueError("refreshed patch did not reproduce the current source") + manifest["patchSha256"] = digest(patch_bytes) + manifest["files"] = {local: {"upstreamSha256": digest(sources[local]), "localSha256": digest(data)} for local, data in current.items()} + (root / PATCH).write_bytes(patch_bytes) + (root / MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n") + + +def sync(root, manifest, sha, write): + verify_local(root, manifest) + upstream = fetch_sources(sha) + if sha == manifest["revision"]: + for local, data in upstream.items(): + if digest(data) != manifest["files"][local]["upstreamSha256"]: + raise ValueError(f"pinned upstream integrity mismatch: {local}") + patched = apply_patch(upstream, (root / PATCH).read_bytes()) + changed = any(digest(data) != manifest["files"][local]["upstreamSha256"] for local, data in upstream.items()) + if not changed: + if any(digest(data) != manifest["files"][local]["localSha256"] for local, data in patched.items()): + raise ValueError("adapter patch does not reproduce the recorded local source") + return False + if write: + updated = dict(manifest, revision=sha) + updated["files"] = {local: {"upstreamSha256": digest(upstream[local]), "localSha256": digest(data)} for local, data in patched.items()} + write_import(root, patched, updated) + return True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--verify", action="store_true", help="offline checksum verification") + mode.add_argument("--check", action="store_true", help="check and stage upstream changes without modifying files") + mode.add_argument("--sync", action="store_true", help="import upstream and reapply the reviewed adapter patch") + mode.add_argument("--refresh-patch", action="store_true", help="record reviewed local source changes against the pinned revision") + parser.add_argument("--revision", type=revision, help="immutable upstream SHA; defaults to current master for check/sync") + parser.add_argument("--github-output", type=Path) + args = parser.parse_args() + manifest = load_manifest(APP_ROOT) + sha = manifest["revision"] + changed = False + if args.verify: + verify_local(APP_ROOT, manifest) + elif args.refresh_patch: + refresh_patch(APP_ROOT, manifest) + else: + sha = args.revision or revision(json.loads(download(f"https://api.github.com/repos/{REPOSITORY}/commits/master"))["sha"]) + changed = sync(APP_ROOT, manifest, sha, args.sync) + print(json.dumps({"revision": sha, "changed": changed, "mode": "verify" if args.verify else "refresh" if args.refresh_patch else "sync" if args.sync else "check"})) + if args.github_output: + with args.github_output.open("a") as output: + output.write(f"changed={str(changed).lower()}\nrevision={sha}\n") + + +if __name__ == "__main__": + main() diff --git a/openless-all/app/scripts/test-recorder-module.py b/openless-all/app/scripts/test-recorder-module.py index 2fac17a0e..c7e25910f 100644 --- a/openless-all/app/scripts/test-recorder-module.py +++ b/openless-all/app/scripts/test-recorder-module.py @@ -25,12 +25,14 @@ serde_json = "1" thiserror = "1" log = "0.4" -""") +""" + "micyou-dsp = { path = " + json.dumps((recorder.parents[2] / "crates/micyou-dsp").as_posix()) + " }\n") (root / "src/lib.rs").write_text( "extern crate self as openless_core;\n" "#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackendErrorCode { InvalidArgument }\n" "#[derive(Debug, Clone)] pub struct BackendError { pub code: BackendErrorCode, pub message: String }\n" "impl BackendError { pub fn new(code: BackendErrorCode, message: impl Into) -> Self { Self { code, message: message.into() } } }\n" + + "#[path = " + json.dumps(str(recorder.parents[2] / "crates/openless-core/src/audio_dsp.rs"), ensure_ascii=False) + "]\npub mod audio_dsp;\n" + "pub use audio_dsp::{AudioDspConfig, AudioDspStage, NoiseReduction};\n" + "#[path = " + json.dumps(str(recorder.parents[2] / "crates/openless-core/src/audio.rs"), ensure_ascii=False) + "]\npub mod audio;\n" "pub use audio::{AudioProcessingConfig, AudioProcessor};\n" "pub trait AudioConsumer: Send + Sync { fn consume_pcm_chunk(&self, pcm: &[u8]); }\n" diff --git a/openless-all/app/scripts/test_sync_micyou.py b/openless-all/app/scripts/test_sync_micyou.py new file mode 100644 index 000000000..78a621fd9 --- /dev/null +++ b/openless-all/app/scripts/test_sync_micyou.py @@ -0,0 +1,97 @@ +import difflib +import importlib.util +import json +from pathlib import Path +import tempfile +import os +import unittest +from unittest import mock + +spec = importlib.util.spec_from_file_location("sync_micyou", Path(__file__).with_name("sync-micyou.py")) +sync = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sync) + + +class SyncTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.sources = {local: b"upstream\n" for local in sync.IMPORTS} + self.path = "crates/micyou-dsp/src/lib.rs" + self.sources[self.path] = "".join(f"line {i}\n" for i in range(20)).encode() + local = self.sources[self.path].replace(b"line 5\n", b"line 5 adapted\n") + patch = "".join(difflib.unified_diff(self.sources[self.path].decode().splitlines(True), local.decode().splitlines(True), fromfile="a/"+self.path, tofile="b/"+self.path)).encode() + current = dict(self.sources, **{self.path: local}) + for path, data in {**current, sync.PATCH: patch}.items(): + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + self.manifest = { + "repository": sync.REPOSITORY, "revision": "a"*40, + "patchSha256": sync.digest(patch), + "files": {path: {"upstreamSha256": sync.digest(self.sources[path]), "localSha256": sync.digest(data)} for path, data in current.items()}, + } + (self.root / sync.MANIFEST).write_text(json.dumps(self.manifest)) + + def test_same_pin_reproduces_exact_source_without_modifying_manifest(self): + before = (self.root / sync.MANIFEST).read_bytes() + with mock.patch.object(sync, "fetch_sources", return_value=self.sources): + self.assertFalse(sync.sync(self.root, self.manifest, "a"*40, True)) + self.assertEqual(before, (self.root / sync.MANIFEST).read_bytes()) + + def test_git_autocrlf_does_not_change_imported_bytes(self): + # Git for Windows commonly enables autocrlf globally. The staging + # operation must still produce the canonical bytes in the manifest. + with mock.patch.dict(os.environ, {"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "core.autocrlf", "GIT_CONFIG_VALUE_0": "true"}): + with mock.patch.object(sync, "fetch_sources", return_value=self.sources): + self.assertFalse(sync.sync(self.root, self.manifest, "a"*40, True)) + + def test_new_upstream_fix_preserves_adapter_and_advances_pin(self): + changed = dict(self.sources) + changed[self.path] = changed[self.path].replace(b"line 15\n", b"upstream bugfix 15\n") + with mock.patch.object(sync, "fetch_sources", return_value=changed): + self.assertTrue(sync.sync(self.root, self.manifest, "b"*40, True)) + content = (self.root / self.path).read_bytes() + self.assertIn(b"line 5 adapted", content) + self.assertIn(b"upstream bugfix 15", content) + manifest = sync.load_manifest(self.root) + self.assertEqual(manifest["revision"], "b"*40) + sync.verify_local(self.root, manifest) + + def test_patch_conflict_leaves_every_file_untouched(self): + before = {p: (self.root / p).read_bytes() for p in [*sync.IMPORTS, sync.MANIFEST]} + changed = dict(self.sources) + changed[self.path] = b"incompatible replacement\n" + with mock.patch.object(sync, "fetch_sources", return_value=changed): + with self.assertRaisesRegex(ValueError, "no longer applies"): + sync.sync(self.root, self.manifest, "b"*40, True) + self.assertEqual(before, {p: (self.root / p).read_bytes() for p in before}) + + def test_check_never_changes_files(self): + before = (self.root / sync.MANIFEST).read_bytes() + changed = dict(self.sources) + changed[self.path] += b"upstream addition\n" + with mock.patch.object(sync, "fetch_sources", return_value=changed): + self.assertTrue(sync.sync(self.root, self.manifest, "b"*40, False)) + self.assertEqual(before, (self.root / sync.MANIFEST).read_bytes()) + + def test_local_changes_and_pinned_corruption_are_not_overwritten(self): + path = self.root / self.path + path.write_bytes(path.read_bytes()+b"unrecorded change\n") + with self.assertRaisesRegex(ValueError, "local imported file changed"): + sync.verify_local(self.root, self.manifest) + changed = dict(self.sources) + changed["crates/micyou-dsp/src/assets/aec7.mcy"] = b"corrupted model" + path.write_bytes(path.read_bytes().removesuffix(b"unrecorded change\n")) + with mock.patch.object(sync, "fetch_sources", return_value=changed): + with self.assertRaisesRegex(ValueError, "integrity mismatch"): + sync.sync(self.root, self.manifest, "a"*40, True) + + def test_patch_cannot_escape_source_allowlist(self): + with self.assertRaisesRegex(ValueError, "only the three"): + sync.apply_patch(self.sources, b"--- a/../../escape\n+++ b/../../escape\n@@ -1 +1 @@\n-old\n+new\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index 3223fda2e..d6c9d307d 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -151,6 +151,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "arbitrary" version = "1.5.0" @@ -193,6 +199,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "asn1-rs" version = "0.6.2" @@ -1739,6 +1751,19 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "easyfft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f" +dependencies = [ + "array-init", + "generic_singleton", + "num-complex", + "realfft", + "rustfft", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -2353,6 +2378,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic_singleton" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6e923c8e978e57cf63e2e200ca967d1d20f0ea2662b28f6d4e11c44aa6ab16" +dependencies = [ + "anymap3", + "parking_lot", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -3574,6 +3609,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "micyou-dsp" +version = "0.1.0" +dependencies = [ + "log", + "micyou-infer", + "nnnoiseless", + "rustfft", + "serde", +] + +[[package]] +name = "micyou-infer" +version = "0.1.0" + [[package]] name = "mime" version = "0.3.17" @@ -3813,6 +3863,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nnnoiseless" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c" +dependencies = [ + "easyfft", + "once_cell", +] + [[package]] name = "nom" version = "7.1.3" @@ -3833,6 +3893,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -4457,6 +4527,7 @@ dependencies = [ "jsonwebtoken", "log", "md-5 0.11.0", + "micyou-dsp", "once_cell", "parking_lot", "regex", @@ -4929,6 +5000,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -5204,6 +5284,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -5484,6 +5573,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -6178,6 +6281,12 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff7589a1859b09232522bb9edf076ca7fcf14ded96685d3fa629db73023ffbb7" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -7217,6 +7326,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.25.1" diff --git a/openless-all/app/src-tauri/backend-tests/Cargo.lock b/openless-all/app/src-tauri/backend-tests/Cargo.lock index fb37a2b57..8b9df454f 100644 --- a/openless-all/app/src-tauri/backend-tests/Cargo.lock +++ b/openless-all/app/src-tauri/backend-tests/Cargo.lock @@ -32,6 +32,28 @@ dependencies = [ "memchr", ] +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -47,6 +69,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "argon2" version = "0.6.0" @@ -60,6 +88,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "asn1-rs" version = "0.6.2" @@ -158,6 +192,30 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.2" @@ -254,7 +312,22 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", ] [[package]] @@ -315,6 +388,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cmake" version = "0.1.58" @@ -387,6 +471,49 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpubits" version = "0.1.1" @@ -462,6 +589,12 @@ dependencies = [ "cmov", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -574,6 +707,25 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "easyfft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767e39eef2ad8a3b6f1d733be3ec70364d21d437d06d4f18ea76ce08df20b75f" +dependencies = [ + "array-init", + "generic_singleton", + "num-complex", + "realfft", + "rustfft", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + [[package]] name = "ecdsa" version = "0.16.9" @@ -817,6 +969,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic_singleton" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6e923c8e978e57cf63e2e200ca967d1d20f0ea2662b28f6d4e11c44aa6ab16" +dependencies = [ + "anymap3", + "parking_lot", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -844,6 +1006,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "group" version = "0.13.0" @@ -1155,12 +1323,37 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -1170,7 +1363,7 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys", + "jni-sys 0.4.1", "log", "simd_cesu8", "thiserror 2.0.21", @@ -1191,6 +1384,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + [[package]] name = "jni-sys" version = "0.4.1" @@ -1274,6 +1476,16 @@ version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -1322,6 +1534,15 @@ dependencies = [ "sha2 0.11.1", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "md-5" version = "0.11.0" @@ -1338,6 +1559,21 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "micyou-dsp" +version = "0.1.0" +dependencies = [ + "log", + "micyou-infer", + "nnnoiseless", + "rustfft", + "serde", +] + +[[package]] +name = "micyou-infer" +version = "0.1.0" + [[package]] name = "mime" version = "0.3.17" @@ -1391,18 +1627,57 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "nix" version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", ] +[[package]] +name = "nnnoiseless" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805d5964d1e7a0006a7fdced7dae75084d66d18b35f1dfe81bd76929b1f8da0c" +dependencies = [ + "easyfft", + "once_cell", +] + [[package]] name = "nom" version = "7.1.3" @@ -1423,6 +1698,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -1445,6 +1730,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-integer" version = "0.1.47" @@ -1474,6 +1770,28 @@ dependencies = [ "libm", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc-sys" version = "0.3.5" @@ -1499,6 +1817,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni 0.21.1", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -1521,6 +1862,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "block2", + "cpal", "futures-util", "libc", "log", @@ -1537,7 +1879,7 @@ dependencies = [ "time", "tokio", "uuid", - "windows", + "windows 0.58.0", "x509-parser", ] @@ -1559,6 +1901,7 @@ dependencies = [ "jsonwebtoken", "log", "md-5", + "micyou-dsp", "once_cell", "parking_lot", "regex", @@ -1804,6 +2147,24 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "494b4379484fc92217cc7d9e3a532375b1d01a61889ec8333025dc3b0377af56" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2002,13 +2363,22 @@ dependencies = [ "yasna", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.2", ] [[package]] @@ -2180,6 +2550,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2195,7 +2579,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -2247,7 +2631,7 @@ checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls", @@ -2334,7 +2718,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2445,6 +2829,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -2533,6 +2923,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "subtle" version = "2.6.1" @@ -2598,7 +2994,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2789,6 +3185,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -2810,7 +3236,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.2", "bytes", "futures-util", "http", @@ -2853,6 +3279,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -3113,6 +3549,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.58.0" @@ -3120,7 +3566,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" dependencies = [ "windows-core 0.58.0", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", ] [[package]] @@ -3133,7 +3589,7 @@ dependencies = [ "windows-interface 0.58.0", "windows-result 0.2.0", "windows-strings 0.1.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3210,13 +3666,22 @@ dependencies = [ "windows-strings 0.5.1", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3235,7 +3700,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ "windows-result 0.2.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3247,13 +3712,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3265,34 +3739,67 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3305,30 +3812,63 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/openless-all/app/src-tauri/backend-tests/Cargo.toml b/openless-all/app/src-tauri/backend-tests/Cargo.toml index 6078b3e91..1b3248046 100644 --- a/openless-all/app/src-tauri/backend-tests/Cargo.toml +++ b/openless-all/app/src-tauri/backend-tests/Cargo.toml @@ -36,3 +36,8 @@ futures-util = "0.3" [target.'cfg(windows)'.dev-dependencies] windows = { version = "0.58", features = ["Win32_Foundation", "Win32_System_Threading", "Win32_UI_WindowsAndMessaging", "Win32_UI_Input_KeyboardAndMouse"] } + +[target.'cfg(target_os = "linux")'.dev-dependencies] +# Exercise the production recorder worker without the retired Linux Tauri host. +cpal = "0.15.3" +parking_lot = "0.12" diff --git a/openless-all/app/src-tauri/backend-tests/tests/audio_processing.rs b/openless-all/app/src-tauri/backend-tests/tests/audio_processing.rs new file mode 100644 index 000000000..b7dfdf8dd --- /dev/null +++ b/openless-all/app/src-tauri/backend-tests/tests/audio_processing.rs @@ -0,0 +1,4 @@ +#![cfg(target_os = "linux")] +#![allow(dead_code)] +#[path = "../../src/recorder.rs"] +mod recorder; diff --git a/openless-all/app/src-tauri/src/commands/settings.rs b/openless-all/app/src-tauri/src/commands/settings.rs index e3c989a03..2b869abce 100644 --- a/openless-all/app/src-tauri/src/commands/settings.rs +++ b/openless-all/app/src-tauri/src/commands/settings.rs @@ -231,6 +231,7 @@ pub(crate) fn sync_low_latency_capture_preferences(prefs: &UserPreferences) { .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + dsp: prefs.audio_dsp, }, ); } diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index 98af79c56..22c8f12eb 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -150,6 +150,7 @@ pub(crate) fn backend_dependencies( .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + dsp: prefs.audio_dsp, }, ); } @@ -2000,6 +2001,7 @@ impl openless_core::VoiceActivationRuntime for TauriVoiceActivationRuntime { ); let audio_processing = openless_core::AudioProcessingConfig { enhancement_enabled: preferences_snapshot.audio_enhancement_enabled, + dsp: preferences_snapshot.audio_dsp, }; if let Some(wake_asr) = wake_asr.as_ref() { warm_hub.set_activation_generation(config.activation_generation); @@ -2264,6 +2266,7 @@ impl openless_core::VoiceActivationHandle for TauriVoiceActivationHandle { .then(|| prefs.microphone_device_name.clone()), openless_core::AudioProcessingConfig { enhancement_enabled: prefs.audio_enhancement_enabled, + dsp: prefs.audio_dsp, }, ); } @@ -3573,6 +3576,11 @@ impl TauriAudioRecorder { request.microphone_device_name.clone(), request.audio_processing, ) { + log::info!( + "[audio-startup] press={} mode=warm_hub arm_done_us={}", + request.press_id, + request.pressed_at.elapsed().as_micros() + ); return Ok(Box::new(TauriPendingAudioCapture { app: Arc::clone(&self.app), backend: Arc::clone(&self.backend), @@ -3606,6 +3614,12 @@ impl TauriAudioRecorder { return Err(map_recorder_error(error)); } }; + log::info!( + "[audio-startup] press={} capture={} mode=cold arm_done_us={}", + request.press_id, + staged.capture_id(), + request.pressed_at.elapsed().as_micros() + ); Ok(Box::new(TauriPendingAudioCapture { app: Arc::clone(&self.app), backend: Arc::clone(&self.backend), diff --git a/openless-all/app/src-tauri/src/recorder.rs b/openless-all/app/src-tauri/src/recorder.rs index d5acb8839..eb826ea61 100644 --- a/openless-all/app/src-tauri/src/recorder.rs +++ b/openless-all/app/src-tauri/src/recorder.rs @@ -33,6 +33,7 @@ const LOG_EVERY_N_CALLBACKS: usize = 50; const LEVEL_RMS_GAIN: f32 = 4.0; /// Maximum number of native callback blocks waiting for the processing worker. const AUDIO_PROCESSING_QUEUE_CAPACITY: usize = 64; +static NEXT_CAPTURE_ID: AtomicU64 = AtomicU64::new(1); /// Max queued PCM chunks for the archive writer thread. When full, new archive chunks are /// dropped instead of blocking the realtime callback; ASR still receives the full PCM, and /// archive messages accepted before stop are fully flushed. @@ -1712,6 +1713,13 @@ impl Recorder { } impl StagedRecorder { + pub(crate) fn capture_id(&self) -> u64 { + self.recorder + .as_ref() + .map(|recorder| recorder.state.capture_id) + .unwrap_or(0) + } + pub(crate) fn commit( mut self, consumer: Arc, @@ -1989,8 +1997,10 @@ fn build_input_stream( stop_flag: Arc, runtime_error_tx: Sender, ) -> Result<(cpal::Stream, Arc), RecorderError> { + let selecting_at = std::time::Instant::now(); let host = cpal::default_host(); let selected = select_input_device(&host, microphone_device_name.as_deref())?; + state.log_startup_phase("device_selected", selecting_at.elapsed()); let selected_name = selected.name().ok(); let start = |device: &cpal::Device| { start_device_stream(device, &frame_tx, &state, &stop_flag, &runtime_error_tx) @@ -2101,6 +2111,7 @@ fn start_native_config( runtime_error_tx: &Sender, ) -> Result<(cpal::Stream, Arc), RecorderError> { let config = supported.config(); + let building_at = std::time::Instant::now(); state.accepting_audio.store(false, Ordering::Release); let stream = build_stream_for_format( device, @@ -2113,10 +2124,13 @@ fn start_native_config( config.channels as usize, runtime_error_tx.clone(), )?; + state.log_startup_phase("stream_built", building_at.elapsed()); + let playing_at = std::time::Instant::now(); stream .play() .map_err(|error| classify_default_config_err(format!("play: {error}")))?; state.accepting_audio.store(true, Ordering::Release); + state.log_startup_phase("stream_playing", playing_at.elapsed()); log::info!( "[recorder] inputDevice={} inputFormat sampleRate={} channels={} fmt={:?}", device.name().unwrap_or_else(|_| "".into()), @@ -2242,6 +2256,9 @@ fn build_stream_for_format( if !state.accepting_audio.load(Ordering::Acquire) { return; } + if !data.is_empty() { + state.note_first_pcm(false); + } let mut floats = Vec::with_capacity(data.len()); for s in data { floats.push($to_f32(*s)); @@ -2339,6 +2356,7 @@ impl AudioProcessingWorker { frame.channels, frame.input_sample_rate, ) { + state.note_first_pcm(true); consumer.consume_pcm_chunk(&chunk.pcm_i16_le); if let Some(archiver) = &archiver { archiver.append(&chunk.pcm_i16_le); @@ -2348,9 +2366,30 @@ impl AudioProcessingWorker { } } AudioProcessingMessage::Flush(done) => { + if let Some(chunk) = processor.finish() { + state.note_first_pcm(true); + consumer.consume_pcm_chunk(&chunk.pcm_i16_le); + if let Some(archiver) = &archiver { + archiver.append(&chunk.pcm_i16_le); + } + level_handler(chunk.level); + update_peak(&state.peak_output_rms_milli, chunk.level); + } + processor.reset(); let _ = done.send(()); } - AudioProcessingMessage::Shutdown => break, + AudioProcessingMessage::Shutdown => { + if let Some(chunk) = processor.finish() { + state.note_first_pcm(true); + consumer.consume_pcm_chunk(&chunk.pcm_i16_le); + if let Some(archiver) = &archiver { + archiver.append(&chunk.pcm_i16_le); + } + level_handler(chunk.level); + update_peak(&state.peak_output_rms_milli, chunk.level); + } + break; + } } } }) @@ -2382,6 +2421,10 @@ impl AudioProcessingWorker { /// State carried across callbacks: resample leftovers, diagnostic counters and peak. struct StreamState { + capture_id: u64, + created_at: std::time::Instant, + first_raw_pcm_us: AtomicU64, + first_output_pcm_us: AtomicU64, processor_sender: Mutex>>, /// Fractional position left unconsumed by the previous callback; linear-interpolation /// resampling spans buffers. @@ -2400,6 +2443,10 @@ struct StreamState { impl StreamState { fn new() -> Self { Self { + capture_id: NEXT_CAPTURE_ID.fetch_add(1, Ordering::Relaxed), + created_at: std::time::Instant::now(), + first_raw_pcm_us: AtomicU64::new(0), + first_output_pcm_us: AtomicU64::new(0), processor_sender: Mutex::new(None), resample_phase: Mutex::new(0.0), last_sample: Mutex::new(0.0), @@ -2412,6 +2459,40 @@ impl StreamState { last_callback_time: Mutex::new(None), } } + + fn log_startup_phase(&self, phase: &'static str, duration: std::time::Duration) { + log::info!( + "[audio-startup] capture={} phase={} duration_us={} since_open_us={}", + self.capture_id, + phase, + duration.as_micros(), + self.created_at.elapsed().as_micros() + ); + } + + fn note_first_pcm(&self, processed: bool) { + let slot = if processed { + &self.first_output_pcm_us + } else { + &self.first_raw_pcm_us + }; + let elapsed = self.created_at.elapsed().as_micros().max(1) as u64; + if slot + .compare_exchange(0, elapsed, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + log::info!( + "[audio-startup] capture={} phase={} since_open_us={}", + self.capture_id, + if processed { + "first_output_pcm" + } else { + "first_raw_pcm" + }, + elapsed + ); + } + } } /// Per-callback pipeline: downmix -> resample -> quantize to i16 -> compute RMS -> feed downstream. @@ -2427,6 +2508,7 @@ fn process_callback( if !state.accepting_audio.load(Ordering::Acquire) || interleaved.is_empty() || channels == 0 { return; } + state.note_first_pcm(false); let mono = downmix_to_mono(interleaved, channels); let input_rms = rms(&mono); @@ -2439,6 +2521,7 @@ fn process_callback( let (pcm_bytes, output_rms) = quantize_to_i16_le(&resampled); let level = (output_rms * LEVEL_RMS_GAIN).clamp(0.0, 1.0); + state.note_first_pcm(true); consumer.consume_pcm_chunk(&pcm_bytes); if let Some(arch) = archiver { arch.append(&pcm_bytes); @@ -2715,6 +2798,7 @@ mod tests { let (worker, sender) = AudioProcessingWorker::start( openless_core::AudioProcessingConfig { enhancement_enabled: true, + ..Default::default() }, Arc::clone(&consumer) as Arc, Arc::new(|_| {}), @@ -2734,7 +2818,75 @@ mod tests { let chunks = consumer.chunks.lock().unwrap(); assert_eq!(chunks.len(), 1); - assert_eq!(decode_i16_le(&chunks[0]), vec![12287, -12287]); + assert_eq!(decode_i16_le(&chunks[0]).len(), 2); + } + + #[test] + fn audio_processing_flush_delivers_tail_to_asr_and_archive_before_reset() { + let consumer = Arc::new(RecordingConsumer::default()); + let path = std::env::temp_dir().join(format!( + "openless-dsp-tail-{}-{}.wav", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let archive = Arc::new(WavArchiveWriter::create(&path).unwrap()); + let (worker, sender) = AudioProcessingWorker::start( + openless_core::AudioProcessingConfig { + enhancement_enabled: true, + ..Default::default() + }, + Arc::clone(&consumer) as Arc, + Arc::new(|_| {}), + Some(archive.clone()), + Arc::new(StreamState::new()), + ) + .unwrap(); + sender + .send(AudioProcessingMessage::Frame(RawAudioFrame { + samples: vec![0.15; 317], + channels: 1, + input_sample_rate: TARGET_SAMPLE_RATE, + })) + .unwrap(); + let (done_tx, done_rx) = channel(); + sender.send(AudioProcessingMessage::Flush(done_tx)).unwrap(); + done_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert_eq!( + consumer + .chunks + .lock() + .unwrap() + .iter() + .map(Vec::len) + .sum::(), + 317 * 2 + ); + sender + .send(AudioProcessingMessage::Frame(RawAudioFrame { + samples: vec![-0.15; 211], + channels: 1, + input_sample_rate: TARGET_SAMPLE_RATE, + })) + .unwrap(); + worker.finish(); + archive.finish(); + let delivered: Vec = consumer + .chunks + .lock() + .unwrap() + .iter() + .flatten() + .copied() + .collect(); + assert_eq!(delivered.len(), (317 + 211) * 2); + let wav = std::fs::read(&path).unwrap(); + assert_eq!(&wav[44..], delivered); + std::fs::remove_file(&path).unwrap(); } #[test] @@ -2743,6 +2895,7 @@ mod tests { let (worker, sender) = AudioProcessingWorker::start( openless_core::AudioProcessingConfig { enhancement_enabled: false, + ..Default::default() }, Arc::clone(&consumer) as Arc, Arc::new(|_| {}), diff --git a/openless-all/app/src-tauri/tauri.conf.json b/openless-all/app/src-tauri/tauri.conf.json index 2bf2a484c..aa01ef427 100644 --- a/openless-all/app/src-tauri/tauri.conf.json +++ b/openless-all/app/src-tauri/tauri.conf.json @@ -67,6 +67,13 @@ "bundle": { "active": true, "targets": "all", + "resources": { + "../crates/micyou-dsp/LICENSE": "licenses/micyou/LICENSE", + "../crates/micyou-dsp/LICENSE-RNNoise.txt": "licenses/micyou/LICENSE-RNNoise.txt", + "../crates/micyou-dsp/LICENSE-PureVox.txt": "licenses/micyou/LICENSE-PureVox.txt", + "../crates/micyou-dsp/LICENSE-AEC7.txt": "licenses/micyou/LICENSE-AEC7.txt", + "../crates/micyou-dsp/UPSTREAM.md": "licenses/micyou/UPSTREAM.md" + }, "icon": [ "icons/32x32.png", "icons/128x128.png", diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index e05aff4fa..f78c03616 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1553,9 +1553,37 @@ export const de: typeof zhCN = { lowLatencyDictationLabel: 'Aufnahme mit niedriger Latenz', lowLatencyDictationDesc: 'Hält einen Mikrofonstream bereit und bewahrt bis zu 500 ms im Speicher, damit Sprache direkt nach dem Kurzbefehl seltener verloren geht. Vor der Bestätigung wird Audio weder gespeichert noch hochgeladen; der Stromverbrauch steigt und die Mikrofonanzeige bleibt aktiv.', - audioEnhancementLabel: 'Leichte Mikrofonverbesserung', + dsp: { + noise: 'Noise reduction', + dereverb: 'Comb filter', + equalizer: 'Equalizer', + gain: 'Input gain', + agc: 'Automatic gain', + vad: 'Speech gate', + title: 'Speech enhancement settings', + description: + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + off: 'Off', + spectral: 'Spectral subtraction', + strength: 'Noise reduction strength', + target: 'Target level', + attack: 'Attack speed', + decay: 'Release speed', + vadHint: + 'Attenuates quiet passages; it does not stop recording. A high threshold can remove quiet speech.', + threshold: 'Speech gate threshold', + tone: 'Tone and room adjustments', + combHint: + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + preamp: 'Equalizer preamp', + order: 'Processing order', + up: 'Move up', + down: 'Move down', + reset: 'Reset enhancement settings', + }, + audioEnhancementLabel: 'Sprachverbesserung', audioEnhancementDesc: - 'Wendet eine leichte Verstärkung, weiche Begrenzung und konservative Rauschunterdrückung auf das lokale Mikrofon an. Keine Echounterdrückung. Standardmäßig aus.', + 'RNNoise- oder PureVox-Rauschunterdrückung, automatische Verstärkung und Begrenzung. Standardmäßig aus; Änderungen gelten ab der nächsten Aufnahme.', muteDuringRecordingLabel: 'Während der Aufnahme stummschalten', muteDuringRecordingDesc: 'Schaltet die Systemausgabe während der Spracheingabe vorübergehend stumm, um Lautsprecherechos zu vermeiden.', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index d876c5440..dff65f894 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1523,9 +1523,37 @@ export const en: typeof zhCN = { lowLatencyDictationLabel: 'Low-latency capture', lowLatencyDictationDesc: 'Keep a microphone stream ready and retain up to 500 ms in memory so speech immediately after the hotkey is less likely to be missed. Audio is not saved or uploaded until you confirm recording; this uses more power and keeps the microphone indicator active.', - audioEnhancementLabel: 'Lightweight microphone enhancement', + dsp: { + noise: 'Noise reduction', + dereverb: 'Comb filter', + equalizer: 'Equalizer', + gain: 'Input gain', + agc: 'Automatic gain', + vad: 'Speech gate', + title: 'Speech enhancement settings', + description: + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + off: 'Off', + spectral: 'Spectral subtraction', + strength: 'Noise reduction strength', + target: 'Target level', + attack: 'Attack speed', + decay: 'Release speed', + vadHint: + 'Attenuates quiet passages; it does not stop recording. A high threshold can remove quiet speech.', + threshold: 'Speech gate threshold', + tone: 'Tone and room adjustments', + combHint: + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + preamp: 'Equalizer preamp', + order: 'Processing order', + up: 'Move up', + down: 'Move down', + reset: 'Reset enhancement settings', + }, + audioEnhancementLabel: 'Speech enhancement', audioEnhancementDesc: - 'Applies light gain, soft limiting, and conservative noise suppression to the local microphone. It does not include echo cancellation. Off by default.', + 'Apply RNNoise or PureVox noise reduction, automatic gain and limiting. Off by default; changes apply to the next recording.', muteDuringRecordingLabel: 'Mute while recording', muteDuringRecordingDesc: 'Temporarily mute system output during voice input to avoid speaker echo.', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 50c5639fb..9f5fa38e3 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1544,9 +1544,37 @@ export const es: typeof zhCN = { lowLatencyDictationLabel: 'Captura de baja latencia', lowLatencyDictationDesc: 'Mantén preparado el flujo del micrófono y conserva hasta 500 ms en memoria para reducir la pérdida de voz justo después del atajo. El audio no se guarda ni se sube antes de confirmar la grabación; consume más energía y mantiene activo el indicador del micrófono.', - audioEnhancementLabel: 'Mejora ligera del micrófono', + dsp: { + noise: 'Noise reduction', + dereverb: 'Comb filter', + equalizer: 'Equalizer', + gain: 'Input gain', + agc: 'Automatic gain', + vad: 'Speech gate', + title: 'Speech enhancement settings', + description: + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + off: 'Off', + spectral: 'Spectral subtraction', + strength: 'Noise reduction strength', + target: 'Target level', + attack: 'Attack speed', + decay: 'Release speed', + vadHint: + 'Attenuates quiet passages; it does not stop recording. A high threshold can remove quiet speech.', + threshold: 'Speech gate threshold', + tone: 'Tone and room adjustments', + combHint: + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + preamp: 'Equalizer preamp', + order: 'Processing order', + up: 'Move up', + down: 'Move down', + reset: 'Reset enhancement settings', + }, + audioEnhancementLabel: 'Mejora de voz', audioEnhancementDesc: - 'Aplica una ganancia ligera, limitación suave y reducción de ruido conservadora al micrófono local. No incluye cancelación de eco. Desactivada de forma predeterminada.', + 'Reducción de ruido RNNoise o PureVox, ganancia automática y limitación. Desactivada por defecto; los cambios se aplican a la próxima grabación.', muteDuringRecordingLabel: 'Silenciar durante la grabación', muteDuringRecordingDesc: 'Silencia temporalmente el sonido del sistema durante la entrada de voz para evitar el eco de los altavoces.', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index c75fd0462..432bfa669 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1564,9 +1564,37 @@ export const fr: typeof zhCN = { lowLatencyDictationLabel: 'Capture à faible latence', lowLatencyDictationDesc: 'Garde un flux microphone prêt et conserve jusqu’à 500 ms en mémoire afin de réduire les mots perdus juste après le raccourci. Aucun audio n’est enregistré ni envoyé avant confirmation ; ce mode consomme davantage et maintient l’indicateur du microphone actif.', - audioEnhancementLabel: 'Amélioration légère du microphone', + dsp: { + noise: 'Noise reduction', + dereverb: 'Comb filter', + equalizer: 'Equalizer', + gain: 'Input gain', + agc: 'Automatic gain', + vad: 'Speech gate', + title: 'Speech enhancement settings', + description: + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + off: 'Off', + spectral: 'Spectral subtraction', + strength: 'Noise reduction strength', + target: 'Target level', + attack: 'Attack speed', + decay: 'Release speed', + vadHint: + 'Attenuates quiet passages; it does not stop recording. A high threshold can remove quiet speech.', + threshold: 'Speech gate threshold', + tone: 'Tone and room adjustments', + combHint: + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + preamp: 'Equalizer preamp', + order: 'Processing order', + up: 'Move up', + down: 'Move down', + reset: 'Reset enhancement settings', + }, + audioEnhancementLabel: 'Amélioration de la voix', audioEnhancementDesc: - 'Applique un gain léger, une limitation douce et une réduction prudente du bruit au microphone local. N’inclut pas l’annulation d’écho. Désactivée par défaut.', + 'Réduction du bruit RNNoise ou PureVox, gain automatique et limitation. Désactivée par défaut ; les changements prennent effet au prochain enregistrement.', muteDuringRecordingLabel: 'Couper le son pendant l’enregistrement', muteDuringRecordingDesc: 'Coupe temporairement le son système pendant la saisie vocale pour éviter l’écho des haut-parleurs.', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 42b5276b4..62197d107 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1511,9 +1511,37 @@ export const ja: typeof zhCN = { lowLatencyDictationLabel: '低遅延キャプチャ', lowLatencyDictationDesc: 'マイクストリームを準備し、最大 500 ミリ秒をメモリに保持して、ホットキー直後の音声が欠落しにくくします。録音を確定するまで音声は保存・アップロードされません。電力を多く使い、マイク使用中の表示が続きます。', - audioEnhancementLabel: '軽量マイク拡張', + dsp: { + noise: 'Noise reduction', + dereverb: 'Comb filter', + equalizer: 'Equalizer', + gain: 'Input gain', + agc: 'Automatic gain', + vad: 'Speech gate', + title: 'Speech enhancement settings', + description: + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + off: 'Off', + spectral: 'Spectral subtraction', + strength: 'Noise reduction strength', + target: 'Target level', + attack: 'Attack speed', + decay: 'Release speed', + vadHint: + 'Attenuates quiet passages; it does not stop recording. A high threshold can remove quiet speech.', + threshold: 'Speech gate threshold', + tone: 'Tone and room adjustments', + combHint: + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + preamp: 'Equalizer preamp', + order: 'Processing order', + up: 'Move up', + down: 'Move down', + reset: 'Reset enhancement settings', + }, + audioEnhancementLabel: '音声強調', audioEnhancementDesc: - 'ローカルマイクに軽いノイズ抑制、ゲイン、ソフトリミッターを適用します。エコーキャンセルは含みません。既定ではオフです。', + 'RNNoise または PureVox のノイズ抑制、自動ゲイン、リミッターを適用します。既定ではオフ。変更は次の録音から有効です。', muteDuringRecordingLabel: '録音中はミュート', muteDuringRecordingDesc: '録音中にシステム出力を一時的にミュートし、スピーカーのエコーを防ぎます。', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 918e39e27..37fa2265b 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1499,9 +1499,37 @@ export const ko: typeof zhCN = { lowLatencyDictationLabel: '저지연 캡처', lowLatencyDictationDesc: '마이크 스트림을 준비하고 최대 500ms를 메모리에 보관해 단축키 직후의 음성이 누락될 가능성을 줄입니다. 녹음을 확정하기 전에는 오디오를 저장하거나 업로드하지 않습니다. 전력 사용량이 늘고 마이크 표시가 계속 켜져 있습니다.', - audioEnhancementLabel: '가벼운 마이크 향상', + dsp: { + noise: 'Noise reduction', + dereverb: 'Comb filter', + equalizer: 'Equalizer', + gain: 'Input gain', + agc: 'Automatic gain', + vad: 'Speech gate', + title: 'Speech enhancement settings', + description: + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + off: 'Off', + spectral: 'Spectral subtraction', + strength: 'Noise reduction strength', + target: 'Target level', + attack: 'Attack speed', + decay: 'Release speed', + vadHint: + 'Attenuates quiet passages; it does not stop recording. A high threshold can remove quiet speech.', + threshold: 'Speech gate threshold', + tone: 'Tone and room adjustments', + combHint: + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + preamp: 'Equalizer preamp', + order: 'Processing order', + up: 'Move up', + down: 'Move down', + reset: 'Reset enhancement settings', + }, + audioEnhancementLabel: '음성 향상', audioEnhancementDesc: - '로컬 마이크에 가벼운 소음 억제, 게인과 소프트 리미터를 적용합니다. 에코 제거는 포함하지 않으며 기본값은 꺼짐입니다.', + 'RNNoise 또는 PureVox 소음 억제, 자동 게인 및 리미터를 적용합니다. 기본값은 꺼짐이며 다음 녹음부터 적용됩니다.', muteDuringRecordingLabel: '녹음 중 음소거', muteDuringRecordingDesc: '녹음 중 시스템 출력을 일시적으로 음소거하여 스피커 에코를 방지합니다.', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 1314e4288..ffe1cd040 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1450,8 +1450,34 @@ export const zhCN = { lowLatencyDictationLabel: '低延迟录音', lowLatencyDictationDesc: '保持麦克风流就绪，并在内存中保留最多 500 毫秒音频，减少按下热键后第一句话被漏识别的情况。确认录音前不会保存或上传音频；此模式会增加功耗并保持系统麦克风指示灯亮起。', - audioEnhancementLabel: '轻量麦克风增强', - audioEnhancementDesc: '对本机麦克风应用轻量降噪、增益与软限幅，不包含回声消除。默认关闭。', + dsp: { + title: '语音增强设置', + description: 'RNNoise 更轻量；PureVox 模型降噪耗时较高。设置在下一次录音生效。', + noise: '降噪算法', + dereverb: '梳状滤波', + equalizer: '均衡器', + gain: '输入增益', + agc: '自动增益', + vad: '语音门限', + off: '关闭降噪', + spectral: '频谱减法', + strength: '降噪强度', + target: '目标电平', + attack: '启动速度', + decay: '释放速度', + vadHint: '衰减低音量片段，不会停止录音。门限过高可能丢失轻声。', + threshold: '语音门限', + tone: '音色与房间调整', + combHint: '梳状滤波沿用 MicYou 的简单延迟相减，可能改变音色，建议从 0% 开始。', + preamp: '均衡器前置增益', + order: '处理顺序', + up: '上移', + down: '下移', + reset: '恢复增强默认设置', + }, + audioEnhancementLabel: '语音增强', + audioEnhancementDesc: + '应用 RNNoise 或 PureVox 降噪、自动增益和限幅。默认关闭，设置在下一次录音生效。', voiceActivationLabel: '语音唤醒听写', voiceActivationDesc: '仅在 Windows 桌面端启用本地语音命令唤醒听写。待机音频只在内存中处理，不会上传或保存；会增加功耗并保持麦克风指示灯亮起。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index aed89b600..388b17a85 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1452,8 +1452,34 @@ export const zhTW: typeof zhCN = { lowLatencyDictationLabel: '低延遲錄音', lowLatencyDictationDesc: '保持麥克風串流就緒，並在記憶體中保留最多 500 毫秒音訊，減少按下熱鍵後第一句話被漏辨識的情況。確認錄音前不會儲存或上傳音訊；此模式會增加耗電並保持系統麥克風指示燈亮起。', - audioEnhancementLabel: '輕量麥克風增強', - audioEnhancementDesc: '對本機麥克風套用輕量降噪、增益與軟限幅，不包含回聲消除。預設關閉。', + dsp: { + title: '語音增強設定', + description: 'RNNoise 更輕量；PureVox 模型降噪耗時較高。設定於下一次錄音生效。', + noise: '降噪演算法', + dereverb: '梳狀濾波', + equalizer: '等化器', + gain: '輸入增益', + agc: '自動增益', + vad: '語音閾值', + off: '關閉降噪', + spectral: '頻譜減法', + strength: '降噪強度', + target: '目標電平', + attack: '啟動速度', + decay: '釋放速度', + vadHint: '衰減低音量片段，不會停止錄音。閾值過高可能丟失輕聲。', + threshold: '語音閾值', + tone: '音色與房間調整', + combHint: '梳狀濾波沿用 MicYou 的簡單延遲相減，可能改變音色，建議從 0% 開始。', + preamp: '等化器前置增益', + order: '處理順序', + up: '上移', + down: '下移', + reset: '恢復增強預設設定', + }, + audioEnhancementLabel: '語音增強', + audioEnhancementDesc: + '套用 RNNoise 或 PureVox 降噪、自動增益和限幅。預設關閉，設定於下一次錄音生效。', muteDuringRecordingLabel: '錄音時靜音', muteDuringRecordingDesc: '錄音期間臨時靜音系統輸出，避免揚聲器迴音。', audioCueLabel: '錄音提示音', diff --git a/openless-all/app/src/lib/audioDsp.ts b/openless-all/app/src/lib/audioDsp.ts new file mode 100644 index 000000000..9bce1b732 --- /dev/null +++ b/openless-all/app/src/lib/audioDsp.ts @@ -0,0 +1,31 @@ +import type { AudioDspConfig } from './types'; + +export const defaultAudioDsp: AudioDspConfig = { + noiseReduction: 'rnnoise', + noiseStrength: 80, + gainDb: 0, + agcEnabled: true, + agcTarget: 35, + agcAttack: 50, + agcDecay: 50, + vadEnabled: false, + vadThresholdDb: -50, + dereverbLevel: 0, + equalizerEnabled: false, + equalizerPreampDb: 0, + equalizerBandsDb: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + processingOrder: ['noiseReduction', 'dereverb', 'equalizer', 'amplifier', 'agc', 'vad'], +}; + +export const equalizerFrequencies = [ + '31 Hz', + '63 Hz', + '125 Hz', + '250 Hz', + '500 Hz', + '1 kHz', + '2 kHz', + '4 kHz', + '8 kHz', + '16 kHz', +]; diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index e45098ce0..707497848 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -340,6 +340,25 @@ export interface StylePackRuntimeDiagnostics { previewOmitsFrontApp: boolean; } +export type AudioDspStage = + 'noiseReduction' | 'dereverb' | 'equalizer' | 'amplifier' | 'agc' | 'vad'; +export interface AudioDspConfig { + noiseReduction: 'off' | 'rnnoise' | 'purevox' | 'spectral'; + noiseStrength: number; + gainDb: number; + agcEnabled: boolean; + agcTarget: number; + agcAttack: number; + agcDecay: number; + vadEnabled: boolean; + vadThresholdDb: number; + dereverbLevel: number; + equalizerEnabled: boolean; + equalizerPreampDb: number; + equalizerBandsDb: number[]; + processingOrder: AudioDspStage[]; +} + /** Runtime health of the disposable native capsule sidecar (desktop only). */ export interface NativeCapsuleStatus { /** Whether the launch-time low-memory background switch is active. */ @@ -371,10 +390,11 @@ export interface UserPreferences { stableTranscriptionEnabled: boolean; /** Keep a bounded microphone pre-roll in memory to reduce the delay before first speech is captured. */ lowLatencyDictationEnabled: boolean; + /** Enable the session-frozen speech enhancement chain. Disabled by default. */ + audioEnhancementEnabled: boolean; + audioDsp?: AudioDspConfig; /** Desktop resource policy, applied after restarting the application. */ lowMemoryBackgroundEnabled: boolean; - /** Apply conservative noise suppression, lightweight gain, and soft limiting to local microphone input. Disabled by default. */ - audioEnhancementEnabled: boolean; /** Enable local-only voice activation while idle (Windows desktop only). */ voiceActivationEnabled: boolean; voiceActivationWakeEnabled: boolean; diff --git a/openless-all/app/src/pages/settings/AudioDspSettings.tsx b/openless-all/app/src/pages/settings/AudioDspSettings.tsx new file mode 100644 index 000000000..971e63c1a --- /dev/null +++ b/openless-all/app/src/pages/settings/AudioDspSettings.tsx @@ -0,0 +1,204 @@ +import { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import type { AudioDspConfig, AudioDspStage } from '../../lib/types'; +import { defaultAudioDsp, equalizerFrequencies } from '../../lib/audioDsp'; +import { SelectLite } from '../../components/ui/SelectLite'; +import { SettingRow, Toggle } from './shared'; + +export function AudioDspSettings({ + value, + onChange, +}: { + value?: AudioDspConfig; + onChange: (value: AudioDspConfig) => void; +}) { + const { t } = useTranslation(); + const [draft, setDraft] = useState({ ...defaultAudioDsp, ...value }); + useEffect(() => setDraft({ ...defaultAudioDsp, ...value }), [value]); + const label = (key: string, fallback: string) => + t(`settings.recording.dsp.${key}`, { defaultValue: fallback }); + const save = (patch: Partial) => { + const next = { ...draft, ...patch }; + setDraft(next); + onChange(next); + }; + const slider = ( + field: keyof AudioDspConfig, + title: string, + min: number, + max: number, + suffix = '', + ) => ( + +
+ + setDraft((previous) => ({ ...previous, [field]: Number(event.target.value) })) + } + onPointerUp={() => onChange(draft)} + onKeyUp={() => onChange(draft)} + /> + + {draft[field] as number} + {suffix} + +
+ + ); + const stageLabels: Record = { + noiseReduction: label('noise', 'Noise reduction'), + dereverb: label('dereverb', 'Comb filter'), + equalizer: label('equalizer', 'Equalizer'), + amplifier: label('gain', 'Input gain'), + agc: label('agc', 'Automatic gain'), + vad: label('vad', 'Speech gate'), + }; + const moveStage = (index: number, direction: number) => { + const order = [...draft.processingOrder]; + const target = index + direction; + if (target < 0 || target >= order.length) return; + [order[index], order[target]] = [order[target], order[index]]; + save({ processingOrder: order }); + }; + return ( +
+ + {label('title', 'Speech enhancement settings')} + +

+ {label( + 'description', + 'RNNoise is lightweight; PureVox offers model-based noise reduction with higher CPU use. Changes apply to the next recording.', + )} +

+ + save({ noiseReduction: v as AudioDspConfig['noiseReduction'] })} + options={[ + { value: 'off', label: label('off', 'Off') }, + { value: 'rnnoise', label: 'RNNoise' }, + { value: 'purevox', label: 'PureVox 6' }, + { value: 'spectral', label: label('spectral', 'Spectral subtraction') }, + ]} + /> + + {draft.noiseReduction !== 'off' && + slider('noiseStrength', label('strength', 'Noise reduction strength'), 0, 100, '%')} + {slider('gainDb', stageLabels.amplifier, -30, 30, ' dB')} + + save({ agcEnabled: !draft.agcEnabled })} /> + + {draft.agcEnabled && ( + <> + {slider('agcTarget', label('target', 'Target level'), 5, 90, '%')} + {slider('agcAttack', label('attack', 'Attack speed'), 1, 100)} + {slider('agcDecay', label('decay', 'Release speed'), 1, 100)} + + )} + + save({ vadEnabled: !draft.vadEnabled })} /> + + {draft.vadEnabled && + slider('vadThresholdDb', label('threshold', 'Speech gate threshold'), -80, -10, ' dB')} +
+ + {label('tone', 'Tone and room adjustments')} + +

+ {label( + 'combHint', + 'The comb filter follows MicYou’s simple delay subtraction. It can color speech; start at 0%.', + )} +

+ {slider('dereverbLevel', stageLabels.dereverb, 0, 100, '%')} + + save({ equalizerEnabled: !draft.equalizerEnabled })} + /> + + {draft.equalizerEnabled && ( + <> + {slider('equalizerPreampDb', label('preamp', 'Equalizer preamp'), -20, 20, ' dB')} +
+ {equalizerFrequencies.map((frequency, index) => ( + + ))} +
+ + )} +
+
+ + {label('order', 'Processing order')} + +
    + {draft.processingOrder.map((stage, index) => ( +
  1. + {stageLabels[stage]} + {' '} + +
    2. + ))} +
+
+ +
+ ); +} diff --git a/openless-all/app/src/pages/settings/RecordingInputSection.tsx b/openless-all/app/src/pages/settings/RecordingInputSection.tsx index 81fb37227..5ab942b5f 100644 --- a/openless-all/app/src/pages/settings/RecordingInputSection.tsx +++ b/openless-all/app/src/pages/settings/RecordingInputSection.tsx @@ -45,6 +45,7 @@ import { Card, Collapsible, Btn } from '../_atoms'; import { restartApp } from '../../lib/ipc'; import { SectionTitle, SettingRow, Toggle, inputStyle, segmentedTrackStyle } from './shared'; import { MicrophoneSelect } from './MicrophoneSelect'; +import { AudioDspSettings } from './AudioDspSettings'; import { detectOS, type OS } from '../../components/WindowChrome'; /** Selectable silence durations (seconds) for "auto-stop after silence"; see issue #860. */ @@ -544,14 +545,20 @@ export function RecordingInputSection({ )} - {!isAndroid && ( + <> - )} + {prefs.audioEnhancementEnabled && ( + void savePrefs({ ...prefs, audioDsp })} + /> + )} + {!isAndroid && ( ({ // tailwindcss: serves only the official shadcn components of the unified chat panel // (components/chat/chat.css entry; sources explicitly scoped to chat/ and the two panel // files); all other global styles bypass tailwind. - plugins: [react(), tailwindcss()], + plugins: [react(), tailwindcss(), micyouNotices()], resolve: { alias: { '@android': path.resolve(appRoot, 'android/frontend'),