feat(audio): native playback fills declared silence with comfort noise

doc/bin/cli.md

@@ -260,7 +260,9 @@ the one playing. With `--video off`, `--metrics <PATH>` writes a JSON line every
 250 ms with the playhead, the delay, and every gap and stall the speaker hit, in
 the schema the
 [audio quality harness](https://github.com/moq-dev/moq/tree/main/test/audio-quality)
-reduces to underruns, skips, and silence.
+reduces to underruns, skips, and silence. A silence the publisher declares, such
+as a push-to-talk mute, plays as comfort noise at the level of the quietest audio
+heard before it, and counts as no gap or stall.
 
 The broadcast's announcement is its online signal, followed until the window
 closes. Playback starts when the name is announced. A

doc/concept/audio-jitter.md

@@ -733,3 +733,54 @@ cooldown, fast-mode bypass, linked stereo, passive noise, rejected active
 noise, and 8/44.1/48 kHz seams. Both implementations consume these exact
 vectors; generating expected output from the implementation under test is
 not conformance.
+
+## Comfort noise
+
+A [declared silence](#declared-silence) is not an underrun. A receiver whose
+playback runs dry at a declared end counts no underrun or stall, and fills the
+silence with comfort noise instead of plain silence, so a muted talker's
+background carries on rather than the line going dead. It keeps doing so
+through the refill before the next utterance plays.
+
+The publisher sends nothing while muted, so the level is measured from what the
+receiver itself played since playback last resumed:
+
+```
+played(samples):                  # audio just played, not comfort noise or startup padding
+    if filling: filling = false; quietest = infinity; sum = 0; frames = 0
+    for each frame:
+        sum += every channel's sample squared
+        frames += 1
+        if frames == BLOCK:
+            quietest = min(quietest, sum / (BLOCK * channels))
+            sum = 0; frames = 0
+
+fill(samples):
+    if not filling:
+        filling = true
+        if quietest is finite: level = min(sqrt(quietest), COMFORT_MAX)
+    each sample = uniform(-1, 1) * level * sqrt(3)
+```
+
+`BLOCK` is 20 ms at the output rate, rounded, and blocks count from the first
+frame played after a silence. `COMFORT_MAX` is 0.001 RMS, -60 dBFS: audio that
+never falls quiet, such as music, would otherwise fill a pause with noise as
+loud as itself. `level` starts at 0, so a source that was already silent, such
+as a noise-suppressed microphone, gets none, and a silence after too little
+audio to fill one block keeps the previous level. Uniform noise on `[-a, a]`
+has an RMS of `a / sqrt(3)`, so the noise sits at `level`. Sums are `f64` in
+the order written; the noise itself is not part of conformance.
+
+Decoded audio stops a codec's pre-skip short of the declared end when the
+publisher leaves its encoder's lookahead unflushed, so the browser's ring takes
+the end that early. Natively, `moq play` declares the end on the playback sink
+when it reads the marker, and the stopped encoder's final frames that follow
+the marker still play before the silence.
+
+### Comfort corpus
+
+`audio-jitter/comfort/corpus.json` holds cases of constant runs played, each
+followed by the level the next silence must fill at, within 1e-9. It covers the
+quietest block rather than the last, blocks straddling a level change, a stereo
+mean, the -60 dBFS cap, an already silent source, and a silence that keeps or
+remeasures the level.

doc/concept/audio-jitter/comfort/corpus.json

@@ -0,0 +1,62 @@
+[
+	{
+		"name": "quietest",
+		"about": "The quietest whole block sets the level, not the last or the mean",
+		"rate": 1000,
+		"channels": 1,
+		"steps": [
+			{ "play": [0.5], "frames": 40 },
+			{ "play": [0.0002], "frames": 20 },
+			{ "play": [0.3], "frames": 20 },
+			{ "fill": 0.0002 }
+		]
+	},
+	{
+		"name": "straddle",
+		"about": "Blocks are counted from the first frame played, so a quiet run that straddles two blocks is diluted",
+		"rate": 1000,
+		"channels": 1,
+		"steps": [
+			{ "play": [0.0002], "frames": 15 },
+			{ "play": [0.0004], "frames": 25 },
+			{ "fill": 0.00026457513110645906 }
+		]
+	},
+	{
+		"name": "stereo",
+		"about": "A block's mean square spans every channel",
+		"rate": 1000,
+		"channels": 2,
+		"steps": [{ "play": [0.0002, 0.0004], "frames": 20 }, { "fill": 0.00031622776601683794 }]
+	},
+	{
+		"name": "cap",
+		"about": "Audio that never falls quiet fills a silence at -60 dBFS, not at its own level",
+		"rate": 1000,
+		"channels": 1,
+		"steps": [{ "play": [0.5], "frames": 40 }, { "fill": 0.001 }]
+	},
+	{
+		"name": "silent",
+		"about": "A source that was already silent gets no noise",
+		"rate": 1000,
+		"channels": 1,
+		"steps": [{ "play": [0], "frames": 40 }, { "fill": 0 }]
+	},
+	{
+		"name": "remeasure",
+		"about": "Too little audio to fill a block keeps the previous level, and each silence measures afresh from the audio after the last one",
+		"rate": 1000,
+		"channels": 1,
+		"steps": [
+			{ "play": [0.0002], "frames": 19 },
+			{ "fill": 0 },
+			{ "play": [0.0003], "frames": 20 },
+			{ "fill": 0.0003 },
+			{ "play": [0.5], "frames": 10 },
+			{ "fill": 0.0003 },
+			{ "play": [0.0005], "frames": 20 },
+			{ "fill": 0.0005 }
+		]
+	}
+]

js/watch/src/audio/comfort.test.ts

@@ -0,0 +1,33 @@
+import { expect, it } from "bun:test";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { ComfortNoise } from "./comfort";
+
+type Step = { play: number[]; frames: number } | { fill: number };
+type Case = { name: string; rate: number; channels: number; steps: Step[] };
+
+// Shared with moq-audio, so neither implementation is graded against the other.
+const corpus = join(
+	dirname(fileURLToPath(import.meta.url)),
+	"../../../../doc/concept/audio-jitter/comfort/corpus.json",
+);
+const cases = JSON.parse(readFileSync(corpus, "utf8")) as Case[];
+
+for (const { name, rate, channels, steps } of cases) {
+	it(`fills a silence at the corpus level: ${name}`, () => {
+		const comfort = new ComfortNoise(rate);
+		for (const step of steps) {
+			if ("play" in step) {
+				const output = step.play.map((value) => new Float32Array(step.frames).fill(value));
+				comfort.played(output, 0, step.frames);
+			} else {
+				comfort.fill(
+					Array.from({ length: channels }, () => new Float32Array(2)),
+					0,
+				);
+				expect(Math.abs(comfort.level - step.fill)).toBeLessThan(1e-9);
+			}
+		}
+	});
+}

js/watch/src/audio/comfort.ts

@@ -24,6 +24,11 @@ export class ComfortNoise {
 		this.#block = Math.max(1, Math.round(rate / 50));
 	}
 
+	/** The RMS of the noise the current or last silence was filled with. */
+	get level(): number {
+		return this.#level;
+	}
+
 	/** Measure `output` from `from` up to `to`, just played. */
 	played(output: Float32Array[], from: number, to: number): void {
 		if (to <= from) return;

quest/m1/push-to-talk/README.md

@@ -93,10 +93,6 @@ in [Container reads drop backfill](/quest/m1/read-backfill.md).
 The runner fails on a head that is received but never decoded, so the proof
 catches this. A complete set of runs finishes this epic.
 
-## Required
-
-- [Native playback fills declared silence with comfort noise](/quest/m1/push-to-talk/native-comfort-noise.md) - `moq play` plays comfort noise over declared silence and counts no underrun, matching the browser
-
 ## Related
 
 - [Container reads drop backfill](/quest/m1/read-backfill.md) - changes how the ordered read treats a group below its cursor, where this head was lost

rs/moq-audio/MIGRATE.md

@@ -4,6 +4,16 @@ How to update your code for each breaking change, newest first.
 
 ## Unreleased
 
+### Playback fills a declared silence with comfort noise
+
+`Sink::end()` declares that the audio written so far is followed by a silence
+its publisher announced, such as a push-to-talk mute: once it has played, the
+sink plays comfort noise rather than running dry, and the `Observer` counts no
+gap. `Sink::next()` starts the next utterance with the next write, one latency
+out, as a new sink's first write would. A player that opened a new sink at each
+declared end can keep the one it has, calling `end()` at the marker and `next()`
+before the first frame after it. Existing callers need no change.
+
 ### Playback follows a changing latency with received-audio stretch
 
 `Sink::set_latency(duration)?` updates the device callback's playout target

rs/moq-audio/src/comfort.rs

@@ -0,0 +1,144 @@
+//! Comfort noise for a silence the publisher declared, by the rule in
+//! `doc/concept/audio-jitter.md`, which `@moq/watch` follows too.
+
+/// The loudest comfort noise, in RMS: -60 dBFS. Audio that never falls quiet,
+/// such as music, would otherwise fill a pause with noise as loud as itself.
+const MAX_LEVEL: f64 = 0.001;
+
+/// White noise at the level of the quietest stretch played before a declared
+/// silence, so a muted talker's background continues instead of the line going
+/// dead. The publisher sends nothing while muted, so the level is measured from
+/// what this reader played: the quietest 20 ms block since playback last
+/// resumed. A source that was already silent there, such as a noise-suppressed
+/// microphone, gets none.
+///
+/// Allocates nothing, so it runs on the audio thread.
+pub(crate) struct Comfort {
+	channels: usize,
+	/// Frames in a block.
+	block: usize,
+	/// The block being measured: summed squares across channels, and the frames in it so far.
+	sum: f64,
+	frames: usize,
+	/// The mean square of the quietest whole block since playback last resumed.
+	quietest: f64,
+	/// The RMS of the noise for the current silence.
+	level: f64,
+	filling: bool,
+	/// xorshift32 state, never zero.
+	seed: u32,
+}
+
+impl Comfort {
+	pub(crate) fn new(rate: u32, channels: usize) -> Self {
+		Self {
+			channels,
+			block: ((rate as f64 / 50.0).round() as usize).max(1),
+			sum: 0.0,
+			frames: 0,
+			quietest: f64::INFINITY,
+			level: 0.0,
+			filling: false,
+			seed: 0x9e37_79b9,
+		}
+	}
+
+	/// Measure interleaved `output`, just played.
+	pub(crate) fn played(&mut self, output: &[f32]) {
+		if output.len() < self.channels {
+			return;
+		}
+		if std::mem::take(&mut self.filling) {
+			self.quietest = f64::INFINITY;
+			self.sum = 0.0;
+			self.frames = 0;
+		}
+		for frame in output.chunks_exact(self.channels) {
+			for &sample in frame {
+				let sample = sample as f64;
+				self.sum += sample * sample;
+			}
+			self.frames += 1;
+			if self.frames < self.block {
+				continue;
+			}
+			self.quietest = self.quietest.min(self.sum / (self.block * self.channels) as f64);
+			self.sum = 0.0;
+			self.frames = 0;
+		}
+	}
+
+	/// Fill interleaved `output` with comfort noise.
+	pub(crate) fn fill(&mut self, output: &mut [f32]) {
+		if !std::mem::replace(&mut self.filling, true) && self.quietest.is_finite() {
+			// A silence after too little audio to measure keeps the previous level.
+			self.level = self.quietest.sqrt().min(MAX_LEVEL);
+		}
+		// Uniform noise on [-a, a] has an RMS of a / sqrt(3).
+		let amplitude = (self.level * 3f64.sqrt()) as f32;
+		for sample in output {
+			self.seed ^= self.seed << 13;
+			self.seed ^= self.seed >> 17;
+			self.seed ^= self.seed << 5;
+			*sample = (self.seed as f32 / u32::MAX as f32 * 2.0 - 1.0) * amplitude;
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[derive(serde::Deserialize)]
+	struct Case {
+		name: String,
+		rate: u32,
+		channels: usize,
+		steps: Vec<Step>,
+	}
+
+	#[derive(serde::Deserialize)]
+	#[serde(untagged)]
+	enum Step {
+		Play { play: Vec<f32>, frames: usize },
+		Fill { fill: f64 },
+	}
+
+	/// Each case plays constant runs and checks the level the next silence fills at.
+	#[test]
+	fn the_level_conforms_to_the_corpus() {
+		let cases: Vec<Case> =
+			serde_json::from_str(include_str!("../../../doc/concept/audio-jitter/comfort/corpus.json")).unwrap();
+		for case in cases {
+			let mut comfort = Comfort::new(case.rate, case.channels);
+			for (index, step) in case.steps.iter().enumerate() {
+				match step {
+					Step::Play { play, frames } => comfort.played(&play.repeat(*frames)),
+					Step::Fill { fill } => {
+						let mut noise = vec![0.0; 2 * case.channels];
+						comfort.fill(&mut noise);
+						assert!(
+							(comfort.level - fill).abs() < 1e-9,
+							"{} step {index}: level {}, want {fill}",
+							case.name,
+							comfort.level
+						);
+					}
+				}
+			}
+		}
+	}
+
+	/// The noise is white and uniform, so its RMS is the level and nothing exceeds a / sqrt(3).
+	#[test]
+	fn the_noise_has_the_level_it_was_measured_at() {
+		let mut comfort = Comfort::new(48_000, 2);
+		comfort.played(&[0.0002; 960 * 2]);
+		let mut noise = vec![0.0; 48_000 * 2];
+		comfort.fill(&mut noise);
+		let peak = noise.iter().fold(0.0f32, |peak, sample| peak.max(sample.abs()));
+		let rms = (noise.iter().map(|&sample| sample as f64 * sample as f64).sum::<f64>() / noise.len() as f64).sqrt();
+		assert!(peak <= 0.0002 * 3f32.sqrt() * 1.0001, "peak {peak}");
+		assert!((rms - 0.0002).abs() < 0.000_005, "rms {rms}");
+	}
+}

rs/moq-audio/src/lib.rs

@@ -56,6 +56,8 @@ mod allocation;
 #[cfg(feature = "aac")]
 mod aac;
 mod activity;
+#[cfg(any(feature = "bus", feature = "playback"))]
+mod comfort;
 mod error;
 mod format;
 mod frame;

rs/moq-audio/src/playback/mixer.rs

@@ -335,7 +335,8 @@ impl Mixer {
 				let read = match status {
 					Read::NotReady => None,
 					Read::Underflow { frames: read } => Some(read),
-					Read::Ok | Read::Trimmed { .. } => Some(frames),
+					// A declared silence is played on purpose, so it is no gap.
+					Read::Ok | Read::Trimmed { .. } | Read::Comfort { .. } => Some(frames),
 				};
 				match read {
 					Some(read) if entry.completion.writing() => {