From c95a662dc4757c6f3c87bef6f710f32d2a2c0184 Mon Sep 17 00:00:00 2001 From: t0ms Date: Fri, 16 Oct 2026 22:23:31 +0100 Subject: [PATCH 01/12] feat(cli): report import stats and export feedback `moq import --stats` names a stats track in the catalog the import builds and fills it with what each rendition has written: frames, payload bytes, and video keyframes, with the connection's transport. `moq export --echo ` publishes the feedback a catalog solicits, counting what the export reads and its newest arrival. `moq export stats` and `moq export echo` print either as JSON lines. moq-mux counts at the container producer and the source, so every container import and export shares the counters. Capture encoders, gateways, `archive`, and `ts --program all` refuse `--stats`; MSF catalogs refuse `--echo`. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 31 ++ doc/concept/stats.md | 9 +- quest/m2/stats/README.md | 4 +- quest/m2/stats/rust.md | 67 ++-- quest/m3/msf-publish-tracks.md | 4 +- rs/moq-cli/Cargo.toml | 4 +- rs/moq-cli/src/args.rs | 140 ++++++- rs/moq-cli/src/main.rs | 76 +++- rs/moq-cli/src/publish.rs | 66 ++- rs/moq-cli/src/report.rs | 443 +++++++++++++++++++++ rs/moq-cli/src/subscribe.rs | 5 + rs/moq-cli/tests/stats.rs | 271 +++++++++++++ rs/moq-mux/src/catalog/producer.rs | 28 +- rs/moq-mux/src/container/fmp4/import.rs | 8 + rs/moq-mux/src/container/producer.rs | 22 + rs/moq-mux/src/container/source.rs | 6 + rs/moq-mux/src/container/ts/export_test.rs | 47 +++ rs/moq-mux/src/lib.rs | 1 + rs/moq-mux/src/source.rs | 24 +- rs/moq-mux/src/stats.rs | 168 ++++++++ 20 files changed, 1345 insertions(+), 79 deletions(-) create mode 100644 rs/moq-cli/src/report.rs create mode 100644 rs/moq-cli/tests/stats.rs create mode 100644 rs/moq-mux/src/stats.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 8d42bf3687c..aacd0868e9c 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -398,6 +398,37 @@ is a replacement. An export that fails while the broadcast stays up, such as on a codec TS cannot carry, exits 1 without lingering. Only `ts` can mark a break, so the other formats refuse `--linger`. +## Stats and feedback + +`import --stats` names a stats track in the catalog and reports what each +rendition has written, about once a second while someone subscribes, with the +connection's RTT, estimated send rate, and loss. It covers the stdin imports; +capture, the gateways, `archive`, and `ts --program all` refuse it. + +`export --echo ` answers a publisher that solicits feedback: under the +catalog's `echo` path it publishes `.echo`, reporting what each rendition +the export reads has received and its newest arrival, until the export ends. +The name is one path segment. It covers the stdout container exports and reads +the solicitation from the hang catalog, so `--catalog-format msf` refuses it. + +The `stats` and `echo` sinks print reports as JSON lines. `export stats` +follows one broadcast's stats track until the broadcast ends; `export echo` +prints `{"broadcast", "feedback"}` for every `.echo` broadcast under +`--broadcast` as they come and go. + +```bash +moq ... --broadcast event import --stats ts < feed.ts +moq ... --broadcast event export stats +moq ... --broadcast event export --echo alice ts > out.ts +moq ... --broadcast viewers export echo +``` + +Counters are cumulative for the life of the process: a rendition that ends +keeps its final counts, and one that returns under the same ID continues them. +An fMP4 import counts samples as frames. Encoder counters (`skipped`, +`bitrate`) and playback feedback (`decoded`, `late`, `stalls`) are not reported +yet. [Stats](/concept/stats#media-reports) has the schema. + ## Debugging `RUST_LOG=debug` prints the negotiated version and every subscription. diff --git a/doc/concept/stats.md b/doc/concept/stats.md index fa1b349ac3a..19a79b97566 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -32,9 +32,12 @@ with unavailable transport measurements absent. Missing rendition counters mean zero, and JavaScript preserves omission. A known-zero gauge or transport measurement stays explicit, so it remains distinct from unavailable. -These types do not enable automatic reporting in a publisher or player. -See the [hang draft](/draft/moq-hang#media-stats) for track framing, -field units, and the feedback convention. +`moq import --stats` and `moq export --echo` report them from the +[CLI](/bin/cli#stats-and-feedback). An import counts the frames and payload +bytes it writes to each rendition, and video keyframes; an export counts what +it reads and records its newest arrival. See the +[hang draft](/draft/moq-hang#media-stats) for track framing, field units, and +the feedback convention. ## Broadcasts diff --git a/quest/m2/stats/README.md b/quest/m2/stats/README.md index 45195f92ee9..afec56dcf89 100644 --- a/quest/m2/stats/README.md +++ b/quest/m2/stats/README.md @@ -111,8 +111,8 @@ connection view and the dashboard flow. ## Required -- [Rust reporters](/quest/m2/stats/rust.md) - the CLI, players, encoders, and - moq-mux remuxes publish stats and feedback +- [Rust reporters](/quest/m2/stats/rust.md) - the player, encoders, and + gateways publish stats and feedback, as the CLI remuxes do - [Browser reporters](/quest/m2/stats/js.md) - `` publishes stats and `` publishes feedback - [Client health](/quest/m2/stats/health.md) - two snapshots become a diff --git a/quest/m2/stats/rust.md b/quest/m2/stats/rust.md index bc9e18e0bd5..985ade5a447 100644 --- a/quest/m2/stats/rust.md +++ b/quest/m2/stats/rust.md @@ -1,52 +1,39 @@ -# [M] The Rust CLI, players, encoders, and remuxes report stats and feedback +# [M] The Rust player, encoders, and gateways report stats and feedback ## Goal -- `moq import --stats` and `moq publish --stats` announce a stats track in - the catalog and fill it with what they send. -- `moq play` and `moq export`, given `--echo `, publish an `.echo` - broadcast under each watched catalog's echo path and fill its feedback - track. -- `moq export stats ` prints a broadcast's stats snapshots, and - `moq export echo ` prints the feedback of every `.echo` broadcast - under a prefix, as JSON lines. +- `moq import capture --stats` announces a stats track in the catalog and + fills it with what its encoders send, `skipped` and `bitrate` included. +- `moq play`, given `--echo `, publishes an `.echo` broadcast under each + watched catalog's echo path and fills its feedback track with what it + received, decoded, and played. +- The gateway imports (`hls`, `rtmp`, `srt`, `rtc`) and `archive` take + `--stats`. ## Plan +- `moq import --stats` for the stdin imports, `moq export --echo`, and the + `export stats` and `export echo` sinks already report what the `moq-mux` + importers write and exporters read (`rs/moq-mux/src/stats.rs`, + `rs/moq-cli/src/report.rs`). This quest adds the counters they lack and + reuses the rest; `import --stats` refuses capture and the gateways until it + lands. - Counters come from the source, per track, as cumulative `stats()` snapshots: no callback, no per-frame lock. - - Publisher: the `moq-video` and `moq-audio` encode producers count frames, - bytes, keyframes, and drops. The `moq-mux` importers count the frames and - bytes they write per track. A remux never encodes, so without them the - CLI would have nothing to report for `moq import`. + - Publisher: the `moq-video` and `moq-audio` encode producers create their + tracks directly on the broadcast, so they count frames, bytes, keyframes, + and drops themselves, and expose the target bitrate. - Viewer: the decode and render paths count received, decoded, late, - stalled, underruns, and errors. The `moq-mux` exporters count the frames - and bytes they receive per track, so `moq export --echo` reports a row - for each rendition it remuxes. `underruns` in + stalled, underruns, and errors. `underruns` in `rs/moq-audio/src/playback` is already counted privately. -- `moq-cli` folds the counters into `hang::stats::Snapshot` on the stats - interval and writes it through `moq_json::snapshot`, with the `.z` sibling. - `transport` comes from the connection's `ConnectionStats`. - Decided in the quest audit: this quest owns generic per-rendition reporting; - [TS health stats](/quest/m2/ts-health-stats.md) owns serializing and flattening - TS container counters under `mpegts` in publisher stats and export `.echo`. - That integration stays in m2 so this quest does not duplicate its work. -- Feedback: after reading a catalog with an `echo` section, the player - resolves the echo path against the broadcast, appends `.echo`, and - publishes there with the fixed feedback - track, keyed by the catalog's rendition IDs. Each catalog update - reconciles it: a removed `echo` section, a changed path, or no longer - watching unannounces the old broadcast. A name that is not a single path segment is refused at - parse time. -- A viewer given `--echo` with `--catalog-format msf` refuses: the MSF - catalog does not carry an `echo` solicitation. Imports always emit the - MSF catalog beside hang, omitting `stats` and `echo` from MSF. -- `moq export stats` and `moq export echo` are sinks that skip `.hang` media - discovery, so they route around `catalog_format`. -- `doc/bin/cli.md` documents the flags and sinks. -- The media test publishes with `--stats` and plays with `--echo` against a - publisher that solicits feedback. It asserts that the publisher's frame - count matches what was sent and that the viewer's newest arrival advances. +- The gateways build their catalogs in their own crates, so each needs a way + to name the stats track before its first catalog. +- `moq play --echo` reconciles each watched catalog's `echo` section as + `export --echo` does, and refuses `--catalog-format msf` the same way: the + MSF catalog does not carry an `echo` solicitation. +- The media test captures with `--stats` and plays with `--echo` against a + publisher that solicits feedback. It asserts the encoder's frame count and + that the player's newest arrival advances. - Decided 2026-10-08: no E2EE refusal here. The CLI has no encrypted broadcast yet, so [Encrypted native CLI](/quest/m2/e2ee/cli.md) refuses `--stats` and `--echo` when it adds one: they would publish rendition IDs and per-track @@ -54,5 +41,5 @@ ## Related -- [TS health stats](/quest/m2/ts-health-stats.md) - adds TS-specific counters after this generic reporting plumbing lands +- [TS health stats](/quest/m2/ts-health-stats.md) - adds TS-specific counters to the reports - [Encrypted native CLI](/quest/m2/e2ee/cli.md) - refuses stats and feedback on an encrypted broadcast diff --git a/quest/m3/msf-publish-tracks.md b/quest/m3/msf-publish-tracks.md index 17ff695c5a3..7b31c2fdc43 100644 --- a/quest/m3/msf-publish-tracks.md +++ b/quest/m3/msf-publish-tracks.md @@ -11,8 +11,8 @@ consumer asks. Raised in [#5360](https://github.com/moq-dev/moq/issues/5360). Until then the [media stats schema](/doc/concept/stats.md#media-reports) leaves both sections out of -the MSF catalog. The [Rust reporters](/quest/m2/stats/rust.md) quest refuses a -viewer's `--echo` under `--catalog-format msf` when it adds that flag +the MSF catalog. `moq export --echo` refuses `--catalog-format msf`, and the +[Rust reporters](/quest/m2/stats/rust.md) quest's `moq play --echo` will (decided 2026-10-12). Open: how the entry names the hang payload, since MSF fixes its packaging diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 14bbd7016de..4de0a8d5f67 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -109,6 +109,8 @@ moq-auth = { workspace = true, features = ["serve"] } moq-clock = { workspace = true } # `server` enables the HTTP egress server for `moq export hls`; the importer is always available. moq-hls = { workspace = true, features = ["server"] } +# Writes and reads the stats and feedback snapshots. +moq-json = { workspace = true } # Names the MSF catalog track `export archive` records. moq-msf = { workspace = true } moq-mux = { workspace = true } @@ -140,8 +142,6 @@ winit = { version = "0.30.13", optional = true } sd-notify = { workspace = true } [dev-dependencies] -# Decodes the timeline the archive round-trip test checks. -moq-json = { path = "../moq-json" } # `test_relay` stands up a real relay for the `fetch` and `announced` tests to read through. moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } # Writes the PAT and PMTs, CRCs included, of the two-program input `import ts --program all` splits. diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index d89b157ba2b..e295e436819 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -339,9 +339,38 @@ impl Invocation { } for command in &self.stages { + if let Command::Import(import) = command + && import.stats + && let Some(reason) = import.source.stats_refusal() + { + anyhow::bail!("--stats names a track in the catalog the import builds, but {reason}"); + } let Command::Export(export) = command else { continue; }; + if let Some(sink) = export.sink.report() { + anyhow::ensure!( + export.catalog_format.is_none(), + "`export {sink}` reads no media, so it takes no --catalog-format" + ); + anyhow::ensure!( + export.echo.is_none(), + "`export {sink}` reads no media to report on; remove --echo" + ); + } + if export.echo.is_some() { + if export.sink.stdout().is_none() + && let Some(sink) = export.sink.ignores_selection() + { + anyhow::bail!( + "--echo reports what a stdout container export reads, and `export {sink}` is not one" + ); + } + anyhow::ensure!( + !matches!(export.catalog_format, Some(CatalogFormatArg::Msf)), + "--echo follows the hang catalog's `echo` section, which the MSF catalog does not carry" + ); + } if let Some(stdout) = export.sink.stdout() { anyhow::ensure!( stdout.linger.is_zero() || matches!(stdout.format, SubscribeFormat::Ts), @@ -745,6 +774,11 @@ pub struct Import { #[usage(long = "latency-max", hide = true)] latency_max: Option, + /// Name a stats track in the catalog and fill it with what each track has written, about + /// once a second while subscribed. The stdin imports only. + #[usage(long)] + pub stats: bool, + /// The single source feeding the Origin. #[usage(subcommand)] pub source: ImportSource, @@ -827,6 +861,22 @@ impl ImportSource { Self::Ts(_) | Self::Fmp4 | Self::Flv | Self::Hls(_) => None, } } + + /// Why `--stats` can't report this source, or `None` when it can: the CLI builds the one + /// catalog of a stdin import, whose tracks `moq-mux` counts. + fn stats_refusal(&self) -> Option<&'static str> { + match self { + Self::Ts(args) if args.program == Some(TsProgram::All) => { + Some("`import ts --program all` builds a catalog per program") + } + Self::Hls(_) | Self::Rtmp(_) | Self::Srt(_) | Self::Rtc(_) | Self::Archive(_) => { + Some("the gateway and archive imports build their own catalogs") + } + #[cfg(feature = "capture")] + Self::Capture(_) => Some("`import capture` encodes, and its encoders do not count yet"), + _ => None, + } + } } /// The MPEG-TS stdin container: which programs of a multiplex to publish. @@ -930,6 +980,12 @@ pub struct Export { #[usage(flatten)] pub select: crate::subscribe::SelectArgs, + /// Publish feedback where the broadcast's catalog solicits it: a `.echo` broadcast + /// under the catalog's `echo` path, reporting what each rendition this export reads has + /// received. One path segment. The stdout container exports only. + #[usage(long)] + pub echo: Option, + /// The single sink draining the Origin. #[usage(subcommand)] pub sink: ExportSink, @@ -981,6 +1037,10 @@ pub enum ExportSink { Rtc(crate::rtc::Args), /// Record the broadcast into an object store until it ends. Archive(crate::archive::ExportArgs), + /// Write the broadcast's stats snapshots to stdout as JSON lines. + Stats, + /// Write the feedback of every `.echo` broadcast under `--broadcast` to stdout as JSON lines. + Echo, } impl ExportSink { @@ -994,12 +1054,23 @@ impl ExportSink { Self::Srt(_) => "srt", Self::Rtc(_) => "rtc", Self::Archive(_) => "archive", + Self::Stats => "stats", + Self::Echo => "echo", }) } - /// Whether this sink writes to stdout (the container formats). + /// The report sink's name: these read no media, so they skip catalog discovery. + fn report(&self) -> Option<&'static str> { + match self { + Self::Stats => Some("stats"), + Self::Echo => Some("echo"), + _ => None, + } + } + + /// Whether this sink writes to stdout (the container formats and the report sinks). pub fn is_stdout(&self) -> bool { - self.stdout().is_some() + self.stdout().is_some() || self.report().is_some() } /// The stdout container format and its options, when this sink writes to @@ -1395,6 +1466,71 @@ mod tests { } } + /// `--stats` goes where the CLI builds the one catalog it names a track in. + #[test] + fn stats_needs_a_catalog_the_cli_builds() { + let import = |source: &[&str]| { + let argv = [ + "moq", + "--connect", + "http://relay", + "--broadcast", + "room", + "import", + "--stats", + ]; + Invocation::try_parse_from(argv.iter().chain(source).copied()).unwrap() + }; + for source in [ + ["ts"].as_slice(), + &["fmp4"], + &["flv"], + &["avc3"], + &["ts", "--program", "1"], + ] { + import(source).validate().unwrap(); + } + for source in [ + ["ts", "--program", "all"].as_slice(), + &["hls", "http://origin/master.m3u8"], + &["rtmp", "--listen", "127.0.0.1:1935"], + ] { + let err = import(source).validate().unwrap_err().to_string(); + assert!(err.contains("--stats"), "{source:?}: {err}"); + } + } + + /// `--echo` reports what a stdout container export reads from a hang catalog, under one + /// path segment, and the report sinks read no media to report on. + #[test] + fn echo_needs_a_container_export_and_a_hang_catalog() { + for sink in ["ts", "fmp4", "mkv", "flv", "h264", "h265"] { + export(&["--echo", "alice", sink]).unwrap().validate().unwrap(); + } + for sink in [["hls", "--listen", "127.0.0.1:8080"].as_slice(), &["stats"], &["echo"]] { + let err = export(&[["--echo", "alice"].as_slice(), sink].concat()) + .unwrap() + .validate() + .unwrap_err(); + assert!(err.to_string().contains("--echo"), "{sink:?}: {err}"); + } + let err = export(&["--echo", "alice", "--catalog-format", "msf", "fmp4"]) + .unwrap() + .validate() + .unwrap_err(); + assert!(err.to_string().contains("MSF"), "{err}"); + for name in ["", "a/b", ".", "..", "*"] { + assert!(export(&["--echo", name, "fmp4"]).is_err(), "`{name}` must be refused"); + } + for sink in ["stats", "echo"] { + let err = export(&["--catalog-format", "hang", sink]) + .unwrap() + .validate() + .unwrap_err(); + assert!(err.to_string().contains("--catalog-format"), "{sink}: {err}"); + } + } + #[test] fn import_ts_takes_a_program_number_or_all() { // `None` when the command line is refused. diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 6f6091c2505..46ded2ad7af 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -17,6 +17,7 @@ mod hls; mod moq; mod play; mod publish; +mod report; mod rtc; mod rtmp; mod srt; @@ -387,7 +388,7 @@ async fn run() -> anyhow::Result<()> { /// Which directions the stages need on the shared MoQ attachment. #[derive(Clone, Copy, Default)] pub struct Directions { - /// Any `import`: the Origin is published outward. + /// Any `import`, or an `export --echo`: the Origin is published outward. pub publish: bool, /// Any `export` or `play`: the Origin is filled from the network. pub consume: bool, @@ -396,8 +397,13 @@ pub struct Directions { impl Directions { /// The union of what the stages need, so one attachment serves them all. fn of(stages: &[Command]) -> Self { + let publishes = |stage: &Command| match stage { + Command::Import(_) => true, + Command::Export(export) => export.echo.is_some(), + _ => false, + }; Self { - publish: stages.iter().any(|stage| matches!(stage, Command::Import(_))), + publish: stages.iter().any(publishes), consume: stages.iter().any(|stage| matches!(stage, Command::Export(_))), } } @@ -563,9 +569,16 @@ async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Res if import.source.stdin_format().is_some() { claim("stdin", &mut stdin, &name)?; } - if let Some(publish) = - spawn_import(&origin, import, name, epoch.clone(), bandwidth.clone(), &mut tasks)? - { + let monitor = connection.as_ref().map(moq_tokio::Connection::monitor); + if let Some(publish) = spawn_import( + &origin, + import, + name, + epoch.clone(), + bandwidth.clone(), + monitor, + &mut tasks, + )? { locals.push(publish); } } @@ -573,7 +586,8 @@ async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Res if export.sink.is_stdout() { claim("stdout", &mut stdout, &name)?; } - spawn_export(&origin, export, name, &mut tasks)?; + let monitor = connection.as_ref().map(moq_tokio::Connection::monitor); + spawn_export(&origin, export, name, monitor, &mut tasks)?; } other => unreachable!("`{}` is not a stage", other.name()), } @@ -644,13 +658,15 @@ fn display_name(name: &str) -> &str { /// `ts --program all` mint their own per connection or program. /// /// Returns the pipeline that has to run on the caller's thread, for the sources -/// that have one (the stdin containers and capture). +/// that have one (the stdin containers and capture). `connection` supplies a +/// `--stats` report's transport measurements. fn spawn_import( origin: &moq_net::origin::Producer, import: Import, name: String, epoch: moq_net::Epoch, bandwidth: moq_net::bandwidth::Allocator, + connection: Option, tasks: &mut JoinSet>, ) -> anyhow::Result> { if let ImportSource::Rtc(rtc) = &import.source @@ -687,6 +703,11 @@ fn spawn_import( let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; Publish::new(broadcast, &format, config)? }; + let publish = if import.stats { + publish.with_stats(connection)? + } else { + publish + }; // Media stays servable at EOF until it goes stale, the same max-age relays retain it for. let mut publish = publish.linger(max_age.unwrap_or(hang::container::MAX_AGE)); publish.announce(epoch)?; @@ -752,10 +773,12 @@ fn spawn_import( } /// Route the shared Origin OUT to one stage's sink, filling it from the MoQ network. +/// `connection` supplies an `--echo` report's transport measurements. fn spawn_export( origin: &moq_net::origin::Producer, export: Export, name: String, + connection: Option, tasks: &mut JoinSet>, ) -> anyhow::Result<()> { if let ExportSink::Rtc(rtc) = &export.sink @@ -777,7 +800,8 @@ fn spawn_export( select: export.select, }; let consumer = origin.consume(); - tasks.spawn(async move { run_stdout(consumer, name, args).await }); + let echo = export.echo.map(|echo| (origin.clone(), echo, connection)); + tasks.spawn(async move { run_stdout(consumer, name, args, echo).await }); } else { match export.sink { ExportSink::Hls(args) => { @@ -826,6 +850,13 @@ fn spawn_export( .unwrap_or_default(); tasks.spawn(archive::export(origin.consume(), name, format, args)); } + ExportSink::Stats => { + let name = require_broadcast(name, "export stats")?; + tasks.spawn(report::print_stats(origin.consume(), name)); + } + ExportSink::Echo => { + tasks.spawn(report::print_echo(origin.consume(), name)); + } _ => unreachable!("container formats are handled by stdout_format above"), } } @@ -833,8 +864,21 @@ fn spawn_export( Ok(()) } -/// Subscribe to `name` from the Origin and write it to stdout. -async fn run_stdout(consumer: moq_net::origin::Consumer, name: String, args: SubscribeArgs) -> anyhow::Result<()> { +/// Where an `export --echo` publishes its feedback, under what name, and over which connection. +type EchoTarget = ( + moq_net::origin::Producer, + report::EchoName, + Option, +); + +/// Subscribe to `name` from the Origin and write it to stdout, publishing feedback beside it +/// where the catalog solicits it when `echo` is set. +async fn run_stdout( + consumer: moq_net::origin::Consumer, + name: String, + args: SubscribeArgs, + echo: Option, +) -> anyhow::Result<()> { let catalog = args.catalog_format(&name); // Confirm the broadcast is reachable and wait for it to be announced; `Subscribe` then @@ -845,7 +889,17 @@ async fn run_stdout(consumer: moq_net::origin::Consumer, name: String, args: Sub .await .ok_or_else(|| anyhow::anyhow!("origin closed before broadcast `{name}` was announced"))?; - Subscribe::new(consumer, &name, catalog, args).await?.run().await + let subscribe = Subscribe::new(consumer, &name, catalog, args).await?; + let Some((origin, echo, connection)) = echo else { + return subscribe.run().await; + }; + let mut echo = report::Echo::new(origin, subscribe.source().clone(), catalog, echo, connection); + let result = tokio::select! { + result = subscribe.run() => result, + never = echo.run() => match never {}, + }; + echo.finish(); + result } /// Run until an endpoint ends, keeping all tasks alive while a signalled listener drains. diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 7e7cf481a8c..e2e45e5c962 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -303,6 +303,8 @@ pub struct Publish { linger: std::time::Duration, // A stdin import's route, announced once its catalog lists the tracks; see [`Self::announce`]. route: Option, + // The `--stats` report, written beside the decode loop; see [`Self::with_stats`]. + stats: Option, } /// A broadcast's announcement, held until its catalog first goes out. @@ -362,6 +364,7 @@ impl Publish { broadcast: Some(broadcast), linger: Default::default(), route: None, + stats: None, }); } @@ -404,6 +407,7 @@ impl Publish { broadcast: Some(broadcast), linger: Default::default(), route: None, + stats: None, }) } @@ -422,6 +426,7 @@ impl Publish { broadcast: None, linger: Default::default(), route: None, + stats: None, } } @@ -462,9 +467,27 @@ impl Publish { broadcast: Some(broadcast), linger: Default::default(), route: None, + stats: None, }) } + /// Name a stats track in the catalog and write it beside the decode loop: what each track + /// has written, and the transport of `connection` when there is one. + /// + /// Call it before the importer declares its tracks, so the section rides the first catalog. + pub fn with_stats(mut self, connection: Option) -> anyhow::Result { + let (Some(broadcast), Source::Stream { catalog, .. }) = (self.broadcast.as_mut(), &self.source) else { + anyhow::bail!("--stats reports a stdin import's one catalog"); + }; + let stats = match catalog { + PublishCatalog::Media(catalog) => crate::report::Stats::new(broadcast, catalog, connection)?, + PublishCatalog::Ts(catalog) => crate::report::Stats::new(broadcast, catalog, connection)?, + PublishCatalog::TsPrograms => anyhow::bail!("`import ts --program all` builds a catalog per program"), + }; + self.stats = Some(stats); + Ok(self) + } + /// Keep a finished stdin import up for `linger` after EOF, so a subscriber that learns the /// final catalog late can still request and drain its tracks. Pass the media max-age: the /// content is servable until it goes stale. SIGINT or SIGTERM ends it early. @@ -496,7 +519,7 @@ impl Publish { (Some(broadcast), Some(route)) => Some(Pending::new(broadcast.clone(), route)?), _ => None, }; - decode(decoder, catalog, tokio::io::stdin(), self.linger, pending).await + decode(decoder, catalog, tokio::io::stdin(), self.linger, pending, self.stats).await } #[cfg(feature = "capture")] Source::Capture { @@ -556,12 +579,16 @@ impl Publish { /// renditions retire from the catalog as the decoder drops, which a subscriber would read /// as removed tracks, and a catalog dropped unfinished reads as a publisher that vanished. /// The finished tracks then stay servable for `linger`. +/// +/// `stats` reports beside the read loop, so a stalled input keeps reporting, and ends with its +/// final counts when the media tracks do. async fn decode( mut decoder: PublishDecoder, mut catalog: PublishCatalog, mut input: impl tokio::io::AsyncRead + Unpin, linger: std::time::Duration, mut pending: Option, + mut stats: Option, ) -> anyhow::Result<()> { // An empty BytesMut grows to only 64 bytes and clearing it keeps that capacity. // Stdin dispatches every read to a blocking thread, so those tiny reads can @@ -577,7 +604,7 @@ async fn decode( // Run the read/decode loop so an error surfaces here rather than // dropping the decoder (and its tracks) with a bare Error::Dropped. - let result: anyhow::Result<()> = async { + let read = async { loop { buffer.clear(); let n = tokio::io::AsyncReadExt::read_buf(&mut input, &mut buffer).await?; @@ -598,8 +625,14 @@ async fn decode( } } } - } - .await; + }; + let result: anyhow::Result<()> = match stats.as_mut() { + Some(stats) => tokio::select! { + result = read => result, + never = stats.run() => match never {}, + }, + None => read.await, + }; // Flush on a clean EOF; on any error (read, decode, or the flush // itself) abort with the real cause so subscribers see it instead of @@ -618,10 +651,13 @@ async fn decode( } match outcome { Ok(()) => { + if let Some(stats) = &mut stats { + stats.finish(); + } catalog.finish()?; - // The decoder and catalog own the tracks, so holding them keeps the finished - // broadcast servable: a subscriber that saw the final catalog only at EOF still has - // to request its media. + // The decoder, catalog, and stats report own the tracks, so holding them keeps the + // finished broadcast servable: a subscriber that saw the final catalog only at EOF + // still has to request its media. tracing::info!( ?linger, "input ended; serving the finished broadcast until its media is stale" @@ -1375,7 +1411,7 @@ pub(crate) mod tests { remaining: BBB, reads: 0, }; - decode(decoder, catalog, &mut input, Duration::ZERO, None) + decode(decoder, catalog, &mut input, Duration::ZERO, None, None) .await .unwrap(); assert_eq!( @@ -1410,7 +1446,9 @@ pub(crate) mod tests { panic!("expected a stream source"); }; let start = std::time::Instant::now(); - decode(decoder, catalog, input, Duration::ZERO, None).await.unwrap(); + decode(decoder, catalog, input, Duration::ZERO, None, None) + .await + .unwrap(); let elapsed = start.elapsed(); let bytes = INPUT.len() * copies; println!( @@ -1433,7 +1471,7 @@ pub(crate) mod tests { let Source::Stream { decoder, catalog } = publish.source else { panic!("expected a stream source"); }; - decode(decoder, catalog, BBB, Duration::ZERO, None).await.unwrap(); + decode(decoder, catalog, BBB, Duration::ZERO, None, None).await.unwrap(); drop(publish.broadcast); let mut last = None; @@ -1466,7 +1504,7 @@ pub(crate) mod tests { panic!("expected a stream source"); }; let started = tokio::time::Instant::now(); - let mut import = std::pin::pin!(decode(decoder, catalog, BBB, LINGER, None)); + let mut import = std::pin::pin!(decode(decoder, catalog, BBB, LINGER, None, None)); let name = tokio::select! { res = &mut import => panic!("the import returned before its linger: {res:?}"), name = async { @@ -1520,7 +1558,7 @@ pub(crate) mod tests { let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); let pending = Pending::new(broadcast, route).unwrap(); let (mut stdin, input_end) = tokio::io::duplex(input.len()); - let mut import = std::pin::pin!(decode(decoder, catalog, input_end, Duration::ZERO, Some(pending))); + let mut import = std::pin::pin!(decode(decoder, catalog, input_end, Duration::ZERO, Some(pending), None)); tokio::select! { res = &mut import => panic!("the import returned before its input ended: {res:?}"), @@ -1584,7 +1622,7 @@ pub(crate) mod tests { let Source::Stream { decoder, catalog } = publish.source else { panic!("stream source"); }; - let err = decode(decoder, catalog, Broken, Duration::ZERO, None) + let err = decode(decoder, catalog, Broken, Duration::ZERO, None, None) .await .unwrap_err(); assert_eq!(err.to_string(), "stdin failed"); @@ -1616,7 +1654,7 @@ pub(crate) mod tests { panic!("expected a stream source"); }; let pending = Pending::new(broadcast, Default::default()).unwrap(); - let err = decode(decoder, catalog, &input[..], Duration::ZERO, Some(pending)) + let err = decode(decoder, catalog, &input[..], Duration::ZERO, Some(pending), None) .await .unwrap_err(); assert!(err.to_string().contains("declared"), "{err:#}"); diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs new file mode 100644 index 00000000000..7324cec862d --- /dev/null +++ b/rs/moq-cli/src/report.rs @@ -0,0 +1,443 @@ +//! Media reports: the stats track `import --stats` names in its catalog, the feedback +//! `export --echo` publishes where a catalog solicits it, and the `export stats` and +//! `export echo` sinks that print them. +//! +//! Each report is one snapshot track and its compressed `.z` sibling, written about once a +//! second while someone subscribes. See the hang draft's media statistics section for the wire. + +use std::collections::{BTreeMap, HashMap}; +use std::convert::Infallible; +use std::time::Duration; + +use anyhow::Context; +use hang::moq_net; +use moq_mux::catalog::{CatalogFormat, Stream}; +use moq_tokio::connection::Monitor; +use tokio::io::AsyncWriteExt; + +/// How often a report is written: about once a second, as the hang draft asks of a producer. +const INTERVAL: Duration = Duration::from_secs(1); + +/// The publisher snapshot track an import names in its catalog's `stats` section. +pub const STATS_TRACK: &str = "stats.json"; + +/// The suffix naming a snapshot track's compressed sibling. +const COMPRESSED: &str = ".z"; + +/// Below every media track, so a report never delays the media it describes. +const PRIORITY: u8 = 0; + +/// A snapshot track and its compressed `.z` sibling, written together. +struct Pair { + plain: moq_json::snapshot::Producer, + compressed: moq_json::snapshot::Producer, +} + +impl Pair { + /// Create `name` and `name.z` on `broadcast`. + fn create(broadcast: &mut moq_net::broadcast::Producer, name: &str) -> anyhow::Result { + let info = moq_net::track::Info::default().with_priority(PRIORITY); + let plain = broadcast.create_track(name, info.clone())?; + let compressed = broadcast.create_track(format!("{name}{COMPRESSED}").as_str(), info)?; + let mut deflate = moq_json::snapshot::Config::default(); + deflate.compression = moq_json::Compression::Deflate; + Ok(Self { + plain: moq_json::snapshot::Producer::new(plain, moq_json::snapshot::Config::default().with_delta_ratio(0)), + compressed: moq_json::snapshot::Producer::new(compressed, deflate), + }) + } + + /// Publish `value` on both tracks, unless nobody subscribes to either. + fn update(&mut self, value: &T) { + if self.plain.demand().is_used() || self.compressed.demand().is_used() { + self.write(value); + } + } + + /// Publish `value` on both tracks. + fn write(&mut self, value: &T) { + for track in [&mut self.plain, &mut self.compressed] { + if let Err(err) = track.update(value) { + tracing::debug!(%err, "failed to write a report"); + } + } + } + + /// End both tracks cleanly, keeping the last snapshot. + fn finish(&mut self) { + for track in [&mut self.plain, &mut self.compressed] { + if let Err(err) = track.finish() { + tracing::debug!(%err, "failed to finish a report"); + } + } + } +} + +/// Which way the reported media crosses the connection, which picks its rate estimate. +#[derive(Clone, Copy)] +enum Direction { + Send, + Receive, +} + +/// The connection's transport measurements now, or none without a dialed connection or while +/// it reconnects. +fn transport(connection: Option<&Monitor>, direction: Direction) -> hang::stats::Transport { + let mut transport = hang::stats::Transport::default(); + if let Some(stats) = connection.and_then(Monitor::stats) { + let rate = match direction { + Direction::Send => stats.estimated_send_rate, + Direction::Receive => stats.estimated_recv_rate, + }; + transport.rtt = stats.rtt; + transport.rate = rate.map(moq_net::bandwidth::Rate::as_bps); + transport.lost_bytes = stats.bytes_lost; + transport.lost_packets = stats.packets_lost; + } + transport +} + +/// An import's stats track: what each of its tracks has written, from its catalog's counters, +/// and the connection's transport. +pub struct Stats { + tracks: Pair, + renditions: Box BTreeMap + Send>, + connection: Option, +} + +impl Stats { + /// Create the stats track on `broadcast` and name it in `catalog`'s `stats` section. + /// + /// Call it while an importer still holds its reservation, so the section rides the first + /// catalog rather than announcing a catalog with no tracks. + pub fn new( + broadcast: &mut moq_net::broadcast::Producer, + catalog: &moq_mux::catalog::Producer, + connection: Option, + ) -> anyhow::Result { + let tracks = Pair::create(broadcast, STATS_TRACK)?; + let mut catalog = catalog.clone(); + let mut guard = catalog.modify()?; + guard.stats = Some(hang::catalog::Stats::new(STATS_TRACK)); + guard.commit()?; + Ok(Self { + tracks, + renditions: Box::new(move || catalog.stats()), + connection, + }) + } + + fn snapshot(&self) -> hang::stats::Snapshot { + let mut snapshot = hang::stats::Snapshot::default(); + snapshot.transport = transport(self.connection.as_ref(), Direction::Send); + snapshot.renditions = (self.renditions)(); + snapshot + } + + fn write(&mut self) { + let snapshot = self.snapshot(); + self.tracks.update(&snapshot); + } + + /// Write a snapshot every interval, for as long as the import runs beside it. + pub async fn run(&mut self) -> Infallible { + let mut interval = tokio::time::interval(INTERVAL); + loop { + interval.tick().await; + self.write(); + } + } + + /// Write the final counts, subscribed or not, and end the track. Hold on to it while the + /// import lingers, so a late subscriber still reads them. + pub fn finish(&mut self) { + let last = self.snapshot(); + self.tracks.write(&last); + self.tracks.finish(); + } +} + +/// An `export --echo` name: the one path segment a feedback broadcast is published at, before +/// the `.echo` suffix. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EchoName(String); + +impl std::str::FromStr for EchoName { + type Err = String; + + fn from_str(name: &str) -> Result { + if name.is_empty() || name.contains('/') || matches!(name, "." | ".." | "*" | "**") { + return Err(format!("`{name}` is not a single path segment")); + } + Ok(Self(name.to_string())) + } +} + +/// The feedback broadcast in force: where it is published and its snapshot tracks. +struct Feedback { + path: moq_net::PathOwned, + // Held for the route: dropping it retracts the announcement. + _broadcast: moq_net::broadcast::Producer, + tracks: Pair, +} + +/// An export's solicited feedback: once its catalog carries an `echo` section, a +/// `/.echo` broadcast whose `feedback.json` reports what each rendition the export +/// reads has received, and the connection's transport. Each catalog update reconciles it: a +/// removed section or a changed path retracts the broadcast it replaces. It follows the catalog +/// of the instance the export first reads, and ends when the export does. +pub struct Echo { + origin: moq_net::origin::Producer, + source: moq_mux::Source, + format: CatalogFormat, + name: EchoName, + connection: Option, + feedback: Option, +} + +impl Echo { + /// Feedback for the export reading `source`, published into `origin`. + pub fn new( + origin: moq_net::origin::Producer, + source: moq_mux::Source, + format: CatalogFormat, + name: EchoName, + connection: Option, + ) -> Self { + Self { + origin, + source, + format, + name, + connection, + feedback: None, + } + } + + /// Follow the catalog's `echo` section and write a snapshot every interval, for as long as the + /// export runs beside it. + pub async fn run(&mut self) -> Infallible { + let mut catalog = match self.source.catalog::<()>(self.format).await { + Ok(catalog) => Some(catalog), + Err(err) => { + tracing::warn!(%err, "no catalog to read a feedback solicitation from"); + None + } + }; + let mut interval = tokio::time::interval(INTERVAL); + loop { + tokio::select! { + update = next(&mut catalog) => match update { + Ok(Some(update)) => self.reconcile(update.echo.as_ref()), + // The export still writes what it has read once the catalog ends, so the + // feedback in force stays up until the export finishes it. + Ok(None) => catalog = None, + Err(err) => { + tracing::warn!(%err, "stopped reading the catalog's feedback solicitation"); + catalog = None; + } + }, + _ = interval.tick() => self.write(), + } + } + } + + /// Publish at the path `echo` resolves to, retracting a broadcast at any other. + fn reconcile(&mut self, echo: Option<&hang::catalog::Echo>) { + let target = echo.and_then(|echo| { + let prefix = self.source.resolve_reference(Some(&echo.path)); + if prefix.is_none() { + tracing::warn!(path = %echo.path.as_str(), "the catalog solicits feedback above the connection root"); + } + Some(prefix?.join(format!("{}{}", self.name.0, hang::echo::SUFFIX).as_str())) + }); + if self.feedback.as_ref().map(|feedback| &feedback.path) == target.as_ref() { + return; + } + self.retire(); + let Some(path) = target else { + return; + }; + match self.publish(&path) { + Ok(feedback) => { + tracing::info!(path = %path.as_str(), "publishing feedback"); + self.feedback = Some(feedback); + self.write(); + } + Err(err) => tracing::warn!(%err, path = %path.as_str(), "failed to publish feedback"), + } + } + + fn publish(&self, path: &moq_net::PathOwned) -> anyhow::Result { + let mut broadcast = self.origin.create_broadcast(path)?; + let tracks = Pair::create(&mut broadcast, hang::echo::TRACK)?; + broadcast.announce(moq_net::origin::Route::default())?; + Ok(Feedback { + path: path.clone(), + _broadcast: broadcast, + tracks, + }) + } + + fn write(&mut self) { + let Some(feedback) = &mut self.feedback else { + return; + }; + let mut snapshot = hang::echo::Snapshot::default(); + snapshot.transport = transport(self.connection.as_ref(), Direction::Receive); + snapshot.renditions = self.source.stats(); + feedback.tracks.update(&snapshot); + } + + /// End the feedback broadcast in force, if any. + fn retire(&mut self) { + if let Some(mut feedback) = self.feedback.take() { + feedback.tracks.finish(); + } + } + + /// Write the final counts and retract the feedback broadcast. + pub fn finish(mut self) { + self.write(); + self.retire(); + } +} + +/// The next catalog update, or never once there is no catalog to read. +async fn next(catalog: &mut Option) -> moq_mux::Result>> { + match catalog { + Some(catalog) => catalog.next().await, + None => std::future::pending().await, + } +} + +/// The next snapshot of a report track, or never while there is none. +async fn snapshot( + track: &mut Option>, +) -> moq_json::Result> { + match track { + Some(track) => track.next().await, + None => std::future::pending().await, + } +} + +/// `export stats`: write the broadcast's stats snapshots to stdout as JSON lines, following the +/// track its catalog's `stats` section names, until the catalog and that track have both ended. +pub async fn print_stats(origin: moq_net::origin::Consumer, path: String) -> anyhow::Result<()> { + let mut stdout = tokio::io::stdout(); + origin + .routed(&path) + .await + .ok_or_else(|| anyhow::anyhow!("origin closed before broadcast `{path}` was announced"))?; + let broadcast = origin + .request_broadcast(path.as_str(), None) + .await + .with_context(|| format!("failed to resolve broadcast `{path}`"))?; + let source = moq_mux::Source::new(origin, broadcast.clone()); + let mut catalog = Some(source.catalog::<()>(CatalogFormat::Hang).await?); + + let mut named = None; + let mut track = None; + loop { + tokio::select! { + update = next(&mut catalog) => { + // A finished import ends its stats track just before its catalog: read it out. + let Some(update) = update? else { + if track.is_none() { + return Ok(()); + } + catalog = None; + continue; + }; + let stats = update.stats.map(|stats| stats.track); + if stats != named { + track = match &stats { + Some(name) => { + let subscriber = broadcast.track(name)?.subscribe(None).await?; + Some(moq_json::snapshot::Consumer::new(subscriber, Default::default())) + } + None => None, + }; + named = stats; + } + } + value = snapshot(&mut track) => match value? { + Some(value) => { + stdout.write_all(format!("{value}\n").as_bytes()).await?; + stdout.flush().await?; + } + None if catalog.is_none() => return Ok(()), + None => track = None, + }, + } + } +} + +/// One line of `export echo`: the feedback broadcast and its latest snapshot. +#[derive(serde::Serialize)] +struct EchoLine<'a> { + broadcast: &'a str, + feedback: serde_json::Value, +} + +/// `export echo`: write the feedback of every `.echo` broadcast under `prefix` to stdout as JSON +/// lines, following them as they start and end. +pub async fn print_echo(origin: moq_net::origin::Consumer, prefix: String) -> anyhow::Result<()> { + let mut stdout = tokio::io::stdout(); + let prefix = moq_net::Path::new(&prefix).to_owned(); + let mut announced = origin.announced(); + let (lines, mut written) = tokio::sync::mpsc::unbounded_channel::(); + let mut readers = HashMap::new(); + loop { + tokio::select! { + event = announced.next() => { + let event = event.context("announcements ended")?; + let (announce, active) = match event { + moq_net::announce::Event::Start(announce) => (announce, true), + moq_net::announce::Event::End(announce) => (announce, false), + _ => continue, + }; + let path = announce.prefix; + if !path.has_prefix(&prefix) || !hang::echo::is_echo(&path) { + continue; + } + if active { + let reader = tokio::spawn(read_echo(origin.clone(), path.clone(), lines.clone())); + readers.insert(path, reader); + } else if let Some(reader) = readers.remove(&path) { + reader.abort(); + } + } + Some(line) = written.recv() => { + stdout.write_all(line.as_bytes()).await?; + stdout.flush().await?; + } + } + } +} + +/// Send a line for each feedback snapshot the broadcast at `path` publishes, until it ends. +async fn read_echo( + origin: moq_net::origin::Consumer, + path: moq_net::PathOwned, + lines: tokio::sync::mpsc::UnboundedSender, +) { + let read = async { + let broadcast = origin.request_broadcast(&path, None).await?; + let subscriber = broadcast.track(hang::echo::TRACK)?.subscribe(None).await?; + let mut track = moq_json::snapshot::Consumer::::new(subscriber, Default::default()); + while let Some(feedback) = track.next().await? { + let line = EchoLine { + broadcast: path.as_str(), + feedback, + }; + if lines.send(format!("{}\n", serde_json::to_string(&line)?)).is_err() { + break; + } + } + anyhow::Ok(()) + }; + if let Err(err) = read.await { + tracing::debug!(%err, path = %path.as_str(), "stopped reading feedback"); + } +} diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 4d0fcd780ee..4b72048eac8 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -279,6 +279,11 @@ impl Subscribe { }) } + /// The resolved broadcast, whose [`stats`](moq_mux::Source::stats) count what the export reads. + pub fn source(&self) -> &moq_mux::Source { + &self.source + } + /// Build the catalog stream, narrowed by the rendition selection flags. The /// catalog source honors the requested format (e.g. compressed `HangZ` or `Msf`). async fn stream(&self) -> anyhow::Result> { diff --git a/rs/moq-cli/tests/stats.rs b/rs/moq-cli/tests/stats.rs new file mode 100644 index 00000000000..37e33067b56 --- /dev/null +++ b/rs/moq-cli/tests/stats.rs @@ -0,0 +1,271 @@ +//! `moq import --stats` and `moq export --echo` over a real relay, read back through the +//! `export stats` and `export echo` sinks: each report counts what its side carried. +#![cfg(unix)] + +use std::collections::BTreeMap; +use std::process::Stdio; +use std::time::Duration; + +use hang::moq_net; +use moq_mux::container::ts; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines}; +use tokio::process::{Child, ChildStdout}; + +const TIMEOUT: Duration = Duration::from_secs(30); +/// Five seconds of HEVC and Opus at a constant 400 kb/s. +const CLIP: &[u8] = include_bytes!("../../moq-mux/src/container/ts/test_data/bbb5s_cbr.ts"); +/// Bytes fed every [`PACE`], which keeps [`CLIP`] near real time. +const CHUNK: usize = 188 * 40; +const PACE: Duration = Duration::from_millis(150); +/// Past the two seconds of PCR a live TS catalog waits for, so the broadcast is up by then. +const HALF: usize = CLIP.len() / 2; + +/// Feed `clip` to `write` at about real time. +async fn feed(clip: &[u8], mut write: impl AsyncFnMut(&[u8])) { + let mut pace = tokio::time::interval(PACE); + for chunk in clip.chunks(CHUNK) { + pace.tick().await; + write(chunk).await; + } +} + +struct Relay { + url: String, + fingerprint: String, + origin: moq_net::origin::Producer, +} + +async fn relay() -> Relay { + let _ = moq_tokio::crypto::install_default(); + let fixture = moq_relay::test_relay().await.expect("test relay"); + let origin = fixture.relay.cluster().origin.clone(); + let ready = fixture.relay.ready(); + tokio::spawn(fixture.relay.run()); + ready.wait().await.expect("relay ready"); + Relay { + url: fixture.url.to_string(), + fingerprint: fixture.fingerprint, + origin, + } +} + +fn moq(relay: &Relay, broadcast: &str, args: &[&str]) -> tokio::process::Command { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_moq")); + // A `MOQ_*` variable in the developer's shell must not reconfigure the child. + for (name, _) in std::env::vars_os() { + if name.to_string_lossy().starts_with("MOQ_") { + command.env_remove(name); + } + } + command + .args(["--connect", &relay.url, "--connect-tls-fingerprint", &relay.fingerprint]) + .args(["--broadcast", broadcast]) + .args(args) + .kill_on_drop(true); + command +} + +/// Start `moq export ` on `broadcast`, reading the JSON lines it prints. +fn sink(relay: &Relay, broadcast: &str, sink: &str) -> (Child, Lines>) { + let mut child = moq(relay, broadcast, &["export", sink]) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn the sink"); + let lines = BufReader::new(child.stdout.take().expect("stdout")).lines(); + (child, lines) +} + +/// The next line the sink prints, or `None` once it exits. +async fn next(lines: &mut Lines>) -> Option { + let line = tokio::time::timeout(TIMEOUT, lines.next_line()) + .await + .expect("the sink printed nothing") + .expect("read the sink"); + line.map(|line| serde_json::from_str(&line).unwrap_or_else(|err| panic!("`{line}` is not a report: {err}"))) +} + +async fn wait(child: &mut Child) -> std::process::ExitStatus { + tokio::time::timeout(TIMEOUT, child.wait()) + .await + .expect("moq never exited") + .expect("wait for moq") +} + +/// What a TS import of `clip` writes to each rendition, from an in-process import. +fn written(clip: &[u8]) -> BTreeMap { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let config = + moq_mux::catalog::Config::default().with_catalog(moq_mux::catalog::hang::Catalog::::default()); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); + let mut import = ts::Import::new(broadcast, catalog.reserve()); + import.decode(clip).unwrap(); + import.finish().unwrap(); + catalog.stats() +} + +/// The stats an import names in its catalog count each rendition while it runs, and its last +/// report holds what the import wrote in full. +#[tokio::test] +async fn import_stats_count_what_each_rendition_wrote() { + let relay = relay().await; + let expected = written(CLIP); + assert_eq!(expected.len(), 2, "the clip carries video and audio: {expected:?}"); + + let mut import = moq(&relay, "demo", &["import", "--stats", "--max-age", "2s", "ts"]) + .stdin(Stdio::piped()) + .spawn() + .expect("spawn the import"); + let mut stdin = import.stdin.take().expect("stdin"); + let (mut sink, mut lines) = sink(&relay, "demo", "stats"); + + // Half the clip, then a report before the rest: the import reports while it runs, and a + // stalled input keeps reporting until the sink has joined. + feed(&CLIP[..HALF], async |chunk| { + stdin.write_all(chunk).await.expect("write stdin") + }) + .await; + let mut reports: Vec = vec![next(&mut lines).await.expect("a report mid-import")]; + feed(&CLIP[HALF..], async |chunk| { + stdin.write_all(chunk).await.expect("write stdin") + }) + .await; + drop(stdin); + while let Some(report) = next(&mut lines).await { + reports.push(report); + } + assert!(wait(&mut import).await.success()); + assert!(wait(&mut sink).await.success(), "the sink ends with the broadcast"); + + let last = reports.last().unwrap(); + assert_eq!(last.renditions, expected, "the last report holds everything written"); + let (first, written) = (&reports[0].renditions, &last.renditions); + assert!( + !first.is_empty() + && first + .iter() + .all(|(name, rendition)| rendition.frames < written[name].frames), + "the first report counts half the clip: {first:?}" + ); + assert!( + reports.iter().any(|report| report.transport.rtt.is_some()), + "the import reports its connection" + ); +} + +/// A sink that joins once the import has finished, while it lingers, still reads its final +/// counts. +#[tokio::test] +async fn a_late_sink_reads_the_final_stats() { + let relay = relay().await; + let expected = written(CLIP); + + let mut import = moq(&relay, "demo", &["import", "--stats", "--max-age", "10s", "ts"]) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn the import"); + let mut stdin = import.stdin.take().expect("stdin"); + stdin.write_all(CLIP).await.expect("write stdin"); + drop(stdin); + let mut stderr = BufReader::new(import.stderr.take().expect("stderr")).lines(); + let ended = tokio::time::timeout(TIMEOUT, async { + while let Some(line) = stderr.next_line().await.expect("read the import's stderr") { + if line.contains("input ended") { + return; + } + } + panic!("the import exited before its input ended"); + }); + ended.await.expect("the import never reached the end of its input"); + + let (mut sink, mut lines) = sink(&relay, "demo", "stats"); + let report: hang::stats::Snapshot = next(&mut lines).await.expect("the final report"); + assert_eq!(report.renditions, expected); + sink.kill().await.unwrap(); + import.kill().await.unwrap(); +} + +/// One line of `export echo`. +#[derive(serde::Deserialize)] +struct Feedback { + broadcast: String, + feedback: hang::echo::Snapshot, +} + +/// An export asked for feedback publishes it where the catalog solicits it, counting what it +/// reads as it reads it, and `export echo` finds it under that prefix. +#[tokio::test] +async fn export_echo_reports_what_the_export_reads() { + let relay = relay().await; + let expected = written(CLIP); + + // A publisher soliciting feedback under `viewers`, a sibling of its broadcast. + let mut broadcast = relay.origin.create_broadcast("demo").expect("create the broadcast"); + let config = moq_mux::catalog::Config::default() + .with_max_age(Some(Duration::from_secs(1))) + .with_catalog(moq_mux::catalog::hang::Catalog::::default()); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, config).unwrap(); + let mut import = ts::Import::new(broadcast.clone(), catalog.reserve()); + let mut solicit = catalog.modify().unwrap(); + solicit.echo = Some(hang::catalog::Echo::new("viewers".to_string())); + solicit.commit().unwrap(); + broadcast.announce(moq_net::origin::Route::default()).unwrap(); + + let (mut sink, mut lines) = sink(&relay, "viewers", "echo"); + let mut export = moq(&relay, "demo", &["export", "--echo", "alice", "ts"]) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn the export"); + let mut output = export.stdout.take().expect("stdout"); + let draining = tokio::spawn(async move { tokio::io::copy(&mut output, &mut tokio::io::sink()).await }); + + // Half the clip, then feedback before the rest, so the export reports while it reads. + feed(&CLIP[..HALF], async |chunk| import.decode(chunk).unwrap()).await; + let mut feedback: Vec = vec![next(&mut lines).await.expect("feedback mid-export")]; + feed(&CLIP[HALF..], async |chunk| import.decode(chunk).unwrap()).await; + import.finish().unwrap(); + catalog.finish().unwrap(); + + let status = loop { + tokio::select! { + line = next(&mut lines) => feedback.push(line.expect("the echo sink runs until stopped")), + status = export.wait() => break status.expect("wait for the export"), + } + }; + assert!(status.success(), "the export finishes cleanly, got {status}"); + // The export writes its final counts as it exits. + while let Ok(Some(line)) = tokio::time::timeout(Duration::from_secs(2), next(&mut lines)).await { + feedback.push(line); + } + draining.await.unwrap().expect("drain the export"); + sink.kill().await.unwrap(); + drop(broadcast); + + assert!( + feedback.iter().all(|line| line.broadcast == "viewers/alice.echo"), + "the feedback is published at the solicited path" + ); + let first = &feedback.first().expect("the sink printed feedback").feedback; + let last = &feedback.last().unwrap().feedback; + assert!(!last.renditions.is_empty(), "the export reports what it reads"); + for (name, read) in &last.renditions { + assert!( + read.frames > 0 && read.frames <= expected[name].frames, + "{name}: read {} of {} frames", + read.frames, + expected[name].frames + ); + let newest = read.arrival.as_ref().expect("an arrival").timestamp; + let earliest = first.renditions.get(name).and_then(|read| read.arrival.as_ref()); + assert!( + earliest.is_none_or(|earliest| earliest.timestamp < newest), + "{name}: the newest arrival advances over {} lines: {earliest:?} then {:?}", + feedback.len(), + read.arrival + ); + } + assert!( + feedback.iter().any(|line| line.feedback.transport.rtt.is_some()), + "the export reports its connection" + ); +} diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 0598dd02a93..dbf85b3209f 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -288,6 +288,8 @@ pub struct Producer { /// The minimum flush lateness across this catalog's renditions, which each rendition's /// advertised `delay` is measured against. baseline: super::estimate::Baseline, + /// What each media track minted under this catalog has written. See [`stats`](Self::stats). + sent: Arc>, } // Manual Clone so a producer is cheaply clonable regardless of whether `E` is. @@ -301,6 +303,7 @@ impl Clone for Producer { max_age: self.max_age, bandwidth: self.bandwidth.clone(), baseline: self.baseline.clone(), + sent: self.sent.clone(), } } } @@ -468,6 +471,7 @@ impl Producer { max_age: config.max_age, bandwidth: config.bandwidth, baseline: Default::default(), + sent: Default::default(), }) } @@ -804,9 +808,11 @@ impl Producer { { let mut catalog = self.clone(); let recorder = catalog.enroll(track.name(), timeline_hint)?; + let sent = self.sent(rendition.name()); let mut producer = crate::container::Producer::with_rendition(track, container, rendition) .with_recorder(recorder) - .with_bandwidth(self.bandwidth.clone()); + .with_bandwidth(self.bandwidth.clone()) + .with_sent(sent); if let Some(config) = config { producer.set(config)?; } @@ -825,9 +831,27 @@ impl Producer { { let mut catalog = self.clone(); let recorder = catalog.enroll(track.name(), None)?; + let sent = self.sent(track.name()); Ok(crate::container::Producer::new(track, container) .with_recorder(recorder) - .with_bandwidth(self.bandwidth.clone())) + .with_bandwidth(self.bandwidth.clone()) + .with_sent(sent)) + } + + /// The counters a track writing under catalog key `name` bumps, for one that writes its groups + /// by hand rather than through a [`container::Producer`](crate::container::Producer). + pub(crate) fn sent(&self, name: &str) -> Arc { + self.sent.track(name) + } + + /// What each media track minted under this catalog has written, keyed by its catalog key: the + /// rendition ID, or the track name for an `m2ts` or verbatim `mpegts` track. + /// + /// Counters are cumulative for the life of the catalog, so a track that ended keeps its final + /// counts and a later track under the same key continues them. Fold this into a + /// [`hang::stats::Snapshot`] on the stats interval. + pub fn stats(&self) -> std::collections::BTreeMap { + self.sent.sent() } /// A fresh estimator whose `delay` is measured against this catalog's other renditions. diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index d1fe0a9f13c..d3fe4a11e24 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -135,6 +135,9 @@ struct Fmp4Track { /// hand, so this sits beside the estimator the same way it does on a /// [`container::Producer`](crate::container::Producer). claim: crate::catalog::Claim, + + /// The samples and sample bytes the catalog reports this track has written. + sent: std::sync::Arc, } impl Fmp4Track { @@ -340,6 +343,7 @@ impl Import { } }; + let sent = self.catalog.sent(track.name()); self.tracks.insert( track_id, Fmp4Track { @@ -355,6 +359,7 @@ impl Import { pending_sequence: None, estimator: Estimator::new(), claim: crate::catalog::Claim::new(self.catalog.bandwidth()), + sent, }, ); } @@ -785,6 +790,7 @@ impl Import { let mut min_timestamp = None; let mut max_end = None; let mut contains_keyframe = false; + let mut keyframes = 0u64; let total_samples: usize = traf.trun.iter().map(|t| t.entries.len()).sum(); let mut sample_index = 0usize; @@ -850,6 +856,7 @@ impl Import { }; contains_keyframe |= keyframe; + keyframes += u64::from(keyframe && track.kind == Kind::Video); if min_timestamp.is_none_or(|min| timestamp <= min) { min_timestamp = Some(timestamp); @@ -1012,6 +1019,7 @@ impl Import { track.group = Some(g); track.estimator.write(timestamp, fragment_len); + track.sent.add(total_samples as u64, track_mdat_data.len(), keyframes); let end = max_end.ok_or(Error::MissingTrun)?; if let Some(recorder) = track.recorder.as_mut() { recorder.end(end); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index be527d22320..9d29539baf2 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -97,6 +97,9 @@ pub struct Producer { /// The catalog rendition this media producer owns, when it was created through a catalog. rendition: Option>>, + + /// The counters the catalog reports for this track, when it was created through a catalog. + sent: Option>, } trait Rendition: Send { @@ -164,6 +167,7 @@ impl Producer { estimator: crate::catalog::Estimator::new(), bandwidth: None, rendition: None, + sent: None, } } } @@ -200,6 +204,7 @@ where estimator: rendition.estimator(), bandwidth: None, rendition: Some(Box::new(rendition)), + sent: None, } } @@ -370,6 +375,21 @@ where self } + /// Count every frame this track writes into `sent`, which its catalog reports. + pub(crate) fn with_sent(mut self, sent: std::sync::Arc) -> Self { + self.sent = Some(sent); + self + } + + /// Count a written frame. Only video keyframes count as keyframes: elsewhere the bit only + /// marks where a group starts. + fn count(&self, bytes: usize, keyframe: bool) { + if let Some(sent) = &self.sent { + let keyframe = keyframe && self.container.kind() == Kind::Video; + sent.add(1, bytes, keyframe.into()); + } + } + /// Raise the standing claim when the catalog estimate has a new peak. fn claim(&mut self) { let Some(bandwidth) = self.bandwidth.as_mut() else { @@ -466,11 +486,13 @@ where // group, a timestamp that won't convert) leaves the producer usable, and the estimate's // extrema never fall, so counting one would inflate the catalog for good. self.estimator.write(timestamp, bytes); + self.count(bytes, keyframe); self.observe_end(timestamp, duration); } else { // Buffered frames are measured on the way in instead. The flush that eventually writes // them takes the track down with it when it fails, so there is nothing to unwind. self.estimator.write(frame.timestamp, frame.payload.len()); + self.count(frame.payload.len(), frame.keyframe); self.observe_end(frame.timestamp, frame.duration); self.buffer.push(frame); diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 09c4a701c2c..06025f01d81 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -96,6 +96,8 @@ pub(crate) struct ExportSource { cuts: u64, // A stopped encoder may still emit final packets in this group on the old timeline. ends: Option<(u64, u64)>, + /// The counters the export source reports for this rendition, as received, before any transform. + received: std::sync::Arc, } impl ExportSource { @@ -182,6 +184,7 @@ impl ExportSource { open: false, cuts: 0, ends: None, + received: source.received(name), }; source.resolve_video_dimensions(&[])?; Ok(Some(source)) @@ -219,6 +222,7 @@ impl ExportSource { open: false, cuts: 0, ends: None, + received: source.received(name), })) } @@ -249,6 +253,7 @@ impl ExportSource { open: false, cuts: 0, ends: None, + received: source.received(name), }) } @@ -413,6 +418,7 @@ impl ExportSource { self.finish_ends(); } self.open = true; + self.received.frame(frame.timestamp, frame.payload.len()); frame } Some(Item::Read(Read::Gap { timestamp, duration })) => { diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 3f7b293e001..22eb93b8aaf 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2116,6 +2116,53 @@ async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str, kind: frames } +/// The import counts what each rendition writes and the export what it reads: both agree with +/// the frames a subscriber reads back, and only video counts keyframes. +#[tokio::test(start_paused = true)] +async fn stats_count_what_the_import_writes_and_the_export_reads() { + let data = include_bytes!("test_data/bbb_cbr.ts"); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve()); + import.decode(&BytesMut::from(&data[..])).unwrap(); + import.finish().unwrap(); + + let snapshot = catalog.snapshot(); + let sent = catalog.stats(); + let video = snapshot.video.renditions.keys().map(|name| (name, Kind::Video)); + let audio = snapshot.audio.renditions.keys().map(|name| (name, Kind::Audio)); + let renditions: Vec<_> = video.chain(audio).collect(); + assert_eq!( + renditions.len(), + 2, + "the clip carries one video and one audio rendition" + ); + assert_eq!(sent.len(), renditions.len(), "a report per rendition: {sent:?}"); + for (name, kind) in renditions { + let frames = read_frames(&consumer, name, kind).await; + let report = &sent[name]; + assert!(!frames.is_empty(), "{name} carries frames"); + assert_eq!(report.frames, frames.len() as u64, "{name} frames"); + assert_eq!( + report.bytes, + frames.iter().map(Vec::len).sum::() as u64, + "{name} bytes" + ); + assert_eq!(report.keyframes > 0, kind == Kind::Video, "{name} keyframes"); + } + + let source = crate::source::announced(&consumer).await; + drain_with(open_export(source.clone()).await.unwrap()).await; + let received = source.stats(); + for (name, report) in &sent { + assert_eq!(received[name].frames, report.frames, "{name} frames read"); + assert_eq!(received[name].bytes, report.bytes, "{name} bytes read"); + assert!(received[name].arrival.is_some(), "{name} records its newest arrival"); + } +} + /// Both real Kyrion MP2 programs must survive TS -> MoQ -> TS byte-for-byte, and /// the PMT must re-announce them as MPEG-1 audio (0x03): the capture is 48 kHz, /// so the half-rate type (0x04) would be unfaithful. This capture is a dirty start diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index c4bb052d27e..81802d8b006 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -41,6 +41,7 @@ mod pace; pub mod rate; pub mod select; mod source; +mod stats; pub mod timeline; pub use clock::Clock; diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 5cb73cb4188..c8f6c9437b1 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -34,6 +34,9 @@ pub struct Source { origin: moq_net::origin::Consumer, path: moq_net::PathOwned, pins: Pins, + /// What each rendition an export reads through this source has received. See + /// [`stats`](Self::stats). + received: Arc>, } impl Source { @@ -52,20 +55,39 @@ impl Source { origin, path, pins: Arc::new(Mutex::new(pins)), + received: Default::default(), } } /// This source holding nothing but `broadcast` at its own path: an export switching to - /// another instance starts over from the broadcast it switched to. + /// another instance starts over from the broadcast it switched to. Its counters carry on. pub(crate) fn switched(&self, broadcast: &moq_net::broadcast::Consumer) -> Self { let pins = HashMap::from([(self.path.clone(), broadcast.clone())]); Self { origin: self.origin.clone(), path: self.path.clone(), pins: Arc::new(Mutex::new(pins)), + received: self.received.clone(), } } + /// The counters an export reading rendition `name` bumps. + pub(crate) fn received(&self, name: &str) -> Arc { + self.received.track(name) + } + + /// What each rendition exported through this source (and its clones) has received, keyed by + /// rendition ID in the catalog that listed it, including renditions referenced from another + /// broadcast. + /// + /// Counters are cumulative for the life of the source, across resubscriptions and a + /// [`Follower`](crate::container::ts::Follower) switching instances, so a rendition no export + /// reads any more keeps its final counts. Fold this into an + /// [`echo::Snapshot`](hang::echo::Snapshot) on the feedback interval. + pub fn stats(&self) -> std::collections::BTreeMap { + self.received.received() + } + /// Serve this source's own path from `broadcast` from now on, as when the same instance /// returns on a new route and the old broadcast's routes are gone. pub(crate) fn pin(&self, broadcast: &moq_net::broadcast::Consumer) { diff --git a/rs/moq-mux/src/stats.rs b/rs/moq-mux/src/stats.rs new file mode 100644 index 00000000000..f877a495f49 --- /dev/null +++ b/rs/moq-mux/src/stats.rs @@ -0,0 +1,168 @@ +//! Cumulative per-rendition counters behind a publisher's stats and a viewer's feedback. +//! +//! Each track bumps its own atomics as it writes or reads a frame, so counting takes no lock. A +//! reporter reads the whole set about once a second through the [`Registry`] the tracks enrolled in: +//! the catalog [`Producer`](crate::catalog::Producer) for what a publisher writes, the export +//! [`Source`](crate::Source) for what a viewer reads. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// What one published track has written. +#[derive(Default)] +pub(crate) struct Sent { + frames: AtomicU64, + bytes: AtomicU64, + keyframes: AtomicU64, +} + +impl Sent { + /// Count `frames` media frames carrying `bytes` of payload, `keyframes` of them keyframes. + pub(crate) fn add(&self, frames: u64, bytes: usize, keyframes: u64) { + self.frames.fetch_add(frames, Relaxed); + self.bytes.fetch_add(bytes as u64, Relaxed); + self.keyframes.fetch_add(keyframes, Relaxed); + } + + fn report(&self) -> hang::stats::Rendition { + let mut report = hang::stats::Rendition::default(); + report.frames = self.frames.load(Relaxed); + report.bytes = self.bytes.load(Relaxed); + report.keyframes = self.keyframes.load(Relaxed); + report + } +} + +/// What one exported track has read. +#[derive(Default)] +pub(crate) struct Received { + frames: AtomicU64, + bytes: AtomicU64, + /// The latest frame's media timestamp, in microseconds. + timestamp: AtomicU64, + /// When the latest frame arrived, in milliseconds since the Unix epoch, or zero before the first. + wall: AtomicU64, +} + +impl Received { + /// Count one frame of `bytes` payload at media `timestamp`, arriving now. + pub(crate) fn frame(&self, timestamp: moq_net::Timestamp, bytes: usize) { + self.frames.fetch_add(1, Relaxed); + self.bytes.fetch_add(bytes as u64, Relaxed); + // A clock before the MoQ epoch has no wire encoding, so such an arrival is never recorded. + let wall = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| u64::try_from(since.as_millis()).unwrap_or(0)); + if wall < hang::catalog::MOQ_EPOCH_UNIX_MILLIS { + return; + } + let micros = u64::try_from(timestamp.as_micros()).unwrap_or(u64::MAX); + self.timestamp.store(micros, Relaxed); + self.wall.store(wall, Relaxed); + } + + fn report(&self) -> hang::echo::Rendition { + let mut report = hang::echo::Rendition::default(); + report.frames = self.frames.load(Relaxed); + report.bytes = self.bytes.load(Relaxed); + // The pair is read without a lock, so it may straddle two consecutive frames: off by one + // frame interval at most, which no latency reading resolves. + let wall = self.wall.load(Relaxed); + if wall != 0 + && let Ok(timestamp) = moq_net::Timestamp::from_micros(self.timestamp.load(Relaxed)) + { + let wall = UNIX_EPOCH + Duration::from_millis(wall); + report.arrival = Some(hang::echo::Arrival::new(timestamp, wall)); + } + report + } +} + +/// Counters keyed by rendition ID, shared by the tracks that bump them and the reporter that reads +/// them. +pub(crate) struct Registry(Mutex>>); + +impl Default for Registry { + fn default() -> Self { + Self(Mutex::default()) + } +} + +impl Registry { + /// The counters for `name`, continuing those of an earlier track under that name. + pub(crate) fn track(&self, name: &str) -> Arc { + let mut entries = self.0.lock().unwrap_or_else(PoisonError::into_inner); + entries.entry(name.to_string()).or_default().clone() + } + + /// Read every entry. A track that ended keeps its final counts for the registry's life, as the + /// catalog a finished import leaves up still lists it. + fn report(&self, read: impl Fn(&T) -> R) -> BTreeMap { + let entries = self.0.lock().unwrap_or_else(PoisonError::into_inner); + entries + .iter() + .map(|(name, counters)| (name.clone(), read(counters))) + .collect() + } +} + +impl Registry { + /// What each enrolled track has written. + pub(crate) fn sent(&self) -> BTreeMap { + self.report(Sent::report) + } +} + +impl Registry { + /// What each enrolled track has read. + pub(crate) fn received(&self) -> BTreeMap { + self.report(Received::report) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_ended_track_keeps_its_final_counts() { + let registry = Registry::::default(); + let track = registry.track("video"); + track.add(3, 300, 1); + drop(track); + + for _ in 0..2 { + let report = registry.sent(); + assert_eq!(report["video"].frames, 3); + assert_eq!(report["video"].bytes, 300); + assert_eq!(report["video"].keyframes, 1); + } + } + + #[test] + fn a_returning_track_continues_its_counts() { + let registry = Registry::::default(); + registry.track("video").add(2, 20, 0); + registry.track("video").add(1, 10, 0); + assert_eq!(registry.sent()["video"].frames, 3); + assert_eq!(registry.sent()["video"].bytes, 30); + } + + #[test] + fn arrival_pairs_the_latest_timestamp_with_its_wall_time() { + let registry = Registry::::default(); + let track = registry.track("hd"); + assert_eq!(registry.received()["hd"].arrival, None); + + let before = SystemTime::now(); + track.frame(moq_net::Timestamp::from_millis(40).unwrap(), 100); + track.frame(moq_net::Timestamp::from_millis(80).unwrap(), 50); + let report = ®istry.received()["hd"]; + assert_eq!((report.frames, report.bytes), (2, 150)); + let arrival = report.arrival.as_ref().unwrap(); + assert_eq!(arrival.timestamp.as_millis(), 80); + assert!(arrival.wall + Duration::from_millis(1) >= before); + } +} From 2c66df395a1fc8419f2b3d66f2d7eb867b1b848c Mon Sep 17 00:00:00 2001 From: t0ms Date: Sat, 17 Oct 2026 12:04:04 +0100 Subject: [PATCH 02/12] fix(cli): address review on the stats reporters - Count a buffered frame once the container writes it, not on the way in: an abort discards the buffer. - Refuse `--echo` for an MSF catalog detected from a `.msf` broadcast, not only one named. - Refuse an echo name starting with `.`, which discovery hides. - `export echo` replaces a reader on a restart, lets one drain after its broadcast ends, and bounds the lines it queues for stdout. - A report track that refuses a write is closed with the cause and reported once, instead of logged at debug every second; a failed import aborts its stats track with the import's error. - The stats integration test joins `tests/integration`; the arrival test fakes the wall clock. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 6 +- rs/moq-cli/src/args.rs | 37 ++++++++-- rs/moq-cli/src/publish.rs | 3 + rs/moq-cli/src/report.rs | 82 ++++++++++++++++----- rs/moq-cli/tests/integration/main.rs | 1 + rs/moq-cli/tests/{ => integration}/stats.rs | 2 +- rs/moq-mux/src/clock.rs | 2 +- rs/moq-mux/src/container/producer.rs | 5 +- rs/moq-mux/src/stats.rs | 27 +++++-- 9 files changed, 127 insertions(+), 38 deletions(-) rename rs/moq-cli/tests/{ => integration}/stats.rs (99%) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index aacd0868e9c..65a173eaed7 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -408,8 +408,10 @@ capture, the gateways, `archive`, and `ts --program all` refuse it. `export --echo ` answers a publisher that solicits feedback: under the catalog's `echo` path it publishes `.echo`, reporting what each rendition the export reads has received and its newest arrival, until the export ends. -The name is one path segment. It covers the stdout container exports and reads -the solicitation from the hang catalog, so `--catalog-format msf` refuses it. +The name is one path segment, not starting with `.`, which would hide it from +discovery. It covers the stdout container exports and reads the solicitation +from the hang catalog, so an MSF catalog, named or detected from a `.msf` +broadcast, refuses it. The `stats` and `echo` sinks print reports as JSON lines. `export stats` follows one broadcast's stats track until the broadcast ends; `export echo` diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index e295e436819..b95db53ca38 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -366,8 +366,15 @@ impl Invocation { "--echo reports what a stdout container export reads, and `export {sink}` is not one" ); } + let msf = match export.catalog_format { + Some(format) => matches!(format, CatalogFormatArg::Msf), + None => matches!( + moq_mux::catalog::CatalogFormat::detect(&command.broadcast(&self.moq)), + Some(moq_mux::catalog::CatalogFormat::Msf) + ), + }; anyhow::ensure!( - !matches!(export.catalog_format, Some(CatalogFormatArg::Msf)), + !msf, "--echo follows the hang catalog's `echo` section, which the MSF catalog does not carry" ); } @@ -982,7 +989,7 @@ pub struct Export { /// Publish feedback where the broadcast's catalog solicits it: a `.echo` broadcast /// under the catalog's `echo` path, reporting what each rendition this export reads has - /// received. One path segment. The stdout container exports only. + /// received. One path segment, not starting with `.`. The stdout container exports only. #[usage(long)] pub echo: Option, @@ -1514,12 +1521,26 @@ mod tests { .unwrap_err(); assert!(err.to_string().contains("--echo"), "{sink:?}: {err}"); } - let err = export(&["--echo", "alice", "--catalog-format", "msf", "fmp4"]) - .unwrap() - .validate() - .unwrap_err(); - assert!(err.to_string().contains("MSF"), "{err}"); - for name in ["", "a/b", ".", "..", "*"] { + for flags in [ + ["--echo", "alice", "--catalog-format", "msf", "fmp4"].as_slice(), + &["--broadcast", "live.msf", "--echo", "alice", "fmp4"], + ] { + let err = export(flags).unwrap().validate().unwrap_err(); + assert!(err.to_string().contains("MSF"), "{flags:?}: {err}"); + } + export(&[ + "--broadcast", + "live.msf", + "--catalog-format", + "hang", + "--echo", + "alice", + "fmp4", + ]) + .unwrap() + .validate() + .unwrap(); + for name in ["", "a/b", ".", "..", "*", ".alice"] { assert!(export(&["--echo", name, "fmp4"]).is_err(), "`{name}` must be refused"); } for sink in ["stats", "echo"] { diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index e2e45e5c962..d1ce8863671 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -667,6 +667,9 @@ async fn decode( } Err(err) => { let reason = moq_net::Error::Transport(err.to_string()); + if let Some(stats) = stats { + stats.abort(reason.clone()); + } catalog.abort(reason.clone()); decoder.abort(reason); Err(err) diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 7324cec862d..4e7f59c417d 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -24,13 +24,17 @@ pub const STATS_TRACK: &str = "stats.json"; /// The suffix naming a snapshot track's compressed sibling. const COMPRESSED: &str = ".z"; +/// Lines `export echo` queues for stdout before its readers wait for it to catch up. +const LINES: usize = 64; + /// Below every media track, so a report never delays the media it describes. const PRIORITY: u8 = 0; -/// A snapshot track and its compressed `.z` sibling, written together. +/// A snapshot track and its compressed `.z` sibling, written together. Either is `None` once a +/// write failed and closed it. struct Pair { - plain: moq_json::snapshot::Producer, - compressed: moq_json::snapshot::Producer, + plain: Option>, + compressed: Option>, } impl Pair { @@ -42,35 +46,56 @@ impl Pair { let mut deflate = moq_json::snapshot::Config::default(); deflate.compression = moq_json::Compression::Deflate; Ok(Self { - plain: moq_json::snapshot::Producer::new(plain, moq_json::snapshot::Config::default().with_delta_ratio(0)), - compressed: moq_json::snapshot::Producer::new(compressed, deflate), + plain: Some(moq_json::snapshot::Producer::new( + plain, + moq_json::snapshot::Config::default().with_delta_ratio(0), + )), + compressed: Some(moq_json::snapshot::Producer::new(compressed, deflate)), }) } /// Publish `value` on both tracks, unless nobody subscribes to either. fn update(&mut self, value: &T) { - if self.plain.demand().is_used() || self.compressed.demand().is_used() { + let used = [&self.plain, &self.compressed] + .into_iter() + .flatten() + .any(|track| track.demand().is_used()); + if used { self.write(value); } } - /// Publish `value` on both tracks. + /// Publish `value` on both tracks. A track that refuses a write would refuse every later one, + /// so it is closed with the cause, which its subscribers then read, and reported once. fn write(&mut self, value: &T) { - for track in [&mut self.plain, &mut self.compressed] { + for slot in [&mut self.plain, &mut self.compressed] { + let Some(track) = slot else { + continue; + }; if let Err(err) = track.update(value) { - tracing::debug!(%err, "failed to write a report"); + tracing::warn!(%err, "a report track failed; closing it"); + if let Some(track) = slot.take() { + let _ = track.abort(moq_net::Error::Transport(err.to_string())); + } } } } /// End both tracks cleanly, keeping the last snapshot. fn finish(&mut self) { - for track in [&mut self.plain, &mut self.compressed] { + for track in [&mut self.plain, &mut self.compressed].into_iter().flatten() { if let Err(err) = track.finish() { - tracing::debug!(%err, "failed to finish a report"); + tracing::warn!(%err, "failed to finish a report"); } } } + + /// Close both tracks with `err`, so subscribers read why the report ended. + fn abort(self, err: moq_net::Error) { + for track in [self.plain, self.compressed].into_iter().flatten() { + let _ = track.abort(err.clone()); + } + } } /// Which way the reported media crosses the connection, which picks its rate estimate. @@ -155,6 +180,11 @@ impl Stats { self.tracks.write(&last); self.tracks.finish(); } + + /// End the track with the import's failure, as its media tracks end. + pub fn abort(self, err: moq_net::Error) { + self.tracks.abort(err); + } } /// An `export --echo` name: the one path segment a feedback broadcast is published at, before @@ -169,6 +199,11 @@ impl std::str::FromStr for EchoName { if name.is_empty() || name.contains('/') || matches!(name, "." | ".." | "*" | "**") { return Err(format!("`{name}` is not a single path segment")); } + if name.starts_with('.') { + return Err(format!( + "`{name}` starts with `.`, which hides its feedback from discovery" + )); + } Ok(Self(name.to_string())) } } @@ -386,14 +421,17 @@ pub async fn print_echo(origin: moq_net::origin::Consumer, prefix: String) -> an let mut stdout = tokio::io::stdout(); let prefix = moq_net::Path::new(&prefix).to_owned(); let mut announced = origin.announced(); - let (lines, mut written) = tokio::sync::mpsc::unbounded_channel::(); + let (lines, mut written) = tokio::sync::mpsc::channel::(LINES); let mut readers = HashMap::new(); loop { tokio::select! { event = announced.next() => { let event = event.context("announcements ended")?; let (announce, active) = match event { - moq_net::announce::Event::Start(announce) => (announce, true), + // A restart is another publisher instance, which a reader still on the old one never sees. + moq_net::announce::Event::Start(announce) | moq_net::announce::Event::Restart(announce) => { + (announce, true) + } moq_net::announce::Event::End(announce) => (announce, false), _ => continue, }; @@ -403,9 +441,13 @@ pub async fn print_echo(origin: moq_net::origin::Consumer, prefix: String) -> an } if active { let reader = tokio::spawn(read_echo(origin.clone(), path.clone(), lines.clone())); - readers.insert(path, reader); - } else if let Some(reader) = readers.remove(&path) { - reader.abort(); + if let Some(previous) = readers.insert(path, reader) { + previous.abort(); + } + } else { + // Detached, not aborted: the final snapshot can still be in flight, and the reader + // ends with its track. + readers.remove(&path); } } Some(line) = written.recv() => { @@ -420,7 +462,7 @@ pub async fn print_echo(origin: moq_net::origin::Consumer, prefix: String) -> an async fn read_echo( origin: moq_net::origin::Consumer, path: moq_net::PathOwned, - lines: tokio::sync::mpsc::UnboundedSender, + lines: tokio::sync::mpsc::Sender, ) { let read = async { let broadcast = origin.request_broadcast(&path, None).await?; @@ -431,7 +473,11 @@ async fn read_echo( broadcast: path.as_str(), feedback, }; - if lines.send(format!("{}\n", serde_json::to_string(&line)?)).is_err() { + if lines + .send(format!("{}\n", serde_json::to_string(&line)?)) + .await + .is_err() + { break; } } diff --git a/rs/moq-cli/tests/integration/main.rs b/rs/moq-cli/tests/integration/main.rs index 831dea1e5e5..87ad4cfcfde 100644 --- a/rs/moq-cli/tests/integration/main.rs +++ b/rs/moq-cli/tests/integration/main.rs @@ -7,3 +7,4 @@ mod auth_serve; mod export; mod import; mod listen; +mod stats; diff --git a/rs/moq-cli/tests/stats.rs b/rs/moq-cli/tests/integration/stats.rs similarity index 99% rename from rs/moq-cli/tests/stats.rs rename to rs/moq-cli/tests/integration/stats.rs index 37e33067b56..a9a5eec6512 100644 --- a/rs/moq-cli/tests/stats.rs +++ b/rs/moq-cli/tests/integration/stats.rs @@ -13,7 +13,7 @@ use tokio::process::{Child, ChildStdout}; const TIMEOUT: Duration = Duration::from_secs(30); /// Five seconds of HEVC and Opus at a constant 400 kb/s. -const CLIP: &[u8] = include_bytes!("../../moq-mux/src/container/ts/test_data/bbb5s_cbr.ts"); +const CLIP: &[u8] = include_bytes!("../../../moq-mux/src/container/ts/test_data/bbb5s_cbr.ts"); /// Bytes fed every [`PACE`], which keeps [`CLIP`] near real time. const CHUNK: usize = 188 * 40; const PACE: Duration = Duration::from_millis(150); diff --git a/rs/moq-mux/src/clock.rs b/rs/moq-mux/src/clock.rs index ae61919463f..1a237b14782 100644 --- a/rs/moq-mux/src/clock.rs +++ b/rs/moq-mux/src/clock.rs @@ -32,7 +32,7 @@ fn moq_micros(wall: SystemTime) -> i128 { } /// The host's wall clock, or `None` while it reads before the moq epoch (2020): unset. -fn host_wall() -> Option { +pub(crate) fn host_wall() -> Option { let now = SystemTime::from(moq_clock::wall()); #[cfg(test)] let now = HOST_WALL.get().unwrap_or(now); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 9d29539baf2..a379a71bd49 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -492,7 +492,6 @@ where // Buffered frames are measured on the way in instead. The flush that eventually writes // them takes the track down with it when it fails, so there is nothing to unwind. self.estimator.write(frame.timestamp, frame.payload.len()); - self.count(frame.payload.len(), frame.keyframe); self.observe_end(frame.timestamp, frame.duration); self.buffer.push(frame); @@ -736,6 +735,10 @@ where if let Some(recorder) = self.recorder.as_mut() { recorder.frame(position, self.buffer[0].timestamp, self.buffer[0].keyframe); } + // Counted only once written: an abort discards the buffer, and those frames were never sent. + for frame in &self.buffer { + self.count(frame.payload.len(), frame.keyframe); + } self.buffer.clear(); Ok(()) diff --git a/rs/moq-mux/src/stats.rs b/rs/moq-mux/src/stats.rs index f877a495f49..45a22db24bf 100644 --- a/rs/moq-mux/src/stats.rs +++ b/rs/moq-mux/src/stats.rs @@ -8,7 +8,7 @@ use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; use std::sync::{Arc, Mutex, PoisonError}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, UNIX_EPOCH}; /// What one published track has written. #[derive(Default)] @@ -52,12 +52,12 @@ impl Received { self.frames.fetch_add(1, Relaxed); self.bytes.fetch_add(bytes as u64, Relaxed); // A clock before the MoQ epoch has no wire encoding, so such an arrival is never recorded. - let wall = SystemTime::now() + let Some(wall) = crate::clock::host_wall() else { + return; + }; + let wall = wall .duration_since(UNIX_EPOCH) .map_or(0, |since| u64::try_from(since.as_millis()).unwrap_or(0)); - if wall < hang::catalog::MOQ_EPOCH_UNIX_MILLIS { - return; - } let micros = u64::try_from(timestamp.as_micros()).unwrap_or(u64::MAX); self.timestamp.store(micros, Relaxed); self.wall.store(wall, Relaxed); @@ -156,13 +156,26 @@ mod tests { let track = registry.track("hd"); assert_eq!(registry.received()["hd"].arrival, None); - let before = SystemTime::now(); + let wall = UNIX_EPOCH + Duration::from_secs(1_800_000_000); + crate::clock::fake_host_wall(wall); track.frame(moq_net::Timestamp::from_millis(40).unwrap(), 100); track.frame(moq_net::Timestamp::from_millis(80).unwrap(), 50); let report = ®istry.received()["hd"]; assert_eq!((report.frames, report.bytes), (2, 150)); let arrival = report.arrival.as_ref().unwrap(); assert_eq!(arrival.timestamp.as_millis(), 80); - assert!(arrival.wall + Duration::from_millis(1) >= before); + assert_eq!(arrival.wall, wall); + } + + /// A host clock before the MoQ epoch has no wire encoding: the frame counts, its arrival doesn't. + #[test] + fn an_unset_host_clock_records_no_arrival() { + let registry = Registry::::default(); + let track = registry.track("hd"); + crate::clock::fake_host_wall(UNIX_EPOCH + Duration::from_secs(86_400)); + track.frame(moq_net::Timestamp::from_millis(40).unwrap(), 100); + let report = ®istry.received()["hd"]; + assert_eq!((report.frames, report.bytes), (1, 100)); + assert_eq!(report.arrival, None); } } From 9974c234b08a5e7ea9492ab21875522cb4ab5f96 Mon Sep 17 00:00:00 2001 From: t0ms Date: Fri, 16 Oct 2026 23:00:32 +0100 Subject: [PATCH 03/12] feat(mux): report TS health and the export's release clock A TS import's stats and a TS export's feedback now carry an `mpegts` section: each elementary stream's access units and quiet time, and the TR 101 290 counters. An export adds `release`, how its output clock keeps up with the source: frames dropped at the deadline, the least margin frames arrived with over the last second, the clock's offset and drift from the source, and measurements past the 30 ppm a system clock may run off. `ts::stats` gains serde and `Report`, the extension both snapshots flatten. `stats::Export` moves its clock fields under `release`, and `ts::Release` reports the same for a passthrough export. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 7 + doc/concept/stats.md | 56 +++++ quest/m1/ts-reliability/README.md | 1 - quest/m2/README.md | 1 - quest/m2/stats/health.md | 3 +- quest/m2/stats/rust.md | 1 - quest/m2/ts-health-stats.md | 59 ----- rs/moq-cli/src/main.rs | 1 + rs/moq-cli/src/publish.rs | 8 + rs/moq-cli/src/report.rs | 47 +++- rs/moq-cli/src/subscribe.rs | 73 ++++-- rs/moq-cli/tests/integration/stats.rs | 232 ++++++++++++++++-- rs/moq-mux/src/container/ts/export.rs | 22 +- rs/moq-mux/src/container/ts/export_test.rs | 8 +- .../src/container/ts/export_timing_test.rs | 11 +- rs/moq-mux/src/container/ts/release.rs | 9 + rs/moq-mux/src/container/ts/stats.rs | 166 ++++++++++++- rs/moq-mux/src/jitter.rs | 90 +++++++ 18 files changed, 659 insertions(+), 136 deletions(-) delete mode 100644 quest/m2/ts-health-stats.md diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 65a173eaed7..355d1f9fe44 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -431,6 +431,13 @@ An fMP4 import counts samples as frames. Encoder counters (`skipped`, `bitrate`) and playback feedback (`decoded`, `late`, `stalls`) are not reported yet. [Stats](/concept/stats#media-reports) has the schema. +A TS import adds an `mpegts` section: each elementary stream's access units +and quiet time, and the TR 101 290 counters it logs. A TS export adds its +streams' rows and a `release` section: frames dropped late, the margin frames +arrive with, and its clock's offset and drift from the source. A passthrough +reports `release` alone at export and neither at import. See +[MPEG-TS sections](/concept/stats#mpeg-ts-sections). + ## Debugging `RUST_LOG=debug` prints the negotiated version and every subscription. diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 19a79b97566..d60bd47a997 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -39,6 +39,62 @@ it reads and records its newest arrival. See the [hang draft](/draft/moq-hang#media-stats) for track framing, field units, and the feedback convention. +### MPEG-TS sections + +A TS import or export adds sections of its own, which +`moq_mux::container::ts::stats::Report` reads. As above, counters are +cumulative and omitted at zero, and a gauge is omitted until it is measured. + +`mpegts` holds `streams`, one row per elementary stream keyed by its PID in +decimal, and the stream-wide counters. A row has `track` (the track suffix), +`class` (`audio`, `video`, or `data`; an unknown class reads as `data`), +`units` (access units delivered), and `quiet`, a gauge: milliseconds of +program clock since its last access unit. Its losses are `resyncs`, bytes +`discarded` scanning for audio sync, `unconfirmed` frames, `damaged` units, +and its shares of `continuityCountError`, `transportError`, and `ptsError`. + +The stream-wide counters count ETSI TR 101 290 checks at that standard's +fixed limits: + +| Field | Check | +| --- | --- | +| `tsSyncLoss` | 1.1 `TS_sync_loss` | +| `syncByteError` | 1.2 `Sync_byte_error` | +| `patError` | 1.3 `PAT_error_2` | +| `continuityCountError` | 1.4 `Continuity_count_error` | +| `pmtError` | 1.5 `PMT_error_2` | +| `transportError` | 2.1 `Transport_error` | +| `crcError` | 2.2 `CRC_error`, on the PAT and PMT only | +| `pcrRepetitionError` | 2.3a `PCR_repetition_error` | +| `pcrDiscontinuityIndicatorError` | 2.3b `PCR_discontinuity_indicator_error` | +| `ptsError` | 2.5 `PTS_error` | + +An import grades the stream as it arrived. An export builds every header +itself and runs no checks, so its counters stay zero and only its rows' +`units` and `quiet` move. Neither grades the wire a receiver sees downstream. +The PCR checks are graded on PCR values, not arrival: they speak for the +encoder's insertion, since the network re-times arrival, and a forward step +over 100 ms counts as both. 1.6 `PID_error` is covered, more strictly, by each +row's `units` and `quiet`: a dead video path behind a live mux still sends +PCR-only packets on its PID, which a packet count reads as live. +`PCR_accuracy_error` is not measured. No verdict is computed: a sparse stream +such as SCTE-35 is quiet for seconds by design, so an alarm reads counter +rates and `quiet` against its own thresholds. + +An export also reports `release`, how it keeps up with the source: what it +drops, trims and never receives, and how its output clock tracks the +source's. `export ts --passthrough` reports it without `mpegts`. + +| Field | Meaning | +| --- | --- | +| `dropped` | Frames lost from the output after their stream began: missed their release deadline, or a video track dropped them waiting for its next keyframe. | +| `trimmed` | Frames dropped before their stream's first went out, so the output starts on a whole group led by a keyframe. Nothing is missing for these. | +| `gaps` | Times the source skipped media it will never deliver: a group lost or expired upstream, or a resubscription. A data track's declared break counts too. | +| `margin` | Gauge, milliseconds: the least time any frame arrived before its deadline over the last second. Negative when one missed. | +| `offset` | Gauge, milliseconds: how far the output clock sits from the source's plus the delay. Positive holds frames longer than the delay. The clock pulls it shut over ten minutes. | +| `drift` | Gauge, parts per million: the source clock's rate against ours, positive running fast. | +| `outOfTolerance` | Measurements that found the source clock past the 30 ppm ISO/IEC 13818-1 allows, which the output cannot follow. | + ## Broadcasts Each node publishes under a prefix, `.stats` by default, which diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index c9d6bf1003c..94b4e0e6a70 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -110,6 +110,5 @@ remain with their current quests. - [Epoch anchor](/quest/m2/epoch-anchor.md) - shared identity across importers requires one source coordinate per epoch, derived from the epoch - [TS over RTP legs](/quest/m2/ts-rtp-egress.md) - TS identity alone is not ST 2022-7, which also needs matching RTP headers -- [TS health counters](/quest/m2/ts-health-stats.md) - ingress and egress evidence need distinct monitoring points - [MSFTS mapping](/quest/m3/msfts-convergence.md) - document the exact equivalence and differences between the carriage modes diff --git a/quest/m2/README.md b/quest/m2/README.md index b366e6e2d82..aefcf39d7f9 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -152,7 +152,6 @@ ranked just below deterministic TS legs, which it requires. - [BBR media study](/quest/m2/quic-bbr-natural-drain.md) - whether bounded drain credit avoids ProbeRTT deadline interference, and where our BBR differs from Google's - [Keep-alive from the idle timeout](/quest/m2/quic-keep-alive.md) - the keep-alive interval defaults to a fraction of the negotiated idle timeout; an explicit `quic.keep_alive` overrides - [Socket close](/quest/m2/noq-socket-close.md) - moq-tokio's QUIC endpoint releases its socket on close, so it drops its wrapper -- [TS health stats](/quest/m2/ts-health-stats.md) - the TS counters ride the stats plumbing beside the media counters - [Windows.Graphics.Capture](/quest/m2/capture-wgc.md) - the WGC display and window backend verified on real Windows hardware - [Windows capture parity](/quest/m2/capture-windows.md) - system audio and a settled app-capture policy - [Linux capture parity](/quest/m2/capture-linux.md) - Wayland window/system-audio capture with explicit display-selection and app-capture limits diff --git a/quest/m2/stats/health.md b/quest/m2/stats/health.md index 99511272a48..155ed2ab86f 100644 --- a/quest/m2/stats/health.md +++ b/quest/m2/stats/health.md @@ -42,7 +42,7 @@ Guidance, to be settled while building: starts over rather than going negative. - Container sections such as `mpegts` are excluded from the verdict; their counters are read by their own consumers - ([TS health counters](/quest/m2/ts-health-stats.md)). + ([MPEG-TS sections](/doc/concept/stats.md#mpeg-ts-sections)). - Thresholds are a documented default the caller can override, not policy baked into the types. Start from a few measured cases rather than guesses. - A verdict carries its evidence: which inputs drove it and who observed them. @@ -65,4 +65,3 @@ Public API: new health types in `hang` and `@moq/hang`. Wire: none. - [QoS](/quest/m2/qos/README.md) - the relay's delivery counters, which a combined downstream verdict also reads - [Broadcast epochs](/quest/m0/broadcast-epoch/README.md) - a new epoch is one way a counter reset shows -- [TS health counters](/quest/m2/ts-health-stats.md) - container counters kept out of the verdict diff --git a/quest/m2/stats/rust.md b/quest/m2/stats/rust.md index 985ade5a447..03b32a5008d 100644 --- a/quest/m2/stats/rust.md +++ b/quest/m2/stats/rust.md @@ -41,5 +41,4 @@ ## Related -- [TS health stats](/quest/m2/ts-health-stats.md) - adds TS-specific counters to the reports - [Encrypted native CLI](/quest/m2/e2ee/cli.md) - refuses stats and feedback on an encrypted broadcast diff --git a/quest/m2/ts-health-stats.md b/quest/m2/ts-health-stats.md deleted file mode 100644 index d72d74cb681..00000000000 --- a/quest/m2/ts-health-stats.md +++ /dev/null @@ -1,59 +0,0 @@ -# [S] TS health counters ride the stats plumbing - -## Goal - -`moq import ts --stats` and `moq export ts --echo` publish the `ts::stats` -counters so a dashboard reads the TR 101 290 counters and per-PID liveness -beside the media and delivery counters, with no new transport. No health -roll-up in-tree: green, amber or red per priority is the consumer's reading of -counter rates and PID quiet times. - -## Plan - -Decided while planning [#1838](https://github.com/moq-dev/moq/issues/1838): - -The quest audit confirmed this stays in m2 and owns TS counter serialization -and integration. Rust reporters supplies only the generic per-rendition -plumbing; it does not implement this quest's `mpegts` payloads. - -- **Surface through the stats plumbing**, as the issue settled. The - [media stats schema](/doc/concept/stats.md#media-reports) settles the shape: an - import flattens `ts::stats::Snapshot` into the publisher's stats snapshot under the - catalog's `mpegts` key, the counters stay owned by moq-mux beside - `ts::Ext`, and `hang` stays TS-free. -- **The export reports through `.echo`** (decided 2026-10-12, from - [#5360](https://github.com/moq-dev/moq/issues/5360)). An export is a viewer, - so `moq export ts --echo ` folds `ts::stats::Snapshot` into its - solicited `.echo` snapshot under `mpegts`, through the `E` flatten the - [media stats schema](/doc/concept/stats.md#media-reports) gives `echo::Snapshot`. - The egress is where an operator otherwise cannot see per-PID outages: a - 57 s video hole passed every TR 101 290 P1 check in the reporter's lab, - and only per-PID access-unit counts caught it. There is no export - `--stats` flag. -- **Re-timing gets its own section beside `mpegts`** (decided 2026-10-16, - on [#5643](https://github.com/moq-dev/moq/pull/5643)). An export that - re-times onto an output clock reports what `ts::stats::Snapshot` cannot - hold: `ts::stats::Export`'s stream-wide `dropped`, `trimmed`, `gaps`, - `drift`, and `out_of_tolerance`, which its conversion to `Snapshot` - discards, plus the release margin and the output clock's offset from the - source. They describe the export's own output - clock rather than viewer playout, so they sit flat beside `transport` and - `renditions` now instead of waiting for the - [latency ledger](/quest/m2/latency-ledger.md)'s `.echo` shape. -- **Counters sum, gauges do not.** Every check is a cumulative counter, which - keeps `.z` deltas small and lets an aggregate sum; the PID's `quiet` is a gauge and - merges newest-wins. -- `ts::stats::Snapshot` gains serde, every field defaulted and unknown fields - ignored. Its rows' `track` is already an owned `String`, so they deserialize. -- Docs: `doc/concept/stats.md` documents each counter's TR 101 290 check, its - monitoring point (ingest grades the feed, egress grades our muxer, and - neither is the groomed wire), the value domain of the PCR checks, and that - the per-PID rows cover `PID_error`; `doc/bin/cli.md` the flags. -- Test: publish a TS clip with `--stats` and export it with `--echo` to a - soliciting catalog, read both back, and assert every ETSI - counter reads zero on a clean clip while every elementary stream's - access-unit count advances. - -## Required - -- [Rust reporters](/quest/m2/stats/rust.md) - `moq import --stats` and the stats interval diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 46ded2ad7af..7ed60c5d1a3 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -894,6 +894,7 @@ async fn run_stdout( return subscribe.run().await; }; let mut echo = report::Echo::new(origin, subscribe.source().clone(), catalog, echo, connection); + let subscribe = subscribe.with_sections(echo.sections()); let result = tokio::select! { result = subscribe.run() => result, never = echo.run() => match never {}, diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index d1ce8863671..d3fc59641f5 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -601,6 +601,12 @@ async fn decode( // they have to surface as they accumulate rather than at exit. let mut log = ts::stats::Log::default(); let mut sampled = tokio::time::Instant::now(); + let sections = stats.as_ref().map(crate::report::Stats::sections); + let report = |snapshot: &ts::stats::Snapshot| { + if let Some(sections) = §ions { + sections.set(|report| report.mpegts = Some(snapshot.clone())); + } + }; // Run the read/decode loop so an error surfaces here rather than // dropping the decoder (and its tracks) with a bare Error::Dropped. @@ -621,6 +627,7 @@ async fn decode( if sampled.elapsed() >= ts::stats::Log::INTERVAL { sampled = tokio::time::Instant::now(); if let Some(stats) = decoder.stats() { + report(&stats); log.sample(stats); } } @@ -647,6 +654,7 @@ async fn decode( // The drain at end of input can publish a frame nothing vouched for, so the // final snapshot is only complete after `finish`. if let Some(stats) = decoder.stats() { + report(&stats); log.finish(&stats); } match outcome { diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 4e7f59c417d..4f34e9c22ad 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -7,11 +7,13 @@ use std::collections::{BTreeMap, HashMap}; use std::convert::Infallible; +use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use anyhow::Context; use hang::moq_net; use moq_mux::catalog::{CatalogFormat, Stream}; +use moq_mux::container::ts; use moq_tokio::connection::Monitor; use tokio::io::AsyncWriteExt; @@ -122,12 +124,28 @@ fn transport(connection: Option<&Monitor>, direction: Direction) -> hang::stats: transport } +/// The MPEG-TS sections a report carries beside its media counters, as the import or export +/// loop that samples them last set them. Empty for every other container. +#[derive(Clone, Default)] +pub struct Sections(Arc>); + +impl Sections { + pub fn set(&self, update: impl FnOnce(&mut ts::stats::Report)) { + update(&mut self.0.lock().unwrap_or_else(PoisonError::into_inner)); + } + + fn get(&self) -> ts::stats::Report { + self.0.lock().unwrap_or_else(PoisonError::into_inner).clone() + } +} + /// An import's stats track: what each of its tracks has written, from its catalog's counters, -/// and the connection's transport. +/// the connection's transport, and any MPEG-TS sections. pub struct Stats { - tracks: Pair, + tracks: Pair>, renditions: Box BTreeMap + Send>, connection: Option, + sections: Sections, } impl Stats { @@ -149,13 +167,20 @@ impl Stats { tracks, renditions: Box::new(move || catalog.stats()), connection, + sections: Sections::default(), }) } - fn snapshot(&self) -> hang::stats::Snapshot { + /// Where the import sets its MPEG-TS sections. + pub fn sections(&self) -> Sections { + self.sections.clone() + } + + fn snapshot(&self) -> hang::stats::Snapshot { let mut snapshot = hang::stats::Snapshot::default(); snapshot.transport = transport(self.connection.as_ref(), Direction::Send); snapshot.renditions = (self.renditions)(); + snapshot.ext = self.sections.get(); snapshot } @@ -213,20 +238,21 @@ struct Feedback { path: moq_net::PathOwned, // Held for the route: dropping it retracts the announcement. _broadcast: moq_net::broadcast::Producer, - tracks: Pair, + tracks: Pair>, } /// An export's solicited feedback: once its catalog carries an `echo` section, a /// `/.echo` broadcast whose `feedback.json` reports what each rendition the export -/// reads has received, and the connection's transport. Each catalog update reconciles it: a -/// removed section or a changed path retracts the broadcast it replaces. It follows the catalog -/// of the instance the export first reads, and ends when the export does. +/// reads has received, the connection's transport, and any MPEG-TS sections. Each catalog update +/// reconciles it: a removed section or a changed path retracts the broadcast it replaces. It +/// follows the catalog of the instance the export first reads, and ends when the export does. pub struct Echo { origin: moq_net::origin::Producer, source: moq_mux::Source, format: CatalogFormat, name: EchoName, connection: Option, + sections: Sections, feedback: Option, } @@ -245,10 +271,16 @@ impl Echo { format, name, connection, + sections: Sections::default(), feedback: None, } } + /// Where the export sets its MPEG-TS sections. + pub fn sections(&self) -> Sections { + self.sections.clone() + } + /// Follow the catalog's `echo` section and write a snapshot every interval, for as long as the /// export runs beside it. pub async fn run(&mut self) -> Infallible { @@ -321,6 +353,7 @@ impl Echo { let mut snapshot = hang::echo::Snapshot::default(); snapshot.transport = transport(self.connection.as_ref(), Direction::Receive); snapshot.renditions = self.source.stats(); + snapshot.ext = self.sections.get(); feedback.tracks.update(&snapshot); } diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 4b72048eac8..7209423a8fc 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -253,6 +253,22 @@ pub struct Subscribe { source: moq_mux::Source, catalog: CatalogFormat, args: SubscribeArgs, + /// Where a TS export sets its MPEG-TS sections, when it reports them. + sections: Option, +} + +/// Log how a TS export's release clock keeps up with the source. +fn log_release(release: &moq_mux::container::ts::stats::Release, message: &'static str) { + tracing::info!( + dropped = release.dropped, + trimmed = release.trimmed, + gaps = release.gaps, + margin_ms = ?release.margin, + offset_ms = ?release.offset, + drift_ppm = ?release.drift, + out_of_tolerance = release.out_of_tolerance, + "{message}" + ); } impl Subscribe { @@ -276,6 +292,7 @@ impl Subscribe { source, catalog, args, + sections: None, }) } @@ -284,6 +301,12 @@ impl Subscribe { &self.source } + /// Set a TS export's per-PID rows and release clock in `sections` as it samples them. + pub fn with_sections(mut self, sections: crate::report::Sections) -> Self { + self.sections = Some(sections); + self + } + /// Build the catalog stream, narrowed by the rendition selection flags. The /// catalog source honors the requested format (e.g. compressed `HangZ` or `Msf`). async fn stream(&self) -> anyhow::Result> { @@ -406,15 +429,14 @@ impl Subscribe { let mut sampled = tokio::time::Instant::now(); // The export's drop, gap and out-of-tolerance counts, as last logged. let mut reported = (0, 0, 0, 0); - let release = |stats: &moq_mux::container::ts::stats::Export| { - tracing::info!( - dropped = stats.dropped, - trimmed = stats.trimmed, - gaps = stats.gaps, - drift_ppm = ?stats.drift, - out_of_tolerance = stats.out_of_tolerance, - "TS export release clock" - ); + let sections = self.sections; + let report = |stats: &moq_mux::container::ts::stats::Export| { + if let Some(sections) = §ions { + sections.set(|report| { + report.mpegts = Some(stats.clone().into()); + report.release = Some(stats.release.clone()); + }); + } }; let end = loop { let frame = match ts.next().await { @@ -428,15 +450,19 @@ impl Subscribe { if sampled.elapsed() >= moq_mux::container::ts::stats::Log::INTERVAL { sampled = tokio::time::Instant::now(); let stats = ts.export().stats(); - let counts = (stats.dropped, stats.trimmed, stats.gaps, stats.out_of_tolerance); + report(&stats); + let release = &stats.release; + let counts = (release.dropped, release.trimmed, release.gaps, release.out_of_tolerance); if counts != reported { reported = counts; - release(&stats); + log_release(release, "TS export release clock"); } log.sample(stats.into()); } }; - release(&ts.export().stats()); + let stats = ts.export().stats(); + report(&stats); + log_release(&stats.release, "TS export release clock"); match end { Err(err @ moq_mux::Error::Replaced(_)) => { Err(anyhow::Error::from(err).context("pass --stitch to follow a replacement as a program switch")) @@ -452,30 +478,31 @@ impl Subscribe { let mut release = moq_mux::container::ts::Release::new(self.source, self.catalog) .await? .with_delay(self.args.max_delay); - let log = |release: &moq_mux::container::ts::Release| { - tracing::info!( - dropped = release.dropped(), - drift_ppm = ?release.drift().map(|drift| drift * 1e6), - out_of_tolerance = release.out_of_tolerance(), - "TS passthrough release clock" - ); + let report = |stats: &moq_mux::container::ts::stats::Release| { + if let Some(sections) = &self.sections { + sections.set(|report| report.release = Some(stats.clone())); + } }; let mut sampled = tokio::time::Instant::now(); - let mut reported = (0, 0); + let mut reported = (0, 0, 0, 0); while let Some(object) = release.next().await? { stdout.write_all(&object.payload).await?; stdout.flush().await?; if sampled.elapsed() >= moq_mux::container::ts::stats::Log::INTERVAL { sampled = tokio::time::Instant::now(); - let counts = (release.dropped(), release.out_of_tolerance()); + let stats = release.stats(); + report(&stats); + let counts = (stats.dropped, stats.trimmed, stats.gaps, stats.out_of_tolerance); if counts != reported { reported = counts; - log(&release); + log_release(&stats, "TS passthrough release clock"); } } } - log(&release); + let stats = release.stats(); + report(&stats); + log_release(&stats, "TS passthrough release clock"); Ok(()) } diff --git a/rs/moq-cli/tests/integration/stats.rs b/rs/moq-cli/tests/integration/stats.rs index a9a5eec6512..acbd1ae7b5d 100644 --- a/rs/moq-cli/tests/integration/stats.rs +++ b/rs/moq-cli/tests/integration/stats.rs @@ -20,9 +20,10 @@ const PACE: Duration = Duration::from_millis(150); /// Past the two seconds of PCR a live TS catalog waits for, so the broadcast is up by then. const HALF: usize = CLIP.len() / 2; -/// Feed `clip` to `write` at about real time. +/// Feed `clip` to `write` at about real time, never catching up in a burst. async fn feed(clip: &[u8], mut write: impl AsyncFnMut(&[u8])) { let mut pace = tokio::time::interval(PACE); + pace.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); for chunk in clip.chunks(CHUNK) { pace.tick().await; write(chunk).await; @@ -91,8 +92,9 @@ async fn wait(child: &mut Child) -> std::process::ExitStatus { .expect("wait for moq") } -/// What a TS import of `clip` writes to each rendition, from an in-process import. -fn written(clip: &[u8]) -> BTreeMap { +/// What a TS import of `clip` writes to each rendition, and its MPEG-TS counters, from an +/// in-process import. +fn written(clip: &[u8]) -> (BTreeMap, ts::stats::Snapshot) { let mut broadcast = moq_net::broadcast::Info::new().produce(); let config = moq_mux::catalog::Config::default().with_catalog(moq_mux::catalog::hang::Catalog::::default()); @@ -100,16 +102,120 @@ fn written(clip: &[u8]) -> BTreeMap { let mut import = ts::Import::new(broadcast, catalog.reserve()); import.decode(clip).unwrap(); import.finish().unwrap(); - catalog.stats() + (catalog.stats(), import.stats()) } -/// The stats an import names in its catalog count each rendition while it runs, and its last -/// report holds what the import wrote in full. +/// Each elementary stream's access units, by PID. +fn units(mpegts: &ts::stats::Snapshot) -> BTreeMap { + mpegts.streams.iter().map(|(pid, row)| (*pid, row.units)).collect() +} + +/// Every TR 101 290 counter, PSI CRC error, and per-PID loss reads zero. +fn assert_clean(mpegts: &ts::stats::Snapshot) { + let mut stream = mpegts.clone(); + stream.streams.clear(); + assert_eq!( + stream, + ts::stats::Snapshot::default(), + "a stream-wide error was counted" + ); + for (pid, row) in &mpegts.streams { + let lost = [ + row.resyncs, + row.discarded, + row.unconfirmed, + row.damaged, + row.continuity_count_error, + row.transport_error, + row.pts_error, + ]; + assert_eq!(lost, [0; 7], "PID {pid} counted a loss: {row:?}"); + } +} + +/// The PCR a packet's adaptation field carries, in 27 MHz ticks. +fn pcr(pkt: &[u8; 188]) -> Option { + if pkt[3] & 0x20 == 0 || pkt[4] < 7 || pkt[5] & 0x10 == 0 { + return None; + } + let base = (u64::from(pkt[6]) << 25) + | (u64::from(pkt[7]) << 17) + | (u64::from(pkt[8]) << 9) + | (u64::from(pkt[9]) << 1) + | (u64::from(pkt[10]) >> 7); + Some(base * 300 + ((u64::from(pkt[10] & 0x01) << 8) | u64::from(pkt[11]))) +} + +/// `clip` with the PES on `pid` suppressed from its first PES start at or after `from` seconds +/// of program clock to its first at or after `to`, as an encoder whose one input died behind a +/// running mux emits it: a packet that carried the PCR keeps it, every other becomes null +/// stuffing, and the counters after the gap are renumbered so continuity stays legal. The +/// moq-mux import tests' stimulus. +fn suppress(clip: &[u8], pid: u16, from: f64, to: f64) -> Vec { + let mut null = [0xff; 188]; + null[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); + + let mut out = Vec::with_capacity(clip.len()); + let (mut first, mut now) = (None, 0.0); + let (mut active, mut done) = (false, false); + let (mut last_cc, mut dropped) = (0, 0u8); + for pkt in clip.as_chunks::<188>().0 { + let mut pkt = *pkt; + if let Some(pcr) = pcr(&pkt) { + now = (pcr - *first.get_or_insert(pcr)) as f64 / 27e6; + } + if (u16::from(pkt[1] & 0x1f) << 8 | u16::from(pkt[2])) == pid { + if pkt[1] & 0x40 != 0 { + if !active && !done && now >= from { + active = true; + } else if active && now >= to { + (active, done) = (false, true); + } + } + let payload = pkt[3] & 0x10 != 0; + if active { + dropped = (dropped + u8::from(payload)) & 0x0f; + pkt = match pcr(&pkt) { + Some(_) => { + let mut clock = [0xff; 188]; + clock[..6].copy_from_slice(&[0x47, pkt[1] & 0x1f, pkt[2], 0x20 | last_cc, 183, 0x10]); + clock[6..12].copy_from_slice(&pkt[6..12]); + clock + } + None => null, + }; + } else { + pkt[3] = (pkt[3] & 0xf0) | (pkt[3].wrapping_sub(dropped) & 0x0f); + if payload { + last_cc = pkt[3] & 0x0f; + } + } + } + out.extend_from_slice(&pkt); + } + assert!(done, "the clip resumes the PID before it ends"); + out +} + +/// Each PID's access units in `first` fell short of `last`'s: the rows advance while it runs. +fn assert_advancing(first: &ts::stats::Snapshot, last: &ts::stats::Snapshot) { + assert!(!last.streams.is_empty(), "no elementary stream reported"); + let first = units(first); + for (pid, units) in units(last) { + let before = first.get(&pid).copied().unwrap_or(0); + assert!(before < units, "PID {pid}: {before} access units, then {units}"); + } +} + +/// The stats an import names in its catalog count each rendition and elementary stream while +/// it runs, and its last report holds what the import wrote in full, with every TR 101 290 +/// counter at zero on a clean clip. #[tokio::test] async fn import_stats_count_what_each_rendition_wrote() { let relay = relay().await; - let expected = written(CLIP); + let (expected, health) = written(CLIP); assert_eq!(expected.len(), 2, "the clip carries video and audio: {expected:?}"); + assert_clean(&health); let mut import = moq(&relay, "demo", &["import", "--stats", "--max-age", "2s", "ts"]) .stdin(Stdio::piped()) @@ -124,7 +230,8 @@ async fn import_stats_count_what_each_rendition_wrote() { stdin.write_all(chunk).await.expect("write stdin") }) .await; - let mut reports: Vec = vec![next(&mut lines).await.expect("a report mid-import")]; + let mut reports: Vec> = + vec![next(&mut lines).await.expect("a report mid-import")]; feed(&CLIP[HALF..], async |chunk| { stdin.write_all(chunk).await.expect("write stdin") }) @@ -150,6 +257,83 @@ async fn import_stats_count_what_each_rendition_wrote() { reports.iter().any(|report| report.transport.rtt.is_some()), "the import reports its connection" ); + + let mpegts = last.ext.mpegts.as_ref().expect("a TS import reports its counters"); + assert_clean(mpegts); + assert_eq!(units(mpegts), units(&health), "the last report holds every access unit"); + let first = reports.iter().find_map(|report| report.ext.mpegts.as_ref()).unwrap(); + assert_advancing(first, mpegts); + assert!( + reports.iter().all(|report| report.ext.release.is_none()), + "an import has no release clock" + ); +} + +/// A video PID whose PES stops while the mux keeps its PCR and continuity legal: the import's +/// reports show that row's `quiet` growing while every other row keeps delivering, the row +/// counting again once it returns, and no TR 101 290 priority 1 check firing. +#[tokio::test] +async fn import_stats_report_a_silent_video_pid() { + let relay = relay().await; + let (_, clean) = written(CLIP); + let pid = |class| { + let mut rows = clean.streams.iter().filter(|(_, row)| row.class == class); + *rows.next().expect("the clip carries one").0 + }; + let (video, audio) = (pid(ts::stats::Class::Video), pid(ts::stats::Class::Audio)); + let stimulus = suppress(CLIP, video, 2.5, 4.5); + let (_, health) = written(&stimulus); + + let mut import = moq(&relay, "demo", &["import", "--stats", "--max-age", "2s", "ts"]) + .stdin(Stdio::piped()) + .spawn() + .expect("spawn the import"); + let mut stdin = import.stdin.take().expect("stdin"); + let (mut sink, mut lines) = sink(&relay, "demo", "stats"); + + // Up to the silence, then a report, so the sink is reading before the video stops. + feed(&stimulus[..HALF], async |chunk| { + stdin.write_all(chunk).await.expect("write stdin") + }) + .await; + let mut reports: Vec> = + vec![next(&mut lines).await.expect("a report mid-import")]; + feed(&stimulus[HALF..], async |chunk| { + stdin.write_all(chunk).await.expect("write stdin") + }) + .await; + drop(stdin); + while let Some(report) = next(&mut lines).await { + reports.push(report); + } + assert!(wait(&mut import).await.success()); + assert!(wait(&mut sink).await.success()); + + let rows: Vec<&ts::stats::Snapshot> = reports.iter().filter_map(|report| report.ext.mpegts.as_ref()).collect(); + let quiet = |mpegts: &ts::stats::Snapshot, pid: u16| mpegts.streams[&pid].quiet.unwrap_or_default(); + let silent = rows.iter().map(|mpegts| quiet(mpegts, video)).max().unwrap(); + // Two seconds of silence, sampled about once a second. + assert!( + silent >= Duration::from_millis(500), + "the video row was quiet for {silent:?} at most" + ); + let busy = rows.iter().map(|mpegts| quiet(mpegts, audio)).max().unwrap(); + assert!( + busy < Duration::from_millis(500), + "the audio row was quiet for {busy:?}" + ); + + let last = rows.last().unwrap(); + assert_eq!(units(last), units(&health), "the last report holds every access unit"); + assert!(quiet(last, video) < silent, "the video row delivered again"); + let priority1 = [ + last.ts_sync_loss, + last.sync_byte_error, + last.pat_error, + last.continuity_count_error, + last.pmt_error, + ]; + assert_eq!(priority1, [0; 5], "the silence is legal on the wire"); } /// A sink that joins once the import has finished, while it lingers, still reads its final @@ -157,7 +341,7 @@ async fn import_stats_count_what_each_rendition_wrote() { #[tokio::test] async fn a_late_sink_reads_the_final_stats() { let relay = relay().await; - let expected = written(CLIP); + let (expected, health) = written(CLIP); let mut import = moq(&relay, "demo", &["import", "--stats", "--max-age", "10s", "ts"]) .stdin(Stdio::piped()) @@ -179,8 +363,9 @@ async fn a_late_sink_reads_the_final_stats() { ended.await.expect("the import never reached the end of its input"); let (mut sink, mut lines) = sink(&relay, "demo", "stats"); - let report: hang::stats::Snapshot = next(&mut lines).await.expect("the final report"); + let report: hang::stats::Snapshot = next(&mut lines).await.expect("the final report"); assert_eq!(report.renditions, expected); + assert_eq!(report.ext.mpegts.as_ref().map(units), Some(units(&health))); sink.kill().await.unwrap(); import.kill().await.unwrap(); } @@ -189,15 +374,16 @@ async fn a_late_sink_reads_the_final_stats() { #[derive(serde::Deserialize)] struct Feedback { broadcast: String, - feedback: hang::echo::Snapshot, + feedback: hang::echo::Snapshot, } /// An export asked for feedback publishes it where the catalog solicits it, counting what it -/// reads as it reads it, and `export echo` finds it under that prefix. +/// reads as it reads it, with a TS export's per-PID rows and release clock, and `export echo` +/// finds it under that prefix. #[tokio::test] async fn export_echo_reports_what_the_export_reads() { let relay = relay().await; - let expected = written(CLIP); + let (expected, _) = written(CLIP); // A publisher soliciting feedback under `viewers`, a sibling of its broadcast. let mut broadcast = relay.origin.create_broadcast("demo").expect("create the broadcast"); @@ -212,7 +398,8 @@ async fn export_echo_reports_what_the_export_reads() { broadcast.announce(moq_net::origin::Route::default()).unwrap(); let (mut sink, mut lines) = sink(&relay, "viewers", "echo"); - let mut export = moq(&relay, "demo", &["export", "--echo", "alice", "ts"]) + // Held longer than the feed pauses for the first feedback line, so the pause is no stall. + let mut export = moq(&relay, "demo", &["export", "--echo", "alice", "ts", "--delay", "4s"]) .stdout(Stdio::piped()) .spawn() .expect("spawn the export"); @@ -268,4 +455,21 @@ async fn export_echo_reports_what_the_export_reads() { feedback.iter().any(|line| line.feedback.transport.rtt.is_some()), "the export reports its connection" ); + + let rows = last.ext.mpegts.as_ref().expect("a TS export reports its rows"); + assert_clean(rows); + let first = feedback + .iter() + .find_map(|line| line.feedback.ext.mpegts.as_ref()) + .unwrap(); + assert_advancing(first, rows); + let release = last + .ext + .release + .as_ref() + .expect("a TS export reports its release clock"); + assert!( + release.margin.is_some(), + "a second of frames gives a margin: {release:?}" + ); } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7fc88686ed4..43ee01ce2e0 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1754,20 +1754,18 @@ where /// empty until the program tables are built. Cheap enough to poll per frame. pub fn stats(&self) -> super::stats::Export { let mut stats = super::stats::Export { - dropped: self.jitter.dropped(), - trimmed: self.jitter.trimmed() + self.trimmed, - gaps: self - .tracks - .values() - .map(|track| { - let skips = track.base.1 + track.source.skips(); - skips - (track.base.0 + track.source.markers()) - }) - .sum(), - drift: self.jitter.drift().map(|drift| drift * 1e6), - out_of_tolerance: self.jitter.out_of_tolerance(), + release: super::stats::Release::of(&self.jitter), ..Default::default() }; + stats.release.trimmed += self.trimmed; + stats.release.gaps = self + .tracks + .values() + .map(|track| { + let skips = track.base.1 + track.source.skips(); + skips - (track.base.0 + track.source.markers()) + }) + .sum(); if self.psi.is_none() { return stats; } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 22eb93b8aaf..cb523a020d0 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1041,7 +1041,7 @@ async fn export_trims_leading_pictures_after_each_delivery_break() { video, expected, "continuous leading pictures survive; only join and later-hole orphans are trimmed" ); - let stats = export.stats(); + let stats = export.stats().release; assert_eq!( (stats.dropped, stats.trimmed), (2, 2), @@ -5745,7 +5745,7 @@ async fn a_start_group_cut_short_starts_at_the_next() { audio.iter().all(|&p| p >= pts(GOP)), "no audio leads the keyframe the export starts at: {audio:?}" ); - let stats = export.stats(); + let stats = export.stats().release; assert_eq!(stats.dropped, 0, "nothing that went out lost a frame"); assert_eq!(stats.gaps, 1, "the cut is counted"); assert!(stats.trimmed >= 5, "the cut-off start is counted: {}", stats.trimmed); @@ -5822,7 +5822,7 @@ async fn audio_dropped_ahead_of_a_moved_start_is_not_started() { audio.iter().all(|&p| p >= pts(GOP)), "no audio leads the keyframe: {audio:?}" ); - let stats = export.stats(); + let stats = export.stats().release; assert_eq!(stats.dropped, 0, "nothing was lost from the output"); assert!(stats.trimmed > 0); } @@ -5866,7 +5866,7 @@ async fn a_stale_start_group_holds_the_audio_behind_the_next_keyframe() { audio.iter().all(|&p| p >= pts(GOP)), "no audio leads the keyframe the export starts at: {audio:?}" ); - let stats = export.stats(); + let stats = export.stats().release; assert_eq!(stats.dropped, 0, "nothing was lost from the output"); assert!(stats.trimmed > 0); } diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 01282108118..05063f7ef14 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -714,8 +714,11 @@ async fn a_slow_source_clock_keeps_the_delay() { let (spread, dropped, stats) = on_a_scaled_clock(1.0 - 30e-6, DRIFT_TICKS).await; assert_eq!(dropped, 0, "frames went late as the source fell behind the anchor"); assert!(spread <= DRIFT_SLACK, "the delay walked by {spread:?}"); - let drift = stats.drift.expect("a measurement"); + let drift = stats.release.drift.expect("a measurement"); assert!((drift + 30.0).abs() < 0.5, "measured {drift} ppm"); + let margin = stats.release.margin.expect("a margin"); + assert!(margin > 0, "a clean path keeps time to spare: {margin} ms"); + assert!(stats.release.offset.is_some(), "the offset is measured with the rate"); } #[tokio::test(start_paused = true)] @@ -723,7 +726,7 @@ async fn a_fast_source_clock_keeps_the_delay() { let (spread, dropped, stats) = on_a_scaled_clock(1.0 + 30e-6, DRIFT_TICKS).await; assert_eq!(dropped, 0); assert!(spread <= DRIFT_SLACK, "the delay walked by {spread:?}"); - let drift = stats.drift.expect("a measurement"); + let drift = stats.release.drift.expect("a measurement"); assert!((drift - 30.0).abs() < 0.5, "measured {drift} ppm"); } @@ -731,9 +734,9 @@ async fn a_fast_source_clock_keeps_the_delay() { #[tokio::test(start_paused = true)] async fn a_source_past_the_limit_is_counted() { let (_, _, stats) = on_a_scaled_clock(1.0004, 300_000_000 / VIDEO_US).await; - let drift = stats.drift.expect("a measurement"); + let drift = stats.release.drift.expect("a measurement"); assert!((drift - 400.0).abs() < 1.0, "measured {drift} ppm"); - assert!(stats.out_of_tolerance > 0, "not counted"); + assert!(stats.release.out_of_tolerance > 0, "not counted"); } /// The same source fails the export once it has used half the delay, about ten minutes in, diff --git a/rs/moq-mux/src/container/ts/release.rs b/rs/moq-mux/src/container/ts/release.rs index 0b1be9eb035..27ce941611b 100644 --- a/rs/moq-mux/src/container/ts/release.rs +++ b/rs/moq-mux/src/container/ts/release.rs @@ -182,6 +182,15 @@ impl Release { self.jitter.out_of_tolerance() } + /// Snapshot how the release clock is keeping up with the source. + pub fn stats(&self) -> super::stats::Release { + let mut stats = super::stats::Release::of(&self.jitter); + if let Some((_, track)) = &self.track { + stats.gaps = track.skips() - track.markers(); + } + stats + } + /// How many objects are waiting in the jitter buffer. #[cfg(test)] pub(super) fn buffered(&self) -> usize { diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index b8b61981b76..f1fc76991e5 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -1,8 +1,33 @@ //! The counters [`Import::stats`](super::Import::stats) and //! [`Export::stats`](super::Export::stats) return, and reporting on them. +//! +//! [`Report`] carries them in a media report, as the `mpegts` and `release` sections beside +//! `transport` and `renditions`. Counters are cumulative and omitted at zero. use std::collections::{BTreeMap, BTreeSet}; +use serde::{Deserialize, Serialize}; + +fn is_zero(value: &u64) -> bool { + *value == 0 +} + +/// The sections an MPEG-TS import or export adds to its media report: the `E` of +/// [`hang::stats::Snapshot`] or [`hang::echo::Snapshot`], flattened beside `transport` and +/// `renditions`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +#[non_exhaustive] +pub struct Report { + /// What each elementary stream delivered and the transport stream's errors: an import's + /// [`Snapshot`], or the rows of an [`Export`]. A passthrough has none. + #[serde(skip_serializing_if = "Option::is_none")] + pub mpegts: Option, + /// How an export's release clock keeps up with the source. An import has none. + #[serde(skip_serializing_if = "Option::is_none")] + pub release: Option, +} + /// What each demuxed elementary stream delivered, and the audio frame sync it lost or could /// not verify, keyed by elementary stream PID; the PSI sections the stream as a whole lost to /// corruption; and its TR 101 290 errors. @@ -20,52 +45,67 @@ use std::collections::{BTreeMap, BTreeSet}; /// /// [`Export`] carries the same rows for the streams an exporter writes, so one schema reads /// both edges. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[serde_with::serde_as] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] #[non_exhaustive] pub struct Snapshot { /// One row per elementary stream PID the importer carries, ordered by PID. A PID the /// importer drops (an undecoded stream without the `mpegts` catalog section) has none. + // Keys read from strings: a flattened section loses JSON's integer-key parsing. + #[serde_as(as = "BTreeMap")] + #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub streams: BTreeMap, /// PAT and PMT sections dropped because their CRC-32 did not match. Each one left the /// table already in force standing, or, before any, delayed the program's start to the /// next good repetition. Counted for the stream rather than per elementary stream, since /// the PSI PIDs have no row of their own. TR 101 290 2.2 `CRC_error`, on the PAT and PMT /// only: the reduced-SI set of its table 5.1b. Captured SI is not CRC-checked. + #[serde(skip_serializing_if = "is_zero")] pub crc_error: u64, /// 1.1 `TS_sync_loss`: sync lost after two consecutive corrupt sync bytes on the packet /// grid, and acquired again after five correct ones (ISO/IEC 13818-1 G.1). + #[serde(skip_serializing_if = "is_zero")] pub ts_sync_loss: u64, /// 1.2 `Sync_byte_error`: a byte other than 0x47 where the grid expects a sync byte, while /// in sync. + #[serde(skip_serializing_if = "is_zero")] pub sync_byte_error: u64, /// 1.3 `PAT_error_2`: each 0.5 s of program clock without a PAT section on PID 0, and each /// packet on PID 0 that is scrambled or starts a section other than a PAT. + #[serde(skip_serializing_if = "is_zero")] pub pat_error: u64, /// 1.4 `Continuity_count_error`, on every PID but the null PID: a counter gap, or a third /// copy of a packet. A gap declared by `discontinuity_indicator`, the one duplicate ISO/IEC /// 13818-1 2.4.3.3 permits, and a payload-less packet repeating its counter are not errors. /// A loss of exactly 16 packets leaves the counter in step, and only a byte comparison /// tells a loss of 15 from a duplicate, so the count is a floor. + #[serde(skip_serializing_if = "is_zero")] pub continuity_count_error: u64, /// 1.5 `PMT_error_2`: each 0.5 s of program clock without a PMT section on a PMT PID the /// PAT names for the imported program, and each scrambled packet on one. + #[serde(skip_serializing_if = "is_zero")] pub pmt_error: u64, /// 2.1 `Transport_error`: packets with `transport_error_indicator` set, on any PID. Such a /// packet counts nothing else. + #[serde(skip_serializing_if = "is_zero")] pub transport_error: u64, /// 2.3a `PCR_repetition_error`: consecutive PCR values on the PCR PID more than 100 ms /// apart. Graded on the values, not on arrival: that speaks for the encoder's insertion, /// which is what ingest can, while the network in front of the importer re-times arrival. /// The interval across a `discontinuity_indicator` is not graded. + #[serde(skip_serializing_if = "is_zero")] pub pcr_repetition_error: u64, /// 2.3b `PCR_discontinuity_indicator_error`: consecutive PCR values outside 0 to 100 ms /// apart, backwards included, without `discontinuity_indicator`. On values a run of missing /// PCRs and an unsignalled jump are the same difference, so a forward step over 100 ms /// counts here and as a repetition error both. + #[serde(skip_serializing_if = "is_zero")] pub pcr_discontinuity_indicator_error: u64, /// 2.5 `PTS_error`: each 700 ms of program clock without a PTS on an elementary stream /// that has carried one. Verbatim streams are not graded: SCTE-35 and subtitles have no /// cadence to hold. + #[serde(skip_serializing_if = "is_zero")] pub pts_error: u64, } @@ -77,44 +117,57 @@ impl Snapshot { } /// What one elementary stream delivered, lost, or could not verify. See [`Snapshot`]. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[serde_with::serde_as] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] #[non_exhaustive] pub struct Stream { /// The current or most recent MoQ track suffix for this PID (`.avc3`, `.mp2`, `.ts`, ...), /// or empty for MPEG-1/2 video, which is read for its clock and not published. At export, /// the suffix an import of the output would give the PID. + #[serde(skip_serializing_if = "String::is_empty")] pub track: String, /// How this PID was classified. See [`Class`]. pub class: Class, /// Access units the stream delivered: frames published for decoded media, PES payloads or /// sections carried verbatim, and PES read on MPEG-1/2 video. At export, the PES or /// sections written for each frame. + #[serde(skip_serializing_if = "is_zero")] pub units: u64, /// Transport time since the stream last delivered an access unit, or since the PMT /// declared it, measured on the program clock (PCR): the one read at import, the one - /// written at export. `None` until the first PCR. + /// written at export. `None` until the first PCR. Whole milliseconds in a report. /// /// No threshold applies: a sparse stream such as SCTE-35 is legitimately quiet for /// seconds, so how long is too long is for whoever alarms to decide. + #[serde_as(as = "Option>")] + #[serde(skip_serializing_if = "Option::is_none")] pub quiet: Option, /// Completed resyncs: the stream lost frame sync and locked onto a confirmed frame /// again. Each one is a gap in the audio. + #[serde(skip_serializing_if = "is_zero")] pub resyncs: u64, /// Bytes thrown away scanning for sync, across every resync and any scan still in /// progress. + #[serde(skip_serializing_if = "is_zero")] pub discarded: u64, /// Frames published that nothing vouched for: the offset came from a scan, or the bytes /// were joined onto a tail carried across a PES boundary, and no successor frame could /// confirm it. That substitutes audio rather than leaving a gap, which is why it is /// counted separately from a resync. + #[serde(skip_serializing_if = "is_zero")] pub unconfirmed: u64, /// Damaged packets, PES, or access units refused whole on this PID. Export stays zero. + #[serde(skip_serializing_if = "is_zero")] pub damaged: u64, /// This PID's share of [`Snapshot::continuity_count_error`]. + #[serde(skip_serializing_if = "is_zero")] pub continuity_count_error: u64, /// This PID's share of [`Snapshot::transport_error`]. + #[serde(skip_serializing_if = "is_zero")] pub transport_error: u64, /// This PID's share of [`Snapshot::pts_error`]. + #[serde(skip_serializing_if = "is_zero")] pub pts_error: u64, } @@ -143,16 +196,19 @@ impl Stream { /// Not the PMT `stream_type`. `0x86` is DTS audio or, with a CUEI registration, an SCTE-35 /// section, and those take different routes. The stopped-stream log grades audio and video /// only; a sparse data PID stays counted. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum Class { - /// Undecoded or sparse data (SCTE-35, ID3, private sections, verbatim PES), or a PCR PID - /// that carries no elementary stream. A quiet second is not a stall. - #[default] - Data, /// Continuous audio. Audio, /// Continuous video, including MPEG-1/2 video read only for its clock. Video, + /// Undecoded or sparse data (SCTE-35, ID3, private sections, verbatim PES), or a PCR PID + /// that carries no elementary stream. A quiet second is not a stall. A class a later + /// version adds reads as data. + #[default] + #[serde(other)] + Data, } impl Class { @@ -174,29 +230,70 @@ impl Class { pub struct Export { /// One row per elementary stream PID the exporter writes, ordered by PID. pub streams: BTreeMap, + /// How the release clock is keeping up with the source. + pub release: Release, +} + +/// How an export keeps up with the source: the frames it drops, trims, and never receives, +/// and how its release clock tracks the source's. What [`Export::stats`](super::Export::stats) +/// reports beside its rows, and [`Release::stats`](super::Release::stats) on its own. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +#[non_exhaustive] +pub struct Release { /// Frames dropped after their stream's first went out: for arriving after their deadline, /// for falling due before a new timeline's clock was acquired, and the frames a video track /// then dropped waiting for its next keyframe or leading the keyframe it resumed at, late /// or after a hole upstream once the output held frames ahead of the hole. + #[serde(skip_serializing_if = "is_zero")] pub dropped: u64, /// Frames dropped before their stream's first went out, so the output starts on a whole /// group led by the video keyframe: what a joiner is handed older than the delay, a start /// group cut short upstream, the first keyframe's leading pictures, and what precedes the /// first keyframe or the codec config. /// Nothing is missing from the output for these. + #[serde(skip_serializing_if = "is_zero")] pub trimmed: u64, /// Times a source skipped media it will never deliver: a group lost or expired upstream, /// or a resubscription. A data track's declared break is a skipped sequence, so it counts /// here too, though nothing of it is lost. Each is logged with the span it skipped. + #[serde(skip_serializing_if = "is_zero")] pub gaps: u64, + /// The least time any frame arrived before its deadline over the last whole second, in + /// milliseconds: how much more latency the path could take before frames drop. Negative + /// when a frame missed its deadline. `None` until a second has passed, and without a delay. + #[serde(skip_serializing_if = "Option::is_none")] + pub margin: Option, + /// How far the output clock sits from the source's plus the delay, in milliseconds, as last + /// measured: positive holds frames longer than the delay, negative shorter. The clock pulls + /// it shut over ten minutes. `None` until there is a measurement. + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, /// How fast the source's clock runs against ours, in parts per million (positive runs /// fast), as last measured. `None` until there is a measurement. + #[serde(skip_serializing_if = "Option::is_none")] pub drift: Option, /// How many measurements found the source's clock further off ours than the 30 ppm an /// MPEG-TS system clock may be (ISO/IEC 13818-1 2.4.2.1), which the output cannot follow. + #[serde(skip_serializing_if = "is_zero")] pub out_of_tolerance: u64, } +impl Release { + pub(crate) fn of(jitter: &crate::jitter::Buffer) -> Self { + let millis = |seconds: f64| (seconds * 1e3).round() as i64; + Self { + dropped: jitter.dropped(), + trimmed: jitter.trimmed(), + gaps: 0, + margin: jitter.margin().map(millis), + offset: jitter.offset().map(millis), + drift: jitter.drift().map(|drift| drift * 1e6), + out_of_tolerance: jitter.out_of_tolerance(), + } + } +} + /// An export's rows with every stream-wide counter zero, so one [`Log`] reads either edge. impl From for Snapshot { fn from(export: Export) -> Self { @@ -507,6 +604,59 @@ mod test { }); } + /// A report carries the counters beside `transport` and `renditions`, camelCase, keyed by + /// PID, `quiet` in milliseconds, and zero counters omitted; it reads back whole, and a class + /// a later version adds reads as data. + #[test] + fn a_report_flattens_beside_the_media_counters() { + let mut mpegts = Snapshot { + continuity_count_error: 2, + ..Default::default() + }; + mpegts.streams.insert( + 256, + Stream { + track: ".avc3".to_string(), + class: Class::Video, + units: 120, + quiet: Some(std::time::Duration::from_micros(40_400)), + continuity_count_error: 2, + ..Default::default() + }, + ); + let release = Release { + margin: Some(-12), + offset: Some(3), + drift: Some(1.5), + ..Default::default() + }; + let mut snapshot = hang::stats::Snapshot::::default(); + snapshot.ext.mpegts = Some(mpegts); + snapshot.ext.release = Some(release); + + let json = serde_json::to_value(&snapshot).unwrap(); + assert_eq!( + json, + serde_json::json!({ + "mpegts": { + "streams": { + "256": { "track": ".avc3", "class": "video", "units": 120, "quiet": 40, "continuityCountError": 2 } + }, + "continuityCountError": 2 + }, + "release": { "margin": -12, "offset": 3, "drift": 1.5 } + }) + ); + let mut read: hang::stats::Snapshot = serde_json::from_value(json).unwrap(); + let quiet = &mut read.ext.mpegts.as_mut().unwrap().streams.get_mut(&256).unwrap().quiet; + assert_eq!(*quiet, Some(std::time::Duration::from_millis(40))); + *quiet = snapshot.ext.mpegts.as_ref().unwrap().streams[&256].quiet; + assert_eq!(read, snapshot); + + let row: Stream = serde_json::from_value(serde_json::json!({ "class": "subtitle", "later": 1 })).unwrap(); + assert_eq!(row.class, Class::Data); + } + /// Damage is reported per PID when its cumulative count moves, including the final sample. #[test] #[tracing_test::traced_test] diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 6ec87cb0532..49192a6659c 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -162,11 +162,18 @@ pub(crate) struct Buffer { trimmed: u64, /// Steps at which the source's clock ran further off ours than the clock may follow. out_of_tolerance: u64, + /// The least time any frame arrived before its deadline in the last whole [`MARGIN`], in + /// seconds: negative for a frame that missed it. + margin: Option, + /// The [`MARGIN`] under way: when its first frame was read, and the least time to spare so far. + spare: Option<(Instant, f64)>, } /// How often the clock is steered, in our time: long enough that the floor covers a /// group's worth of frames sent ahead by different amounts. const STEER: Duration = Duration::from_secs(2); +/// How long [`Buffer::margin`] takes the least of, in our time: a report's interval. +const MARGIN: Duration = Duration::from_secs(1); /// How far back the floors are kept, in decode time: a spell of queueing shorter than half /// of it moves neither the rate nor the slack. const RATE_WINDOW: f64 = 600.0; @@ -256,6 +263,9 @@ struct Steer { steered: f64, /// The source's clock rate off ours, as last measured: positive runs fast. source: Option, + /// How far the slack of the track sent latest sat off the delay, in seconds, as last + /// measured: the gap the clock pulls shut. + offset: Option, /// What is left of the gap between the slack and the delay that the rate's first /// measurement found, in seconds: what anchoring on frames that reached us late or early /// left, which the rate did not cost. @@ -270,6 +280,7 @@ impl Default for Steer { tracks: BTreeMap::new(), steered: 0.0, source: None, + offset: None, startup: None, } } @@ -348,6 +359,8 @@ impl Buffer { dropped: 0, trimmed: 0, out_of_tolerance: 0, + margin: None, + spare: None, } } @@ -541,6 +554,9 @@ impl Buffer { /// group its source cut short) takes everything the track holds with it, so its output starts on a /// whole group at its next sync frame rather than on a group's cut-off start. fn queue(&mut self, key: K, arrival: Arrival, generation: u64, deadline: Option, stale: bool) -> Push { + if let Some(deadline) = deadline.filter(|_| !stale && !self.delay.is_zero()) { + self.spare(arrival.read, since(arrival.arrived, deadline) as f64 / 1e9); + } let track = self.tracks.entry(key).or_insert_with(|| Track { queue: VecDeque::new(), restart: arrival.restart, @@ -597,6 +613,16 @@ impl Buffer { push } + /// Account for a frame read at `read` that arrived `spare` seconds before its deadline. + fn spare(&mut self, read: Instant, spare: f64) { + let window = self.spare.get_or_insert((read, spare)); + window.1 = window.1.min(spare); + if read.saturating_duration_since(window.0) >= MARGIN { + self.margin = Some(window.1); + self.spare = None; + } + } + /// Whether a frame arriving at `arrived` and due at `deadline` is on the clock: not late, /// and held no more than a delay past both its arrival and everything already queued. fn lands(&self, deadline: Option, arrived: Instant) -> bool { @@ -718,6 +744,7 @@ impl Buffer { }); let gap = at.clamp(low, high) + self.steer.steered; self.steer.source = Some(rate); + self.steer.offset = Some(gap); // The clock pulls the startup gap shut, so it shrinks with the gap and never reopens. let startup = self.steer.startup.get_or_insert(gap); *startup = startup.clamp(gap.min(0.0), gap.max(0.0)); @@ -953,6 +980,21 @@ impl Buffer { pub fn out_of_tolerance(&self) -> u64 { self.out_of_tolerance } + + /// The least time any frame arrived before its deadline in the last whole second of ours, + /// in seconds: negative for a frame that missed it. `None` until a second has passed with + /// a delay. + pub fn margin(&self) -> Option { + self.margin + } + + /// How far the clock sits off the source's plus the delay, in seconds, as last measured: + /// positive holds the track sent latest longer than the delay, negative shorter. The + /// clock pulls it shut over [`RESPONSE`]. `None` until the clock has measured the source's + /// rate, as [`Self::drift`]. + pub fn offset(&self) -> Option { + self.steer.offset + } } /// `nanos` after `at`, or before it if negative, if an instant holds it. @@ -1079,6 +1121,39 @@ mod tests { assert!(buffer.is_empty()); } + /// The margin is the least any frame had to spare over a whole second, a frame that missed + /// its deadline included, and holds until the next second has passed. + #[tokio::test(start_paused = true)] + async fn the_margin_is_the_least_spare_over_a_second() { + let start = Instant::now(); + let at = |ms: u64| start + Duration::from_millis(ms); + let mut buffer = Buffer::new(DELAY).replay(); + assert_eq!(buffer.push(1, arrival(at(0), 0, "v0")).unwrap(), Push::Queued); + assert_eq!(buffer.push(1, arrival(at(30), 40, "v40")).unwrap(), Push::Queued); + // Due at 180 ms. + assert_eq!(buffer.push(1, arrival(at(200), 80, "v80")).unwrap(), Push::Late); + assert_eq!(buffer.margin(), None, "no second has passed"); + + assert_eq!( + buffer.push(1, arrival(at(1_000), 1_000, "v1000")).unwrap(), + Push::Queued + ); + let margin = buffer.margin().expect("a second has passed"); + assert!((margin + 0.020).abs() < 1e-6, "margin {margin} s"); + + assert_eq!( + buffer.push(1, arrival(at(1_500), 1_500, "v1500")).unwrap(), + Push::Queued + ); + assert!(buffer.margin().unwrap() < 0.0, "held until the next second has passed"); + assert_eq!( + buffer.push(1, arrival(at(2_500), 2_500, "v2500")).unwrap(), + Push::Queued + ); + let margin = buffer.margin().unwrap(); + assert!((margin - 0.100).abs() < 1e-6, "the next second was clean: {margin} s"); + } + /// Two buffers fed the same frames with different arrival skew, every frame inside /// its deadline, emit the same order. #[tokio::test(start_paused = true)] @@ -1462,6 +1537,9 @@ mod tests { out_of_tolerance: u64, /// How far off the delay the frame the run failed on would have been held, in seconds. off: f64, + /// The buffer's margin and offset at the end. + margin: Option, + offset: Option, } /// Feed `hours` of 2 fps frames from a source whose clock runs `ppm` slower than ours @@ -1481,6 +1559,8 @@ mod tests { clock_before: 0.0, out_of_tolerance: 0, off: 0.0, + margin: None, + offset: None, }; let mut last: Option<(f64, Timestamp)> = None; let mut stepped = false; @@ -1545,6 +1625,8 @@ mod tests { run.drift = buffer.drift(); run.clock = buffer.clock.map_or(0.0, |clock| clock.drift); run.out_of_tolerance = buffer.out_of_tolerance(); + run.margin = buffer.margin(); + run.offset = buffer.offset(); run } @@ -1559,6 +1641,14 @@ mod tests { let run = drift(ppm, 24.0, 16.0, |_| Duration::ZERO).await; assert!(run.err.is_none(), "{ppm} ppm: {:?}", run.err); assert!(run.worst <= bound, "{ppm} ppm: held {} s off the delay", run.worst); + // One track queued nowhere: every frame's spare is its slack, the delay and the offset. + let offset = run.offset.expect("an offset"); + assert!(offset.abs() <= bound, "{ppm} ppm: offset {offset} s"); + let margin = run.margin.expect("a margin"); + assert!( + (margin - 0.5 - offset).abs() < 0.001, + "{ppm} ppm: margin {margin} s, offset {offset} s" + ); let drift = run.drift.expect("a measurement") * 1e6; assert!((drift + ppm).abs() < 0.1, "{ppm} ppm: measured {drift} ppm"); assert!( From 58ffb52e376f316fffe6011f621378bf36fb0bad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 20:42:19 -0700 Subject: [PATCH 04/12] fix(cli): refuse --stats with --epoch and give feedback an epoch `--epoch` makes instances one publisher of identical tracks, and a stats track carries this instance's own counts and transport, so the pair is refused. Each `.echo` feedback broadcast now announces a fresh epoch like every other CLI publish, so a replaced viewer reads as a restart. The echo target is a struct instead of a tuple, and the CLI page drops the edge cases and unshipped fields the API docs and quest already carry. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 12 ++++-------- rs/moq-cli/src/args.rs | 8 +++++++- rs/moq-cli/src/main.rs | 19 ++++++++----------- rs/moq-cli/src/report.rs | 34 ++++++++++++++++------------------ 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 65a173eaed7..bb4a2492a26 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -403,15 +403,13 @@ so the other formats refuse `--linger`. `import --stats` names a stats track in the catalog and reports what each rendition has written, about once a second while someone subscribes, with the connection's RTT, estimated send rate, and loss. It covers the stdin imports; -capture, the gateways, `archive`, and `ts --program all` refuse it. +capture, the gateways, `archive`, `ts --program all`, and `--epoch` refuse it. `export --echo ` answers a publisher that solicits feedback: under the catalog's `echo` path it publishes `.echo`, reporting what each rendition the export reads has received and its newest arrival, until the export ends. -The name is one path segment, not starting with `.`, which would hide it from -discovery. It covers the stdout container exports and reads the solicitation -from the hang catalog, so an MSF catalog, named or detected from a `.msf` -broadcast, refuses it. +It covers the stdout container exports and needs a hang catalog, so MSF +refuses it. The `stats` and `echo` sinks print reports as JSON lines. `export stats` follows one broadcast's stats track until the broadcast ends; `export echo` @@ -427,9 +425,7 @@ moq ... --broadcast viewers export echo Counters are cumulative for the life of the process: a rendition that ends keeps its final counts, and one that returns under the same ID continues them. -An fMP4 import counts samples as frames. Encoder counters (`skipped`, -`bitrate`) and playback feedback (`decoded`, `late`, `stalls`) are not reported -yet. [Stats](/concept/stats#media-reports) has the schema. +[Stats](/concept/stats#media-reports) has the schema. ## Debugging diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index b95db53ca38..ac400122a46 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -323,6 +323,10 @@ impl Invocation { if let Some(reason) = import.source.epoch_refusal() { anyhow::bail!("--epoch makes instances one publisher of identical tracks, but {reason}"); } + anyhow::ensure!( + !import.stats, + "--epoch makes instances one publisher of identical tracks, but --stats reports this instance's own counts" + ); publishes = true; } #[cfg(feature = "transcode")] @@ -782,7 +786,8 @@ pub struct Import { latency_max: Option, /// Name a stats track in the catalog and fill it with what each track has written, about - /// once a second while subscribed. The stdin imports only. + /// once a second while subscribed. The stdin imports only, without --epoch: the counts are + /// this instance's own. #[usage(long)] pub stats: bool, @@ -1731,6 +1736,7 @@ mod tests { &["import", "ts", "--program", "all"], &["import", "avc3"], &["import", "archive", "file:///tmp/recording"], + &["import", "--stats", "ts"], #[cfg(feature = "capture")] &["import", "capture"], #[cfg(feature = "transcode")] diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 46ded2ad7af..06dec8e3363 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -800,7 +800,11 @@ fn spawn_export( select: export.select, }; let consumer = origin.consume(); - let echo = export.echo.map(|echo| (origin.clone(), echo, connection)); + let echo = export.echo.map(|name| report::EchoTarget { + origin: origin.clone(), + name, + connection, + }); tasks.spawn(async move { run_stdout(consumer, name, args, echo).await }); } else { match export.sink { @@ -864,20 +868,13 @@ fn spawn_export( Ok(()) } -/// Where an `export --echo` publishes its feedback, under what name, and over which connection. -type EchoTarget = ( - moq_net::origin::Producer, - report::EchoName, - Option, -); - /// Subscribe to `name` from the Origin and write it to stdout, publishing feedback beside it /// where the catalog solicits it when `echo` is set. async fn run_stdout( consumer: moq_net::origin::Consumer, name: String, args: SubscribeArgs, - echo: Option, + echo: Option, ) -> anyhow::Result<()> { let catalog = args.catalog_format(&name); @@ -890,10 +887,10 @@ async fn run_stdout( .ok_or_else(|| anyhow::anyhow!("origin closed before broadcast `{name}` was announced"))?; let subscribe = Subscribe::new(consumer, &name, catalog, args).await?; - let Some((origin, echo, connection)) = echo else { + let Some(echo) = echo else { return subscribe.run().await; }; - let mut echo = report::Echo::new(origin, subscribe.source().clone(), catalog, echo, connection); + let mut echo = report::Echo::new(echo, subscribe.source().clone(), catalog); let result = tokio::select! { result = subscribe.run() => result, never = echo.run() => match never {}, diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 4e7f59c417d..c7ecda63a20 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -208,6 +208,13 @@ impl std::str::FromStr for EchoName { } } +/// Where an `export --echo` publishes its feedback, under what name, and over which connection. +pub struct EchoTarget { + pub origin: moq_net::origin::Producer, + pub name: EchoName, + pub connection: Option, +} + /// The feedback broadcast in force: where it is published and its snapshot tracks. struct Feedback { path: moq_net::PathOwned, @@ -222,29 +229,19 @@ struct Feedback { /// removed section or a changed path retracts the broadcast it replaces. It follows the catalog /// of the instance the export first reads, and ends when the export does. pub struct Echo { - origin: moq_net::origin::Producer, + target: EchoTarget, source: moq_mux::Source, format: CatalogFormat, - name: EchoName, - connection: Option, feedback: Option, } impl Echo { - /// Feedback for the export reading `source`, published into `origin`. - pub fn new( - origin: moq_net::origin::Producer, - source: moq_mux::Source, - format: CatalogFormat, - name: EchoName, - connection: Option, - ) -> Self { + /// Feedback at `target` for the export reading `source` through a `format` catalog. + pub fn new(target: EchoTarget, source: moq_mux::Source, format: CatalogFormat) -> Self { Self { - origin, + target, source, format, - name, - connection, feedback: None, } } @@ -284,7 +281,7 @@ impl Echo { if prefix.is_none() { tracing::warn!(path = %echo.path.as_str(), "the catalog solicits feedback above the connection root"); } - Some(prefix?.join(format!("{}{}", self.name.0, hang::echo::SUFFIX).as_str())) + Some(prefix?.join(format!("{}{}", self.target.name.0, hang::echo::SUFFIX).as_str())) }); if self.feedback.as_ref().map(|feedback| &feedback.path) == target.as_ref() { return; @@ -304,9 +301,10 @@ impl Echo { } fn publish(&self, path: &moq_net::PathOwned) -> anyhow::Result { - let mut broadcast = self.origin.create_broadcast(path)?; + let mut broadcast = self.target.origin.create_broadcast(path)?; let tracks = Pair::create(&mut broadcast, hang::echo::TRACK)?; - broadcast.announce(moq_net::origin::Route::default())?; + // A fresh instance, so a reader sees a replaced feedback broadcast as a restart. + broadcast.announce(moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()))?; Ok(Feedback { path: path.clone(), _broadcast: broadcast, @@ -319,7 +317,7 @@ impl Echo { return; }; let mut snapshot = hang::echo::Snapshot::default(); - snapshot.transport = transport(self.connection.as_ref(), Direction::Receive); + snapshot.transport = transport(self.target.connection.as_ref(), Direction::Receive); snapshot.renditions = self.source.stats(); feedback.tracks.update(&snapshot); } From 90c4fbfe7a57446995e460a1ad82a070140d388a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 20:54:18 -0700 Subject: [PATCH 05/12] chore(cli): format, and name the --echo value Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/args.rs | 2 +- rs/moq-cli/src/publish.rs | 10 +++++++++- rs/moq-cli/src/report.rs | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 26f37f9d453..1b99f4a7a1d 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -1007,7 +1007,7 @@ pub struct Export { /// Publish feedback where the broadcast's catalog solicits it: a `.echo` broadcast /// under the catalog's `echo` path, reporting what each rendition this export reads has /// received. One path segment, not starting with `.`. The stdout container exports only. - #[usage(long)] + #[usage(long, value_name = "NAME")] pub echo: Option, /// The single sink draining the Origin. diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 9e241664ec8..c4f7a16b52d 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -508,7 +508,15 @@ impl Publish { (Some(broadcast), Some(route)) => Some(Pending::new(broadcast.clone(), route)?), _ => None, }; - decode(decoder, catalog, tokio::io::stdin(), self.broadcast, pending, self.stats).await + decode( + decoder, + catalog, + tokio::io::stdin(), + self.broadcast, + pending, + self.stats, + ) + .await } #[cfg(feature = "capture")] Source::Capture { diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 77a6ba10a7c..577b64b911d 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -211,8 +211,11 @@ impl std::str::FromStr for EchoName { /// Where an `export --echo` publishes its feedback, under what name, and over which connection. pub struct EchoTarget { + /// The origin the feedback broadcast is published into. pub origin: moq_net::origin::Producer, + /// The path segment the feedback is published at, before `.echo`. pub name: EchoName, + /// The connection whose transport the feedback reports, when one was dialed. pub connection: Option, } From c97bc23801b18dc4e5be09876b4accde7fdedd0c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 21:58:57 -0700 Subject: [PATCH 06/12] fix(cli): attach --stats where the import's pipeline is collected spawn_import gained a shared-epoch argument on main, which pushed it past clippy's argument limit; the stats report now attaches to the returned Publish instead, still before any track is declared. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/main.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 285ccb2345b..dbe9e8ce064 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -595,17 +595,20 @@ async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Res if import.source.stdin_format().is_some() { claim("stdin", &mut stdin, &name)?; } - let monitor = connection.as_ref().map(moq_tokio::Connection::monitor); - if let Some(publish) = spawn_import( + let stats = import.stats; + if let Some(mut publish) = spawn_import( &origin, import, name, epoch.clone(), shared, bandwidth.clone(), - monitor, &mut tasks, )? { + // Before `run` declares a track, so the stats section rides the first catalog. + if stats { + publish = publish.with_stats(connection.as_ref().map(moq_tokio::Connection::monitor))?; + } locals.push(publish); } } @@ -699,8 +702,7 @@ fn display_name(name: &str) -> &str { /// one redundant publishers were given, also places the importers' clock. /// /// Returns the pipeline that has to run on the caller's thread, for the sources -/// that have one (the stdin containers and capture). `connection` supplies a -/// `--stats` report's transport measurements. +/// that have one (the stdin containers and capture). fn spawn_import( origin: &moq_net::origin::Producer, import: Import, @@ -708,7 +710,6 @@ fn spawn_import( epoch: moq_net::Epoch, shared: bool, bandwidth: moq_net::bandwidth::Allocator, - connection: Option, tasks: &mut JoinSet>, ) -> anyhow::Result> { if let ImportSource::Rtc(rtc) = &import.source @@ -745,9 +746,6 @@ fn spawn_import( let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; Publish::new(broadcast, &format, config, shared.then_some(&epoch))? }; - if import.stats { - publish = publish.with_stats(connection)?; - } publish.announce(epoch)?; local = Some(publish); } else { From 39fa8f59137426ef72dd777eb55cdda01244228f Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 20 Oct 2026 12:50:29 +0100 Subject: [PATCH 07/12] docs(cli): drop --epoch from what refuses --stats Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 2 +- rs/moq-cli/src/args.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 1f63fce4e41..a91a82b4c04 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -453,7 +453,7 @@ so the other formats refuse `--linger`. `import --stats` names a stats track in the catalog and reports what each rendition has written, about once a second while someone subscribes, with the connection's RTT, estimated send rate, and loss. It covers the stdin imports; -capture, the gateways, `archive`, `ts --program all`, and `--epoch` refuse it. +capture, the gateways, `archive`, and `ts --program all` refuse it. `export --echo ` answers a publisher that solicits feedback: under the catalog's `echo` path it publishes `.echo`, reporting what each rendition diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 0b46749463a..e00348d9481 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -757,8 +757,7 @@ pub struct Import { latency_max: Option, /// Name a stats track in the catalog and fill it with what each track has written, about - /// once a second while subscribed. The stdin imports only, without --epoch: the counts are - /// this instance's own. + /// once a second while subscribed. The stdin imports only. #[usage(long)] pub stats: bool, From 7bc5705fb7a811bba792353eb713ac71ff13917d Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 20 Oct 2026 15:04:01 +0100 Subject: [PATCH 08/12] fix(cli): bound what the report sinks hold - `export stats` and `export echo` read a report with a 1 MiB cap on its reconstructed state, so patches that keep adding keys end the read instead of growing the sink. - `export echo` keeps a reader whose broadcast ended, so a start at the path replaces it, and aborts it after 5 s: the final snapshot still drains, and a track that outlives its route no longer leaves a reader behind. - A stats registry keeps the first 1,024 rendition IDs; a catalog that keeps renaming its renditions no longer grows it, or each report, without bound. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 4 +- rs/moq-cli/src/report.rs | 88 ++++++++++++++++++++++++++++++++++++---- rs/moq-mux/src/stats.rs | 27 +++++++++++- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index a91a82b4c04..09c495363b3 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -464,7 +464,8 @@ refuses it. The `stats` and `echo` sinks print reports as JSON lines. `export stats` follows one broadcast's stats track until the broadcast ends; `export echo` prints `{"broadcast", "feedback"}` for every `.echo` broadcast under -`--broadcast` as they come and go. +`--broadcast` as they come and go, reading one that ends for 5 s more for its +final report. Either sink stops reading a report that grows past 1 MiB. ```bash moq ... --broadcast event import --stats ts < feed.ts @@ -475,6 +476,7 @@ moq ... --broadcast viewers export echo Counters are cumulative for the life of the process: a rendition that ends keeps its final counts, and one that returns under the same ID continues them. +A report keeps the first 1,024 rendition IDs it sees. [Stats](/concept/stats#media-reports) has the schema. ## Debugging diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 1740b1cd9c9..f5ed489a484 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -27,6 +27,14 @@ const COMPRESSED: &str = ".z"; /// Lines `export echo` queues for stdout before its readers wait for it to catch up. const LINES: usize = 64; +/// The most reconstructed state a sink holds for one report. A report is a few KiB, so this +/// only stops a publisher whose patches keep adding keys. +const MAX_REPORT: usize = 1024 * 1024; + +/// How long `export echo` lets a reader drain after its broadcast ends. The final snapshot can +/// still be in flight, but the track can also outlive its route. +const DRAIN: Duration = Duration::from_secs(5); + /// Below every media track, so a report never delays the media it describes. const PRIORITY: u8 = 0; @@ -348,6 +356,13 @@ async fn next(catalog: &mut Option) -> moq_mux::Result moq_json::snapshot::Consumer { + let mut config = moq_json::snapshot::consumer::Config::default(); + config.max_size = Some(MAX_REPORT); + moq_json::snapshot::Consumer::new(subscriber, config) +} + /// The next snapshot of a report track, or never while there is none. async fn snapshot( track: &mut Option>, @@ -391,7 +406,7 @@ pub async fn print_stats(origin: moq_net::origin::Consumer, path: String) -> any track = match &stats { Some(name) => { let subscriber = broadcast.track(name)?.subscribe(None).await?; - Some(moq_json::snapshot::Consumer::new(subscriber, Default::default())) + Some(reader(subscriber)) } None => None, }; @@ -420,7 +435,15 @@ struct EchoLine<'a> { /// `export echo`: write the feedback of every `.echo` broadcast under `prefix` to stdout as JSON /// lines, following them as they start and end. pub async fn print_echo(origin: moq_net::origin::Consumer, prefix: String) -> anyhow::Result<()> { - let mut stdout = tokio::io::stdout(); + write_echo(origin, prefix, tokio::io::stdout()).await +} + +/// [`print_echo`] to any writer. +async fn write_echo( + origin: moq_net::origin::Consumer, + prefix: String, + mut stdout: impl tokio::io::AsyncWrite + Unpin, +) -> anyhow::Result<()> { let prefix = moq_net::Path::new(&prefix).to_owned(); let mut announced = origin.announced(); let (lines, mut written) = tokio::sync::mpsc::channel::(LINES); @@ -446,10 +469,16 @@ pub async fn print_echo(origin: moq_net::origin::Consumer, prefix: String) -> an if let Some(previous) = readers.insert(path, reader) { previous.abort(); } - } else { - // Detached, not aborted: the final snapshot can still be in flight, and the reader - // ends with its track. - readers.remove(&path); + } else if let Some(reader) = readers.get(&path) { + // Left to drain, not aborted: the final snapshot can still be in flight. It stays + // here so a start at the path replaces it, and goes after `DRAIN` even if its + // track stays open. + let reader = reader.abort_handle(); + tokio::spawn(async move { + tokio::time::sleep(DRAIN).await; + reader.abort(); + }); + readers.retain(|_, reader| !reader.is_finished()); } } Some(line) = written.recv() => { @@ -469,7 +498,7 @@ async fn read_echo( let read = async { let broadcast = origin.request_broadcast(&path, None).await?; let subscriber = broadcast.track(hang::echo::TRACK)?.subscribe(None).await?; - let mut track = moq_json::snapshot::Consumer::::new(subscriber, Default::default()); + let mut track = reader(subscriber); while let Some(feedback) = track.next().await? { let line = EchoLine { broadcast: path.as_str(), @@ -489,3 +518,48 @@ async fn read_echo( tracing::debug!(%err, path = %path.as_str(), "stopped reading feedback"); } } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncBufReadExt, BufReader}; + + /// Patches that grow a report past `MAX_REPORT` end the read instead of growing the sink. + #[tokio::test] + async fn a_report_past_the_cap_ends_the_read() { + let track = moq_net::broadcast::Info::new() + .produce() + .create_track(STATS_TRACK, None) + .unwrap(); + let mut read = reader(track.subscribe(None)); + let mut report = moq_json::snapshot::Producer::::new(track, Default::default()); + report + .update(&serde_json::json!({ "pad": "x".repeat(MAX_REPORT) })) + .unwrap(); + assert!(matches!(read.next().await, Err(moq_json::Error::TooLarge(MAX_REPORT)))); + } + + /// A feedback track can outlive its route, so a reader left to drain stops after `DRAIN`. + #[tokio::test(start_paused = true)] + async fn an_ended_feedback_reader_stops_after_draining() { + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast("viewer.echo").unwrap(); + let track = broadcast.create_track(hang::echo::TRACK, None).unwrap(); + let mut feedback = moq_json::snapshot::Producer::::new(track, Default::default()); + broadcast.announce(Default::default()).unwrap(); + + let (out, read) = tokio::io::duplex(4096); + let mut lines = BufReader::new(read).lines(); + tokio::spawn(write_echo(origin.consume(), String::new(), out)); + + feedback.update(&serde_json::json!({ "n": 1 })).unwrap(); + let line = lines.next_line().await.unwrap().unwrap(); + assert!(line.contains(r#""n":1"#), "{line}"); + + broadcast.unannounce(); + tokio::time::sleep(DRAIN * 2).await; + feedback.update(&serde_json::json!({ "n": 2 })).unwrap(); + let late = tokio::time::timeout(Duration::from_secs(1), lines.next_line()).await; + assert!(late.is_err(), "a drained reader wrote {late:?}"); + } +} diff --git a/rs/moq-mux/src/stats.rs b/rs/moq-mux/src/stats.rs index 45a22db24bf..e5e3ca0fed7 100644 --- a/rs/moq-mux/src/stats.rs +++ b/rs/moq-mux/src/stats.rs @@ -80,6 +80,10 @@ impl Received { } } +/// The most rendition IDs a registry keeps. Entries are never evicted, so a remote catalog that +/// keeps renaming its renditions would otherwise grow it, and each report, without bound. +const MAX_ENTRIES: usize = 1024; + /// Counters keyed by rendition ID, shared by the tracks that bump them and the reporter that reads /// them. pub(crate) struct Registry(Mutex>>); @@ -91,9 +95,16 @@ impl Default for Registry { } impl Registry { - /// The counters for `name`, continuing those of an earlier track under that name. + /// The counters for `name`, continuing those of an earlier track under that name. Past + /// [`MAX_ENTRIES`] names, a new one counts into counters no report reads. pub(crate) fn track(&self, name: &str) -> Arc { let mut entries = self.0.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(counters) = entries.get(name) { + return counters.clone(); + } + if entries.len() >= MAX_ENTRIES { + return Arc::default(); + } entries.entry(name.to_string()).or_default().clone() } @@ -150,6 +161,20 @@ mod tests { assert_eq!(registry.sent()["video"].bytes, 30); } + #[test] + fn renditions_past_the_cap_are_not_kept() { + let registry = Registry::::default(); + for n in 0..=MAX_ENTRIES { + registry.track(&format!("r{n}")).add(1, 1, 0); + } + let report = registry.sent(); + assert_eq!(report.len(), MAX_ENTRIES); + assert!(!report.contains_key(&format!("r{MAX_ENTRIES}"))); + + registry.track("r0").add(1, 1, 0); + assert_eq!(registry.sent()["r0"].frames, 2); + } + #[test] fn arrival_pairs_the_latest_timestamp_with_its_wall_time() { let registry = Registry::::default(); From 0f0d474668f2947d37fb69c00803585eca095570 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 20 Oct 2026 15:33:37 +0100 Subject: [PATCH 09/12] fix(mux): pin `Class`'s discriminants; note the `Export` move in MIGRATE `Data` stays last for `#[serde(other)]`, and explicit discriminants keep the `as` values `main` gave each class. MIGRATE.md tells callers where `Export`'s release counters went. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/MIGRATE.md | 5 +++++ rs/moq-mux/src/container/ts/stats.rs | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index f3477c78df7..ecaee5131d9 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `ts::stats::Export`'s release counters move into `release` (#5849) + +`dropped`, `trimmed`, `gaps`, `drift`, and `out_of_tolerance` are now fields of `Export::release`, a `ts::stats::Release`, which `ts::Release::stats` also returns on its own. +Read `export.release.dropped` where you read `export.dropped`, and likewise for the rest. + ### `ts::Passthrough` describes the whole track in its `m2ts` section (#5967) The section's `randomAccess` turns false at the first group that does not start at a random access point and stays false, instead of recovering with the next two-second window; its `muxRate` follows the source's measured rate. diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index efe5aa4b33b..a760599e5e4 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -213,15 +213,16 @@ impl Stream { #[serde(rename_all = "lowercase")] pub enum Class { /// Continuous audio. - Audio, + Audio = 1, /// Continuous video, including MPEG-1/2 video read only for its clock. - Video, + Video = 2, /// Undecoded or sparse data (SCTE-35, ID3, private sections, private PES), or a PCR PID /// that carries no elementary stream. A quiet second is not a stall. A class a later - /// version adds reads as data. + /// version adds reads as data. It must stay last for `#[serde(other)]`; the explicit + /// discriminants keep every variant's `as` value. #[default] #[serde(other)] - Data, + Data = 0, } impl Class { @@ -438,6 +439,14 @@ impl Log { mod test { use super::*; + /// `Class` keeps its `as` values with `Data` last, and an unknown name reads as data. + #[test] + fn class_keeps_its_discriminants() { + assert_eq!([Class::Data as u8, Class::Audio as u8, Class::Video as u8], [0, 1, 2]); + let class: Class = serde_json::from_str(r#""subtitle""#).unwrap(); + assert_eq!(class, Class::Data); + } + /// A stream whose count stops across a sample is logged once, with its silence, and again /// only after it has delivered in between; a stream that keeps counting never is, and /// neither reports frame sync it never lost. From d3c4b727d4ef7dd7e2b47dffc6f4255f87bafb62 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 20 Oct 2026 16:20:00 +0100 Subject: [PATCH 10/12] fix(cli): report live renditions past the cap; read the root and hidden prefixes At its 1,024 names a stats registry now drops the renditions no track counts into before it refuses a name, and logs once when every entry is live. `export stats` reads the connection root when `--connect` names the broadcast, as the other point exports do. `export echo` asks for hidden routes when its prefix names one. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- rs/moq-cli/src/main.rs | 16 ++++++++- rs/moq-cli/src/report.rs | 23 +++++++++++- rs/moq-mux/src/catalog/producer.rs | 3 +- rs/moq-mux/src/source.rs | 3 +- rs/moq-mux/src/stats.rs | 57 ++++++++++++++++++++++-------- 6 files changed, 86 insertions(+), 19 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 09c495363b3..0d4a046c500 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -476,7 +476,8 @@ moq ... --broadcast viewers export echo Counters are cumulative for the life of the process: a rendition that ends keeps its final counts, and one that returns under the same ID continues them. -A report keeps the first 1,024 rendition IDs it sees. +A report holds 1,024 rendition IDs at most: past that, ended renditions make +way for new ones. [Stats](/concept/stats#media-reports) has the schema. ## Debugging diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index c405e001f36..1585606677d 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -885,7 +885,6 @@ fn spawn_export( tasks.spawn(archive::export(origin.consume(), name, format, args)); } ExportSink::Stats => { - let name = require_broadcast(name, "export stats")?; tasks.spawn(report::print_stats(origin.consume(), name)); } ExportSink::Echo => { @@ -1381,4 +1380,19 @@ mod tests { assert!(!is_public_transport(moq_tokio::Transport::Quic, false)); assert!(is_public_transport(moq_tokio::Transport::Quic, true)); } + + /// A `--connect` URL can name the broadcast itself, leaving the stage's broadcast empty: the + /// connection root, which `export stats` reads like the other point exports. + #[tokio::test] + async fn export_stats_reads_the_connection_root() { + let invocation = + Invocation::try_parse_from(["moq", "--connect", "http://relay/event", "export", "stats"]).unwrap(); + let Some(Command::Export(export)) = invocation.stages.into_iter().next() else { + panic!("parsed as something other than an export"); + }; + let origin = moq_tokio::origin::spawn(); + let mut tasks = JoinSet::new(); + spawn_export(&origin, export, String::new(), None, &mut tasks).expect("the root is a broadcast"); + tasks.abort_all(); + } } diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index f5ed489a484..854b75907a2 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -444,8 +444,10 @@ async fn write_echo( prefix: String, mut stdout: impl tokio::io::AsyncWrite + Unpin, ) -> anyhow::Result<()> { + // Announcements leave hidden routes out unless asked, and a prefix naming one asks. + let hidden = prefix.split('/').any(|segment| segment.starts_with('.')); let prefix = moq_net::Path::new(&prefix).to_owned(); - let mut announced = origin.announced(); + let mut announced = origin.clone().with_hidden(hidden).announced(); let (lines, mut written) = tokio::sync::mpsc::channel::(LINES); let mut readers = HashMap::new(); loop { @@ -562,4 +564,23 @@ mod tests { let late = tokio::time::timeout(Duration::from_secs(1), lines.next_line()).await; assert!(late.is_err(), "a drained reader wrote {late:?}"); } + + /// Feedback under a hidden prefix is found when that prefix is the one asked for. + #[tokio::test(start_paused = true)] + async fn a_hidden_prefix_finds_its_feedback() { + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast(".viewers/alice.echo").unwrap(); + let track = broadcast.create_track(hang::echo::TRACK, None).unwrap(); + let mut feedback = moq_json::snapshot::Producer::::new(track, Default::default()); + broadcast.announce(Default::default()).unwrap(); + + let (out, read) = tokio::io::duplex(4096); + let mut lines = BufReader::new(read).lines(); + tokio::spawn(write_echo(origin.consume(), ".viewers".to_string(), out)); + + feedback.update(&serde_json::json!({ "n": 1 })).unwrap(); + let line = tokio::time::timeout(Duration::from_secs(5), lines.next_line()).await; + let line = line.expect("no feedback read").unwrap().unwrap(); + assert!(line.contains(r#""n":1"#), "{line}"); + } } diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index ff4c37e36fd..65108d16292 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -855,7 +855,8 @@ impl Producer { /// rendition ID, or the track name for an `m2ts` or verbatim `mpegts` track. /// /// Counters are cumulative for the life of the catalog, so a track that ended keeps its final - /// counts and a later track under the same key continues them. Fold this into a + /// counts and a later track under the same key continues them, until 1,024 keys are held: then + /// ended tracks make way for new ones. Fold this into a /// [`hang::stats::Snapshot`] on the stats interval. pub fn stats(&self) -> std::collections::BTreeMap { self.sent.sent() diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index c8f6c9437b1..c619162c924 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -82,7 +82,8 @@ impl Source { /// /// Counters are cumulative for the life of the source, across resubscriptions and a /// [`Follower`](crate::container::ts::Follower) switching instances, so a rendition no export - /// reads any more keeps its final counts. Fold this into an + /// reads any more keeps its final counts, until 1,024 renditions are held: then those no export + /// reads make way for new ones. Fold this into an /// [`echo::Snapshot`](hang::echo::Snapshot) on the feedback interval. pub fn stats(&self) -> std::collections::BTreeMap { self.received.received() diff --git a/rs/moq-mux/src/stats.rs b/rs/moq-mux/src/stats.rs index e5e3ca0fed7..572f6b45edd 100644 --- a/rs/moq-mux/src/stats.rs +++ b/rs/moq-mux/src/stats.rs @@ -6,7 +6,7 @@ //! [`Source`](crate::Source) for what a viewer reads. use std::collections::BTreeMap; -use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, UNIX_EPOCH}; @@ -80,38 +80,57 @@ impl Received { } } -/// The most rendition IDs a registry keeps. Entries are never evicted, so a remote catalog that -/// keeps renaming its renditions would otherwise grow it, and each report, without bound. +/// The most rendition IDs a registry keeps, so that a remote catalog that keeps renaming its +/// renditions cannot grow it, and each report, without bound. const MAX_ENTRIES: usize = 1024; /// Counters keyed by rendition ID, shared by the tracks that bump them and the reporter that reads /// them. -pub(crate) struct Registry(Mutex>>); +pub(crate) struct Registry { + entries: Mutex>>, + /// Whether a name has gone unreported because every entry was live, which is logged once. + full: AtomicBool, +} impl Default for Registry { fn default() -> Self { - Self(Mutex::default()) + Self { + entries: Mutex::default(), + full: AtomicBool::new(false), + } } } impl Registry { - /// The counters for `name`, continuing those of an earlier track under that name. Past - /// [`MAX_ENTRIES`] names, a new one counts into counters no report reads. + /// The counters for `name`, continuing those of an earlier track under that name. At + /// [`MAX_ENTRIES`] names, those no track still counts into make way; with every one live, a new + /// name counts into counters no report reads. pub(crate) fn track(&self, name: &str) -> Arc { - let mut entries = self.0.lock().unwrap_or_else(PoisonError::into_inner); + let mut entries = self.entries.lock().unwrap_or_else(PoisonError::into_inner); if let Some(counters) = entries.get(name) { return counters.clone(); } if entries.len() >= MAX_ENTRIES { + // Each track holds its counters for as long as it runs, so the registry's is the last. + entries.retain(|_, counters| Arc::strong_count(counters) > 1); + } + if entries.len() >= MAX_ENTRIES { + if !self.full.swap(true, Relaxed) { + tracing::warn!( + name, + limit = MAX_ENTRIES, + "every reported rendition is live; not reporting more" + ); + } return Arc::default(); } entries.entry(name.to_string()).or_default().clone() } - /// Read every entry. A track that ended keeps its final counts for the registry's life, as the + /// Read every entry. A track that ended keeps its final counts until the registry fills, as the /// catalog a finished import leaves up still lists it. fn report(&self, read: impl Fn(&T) -> R) -> BTreeMap { - let entries = self.0.lock().unwrap_or_else(PoisonError::into_inner); + let entries = self.entries.lock().unwrap_or_else(PoisonError::into_inner); entries .iter() .map(|(name, counters)| (name.clone(), read(counters))) @@ -161,18 +180,28 @@ mod tests { assert_eq!(registry.sent()["video"].bytes, 30); } + /// With every entry live a new name goes unreported and a known one still counts; once the + /// tracks end, a new name takes their place. #[test] - fn renditions_past_the_cap_are_not_kept() { + fn renditions_past_the_cap_replace_ended_ones() { let registry = Registry::::default(); - for n in 0..=MAX_ENTRIES { - registry.track(&format!("r{n}")).add(1, 1, 0); + let live: Vec<_> = (0..MAX_ENTRIES).map(|n| registry.track(&format!("r{n}"))).collect(); + for counters in &live { + counters.add(1, 1, 0); } + registry.track("late").add(1, 1, 0); let report = registry.sent(); assert_eq!(report.len(), MAX_ENTRIES); - assert!(!report.contains_key(&format!("r{MAX_ENTRIES}"))); + assert!(!report.contains_key("late")); registry.track("r0").add(1, 1, 0); assert_eq!(registry.sent()["r0"].frames, 2); + + drop(live); + registry.track("late").add(1, 1, 0); + let report = registry.sent(); + assert_eq!(report.keys().collect::>(), ["late"]); + assert_eq!(report["late"].frames, 1); } #[test] From 4b1952a79a2c8688694c98180b9e36524c0d148d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 13:40:00 -0700 Subject: [PATCH 11/12] fix(cli): evict one ended rendition at the cap and reap drained echo readers At the registry cap, a new rendition ID now evicts a single ended one, so the others keep their final counts. `export echo` forgets a reader once its drain passes instead of on the next `End`. The sinks read `latest`, unwrapping main's `Timed` snapshots. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 4 +- rs/moq-cli/src/report.rs | 91 +++++++++++++++++++++++++++++++++------- rs/moq-mux/src/stats.rs | 62 ++++++++++++++++++++------- 3 files changed, 125 insertions(+), 32 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 0d4a046c500..44004a3237e 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -476,8 +476,8 @@ moq ... --broadcast viewers export echo Counters are cumulative for the life of the process: a rendition that ends keeps its final counts, and one that returns under the same ID continues them. -A report holds 1,024 rendition IDs at most: past that, ended renditions make -way for new ones. +A report holds 1,024 rendition IDs at most: past that, each new ID replaces +one that has ended. [Stats](/concept/stats#media-reports) has the schema. ## Debugging diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 854b75907a2..9c6a350b4f5 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -368,7 +368,7 @@ async fn snapshot( track: &mut Option>, ) -> moq_json::Result> { match track { - Some(track) => track.next().await, + Some(track) => Ok(track.latest().await?.map(|timed| timed.value)), None => std::future::pending().await, } } @@ -449,7 +449,7 @@ async fn write_echo( let prefix = moq_net::Path::new(&prefix).to_owned(); let mut announced = origin.clone().with_hidden(hidden).announced(); let (lines, mut written) = tokio::sync::mpsc::channel::(LINES); - let mut readers = HashMap::new(); + let mut readers = Readers::default(); loop { tokio::select! { event = announced.next() => { @@ -468,25 +468,64 @@ async fn write_echo( } if active { let reader = tokio::spawn(read_echo(origin.clone(), path.clone(), lines.clone())); - if let Some(previous) = readers.insert(path, reader) { - previous.abort(); - } - } else if let Some(reader) = readers.get(&path) { - // Left to drain, not aborted: the final snapshot can still be in flight. It stays - // here so a start at the path replaces it, and goes after `DRAIN` even if its - // track stays open. - let reader = reader.abort_handle(); - tokio::spawn(async move { - tokio::time::sleep(DRAIN).await; - reader.abort(); - }); - readers.retain(|_, reader| !reader.is_finished()); + readers.start(path, reader); + } else { + readers.end(path); } } Some(line) = written.recv() => { stdout.write_all(line.as_bytes()).await?; stdout.flush().await?; } + () = readers.reap() => {} + } + } +} + +/// The `export echo` reader of each feedback broadcast, live or draining after its `End`. +#[derive(Default)] +struct Readers { + running: HashMap>, + /// One timer per `End`, yielding the path and the reader it ended once `DRAIN` passes. + drains: tokio::task::JoinSet<(moq_net::PathOwned, tokio::task::Id)>, +} + +impl Readers { + /// Read `path` with `reader`, replacing any reader still on it. + fn start(&mut self, path: moq_net::PathOwned, reader: tokio::task::JoinHandle<()>) { + if let Some(previous) = self.running.insert(path, reader) { + previous.abort(); + } + } + + /// Let the reader of `path` drain, not abort it: the final snapshot can still be in flight. It + /// stays so a start at the path replaces it, and goes after `DRAIN` even if its track stays open. + fn end(&mut self, path: moq_net::PathOwned) { + let Some(reader) = self.running.get(&path) else { + return; + }; + let id = reader.id(); + self.drains.spawn(async move { + tokio::time::sleep(DRAIN).await; + (path, id) + }); + } + + /// Wait for the next drain to pass, then abort and forget its reader unless a start has since + /// replaced it. Never returns while none is draining. + async fn reap(&mut self) { + let (path, id) = loop { + match self.drains.join_next().await { + Some(Ok(drained)) => break drained, + // A timer only sleeps, so it never fails. + Some(Err(_)) => continue, + None => std::future::pending().await, + } + }; + if self.running.get(&path).is_some_and(|reader| reader.id() == id) + && let Some(reader) = self.running.remove(&path) + { + reader.abort(); } } } @@ -501,7 +540,7 @@ async fn read_echo( let broadcast = origin.request_broadcast(&path, None).await?; let subscriber = broadcast.track(hang::echo::TRACK)?.subscribe(None).await?; let mut track = reader(subscriber); - while let Some(feedback) = track.next().await? { + while let Some(moq_net::Timed { value: feedback, .. }) = track.latest().await? { let line = EchoLine { broadcast: path.as_str(), feedback, @@ -565,6 +604,26 @@ mod tests { assert!(late.is_err(), "a drained reader wrote {late:?}"); } + /// A reader is forgotten once its drain passes, unless a start at its path replaced it. + #[tokio::test(start_paused = true)] + async fn a_drained_reader_is_forgotten() { + let mut readers = Readers::default(); + let path = moq_net::Path::new("viewer.echo").to_owned(); + + readers.start(path.clone(), tokio::spawn(std::future::pending())); + readers.end(path.clone()); + readers.reap().await; + assert!(readers.running.is_empty()); + + readers.start(path.clone(), tokio::spawn(std::future::pending())); + readers.end(path.clone()); + let replacement = tokio::spawn(std::future::pending()); + let id = replacement.id(); + readers.start(path.clone(), replacement); + readers.reap().await; + assert_eq!(readers.running.get(&path).map(|reader| reader.id()), Some(id)); + } + /// Feedback under a hidden prefix is found when that prefix is the one asked for. #[tokio::test(start_paused = true)] async fn a_hidden_prefix_finds_its_feedback() { diff --git a/rs/moq-mux/src/stats.rs b/rs/moq-mux/src/stats.rs index 572f6b45edd..e626fc18732 100644 --- a/rs/moq-mux/src/stats.rs +++ b/rs/moq-mux/src/stats.rs @@ -103,8 +103,8 @@ impl Default for Registry { impl Registry { /// The counters for `name`, continuing those of an earlier track under that name. At - /// [`MAX_ENTRIES`] names, those no track still counts into make way; with every one live, a new - /// name counts into counters no report reads. + /// [`MAX_ENTRIES`] names, one that no track still counts into makes way; with every one live, a + /// new name counts into counters no report reads. pub(crate) fn track(&self, name: &str) -> Arc { let mut entries = self.entries.lock().unwrap_or_else(PoisonError::into_inner); if let Some(counters) = entries.get(name) { @@ -112,17 +112,22 @@ impl Registry { } if entries.len() >= MAX_ENTRIES { // Each track holds its counters for as long as it runs, so the registry's is the last. - entries.retain(|_, counters| Arc::strong_count(counters) > 1); - } - if entries.len() >= MAX_ENTRIES { - if !self.full.swap(true, Relaxed) { - tracing::warn!( - name, - limit = MAX_ENTRIES, - "every reported rendition is live; not reporting more" - ); - } - return Arc::default(); + // Only one makes way, so the other ended renditions keep their final counts. + let ended = entries + .iter() + .find(|(_, counters)| Arc::strong_count(counters) == 1) + .map(|(ended, _)| ended.clone()); + let Some(ended) = ended else { + if !self.full.swap(true, Relaxed) { + tracing::warn!( + name, + limit = MAX_ENTRIES, + "every reported rendition is live; not reporting more" + ); + } + return Arc::default(); + }; + entries.remove(&ended); } entries.entry(name.to_string()).or_default().clone() } @@ -200,10 +205,39 @@ mod tests { drop(live); registry.track("late").add(1, 1, 0); let report = registry.sent(); - assert_eq!(report.keys().collect::>(), ["late"]); + assert_eq!(report.len(), MAX_ENTRIES); assert_eq!(report["late"].frames, 1); } + /// A new name past the cap evicts one ended rendition, so the rest keep their final counts and + /// a returning one continues them. + #[test] + fn a_new_rendition_evicts_only_one_ended() { + let registry = Registry::::default(); + let live = registry.track("live"); + for n in 1..MAX_ENTRIES { + registry.track(&format!("r{n}")).add(1, 1, 0); + } + + registry.track("late").add(1, 1, 0); + let report = registry.sent(); + assert_eq!(report.len(), MAX_ENTRIES); + assert!(report.contains_key("live")); + assert!(report.contains_key("late")); + let kept = (1..MAX_ENTRIES) + .filter(|n| report.contains_key(&format!("r{n}"))) + .count(); + assert_eq!(kept, MAX_ENTRIES - 2); + + let returning = (1..MAX_ENTRIES) + .map(|n| format!("r{n}")) + .find(|name| report.contains_key(name)) + .unwrap(); + registry.track(&returning).add(1, 1, 0); + assert_eq!(registry.sent()[&returning].frames, 2); + drop(live); + } + #[test] fn arrival_pairs_the_latest_timestamp_with_its_wall_time() { let registry = Registry::::default(); From bda2cf44bc9ffb721ef8587202ff365c45468bed Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 09:37:24 +0100 Subject: [PATCH 12/12] fix(mux): keep a release span's sign in a typed SignedDuration; size the report cap for the largest TS report Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- doc/concept/stats.md | 4 +- rs/moq-cli/src/report.rs | 87 ++++++++++++++- rs/moq-cli/src/subscribe.rs | 5 +- .../src/container/ts/export_timing_test.rs | 7 +- rs/moq-mux/src/container/ts/stats.rs | 104 +++++++++++++++--- 6 files changed, 188 insertions(+), 22 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 6173db3b8c6..02d36e054a9 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -463,7 +463,8 @@ The `stats` and `echo` sinks print reports as JSON lines. `export stats` follows one broadcast's stats track until the broadcast ends; `export echo` prints `{"broadcast", "feedback"}` for every `.echo` broadcast under `--broadcast` as they come and go, reading one that ends for 5 s more for its -final report. Either sink stops reading a report that grows past 1 MiB. +final report. Either sink stops reading a report that grows past 8 MiB, twice +the largest a TS import publishes. ```bash moq ... --broadcast event import --stats ts < feed.ts diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 4724d9d271f..8f1227c393f 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -92,8 +92,8 @@ source's. `export ts --passthrough` reports it without `mpegts`. | `dropped` | Frames lost from the output after their stream began: missed their release deadline, or a video track dropped them waiting for its next keyframe. | | `trimmed` | Frames dropped before their stream's first went out, so the output starts on a whole group led by a keyframe. Nothing is missing for these. | | `gaps` | Times the source skipped media it will never deliver: a group lost or expired upstream, or a resubscription. A declared break is not one. | -| `margin` | Gauge, milliseconds: the least time any frame arrived before its deadline over the last second. Negative when one missed. | -| `offset` | Gauge, milliseconds: how far the output clock sits from the source's plus the delay. Positive holds frames longer than the delay. The clock pulls it shut over ten minutes. | +| `margin` | Gauge, milliseconds rounded away from zero: the least time any frame arrived before its deadline over the last second. Negative when one missed, however narrowly. | +| `offset` | Gauge, milliseconds rounded away from zero: how far the output clock sits from the source's plus the delay. Positive holds frames longer than the delay. The clock pulls it shut over ten minutes. | | `drift` | Gauge, parts per million: the source clock's rate against ours, positive running fast. | | `outOfTolerance` | Measurements that found the source clock past the 30 ppm ISO/IEC 13818-1 allows, which the output cannot follow. | diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index ae13fb52002..eeaac78fbb8 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -29,9 +29,11 @@ const COMPRESSED: &str = ".z"; /// Lines `export echo` queues for stdout before its readers wait for it to catch up. const LINES: usize = 64; -/// The most reconstructed state a sink holds for one report. A report is a few KiB, so this -/// only stops a publisher whose patches keep adding keys. -const MAX_REPORT: usize = 1024 * 1024; +/// The most reconstructed state a sink holds for one report. A report is a few KiB, but a TS +/// import keeps a row for each PID it has carried, and the largest it can publish (a row for +/// each of the 8,192 PIDs, every counter at its limit) is under half this. So this only stops +/// a publisher whose patches keep adding keys. +const MAX_REPORT: usize = 8 * 1024 * 1024; /// How long `export echo` lets a reader drain after its broadcast ends. The final snapshot can /// still be in flight, but the track can also outlive its route. @@ -613,6 +615,85 @@ mod tests { assert!(matches!(read.next().await, Err(moq_json::Error::TooLarge(MAX_REPORT)))); } + /// The largest report a TS import can publish reads whole: a row for each of the 8,192 + /// PIDs, every counter at its limit, beside the 1,024 renditions a registry holds. + #[tokio::test] + async fn the_largest_ts_report_reads_whole() { + const SAFE: u64 = hang::catalog::MAX_SAFE_INTEGER; + let mut mpegts = ts::stats::Snapshot::default(); + for counter in [ + &mut mpegts.crc_error, + &mut mpegts.ts_sync_loss, + &mut mpegts.sync_byte_error, + &mut mpegts.pat_error, + &mut mpegts.continuity_count_error, + &mut mpegts.pmt_error, + &mut mpegts.transport_error, + &mut mpegts.pcr_repetition_error, + &mut mpegts.pcr_discontinuity_indicator_error, + &mut mpegts.pts_error, + ] { + *counter = u64::MAX; + } + for pid in 0..8192 { + let mut row = ts::stats::Stream::default(); + row.track = ".avc3".to_string(); + row.class = ts::stats::Class::Video; + row.quiet = Some(Duration::from_millis(u64::MAX)); + for counter in [ + &mut row.units, + &mut row.resyncs, + &mut row.discarded, + &mut row.unconfirmed, + &mut row.damaged, + &mut row.unplaced, + &mut row.refused, + &mut row.continuity_count_error, + &mut row.transport_error, + &mut row.pts_error, + ] { + *counter = u64::MAX; + } + mpegts.streams.insert(pid, row); + } + let mut report = hang::stats::Snapshot::::default(); + report.ext.mpegts = Some(mpegts); + report.transport.rtt = Some(Duration::from_millis(SAFE)); + report.transport.age = Some(Duration::from_millis(SAFE)); + ( + report.transport.rate, + report.transport.lost_bytes, + report.transport.lost_packets, + ) = (Some(SAFE), Some(SAFE), Some(SAFE)); + for n in 0..1024 { + let mut rendition = hang::stats::Rendition::default(); + ( + rendition.frames, + rendition.bytes, + rendition.keyframes, + rendition.skipped, + ) = (SAFE, SAFE, SAFE, SAFE); + rendition.bitrate = Some(SAFE); + report.renditions.insert(format!("0x{n:04x}-0x1b.avc3"), rendition); + } + let value = serde_json::to_value(&report).unwrap(); + + let track = moq_net::broadcast::Info::new() + .produce() + .create_track(STATS_TRACK, None) + .unwrap(); + let mut read = reader(track.subscribe(None)); + let mut producer = moq_json::snapshot::Producer::::new(track, Default::default()); + producer.update(&value).unwrap(); + assert_eq!(read.latest().await.unwrap().unwrap().value, value); + + let size = serde_json::to_vec(&value).unwrap().len(); + assert!( + size <= MAX_REPORT / 2, + "a {size}-byte report leaves the cap too little room" + ); + } + /// A feedback track can outlive its route, so a reader left to drain stops after `DRAIN`. #[tokio::test(start_paused = true)] async fn an_ended_feedback_reader_stops_after_draining() { diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index fb93a4c72b9..ff9c031e51e 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -259,12 +259,13 @@ pub struct Subscribe { /// Log how a TS export's release clock keeps up with the source. fn log_release(release: &moq_mux::container::ts::stats::Release, message: &'static str) { + use moq_mux::container::ts::stats::SignedDuration; tracing::info!( dropped = release.dropped, trimmed = release.trimmed, gaps = release.gaps, - margin_ms = ?release.margin, - offset_ms = ?release.offset, + margin_ms = ?release.margin.map(SignedDuration::as_millis), + offset_ms = ?release.offset.map(SignedDuration::as_millis), drift_ppm = ?release.drift, out_of_tolerance = release.out_of_tolerance, "{message}" diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 39447181a5b..37d0ff918ce 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -20,6 +20,7 @@ use crate::catalog::hang::Container as HangContainer; use crate::container::ts::Export; use crate::container::ts::export::PCR_INTERVAL; use crate::container::ts::stats::Export as Stats; +use crate::container::ts::stats::SignedDuration; use crate::container::{Container as _, Frame, Producer}; const SPS: &[u8] = &[0x67, 0x42, 0xc0, 0x1f, 0xde]; @@ -717,7 +718,11 @@ async fn a_slow_source_clock_keeps_the_delay() { let drift = stats.release.drift.expect("a measurement"); assert!((drift + 30.0).abs() < 0.5, "measured {drift} ppm"); let margin = stats.release.margin.expect("a margin"); - assert!(margin > 0, "a clean path keeps time to spare: {margin} ms"); + assert!( + margin > SignedDuration::default(), + "a clean path keeps time to spare: {} ms", + margin.as_millis() + ); assert!(stats.release.offset.is_some(), "the offset is measured with the rate"); } diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index a760599e5e4..b2f4a97fe9d 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -251,6 +251,7 @@ pub struct Export { /// How an export keeps up with the source: the frames it drops, trims, and never receives, /// and how its release clock tracks the source's. What [`Export::stats`](super::Export::stats) /// reports beside its rows, and [`Release::stats`](super::Release::stats) on its own. +#[serde_with::serde_as] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase")] #[non_exhaustive] @@ -273,16 +274,18 @@ pub struct Release { /// skipped. #[serde(skip_serializing_if = "is_zero")] pub gaps: u64, - /// The least time any frame arrived before its deadline over the last whole second, in - /// milliseconds: how much more latency the path could take before frames drop. Negative - /// when a frame missed its deadline. `None` until a second has passed, and without a delay. + /// The least time any frame arrived before its deadline over the last whole second: how + /// much more latency the path could take before frames drop. Negative when a frame missed + /// its deadline. `None` until a second has passed, and without a delay. + #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] - pub margin: Option, - /// How far the output clock sits from the source's plus the delay, in milliseconds, as last - /// measured: positive holds frames longer than the delay, negative shorter. The clock pulls - /// it shut over ten minutes. `None` until there is a measurement. + pub margin: Option, + /// How far the output clock sits from the source's plus the delay, as last measured: + /// positive holds frames longer than the delay, negative shorter. The clock pulls it shut + /// over ten minutes. `None` until there is a measurement. + #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] - pub offset: Option, + pub offset: Option, /// How fast the source's clock runs against ours, in parts per million (positive runs /// fast), as last measured. `None` until there is a measurement. #[serde(skip_serializing_if = "Option::is_none")] @@ -295,19 +298,72 @@ pub struct Release { impl Release { pub(crate) fn of(jitter: &crate::jitter::Buffer) -> Self { - let millis = |seconds: f64| (seconds * 1e3).round() as i64; Self { dropped: jitter.dropped(), trimmed: jitter.trimmed(), gaps: 0, - margin: jitter.margin().map(millis), - offset: jitter.offset().map(millis), + margin: jitter.margin().map(SignedDuration::from_secs_f64), + offset: jitter.offset().map(SignedDuration::from_secs_f64), drift: jitter.drift().map(|drift| drift * 1e6), out_of_tolerance: jitter.out_of_tolerance(), } } } +/// A span of time that can fall below zero, which [`Duration`](std::time::Duration) cannot: a +/// [`Release`]'s margin and offset. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SignedDuration { + nanos: i64, +} + +impl SignedDuration { + /// `millis` milliseconds, below zero when negative. + pub const fn from_millis(millis: i64) -> Self { + Self { + nanos: millis.saturating_mul(1_000_000), + } + } + + /// `seconds`, below zero when negative, saturating at about 292 years either way. + pub(crate) fn from_secs_f64(seconds: f64) -> Self { + Self { + nanos: (seconds * 1e9).round() as i64, + } + } + + /// Whether the span falls below zero. + pub const fn is_negative(self) -> bool { + self.nanos < 0 + } + + /// How long the span is, whichever side of zero it falls. + pub const fn unsigned_abs(self) -> std::time::Duration { + std::time::Duration::from_nanos(self.nanos.unsigned_abs()) + } + + /// Whole milliseconds, rounded away from zero, so a span short of one keeps its sign: what + /// a report carries. + pub const fn as_millis(self) -> i64 { + self.nanos / 1_000_000 + (self.nanos % 1_000_000).signum() + } +} + +/// A [`SignedDuration`] in a report: [`SignedDuration::as_millis`]. +struct SignedMillis; + +impl serde_with::SerializeAs for SignedMillis { + fn serialize_as(value: &SignedDuration, serializer: S) -> Result { + value.as_millis().serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, SignedDuration> for SignedMillis { + fn deserialize_as>(deserializer: D) -> Result { + i64::deserialize(deserializer).map(SignedDuration::from_millis) + } +} + /// An export's rows with every stream-wide counter zero, so one [`Log`] reads either edge. impl From for Snapshot { fn from(export: Export) -> Self { @@ -657,8 +713,8 @@ mod test { }, ); let release = Release { - margin: Some(-12), - offset: Some(3), + margin: Some(SignedDuration::from_millis(-12)), + offset: Some(SignedDuration::from_millis(3)), drift: Some(1.5), ..Default::default() }; @@ -689,6 +745,28 @@ mod test { assert_eq!(row.class, Class::Data); } + /// A span short of a millisecond keeps its sign in a report, so a frame that missed its + /// deadline by less still reads as a miss. + #[test] + fn a_release_span_under_a_millisecond_keeps_its_sign() { + let release = Release { + margin: Some(SignedDuration::from_secs_f64(-0.000_3)), + offset: Some(SignedDuration::from_secs_f64(0.000_2)), + ..Default::default() + }; + assert!(release.margin.unwrap().is_negative()); + assert_eq!( + release.margin.unwrap().unsigned_abs(), + std::time::Duration::from_micros(300) + ); + assert_eq!( + serde_json::to_value(&release).unwrap(), + serde_json::json!({ "margin": -1, "offset": 1 }) + ); + assert_eq!(SignedDuration::from_secs_f64(-0.012).as_millis(), -12); + assert_eq!(SignedDuration::default().as_millis(), 0); + } + /// Damage is reported per PID when its cumulative count moves, including the final sample. #[test] #[tracing_test::traced_test]