From 61ae2ad3898ffd3c6544aadbc30b30813afbf521 Mon Sep 17 00:00:00 2001 From: t0ms Date: Sat, 17 Oct 2026 09:19:25 +0100 Subject: [PATCH 01/13] feat(moq-mux): render identical TS legs with `export ts --sync` `moq export ts --sync` lays every packet on fixed per-PID lanes of the constant-rate grid and re-anchors each group at its first unit's release instant, so two exports of one broadcast emit the same packets, continuity counters included, wherever both hold the content; what one leg lacks stays stuffing in its positions. Implements quest/m2/ts-hitless.md. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 13 + quest/m2/README.md | 1 - quest/m2/hop-aligned-import.md | 1 - quest/m2/ts-hitless.md | 89 -- quest/m2/ts-rtp-egress.md | 6 +- rs/moq-cli/src/args.rs | 32 + rs/moq-cli/src/main.rs | 1 + rs/moq-cli/src/subscribe.rs | 21 +- rs/moq-mux/benches/ts_schedule.rs | 4 + rs/moq-mux/src/container/ts/export.rs | 896 +++++++++++++----- .../src/container/ts/export_timing_test.rs | 250 ++++- rs/moq-mux/src/container/ts/lane.rs | 774 +++++++++++++++ rs/moq-mux/src/container/ts/mod.rs | 1 + rs/moq-mux/src/container/ts/schedule.rs | 57 +- rs/moq-mux/src/container/ts/stats.rs | 7 + 15 files changed, 1832 insertions(+), 321 deletions(-) delete mode 100644 quest/m2/ts-hitless.md create mode 100644 rs/moq-mux/src/container/ts/lane.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 4b3455ae6e0..5a579bc7375 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -433,6 +433,19 @@ continuity error, and the rest carry on. It takes neither `--linger`, `--stitch` nor `--mux-rate`. Only the hang catalog names the track, so an `.msf` broadcast needs `--catalog-format hang`. +`export ts --sync` renders every packet from the broadcast alone, so two exports +of it, joined at different moments and fed with different skew, emit identical +TS for the content both receive, continuity counters included: the two legs of +an SMPTE ST 2022-7 pair. Each PID gets a fixed lane of the constant rate, sized +for its codec's most, with video taking the rest; content one leg lacks leaves +its packets as stuffing there. The PCR rides a PID of its own, and each PID's +continuity counter restarts at every group with the break flagged. It needs a +constant rate (`--mux-rate`, or the catalog's), costing headroom over the +source's (a 10 Mb/s channel needs about 12), and takes neither `--stitch` nor +`--passthrough`. A group that starts behind the one before it, on a lane too +narrow to absorb that one, is logged and counted in `overruns`: the legs can +differ there. + A stdout export ends with the broadcast. `export ts --linger 10s` waits that long for the same publisher instance to come back, and carries on with the same stream. A return that cannot be read yet, such as one still routed through a diff --git a/quest/m2/README.md b/quest/m2/README.md index 0b4c6a04840..de1865f1633 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -89,7 +89,6 @@ ranked just below deterministic TS legs, which it requires. - [One port on the io_uring workers](/quest/m2/uring-demux.md) - `moq-uring`'s workers host the UDP demux, so a ring relay keeps WebRTC media and SRT on its QUIC port - [Synced data playback](/quest/m2/watch-data-sync.md) - js/watch releases JSON and binary payloads on the media playhead, and a slow data track holds media back - [Watch decode gate](/quest/m2/watch-decode-gate.md) - video lookahead stays encoded until it is near presentation, like audio -- [Deterministic TS legs](/quest/m2/ts-hitless.md) - two `--sync` legs of one broadcast render every common slot identically, continuity counters included, through complementary loss - [TS over RTP legs](/quest/m2/ts-rtp-egress.md) - two legs send identical RTP packets, headers included, so an ST 2022-7 receiver can merge them - [moqsrc A/V sync](/quest/m2/moqsrc-av-sync.md) - a nightly test measures A/V skew from `moqsrc` through clock-synced sinks - [MP4 export](/quest/m2/mp4-export.md) - `moq export mp4 --output` records crash-safe fragments, then finishes a regular MP4 with moov at the end diff --git a/quest/m2/hop-aligned-import.md b/quest/m2/hop-aligned-import.md index 81e686c5795..f48daf68502 100644 --- a/quest/m2/hop-aligned-import.md +++ b/quest/m2/hop-aligned-import.md @@ -82,4 +82,3 @@ stopping and restarting; repeat it with `--passthrough`. - [Broadcast epochs](/quest/m0/broadcast-epoch/README.md) - a redundant pair shares one epoch - [E2EE](/quest/m2/e2ee/README.md) - an encrypting publisher refuses a shared epoch, so a redundant pair is plaintext only -- [Deterministic TS legs](/quest/m2/ts-hitless.md) - what two exports of the pair render from the common coordinate diff --git a/quest/m2/ts-hitless.md b/quest/m2/ts-hitless.md deleted file mode 100644 index 1ea6cfee2b7..00000000000 --- a/quest/m2/ts-hitless.md +++ /dev/null @@ -1,89 +0,0 @@ -# [L] Redundant TS exports render common content identically - -## Goal - -Two `moq export ts --sync` legs of one broadcast, joined at different -moments and fed with different delivery skew, emit identical TS for every -slot whose content both release, continuity counters included, from their -first common slot, within one epoch. A slot whose content one leg lacks -keeps its position in that leg as stuffing, so complementary losses stay -complementary. - -This is TS identity only. RTP packetization for ST 2022-7 is -[its own quest](/quest/m2/ts-rtp-egress.md), and merging the legs is the -receiver's job. - -Non-goal: identity across a replaced broadcast. A leg ends on a replacement, or -starts a fresh stream with `--stitch`, -so legs may differ after one (#5101, maintainer 2026-10-09: byte-identical -output across restarts is not supported). - -## Plan - -Decided 2026-10-13 by the maintainer: the coordinate the -legs share is the broadcast's media timeline, each object's timestamp on its -catalog clock, with a constant mux rate. Two exports of one broadcast -already read the same objects and timestamps, so this needs no importer -alignment; [same-epoch importers](/quest/m2/hop-aligned-import.md) extend it -to legs fed by different importers of one epoch. Every byte a `--sync` leg -renders from the objects it has (slot, PCR, table, PES header, continuity -counter) is a function of those objects, that coordinate, the catalog, and -the rate, never of what the leg has sent or when it joined. 2022-7 is a differentiator for primary distribution, not -a requirement (t0ms's review of #4645), so this waits until the T-STD line -settles. - -- Release is unchanged: measured acquisition and the fixed-delay release's - clock recovery. This supersedes the 2026-10-01 plan to release at - `clock.wall + pts + delay`. The catalog's wall mapping is fixed for the - epoch, so a wall-scheduled release spends its delay on the source's clock - error (a 1 s delay lasts about 9 h at 30 ppm), and NTP on the hosts does - not change that. The legs' differential delay stays their delivery skew, - which the receiver absorbs. -- Continuity counters restart at each group. Each media PID's first packet - in a group sets `discontinuity_indicator` and is numbered from there. The - PCR moves to its own PID, because the flag on the PCR PID would declare a - time-base break. Loss is still detected within a group. -- Content one leg misses never changes what the other renders, and the leg - that misses it renders stuffing in its slots. Forcing both legs to drop the - same content would defeat redundant paths. -- A leg that lacks an access unit cannot know how many packets it fills, so - no later packet's slot may depend on an earlier access unit's size. A - constant rate fixes capacity, not placement. Settle the placement rule - before building the scheduler: fixed per-PID reservations on the timeline, - a packet plan carried with the objects, or each group re-anchored at a - slot derived from its timestamp, with headroom. -- t0ms's - `two_legs_render_a_skip_the_same_way` - (`rs/moq-mux/src/container/ts/export_timing_test.rs`) becomes this - complementary case and stops being ignored. With the audio late, it also - showed a PES header byte that differs at the skip; find and fix that. -- `--sync` refuses what it cannot keep: an output with no constant rate - (neither `--mux-rate` nor the catalog's recorded rate), since a measured - rate gives each leg its own grid; `--stitch`, since identity is per epoch; - and `--passthrough`, whose output is already the source's packets. - -Tests on mocked time compare slots keyed by their media time, not by output -position, and assert continuity counters too (`assert_same_packets` stops -masking them under `--sync`): - -- legs joined at different source offsets, mid-group and a group apart; -- different delivery skew and read cadence, each track lagging in turn; -- complementary loss, each leg missing groups the other has; -- a broadcast timeline crossing the 33-bit PCR and PTS wrap while both run, - and a leg joining just after it; -- ±30 ppm source drift; -- each refusal. - -The validation work (#5328) found the `just test ts --pair` arm at 86.96% -SDT agreement over 41 seconds of clean delivery, below its 90% gate. Its -grader associates a table with the next PES PTS; check that association -against the slot scheduler before attributing mismatches to table emission. -Do not loosen the threshold, and do not claim packet identity from table -agreement alone. - -Public API: `moq export ts --sync`. - -## Related - -- [TS over RTP legs](/quest/m2/ts-rtp-egress.md) - packetizes these slots with matching RTP headers for ST 2022-7 -- [Same-epoch importers](/quest/m2/hop-aligned-import.md) - extends the coordinate to legs fed by different importers diff --git a/quest/m2/ts-rtp-egress.md b/quest/m2/ts-rtp-egress.md index db3599f4fa5..455974e00fa 100644 --- a/quest/m2/ts-rtp-egress.md +++ b/quest/m2/ts-rtp-egress.md @@ -17,7 +17,7 @@ identity across a replaced broadcast. ## Plan Decided 2026-10-13 by the maintainer: RTP identity is a -function of [TS slot identity](/quest/m2/ts-hitless.md) and the epoch, and of +function of TS slot identity (`moq export ts --sync`) and the epoch, and of nothing a leg mints. - Datagrams are cut on the slot grid, so their boundaries fall at the same @@ -54,7 +54,3 @@ epochless route is refused. Claims of ST 2022-7 compatibility cite the standard RTP-level test, not TS identity. Public API: an RTP output for TS export, in the shape settled above. - -## Required - -- [Deterministic TS legs](/quest/m2/ts-hitless.md) - the slots and counters these datagrams carry diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index aa03a9669bf..881a010d3fd 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -1033,6 +1033,7 @@ impl ExportSink { fragment_duration: None, mux_rate: None, passthrough: false, + sync: false, }; let buffered = |format, args: &Buffered| Stdout { format, @@ -1042,6 +1043,7 @@ impl ExportSink { fragment_duration: None, mux_rate: None, passthrough: false, + sync: false, }; Some(match self { Self::Fmp4(args) => Stdout { @@ -1060,6 +1062,7 @@ impl ExportSink { fragment_duration: None, mux_rate: args.mux_rate, passthrough: args.passthrough, + sync: args.sync, }, Self::Flv(args) => buffered(SubscribeFormat::Flv, args), Self::H264(args) => container(SubscribeFormat::H264, args), @@ -1082,6 +1085,8 @@ pub struct Stdout { pub mux_rate: Option, /// Write the `m2ts` passthrough track back out (`ts` only). pub passthrough: bool, + /// Render identical packets to every other `--sync` export of the broadcast (`ts` only). + pub sync: bool, } /// Staleness options for stdout sinks without a fixed release delay. @@ -1204,6 +1209,12 @@ pub struct Transport { #[usage(long, conflicts("--linger", "--stitch", "--mux-rate"))] pub passthrough: bool, + /// Render every packet from the broadcast alone, on a fixed lane per PID, so two exports + /// of it emit identical TS for the content both receive, continuity counters included: + /// a 1+1 (SMPTE ST 2022-7) pair. Needs a constant rate (`--mux-rate`, or the catalog's). + #[usage(long, conflicts("--stitch", "--passthrough"))] + pub sync: bool, + /// The released spelling of [`Self::delay`]. #[usage(long = "max-age", hide = true)] max_age: Option, @@ -1333,6 +1344,27 @@ mod tests { } } + /// `export ts --sync` keeps its legs identical within one publisher epoch, so it refuses + /// `--stitch`, and `--passthrough`, whose output is already the source's own packets. + #[test] + fn export_ts_sync_refuses_a_stitch_and_a_passthrough() { + let parse = |args: &[&str]| { + let mut argv = vec!["moq", "export", "--broadcast", "b", "ts", "--sync"]; + argv.extend_from_slice(args); + Invocation::try_parse_from(argv) + }; + let cli = parse(&["--mux-rate", "12000000"]).unwrap(); + let Command::Export(export) = &cli.stages[0] else { + panic!("not an export"); + }; + let stdout = export.sink.stdout().unwrap(); + assert!(stdout.sync); + assert_eq!(stdout.mux_rate, Some(12_000_000)); + for flags in [&["--stitch"][..], &["--passthrough"]] { + assert!(parse(flags).is_err(), "{flags:?} must not start a run"); + } + } + fn export(flags: &[&str]) -> Result { let argv = ["moq", "--connect", "http://relay", "--broadcast", "room", "export"]; Invocation::try_parse_from(argv.iter().chain(flags).copied()) diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 63662a1ac5f..502ef95d14e 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -821,6 +821,7 @@ fn spawn_export( fragment_duration: stdout.fragment_duration, mux_rate: stdout.mux_rate, passthrough: stdout.passthrough, + sync: stdout.sync, catalog: export.catalog_format, select: export.select, }; diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 4d0fcd780ee..8bb3db9bf98 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -197,6 +197,9 @@ pub struct SubscribeArgs { /// Write the `m2ts` passthrough track back out instead of muxing the media (TS only). pub passthrough: bool, + /// Render identical packets to every other synced export of the broadcast (TS only). + pub sync: bool, + /// Catalog format for track discovery (default: detect from the broadcast suffix). pub catalog: Option, @@ -386,6 +389,9 @@ impl Subscribe { if let Some(mux_rate) = self.args.mux_rate { ts = ts.with_mux_rate(mux_rate); } + if self.args.sync { + ts = ts.with_sync(); + } // A TS byte stream carries no per-frame timing, so delivery time is the only // carrier of each frame's spacing (#2984). The export lays each slice of the PCR @@ -399,8 +405,8 @@ impl Subscribe { // the way `publish` reports one that stops arriving. let mut log = moq_mux::container::ts::stats::Log::default(); let mut sampled = tokio::time::Instant::now(); - // The export's drop, gap and out-of-tolerance counts, as last logged. - let mut reported = (0, 0, 0, 0); + // The export's drop, gap, out-of-tolerance and `--sync` counts, as last logged. + let mut reported = (0, 0, 0, 0, 0, 0); let release = |stats: &moq_mux::container::ts::stats::Export| { tracing::info!( dropped = stats.dropped, @@ -408,6 +414,8 @@ impl Subscribe { gaps = stats.gaps, drift_ppm = ?stats.drift, out_of_tolerance = stats.out_of_tolerance, + overruns = stats.overruns, + late = stats.late, "TS export release clock" ); }; @@ -423,7 +431,14 @@ impl Subscribe { if sampled.elapsed() >= moq_mux::container::ts::stats::Log::INTERVAL { sampled = tokio::time::Instant::now(); let stats = ts.export().stats(); - let counts = (stats.dropped, stats.trimmed, stats.gaps, stats.out_of_tolerance); + let counts = ( + stats.dropped, + stats.trimmed, + stats.gaps, + stats.out_of_tolerance, + stats.overruns, + stats.late, + ); if counts != reported { reported = counts; release(&stats); diff --git a/rs/moq-mux/benches/ts_schedule.rs b/rs/moq-mux/benches/ts_schedule.rs index dad04ee369d..565ebd33eed 100644 --- a/rs/moq-mux/benches/ts_schedule.rs +++ b/rs/moq-mux/benches/ts_schedule.rs @@ -7,8 +7,12 @@ use std::time::Duration; mod export { pub const PCR_INTERVAL: Duration = Duration::from_millis(25); + pub const PSI_INTERVAL: Duration = Duration::from_millis(500); use std::time::Duration; } +#[path = "../src/container/ts/lane.rs"] +#[allow(dead_code)] +mod lane; #[path = "../src/container/ts/schedule.rs"] #[allow(dead_code)] mod schedule; diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index c3892e71c46..944342a04a8 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -23,6 +23,7 @@ //! inline NALs on every keyframe. CMAF tracks are rejected with a clear error. use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::sync::Arc; use std::task::Poll; use std::time::Duration; @@ -51,6 +52,7 @@ use crate::jitter::{self, Arrival, Push}; use super::adts; use super::catalog; +use super::lane::{self, Grid, Lane, Layout, Need, Unplaced}; use super::schedule::{self, Buffer, Schedule}; /// PID of the single program's PMT. @@ -58,7 +60,7 @@ const PMT_PID: u16 = 0x1000; /// First elementary-stream PID; each track gets the next one. const FIRST_ES_PID: u16 = 0x1001; /// Re-emit PAT/PMT at least this often (wall-clock of the media) for tune-in. -const PSI_INTERVAL: Duration = Duration::from_millis(500); +pub(super) const PSI_INTERVAL: Duration = Duration::from_millis(500); /// The floor between emissions of a *changed* SI snapshot, bounding how fast a /// revising publisher can make the mux re-emit its tables. The import side /// debounces its own snapshot cuts (`si::DEBOUNCE`), but export consumes any @@ -66,6 +68,11 @@ const PSI_INTERVAL: Duration = Duration::from_millis(500); /// enforced on the media timeline. Clamped to the entry's own interval, so a /// table asking for faster repetition than this still gets it. const SI_REVISION_INTERVAL: Duration = Duration::from_secs(1); +/// How many of an SI entry's newest snapshots [`Export::with_sync`] keeps by media time: more +/// than a delay's worth of revisions, which the importer debounces. +const SI_HISTORY: usize = 16; +/// The SI packets a second every [`Export::with_sync`] multiplex keeps clear of its lanes. +const SI_LANE: u64 = 80; /// Emit a PCR on every crossing of this media-time grid ([`Schedule`]). /// TR 101 290 V1.4.1 flags a gap over 100 ms; broadcast muxes emit every 25-40 ms. pub(super) const PCR_INTERVAL: Duration = Duration::from_millis(25); @@ -197,6 +204,10 @@ where trimmed: u64, /// Each elementary stream's access units and silence on the PCR returned ([`Self::stats`]). liveness: super::import::Liveness, + /// Place every packet on a fixed lane of the constant rate ([`Self::with_sync`]). + sync: bool, + /// Under [`Self::with_sync`], the slot whose SI has been queued ([`Self::queue_si`]). + si_slot: Option, } /// What a queued output frame tells [`Export::stats`], applied only once the frame is @@ -222,6 +233,8 @@ struct Pending { arrived: moq_clock::Instant, /// When it was read: the latest it could have arrived. read: moq_clock::Instant, + /// The MoQ group it was read from, and whether it was the first frame read from that group. + group: (u64, bool), } /// A frame waiting in the jitter buffer, with what it needs from the moment it was read. @@ -237,6 +250,22 @@ struct Queued { description: Option, /// The PES it may share ([`Track::pack_start`]). pes: Option, + /// Its packets on the track's lane, under [`Export::with_sync`]. + planned: Option, +} + +/// A frame's packets, numbered, and their positions on the timeline ([`Track::plan`]). +struct Planned { + packets: Vec, + positions: Vec, +} + +/// A track's lane under [`Export::with_sync`]. +struct Synced { + layout: Arc, + lane: Lane, + /// The release slot of the last frame planned ([`Track::plan`]). + released: Option, } /// The PES an audio frame may share ([`Track::pack_start`]): the place of its first frame on @@ -285,6 +314,10 @@ struct Track { /// The last frame's decode time, its jitter buffer key ([`Self::release_key`]) and the PES /// it may share. released: Option<(Timestamp, Timestamp, Option)>, + /// The MoQ group of the last frame read. + group: Option, + /// Its lane, under [`Export::with_sync`]. + sync: Option, } /// Shared program state needed while reading one track. @@ -319,6 +352,8 @@ impl Track { dolby_carry: None, pack: None, released: None, + group: None, + sync: None, } } @@ -333,6 +368,10 @@ impl Track { self.release(name, jitter, true)?; self.restart = pending.restart; self.clock = DecodeClock::default(); + if let Some(sync) = &mut self.sync { + sync.lane.reset(); + sync.released = None; + } } if let Some(codec) = self.dolby() { return self.queue_dolby(codec, name, pending, jitter); @@ -347,8 +386,12 @@ impl Track { let ahead = self.held == Some(pending.cut); self.leading_floor = recover.then_some((pending.frame.timestamp, ahead)); } + // Under sync they go on to be planned and decode-clocked, as on a leg that held what + // they reference, and the jitter buffer drops them as leading: the frames after them + // keep their positions and DTS. if let Some((floor, ahead)) = self.leading_floor && pending.frame.timestamp < floor + && self.sync.is_none() { jitter.lose(&self.pid, ahead); return Ok(()); @@ -376,6 +419,9 @@ impl Track { } match self.source.poll_read(waiter)? { Poll::Ready(Some(frame)) => { + // A group arrives from its first frame, so the first read from it opens it. + let sequence = self.source.group().unwrap_or_default(); + let opens = self.group.replace(sequence) != Some(sequence); if input.waiting_for_header && !self.source.header_ready() { *input.trimmed += 1; continue; @@ -387,6 +433,7 @@ impl Track { cut: self.base.2 + self.source.cuts(), arrived: self.empty, read: moq_clock::now(), + group: (sequence, opens), }; // A new timeline must reach the reset before tune-in alignment can drop it. if let Some(start) = input.video_start @@ -437,9 +484,23 @@ impl Track { cut, arrived, read, + group, } = pending; - let frames = dolby_units(codec, &mut self.dolby_carry, skip, frame.timestamp, &frame.payload)?; - for (timestamp, payload) in frames { + let frames = dolby_units( + codec, + &mut self.dolby_carry, + (skip, group.0), + frame.timestamp, + &frame.payload, + )?; + // A unit begun in the last PES belongs to that PES's group, which a leg lacking it never + // places, so the units that begin here lay from this group's own anchor. + let mut opens = group.1; + for (timestamp, payload, began) in frames { + let group = match began == group.0 { + true => (began, std::mem::take(&mut opens)), + false => (began, false), + }; let frame = Frame { timestamp, duration: None, @@ -454,6 +515,7 @@ impl Track { cut, arrived, read, + group, }; self.push(name, pending, None, None, jitter)?; } @@ -524,6 +586,9 @@ impl Track { /// timeline that is a multiple of them, cut again where they would span more than half the /// delay and at each [`Self::psi_slot`]. fn pack_start(&self, payload: &[u8], timestamp: Timestamp, delay: Duration) -> Option { + if self.sync.is_some() { + return None; + } let (samples, rate) = self.samples(payload).filter(|_| !delay.is_zero())?; let len = match self.kind { Kind::Aac { .. } => adts::MIN_HEADER_LEN + payload.len(), @@ -549,12 +614,152 @@ impl Track { /// as [`schedule::ahead`] lets them, and nothing without a delay or for a placed PES, /// whose timestamp is when it goes out. fn ahead(&self, delay: Duration) -> Duration { + PCR_INTERVAL * u32::try_from(self.ahead_slots(delay)).unwrap_or(u32::MAX) + } + + /// [`Self::ahead`] in slots. Under [`Export::with_sync`] a unit goes out no further ahead + /// than its lane fills its decoder buffer in, less a slot, so the buffer holds whatever + /// the frames' sizes; a verbatim stream with no decoder buffer may use the whole delay to + /// spread a unit over its lane. + fn ahead_slots(&self, delay: Duration) -> u128 { // A placed PES goes out at its placement, never ahead of it. if delay.is_zero() || matches!(self.kind, Kind::PesPacket { .. }) { - return Duration::ZERO; + return 0; + } + let window = schedule::slot(delay.as_nanos()); + let buffer = self.buffer(); + let slots = schedule::ahead(buffer.as_ref(), window); + match (&self.sync, buffer.and_then(|buffer| buffer.size)) { + (Some(sync), Some(size)) => { + let rate = u128::from(sync.layout.payload_rate(sync.lane.index()).max(1)); + let fills = size as u128 * 8 * 1_000_000_000 / rate; + slots.min(schedule::slot(fills).saturating_sub(1)) + } + (Some(_), None) if matches!(self.kind, Kind::Verbatim { .. }) => window, + _ => slots, + } + } + + /// The lane the track takes under [`Export::with_sync`], in packets a second. Lanes are + /// fixed before any frame is seen, so audio and the other streams take the most their + /// codec can send, and video the rest of the rate, up to what its receiver buffers pass on. + fn lane_need(&self) -> Need { + // A frame's packets: its PES header with a PTS, and the adaptation field of a flag. + let packets = |bytes: u64| (bytes + 16).div_ceil(TsBytes::MAX_SIZE as u64); + let per_second = |bytes: u64, samples: u64, rate: u32| (packets(bytes) * u64::from(rate)).div_ceil(samples); + match &self.kind { + Kind::Video(_) => Need::Rest(self.buffer().map_or(u64::MAX, |buffer| { + let payload = buffer + .leak + .map_or(u64::MAX, |leak| leak / (TsBytes::MAX_SIZE as u64 * 8)); + (buffer.rate / (TsPacket::SIZE as u64 * 8)).min(payload) + })), + // At most 6144 bits a channel in a raw data block (ISO 14496-3 4.5.3.2), with the + // ADTS header and the program config element a frame may repeat. + Kind::Aac { config, .. } => { + let channels = match config.channel_config { + 0 | 7 => 8, + n => u64::from(n), + }; + let pce = config.program_config.as_ref().map_or(0, |pce| pce.len() as u64); + Need::Fixed(per_second(768 * channels + 7 + pce, 1024, config.sample_rate)) + } + // Layer II tops out at 384 kb/s, and 160 kb/s at the half rates (ISO 13818-3). + Kind::Mp2 { sample_rate } => { + let top: u64 = if *sample_rate >= 32_000 { 384_000 } else { 160_000 }; + let frame = 144 * top / u64::from((*sample_rate).max(1)) + 1; + Need::Fixed(per_second(frame, 1152, *sample_rate)) + } + Kind::Ac3 => Need::Fixed(AC3_LANE), + Kind::Eac3 => Need::Fixed(EAC3_LANE), + _ if self.dolby() == Some(Dolby::Ac3) => Need::Fixed(AC3_LANE), + _ if self.dolby() == Some(Dolby::Eac3) => Need::Fixed(EAC3_LANE), + Kind::Opus { channel_config_code } => Need::Fixed(opus_lane(*channel_config_code)), + Kind::Verbatim { + framing: catalog::Framing::Section, + .. + } => Need::Fixed(SECTION_LANE), + Kind::Verbatim { .. } if self.descriptors.iter().any(|d| d.tag == TELETEXT_DESCRIPTOR) => { + Need::Fixed(TELETEXT_LANE) + } + Kind::Verbatim { .. } | Kind::PesPacket { .. } => Need::Fixed(VERBATIM_LANE), + } + } + + /// Plan a frame on the track's lane under [`Export::with_sync`]: packetized, numbered on + /// from its group's earlier units, at the positions [`Lane::plan`] gives. `None` places + /// nothing: a group this leg did not hold from its first unit, a malformed section, or + /// verbatim data its lane cannot carry by its deadline. + fn plan( + &mut self, + name: &str, + frame: &Frame, + (decode, dts): (Timestamp, Option), + description: Option<&Bytes>, + group: (u64, bool), + delay: Duration, + ) -> anyhow::Result> { + let by = decode.as_nanos().saturating_sub(self.drain().as_nanos()); + let release = lane::release(by, self.ahead_slots(delay)); + let video = matches!(self.kind, Kind::Video(_)); + let Some(sync) = &mut self.sync else { + return Ok(None); + }; + let opening = sync.lane.opening(group); + // A receiver tunes in at the tables, so an AAC program config element leads the first + // frame its group places and the first released on or after each repetition of them. + let slot = schedule::slot(release); + let pce = opening + || sync + .released + .is_some_and(|last| last / lane::PSI_SLOTS < slot / lane::PSI_SLOTS); + sync.released = Some(slot); + let mut packets = Vec::new(); + match es_payload(&self.kind, description, frame, pce)? { + Some(es) => { + let unit = PesUnit { + pid: self.pid, + is_video: video, + keyframe: frame.keyframe, + timestamp: frame.timestamp, + dts, + stream_id: stream_id(&self.kind), + }; + pes_packets(&mut packets, &unit, &es, opening)?; + } + None if is_complete_section(&frame.payload) => { + section_packets(&mut packets, self.pid, &frame.payload, opening)?; + } + None => { + tracing::warn!(track = %name, len = frame.payload.len(), "dropping malformed private section on export"); + return Ok(None); + } + } + let count = packets.len() / TsPacket::SIZE; + let overruns = sync.lane.overruns(); + let plan = sync.lane.plan(&sync.layout, group, release, by, count); + if sync.lane.overruns() > overruns { + tracing::warn!(track = %name, pid = self.pid, "--sync group began behind the one before it: its lane is too narrow for legs that lose different data to agree here"); + } + match plan { + Ok(placed) => { + lane::number(&mut packets, placed.counter); + Ok(Some(Planned { + packets, + positions: placed.positions, + })) + } + Err(Unplaced::Partial) => Ok(None), + Err(Unplaced::Late) if matches!(self.kind, Kind::Verbatim { .. }) => { + tracing::warn!(track = %name, count, "verbatim unit its lane cannot carry by its deadline; dropped"); + Ok(None) + } + Err(Unplaced::Late) => anyhow::bail!( + "MPEG-TS --sync cannot carry a {count}-packet unit on PID {} by its decode time on its lane at {} b/s; raise --mux-rate or the delay", + self.pid, + sync.layout.rate() + ), } - let slots = schedule::ahead(self.buffer().as_ref(), schedule::slot(delay.as_nanos())); - PCR_INTERVAL * u32::try_from(slots).unwrap_or(u32::MAX) } /// Where the jitter buffer orders a frame decoding at `decode` under `delay`: at its release @@ -632,12 +837,21 @@ impl Track { cut, arrived, read, + group, } = pending; let dts = dts.filter(|&dts| dts != to_ticks(frame.timestamp)); let decode = dts .and_then(|ticks| Timestamp::from_scale(ticks, 90_000).ok()) .unwrap_or(frame.timestamp); let (key, pes) = self.release_key(decode, jitter.delay(), &frame)?; + // Planned as it is read, so a frame the jitter buffer drops still moves its group on. + let planned = match self.sync.is_some() { + true => match self.plan(name, &frame, (decode, dts), description.as_ref(), group, jitter.delay())? { + Some(planned) => Some(planned), + None => return Ok(()), + }, + false => None, + }; let video = matches!(self.kind, Kind::Video(_)); let leading = video && !frame.keyframe && self.keyframe.is_some_and(|keyframe| frame.timestamp < keyframe); if video && frame.keyframe { @@ -670,6 +884,7 @@ impl Track { dts, description, pes, + planned, }, }; if jitter.push(self.pid, arrival)? == Push::Late { @@ -885,9 +1100,12 @@ struct SiTrack { state: SiState, /// The reduced newest *complete* group: what emission re-transmits. active: super::si::Snapshot, - /// A group still being read; swapped into `active` when it ends, so a torn - /// half-received snapshot is never emitted. - pending: Option<(moq_net::group::Consumer, super::si::Snapshot)>, + /// A group still being read, with its first frame's media time; swapped into `active` + /// when it ends, so a torn half-received snapshot is never emitted. + pending: Option<(moq_net::group::Consumer, super::si::Snapshot, Option)>, + /// The newest snapshots' sections by media time, for [`Export::with_sync`] to send each + /// in the slot its time falls in ([`Self::due`]). + history: VecDeque<(Timestamp, Vec)>, /// A promoted snapshot changed `active` and has not hit the wire yet: emit on /// the revision floor ([`SI_REVISION_INTERVAL`]) instead of waiting out the /// repetition interval. For a clock table (TDT/TOT) the cadence is the content, @@ -940,11 +1158,38 @@ impl SiTrack { dirty: !active.is_empty(), active, pending: None, + history: VecDeque::new(), last_emit: None, max_age, } } + /// The sections to send in slot `index` under [`Export::with_sync`], if any: a snapshot in + /// the slot its media time falls in, and the one in force on the absolute repetition grid + /// unless `table_id` is a clock table, whose every value is a revision. An inline entry's + /// fixed snapshot is in force throughout. + fn due(&mut self, index: u128, table_id: u8) -> Option> { + while self + .history + .get(1) + .is_some_and(|(pts, _)| schedule::slot(pts.as_nanos()) <= index) + { + self.history.pop_front(); + } + let interval = schedule::slot(self.interval.unwrap_or(PSI_INTERVAL).as_nanos()).max(1); + let repeat = !is_clock_table(table_id) && index.is_multiple_of(interval); + match self.history.front() { + Some((pts, sections)) => { + let at = schedule::slot(pts.as_nanos()); + (at == index || (at < index && repeat)).then(|| sections.clone()) + } + None if matches!(self.state, SiState::Done) && repeat && !self.active.is_empty() => { + Some(self.active.sections().cloned().collect()) + } + None => None, + } + } + /// The snapshot an inline-form entry's sections reduce to. fn inline(entry: &catalog::SiEntry) -> super::si::Snapshot { let mut snapshot = super::si::Snapshot::default(); @@ -1013,7 +1258,7 @@ impl SiTrack { // one, including a partially-read pending group. match track.poll_next_group(waiter) { Poll::Ready(Ok(Some(group))) => { - self.pending = Some((group, Default::default())); + self.pending = Some((group, Default::default(), None)); } Poll::Ready(Ok(None)) => { ended = true; @@ -1035,9 +1280,10 @@ impl SiTrack { // Read the pending group to its end, then promote it wholesale: emitting a // half-received snapshot would re-introduce the torn state the group // boundary exists to prevent. - while let Some((group, snapshot)) = &mut self.pending { + while let Some((group, snapshot, pts)) = &mut self.pending { let (promote, drop_pending) = match group.poll_read_frame(waiter) { Poll::Ready(Ok(Some(frame))) => { + *pts = pts.or(frame.timestamp); snapshot.apply(&frame.payload); (false, false) } @@ -1049,7 +1295,13 @@ impl SiTrack { Poll::Pending => break, }; if promote { - let (_, snapshot) = self.pending.take().unwrap(); + let (_, snapshot, pts) = self.pending.take().unwrap(); + if let Some(pts) = pts { + self.history.push_back((pts, snapshot.sections().cloned().collect())); + if self.history.len() > SI_HISTORY { + self.history.pop_front(); + } + } // A repeated group (the same section set re-published) is not a change: // treating it as one would turn the source's snapshot cadence into extra // wire repetitions. @@ -1109,9 +1361,30 @@ where trimmed: 0, mux_rate: None, mux_rate_override: None, + sync: false, + si_slot: None, } } + /// Render every packet as a function of the broadcast alone, so two exports of it that + /// joined at different moments and received it with different skew emit identical TS for + /// the content both hold, continuity counters included: a 1+1 (SMPTE ST 2022-7) pair. + /// + /// Each PID gets a fixed lane of the constant multiplex rate, a function of the slot and + /// the catalog, and each MoQ group lays its units from a position its first unit's + /// timestamp gives. What a leg lacks leaves its positions as stuffing, so the other leg's + /// packets are untouched and complementary losses stay complementary. The PCR rides a PID + /// of its own, and each PID's continuity counter restarts at every group, its first packet + /// flagging the break. Audio is not packed: one frame a PES. + /// + /// Needs a constant rate ([`Self::with_mux_rate`], or the catalog's), and fails at the + /// program tables without one, or if the rate cannot carry each track's lane. The lanes + /// take a worst case per codec, and the video lane needs headroom over the video's rate. + pub fn with_sync(mut self) -> Self { + self.sync = true; + self + } + /// Pad the output with null packets to `mux_rate` bits per second, whatever the /// catalog records. Without this the catalog's `mpegts.muxRate` decides, and a /// catalog without one leaves the output unpadded. @@ -1387,7 +1660,7 @@ where } // SI emission rides media frames, so a snapshot that arrived behind the // last one gets one trailing flush before the stream ends. - if self.catalog_ended && !self.si_flushed { + if self.catalog_ended && !self.si_flushed && !self.sync { self.si_flushed = true; if let Some(out) = self.write_si_tail()? { return Poll::Ready(Ok(Some(out))); @@ -1755,6 +2028,13 @@ where .sum(), drift: self.jitter.drift().map(|drift| drift * 1e6), out_of_tolerance: self.jitter.out_of_tolerance(), + overruns: self + .tracks + .values() + .filter_map(|track| track.sync.as_ref()) + .map(|sync| sync.lane.overruns()) + .sum(), + late: self.schedule.grid().map_or(0, |grid| grid.late()), ..Default::default() }; if self.psi.is_none() { @@ -1846,6 +2126,7 @@ where if !same || self.psi.is_none() { let mut next = Self::new(self.source.clone(), catalog); next.replay = self.replay; + next.sync = self.sync; next = next.with_delay(self.delay); if let Some(rate) = self.mux_rate_override { next = next.with_mux_rate(rate); @@ -1952,7 +2233,9 @@ where si.last_emit = None; } self.video_start = None; - self.pcr_discontinuity = true; + // Under sync the grid flags a slot that does not follow the last ([`Grid::index`]). + self.pcr_discontinuity = !self.sync; + self.si_slot = None; self.tables.clear(); for track in self.tracks.values_mut() { track.pack = None; @@ -2010,6 +2293,22 @@ where /// Build the PAT/PMT once every track's PID and codec is known. fn build_psi(&mut self) -> anyhow::Result<()> { + // A measured rate would give each leg a grid of its own. + let sync = match self.sync { + true => Some(self.mux_rate.context( + "MPEG-TS --sync needs a constant multiplex rate: pass --mux-rate, or publish the catalog's mpegts.muxRate", + )?), + false => None, + }; + if sync.is_some() { + // The video lane's reach is its first keyframe's HRD, which every leg reads alike. + for (name, track) in self.tracks.iter_mut() { + if let Kind::Video(stream_type) = track.kind { + track.timing.describe(stream_type, track.source.description(), name); + } + } + } + // Order tracks by PID for a stable layout; first video track carries the PCR. let mut tracks: Vec<&Track> = self.tracks.values().collect(); tracks.sort_by_key(|t| t.pid); @@ -2042,6 +2341,12 @@ where }) .map(|t| t.pid) .context("TS export requires a video or audio track for the PCR")?; + // A media PID flags the break at every group, which on the PCR's PID would declare a + // break in the time base, so under sync the PCR rides a PID of its own. + let pcr_pid = match sync { + Some(_) => self.clock_pid()?, + None => pcr_pid, + }; let streams = tracks .iter() @@ -2134,18 +2439,79 @@ where for track in self.tracks.values() { self.liveness.register(track.pid); } - self.psi = Some(Psi { + let psi = Psi { pat: super::psi::encode_pat(&pat)?, pmt: super::psi::encode_pmt(&pmt, self.version)?, pcr_pid, pmt_pid, - }); + }; + if let Some(rate) = sync { + self.lay_lanes(rate, &psi)?; + } + self.psi = Some(psi); + Ok(()) + } + + /// The PID the PCR rides alone under [`Self::with_sync`]: the first above the program's + /// elementary streams that nothing else in it uses, so every leg picks the same one. + fn clock_pid(&self) -> anyhow::Result { + let used: HashSet = self + .tracks + .values() + .map(|track| track.pid) + .chain([self.pmt_pid()]) + .chain(self.si.keys().map(|(pid, _)| *pid)) + .collect(); + let top = self + .tracks + .values() + .map(|track| track.pid) + .max() + .unwrap_or(FIRST_ES_PID); + (top + 1..0x1FFF) + .find(|pid| !used.contains(pid)) + .context("MPEG-TS --sync found no PID left for the PCR") + } + + /// Give each track its lane of `rate` under [`Self::with_sync`], beside the tables in + /// `psi` and room for the SI, and lay every slot on them from now on. + fn lay_lanes(&mut self, rate: u64, psi: &Psi) -> anyhow::Result<()> { + if let Some((name, _)) = self + .tracks + .iter() + .find(|(_, t)| matches!(t.kind, Kind::PesPacket { .. })) + { + anyhow::bail!("MPEG-TS --sync cannot yet lay whole-PES track '{name}' on a lane"); + } + let mut tables = Vec::new(); + section_packets(&mut tables, Pid::PAT, &psi.pat, false)?; + section_packets(&mut tables, psi.pmt_pid, &psi.pmt, false)?; + let tables = tables.as_chunks::<{ TsPacket::SIZE }>().0.to_vec(); + let mut needs: Vec<(u16, Need)> = self + .tracks + .values() + .map(|track| (track.pid, track.lane_need())) + .collect(); + needs.sort_unstable_by_key(|(pid, _)| *pid); + let layout = Arc::new(Layout::new(rate, tables.len(), SI_LANE, &needs)?); + for track in self.tracks.values_mut() { + let lane = layout.lane(track.pid).context("a track without a lane")?; + track.sync = Some(Synced { + layout: layout.clone(), + lane: Lane::new(lane), + released: None, + }); + } + self.schedule.set_sync(Grid::new(layout, tables)); Ok(()) } /// Packetize one media frame and queue it on the [`Schedule`], re-emitting PAT/PMT /// before video keyframes (and periodically) so receivers can tune in mid-stream. fn mux(&mut self, name: &str, queued: Queued) -> anyhow::Result<()> { + if self.sync { + return self.mux_sync(queued); + } let Queued { frame, decode, @@ -2177,43 +2543,8 @@ where *repeat = false; } let keyframe = frame.keyframe; - - // Build the elementary-stream payload for this frame. Video needs the - // resolved avcC/hvcC to rewrite length-prefixed NALs as Annex-B. Section-framed - // verbatim streams carry no PES payload; the section is written separately below. - let es_payload = match &kind { - Kind::Video(stream_type) => Some(video_es_payload(*stream_type, description.as_ref(), &frame)?), - Kind::Aac { config, repeat } => { - let pce = if *repeat { - config.program_config.as_deref().unwrap_or_default() - } else { - &[] - }; - let raw_len = pce.len() + frame.payload.len(); - let header = - adts::write_header(config.object_type, config.sample_rate, config.channel_config, raw_len)?; - let mut framed = Vec::with_capacity(header.len() + raw_len); - framed.extend_from_slice(&header); - framed.extend_from_slice(pce); - framed.extend_from_slice(&frame.payload); - Some(framed) - } - // Each moq Opus frame is one packet; prefix the Opus-in-TS control header. - Kind::Opus { .. } => Some(opus_es_payload(&frame.payload)), - // Legacy audio frames were ingested whole (framing header included), so - // they pass through untouched. PES-framed verbatim payloads likewise. - Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 => Some(frame.payload.to_vec()), - Kind::Verbatim { - framing: catalog::Framing::Pes, - .. - } => Some(frame.payload.to_vec()), - Kind::Verbatim { - framing: catalog::Framing::Section, - .. - } => None, - // The frame is the PES already, written by [`Self::place`] below. - Kind::PesPacket { .. } => None, - }; + let pce = matches!(kind, Kind::Aac { repeat: true, .. }); + let es_payload = es_payload(&kind, description.as_ref(), &frame, pce)?; let mut out = Vec::with_capacity(TsPacket::SIZE); @@ -2279,22 +2610,13 @@ where // carry the bytes verbatim. None => self.write_section(&mut out, pid, &frame.payload)?, Some(es_payload) => { - // Verbatim PES re-emits its original stream_id (falling back to - // private_stream_1 for an undecoded stream with none recorded); media - // derives it from is_video. - let stream_id = match &kind { - Kind::Verbatim { stream_id, .. } => Some(stream_id.unwrap_or(StreamId::PRIVATE_STREAM_1)), - // Opus is private-data PES, carried under private_stream_1 like ffmpeg. - Kind::Opus { .. } => Some(StreamId::PRIVATE_STREAM_1), - _ => None, - }; let unit = PesUnit { pid, is_video, keyframe, timestamp: frame.timestamp, dts, - stream_id, + stream_id: stream_id(&kind), }; // Only a delay leaves the frames time to wait for the ones after them. let track = self.tracks.get(name).context("missing track")?; @@ -2339,6 +2661,47 @@ where .push_placed(pid, (place, decode.unwrap_or(place)), out, frame.keyframe, frames) } + /// Put a frame's planned packets on their lane under [`Self::with_sync`]. The tables and SI + /// have positions of their own ([`Self::queue_si`]), so nothing rides with it. + fn mux_sync(&mut self, queued: Queued) -> anyhow::Result<()> { + let Some(planned) = queued.planned else { + return Ok(()); + }; + let grid = self.schedule.sync().context("MPEG-TS --sync lanes not laid out")?; + grid.place(&planned.packets, &planned.positions, queued.frame.keyframe); + Ok(()) + } + + /// Queue the SI due in slot `index` under [`Self::with_sync`], in PID and table order, each + /// entry's sections numbered from zero with the first flagging the break: every leg holding + /// the same snapshots sends the same packets ([`SiTrack::due`]). + fn queue_si(&mut self, index: u128) -> anyhow::Result<()> { + if self.si_slot.is_some_and(|slot| slot >= index) { + return Ok(()); + } + self.si_slot = Some(index); + let mut keys: Vec<(u16, u8)> = self.si.keys().copied().collect(); + keys.sort_unstable(); + let mut out = Vec::new(); + for (pid, table_id) in keys { + let Some(sections) = self.si.get_mut(&(pid, table_id)).and_then(|si| si.due(index, table_id)) else { + continue; + }; + let mut packets = Vec::new(); + for section in sections.iter().filter(|section| is_complete_section(section)) { + let first = packets.is_empty(); + section_packets(&mut packets, pid, section, first)?; + } + lane::number(&mut packets, 0); + out.extend_from_slice(&packets); + } + if !out.is_empty() { + let grid = self.schedule.sync().context("MPEG-TS --sync lanes not laid out")?; + grid.push_si(index, &out); + } + Ok(()) + } + /// Mux an audio frame into `name`'s PES, sending the PES on first if the frame cannot join /// it, and leave the tables muxed ahead of it to ride the next unit queued. A PES carries /// frames while they run on from its PTS without a gap and fit the PID's decoder buffer @@ -2592,6 +2955,12 @@ where { self.flush_packs(|_, due| due <= index + lead)?; } + if self.sync + && let Some((index, _)) = self.schedule.upcoming() + && known.is_none_or(|known| index < known) + { + self.queue_si(index)?; + } let Some(slot) = self.schedule.next(known)? else { break; }; @@ -2599,11 +2968,14 @@ where let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); // The slot opens with its PCR, in the clock packet or the clock PID's packet that // carries it, whose adaptation field flags the break. - let discontinuity = std::mem::take(&mut self.pcr_discontinuity); + let discontinuity = std::mem::take(&mut self.pcr_discontinuity) || slot.jump; if discontinuity { payload[5] |= 0x80; } - self.number(&mut payload); + // Under sync every packet was numbered where it was planned. + if !self.sync { + self.number(&mut payload); + } let frame = Frame { timestamp: slot_stamp(slot.index)?, duration: None, @@ -2639,75 +3011,8 @@ where /// Packetize a PES payload into 188-byte TS packets. fn write_pes(&mut self, out: &mut Vec, unit: &PesUnit, payload: &[u8]) -> anyhow::Result<()> { - let pts = to_ts_timestamp(unit.timestamp)?; - // A reordered video frame carries DTS alongside PTS; else PTS-only. The decode clock - // is continuous ticks, so wrap into the 33-bit wire field here, like the PTS. - let dts = unit - .dts - .map(|t| TsTimestamp::new(t & TS_TIMESTAMP_MASK).map_err(anyhow::Error::msg)) - .transpose()?; - let stream_id = match unit.stream_id { - Some(id) => StreamId::new(id), - None if unit.is_video => StreamId::new(StreamId::VIDEO_MIN), - None => StreamId::new(StreamId::AUDIO_MIN), - }; - let header = mpeg2ts::pes::PesHeader { - stream_id, - priority: false, - data_alignment_indicator: true, - copyright: false, - original_or_copy: false, - pts: Some(pts), - dts, - escr: None, - }; - - // The optional PES header grows by 5 bytes when it also carries a DTS. - let optional_len = PES_OPTIONAL_LEN + if dts.is_some() { PES_DTS_LEN } else { 0 }; - - // `pes_packet_len` counts the optional header plus the payload (not the - // 6-byte fixed prefix). Unbounded for video (0); bounded for audio when - // it fits a u16. - let pes_packet_len = if unit.is_video { - 0 - } else { - u16::try_from(optional_len + payload.len()).unwrap_or(0) - }; - - let mut offset = 0; - let mut first = true; - loop { - let flag = first && self.breaks(unit.pid); - let adaptation = ((first && unit.keyframe) || flag).then(|| AdaptationField { - random_access_indicator: first && unit.keyframe, - ..flags(flag) - }); - - let header_len = if first { 6 + optional_len } else { 0 }; - let af_len = adaptation.as_ref().map(adaptation_size).unwrap_or(0); - let avail = TsBytes::MAX_SIZE - header_len - af_len; - let take = avail.min(payload.len() - offset); - let chunk = &payload[offset..offset + take]; - - let ts_payload = if first { - TsPayload::PesStart(Pes { - header: header.clone(), - pes_packet_len, - data: TsBytes::new(chunk).map_err(anyhow::Error::msg)?, - }) - } else { - TsPayload::PesContinuation(TsBytes::new(chunk).map_err(anyhow::Error::msg)?) - }; - - self.write_packet(out, unit.pid, adaptation, ts_payload)?; - - offset += take; - first = false; - if offset >= payload.len() { - break; - } - } - Ok(()) + let flag = self.breaks(unit.pid); + pes_packets(out, unit, payload, flag) } /// Packetize `pes`, a whole PES packet, into TS packets on `pid` without touching a byte of @@ -2741,10 +3046,7 @@ where Ok(()) } - /// Packetize a complete PSI or private section verbatim. The first packet - /// carries the pointer_field plus the section start as a `Section` payload (sets - /// the unit-start bit so the receiver finds the pointer_field); continuations are - /// `Raw`. The section bytes are opaque, so this round-trips byte-for-byte. + /// Packetize a complete PSI or private section verbatim ([`section_packets`]). fn write_section(&mut self, out: &mut Vec, pid: u16, section: &[u8]) -> anyhow::Result<()> { // The verbatim track is public; a non-importer producer could publish a frame // that isn't a complete section. Drop it (with a warning) rather than emit a @@ -2754,65 +3056,145 @@ where tracing::warn!(pid, len = section.len(), "dropping malformed private section on export"); return Ok(()); } + let flag = self.breaks(pid); + section_packets(out, pid, section, flag) + } +} - let mut offset = 0; - let mut first = true; - loop { - let flag = (first && self.breaks(pid)).then(|| flags(true)); - let payload = if first { - // pointer_field (1 byte, written by `Section`) eats one payload byte, and a flag - // for the break its adaptation field. - let room = TsBytes::MAX_SIZE - 1 - flag.as_ref().map_or(0, adaptation_size); - let take = room.min(section.len()); - let chunk = §ion[..take]; - offset = take; - TsPayload::Section(Section { - pointer_field: 0, - data: TsBytes::new(chunk).map_err(anyhow::Error::msg)?, - }) - } else { - let take = TsBytes::MAX_SIZE.min(section.len() - offset); - let chunk = §ion[offset..offset + take]; - offset += take; - TsPayload::Raw(TsBytes::new(chunk).map_err(anyhow::Error::msg)?) - }; +/// Packetize a PES payload into 188-byte TS packets, flagging a break in the first when `flag`. +fn pes_packets(out: &mut Vec, unit: &PesUnit, payload: &[u8], flag: bool) -> anyhow::Result<()> { + let pts = to_ts_timestamp(unit.timestamp)?; + // A reordered video frame carries DTS alongside PTS; else PTS-only. The decode clock + // is continuous ticks, so wrap into the 33-bit wire field here, like the PTS. + let dts = unit + .dts + .map(|t| TsTimestamp::new(t & TS_TIMESTAMP_MASK).map_err(anyhow::Error::msg)) + .transpose()?; + let stream_id = match unit.stream_id { + Some(id) => StreamId::new(id), + None if unit.is_video => StreamId::new(StreamId::VIDEO_MIN), + None => StreamId::new(StreamId::AUDIO_MIN), + }; + let header = mpeg2ts::pes::PesHeader { + stream_id, + priority: false, + data_alignment_indicator: true, + copyright: false, + original_or_copy: false, + pts: Some(pts), + dts, + escr: None, + }; - self.write_packet(out, pid, flag, payload)?; - first = false; - if offset >= section.len() { - break; - } + // The optional PES header grows by 5 bytes when it also carries a DTS. + let optional_len = PES_OPTIONAL_LEN + if dts.is_some() { PES_DTS_LEN } else { 0 }; + + // `pes_packet_len` counts the optional header plus the payload (not the + // 6-byte fixed prefix). Unbounded for video (0); bounded for audio when + // it fits a u16. + let pes_packet_len = if unit.is_video { + 0 + } else { + u16::try_from(optional_len + payload.len()).unwrap_or(0) + }; + + let mut offset = 0; + let mut first = true; + loop { + let flag = first && flag; + let adaptation = ((first && unit.keyframe) || flag).then(|| AdaptationField { + random_access_indicator: first && unit.keyframe, + ..flags(flag) + }); + + let header_len = if first { 6 + optional_len } else { 0 }; + let af_len = adaptation.as_ref().map(adaptation_size).unwrap_or(0); + let avail = TsBytes::MAX_SIZE - header_len - af_len; + let take = avail.min(payload.len() - offset); + let chunk = &payload[offset..offset + take]; + + let ts_payload = if first { + TsPayload::PesStart(Pes { + header: header.clone(), + pes_packet_len, + data: TsBytes::new(chunk).map_err(anyhow::Error::msg)?, + }) + } else { + TsPayload::PesContinuation(TsBytes::new(chunk).map_err(anyhow::Error::msg)?) + }; + + write_packet(out, unit.pid, adaptation, ts_payload)?; + + offset += take; + first = false; + if offset >= payload.len() { + break; } - Ok(()) } + Ok(()) +} - /// Serialize one TS packet into `out`. Its continuity counter is numbered when it goes - /// out ([`Self::number`]), since the schedule decides that order. - fn write_packet( - &mut self, - out: &mut Vec, - pid: u16, - adaptation_field: Option, - payload: TsPayload, - ) -> anyhow::Result<()> { - let continuity_counter = ContinuityCounter::default(); - - let packet = TsPacket { - header: TsHeader { - transport_error_indicator: false, - transport_priority: false, - pid: Pid::new(pid)?, - transport_scrambling_control: TransportScramblingControl::NotScrambled, - continuity_counter, - }, - adaptation_field, - payload: Some(payload), +/// Packetize a complete PSI or private section verbatim, flagging a break in the first packet +/// when `flag`. The first packet carries the pointer_field plus the section start as a +/// `Section` payload (sets the unit-start bit so the receiver finds the pointer_field); +/// continuations are `Raw`. The section bytes are opaque, so this round-trips byte-for-byte. +fn section_packets(out: &mut Vec, pid: u16, section: &[u8], flag: bool) -> anyhow::Result<()> { + let mut offset = 0; + let mut first = true; + loop { + let flag = (first && flag).then(|| flags(true)); + let payload = if first { + // pointer_field (1 byte, written by `Section`) eats one payload byte, and a flag + // for the break its adaptation field. + let room = TsBytes::MAX_SIZE - 1 - flag.as_ref().map_or(0, adaptation_size); + let take = room.min(section.len()); + let chunk = §ion[..take]; + offset = take; + TsPayload::Section(Section { + pointer_field: 0, + data: TsBytes::new(chunk).map_err(anyhow::Error::msg)?, + }) + } else { + let take = TsBytes::MAX_SIZE.min(section.len() - offset); + let chunk = §ion[offset..offset + take]; + offset += take; + TsPayload::Raw(TsBytes::new(chunk).map_err(anyhow::Error::msg)?) }; - let mut writer = TsPacketWriter::new(out); - writer.write_ts_packet(&packet).map_err(anyhow::Error::msg)?; - Ok(()) + write_packet(out, pid, flag, payload)?; + first = false; + if offset >= section.len() { + break; + } } + Ok(()) +} + +/// Serialize one TS packet into `out`. Its continuity counter is numbered when it goes +/// out ([`Export::number`]), since the schedule decides that order. +fn write_packet( + out: &mut Vec, + pid: u16, + adaptation_field: Option, + payload: TsPayload, +) -> anyhow::Result<()> { + let continuity_counter = ContinuityCounter::default(); + + let packet = TsPacket { + header: TsHeader { + transport_error_indicator: false, + transport_priority: false, + pid: Pid::new(pid)?, + transport_scrambling_control: TransportScramblingControl::NotScrambled, + continuity_counter, + }, + adaptation_field, + payload: Some(payload), + }; + + let mut writer = TsPacketWriter::new(out); + writer.write_ts_packet(&packet).map_err(anyhow::Error::msg)?; + Ok(()) } impl Export> { @@ -3281,6 +3663,33 @@ fn audio_buffer(config: &AudioConfig, kind: &Kind) -> Buffer { } } +/// The most packets a second an AC-3 PID takes ([`Track::lane_need`]): 44.1 kHz's largest +/// frame, 2788 bytes in 16 packets, 28.7 times a second (ATSC A/52 Table 5.18). +const AC3_LANE: u64 = 460; +/// E-AC-3's: its largest frame, 4096 bytes in 23 packets, at six blocks of 48 kHz. +const EAC3_LANE: u64 = 719; +/// Teletext's: a nine-packet PES, a frame's full data units (ETSI EN 300 472), at 25 frames. +const TELETEXT_LANE: u64 = 225; +/// Any other verbatim PES's, which nothing bounds: two packets a slot. +const VERBATIM_LANE: u64 = 80; +/// A section-framed stream's (SCTE-35, ...): a packet a slot. +const SECTION_LANE: u64 = 40; + +/// An Opus PID's most packets a second ([`Track::lane_need`]): `code`'s channels at Opus's +/// 510 kb/s ceiling (RFC 6716 2.1.1), in packets as short as 2.5 ms or as long as 20 ms, +/// whichever takes more. +fn opus_lane(code: u8) -> u64 { + let channels = u64::from(code.clamp(1, 8)); + [400u64, 200, 100, 50] + .into_iter() + .map(|per_second| { + let bytes = (channels * 510_000 / 8 / per_second).min(channels * 1275) + 8; + (bytes + 16).div_ceil(TsBytes::MAX_SIZE as u64) * per_second + }) + .max() + .unwrap_or(0) +} + /// The DVB AC-3 descriptor's tag (ETSI EN 300 468 6.2.1), marking AC-3 as private data. const AC3_DESCRIPTOR: u8 = 0x6a; @@ -3383,20 +3792,25 @@ struct DolbyCarry { unit: Option, /// The bytes so far of a sync frame left unfinished. bytes: Vec, + /// The MoQ group of the PES those bytes began in, which their frame belongs to. + group: u64, } -/// An access unit begun: when it is presented, and the blocks and rate its substreams share. +/// An access unit begun: when it is presented, the blocks and rate its substreams share, and +/// the MoQ group of the PES it began in, which it and its substreams belong to. #[derive(Clone, Copy)] struct DolbyUnit { at: Timestamp, samples: u64, rate: u32, + group: u64, } -/// The Dolby access units a PES's `payload` completes or holds, each with its presentation -/// time; none when no sync frame can begin in it. An AC-3 access unit is one sync frame; an -/// E-AC-3 one is independent substream 0 with every substream after it (ETSI TS 102 366 -/// E.1.3.1.2), which share its presentation interval. +/// The Dolby access units a PES's `payload` from MoQ group `group` completes or holds, each +/// with its presentation time and the group of the PES it began in; none when no sync frame can +/// begin in it. An AC-3 access unit is one sync frame; an E-AC-3 one is independent substream 0 +/// with every substream after it (ETSI TS 102 366 E.1.3.1.2), which share its presentation +/// interval. /// /// A PES need not begin on a sync frame: a source that leaves `data_alignment_indicator` /// clear splits frames, and even their headers, across PES. `carry` holds the frame the last @@ -3414,10 +3828,10 @@ struct DolbyUnit { fn dolby_units( codec: Dolby, carry: &mut Option, - skip: u64, + (skip, group): (u64, u64), start: Timestamp, payload: &Bytes, -) -> anyhow::Result> { +) -> anyhow::Result> { let duration = |samples: u64, rate: u32| Timestamp::from_micros(samples * 1_000_000 / u64::from(rate.max(1))); let joined = carry.as_ref().is_none_or(|head| head.skip != skip); let mut units = Vec::new(); @@ -3483,7 +3897,7 @@ fn dolby_units( let frame = Bytes::from(head.bytes); // A unit stays open only through a carried frame that completes without a gap. open = head.unit; - dolby_place(codec, frame, &header, head.next, &mut open, &mut units)?; + dolby_place(codec, frame, &header, (head.next, head.group), &mut open, &mut units)?; offset = Some(need); } else { tracing::warn!(?end, ?start, "dropped a Dolby sync frame cut by a gap between PES"); @@ -3536,7 +3950,7 @@ fn dolby_units( break; } let frame = payload.slice(offset..offset + header.len); - if dolby_place(codec, frame, &header, next, &mut open, &mut units)? { + if dolby_place(codec, frame, &header, (next, group), &mut open, &mut units)? { samples += header.samples; rate = header.sample_rate; } @@ -3547,28 +3961,30 @@ fn dolby_units( next: base.checked_add(duration(samples, rate)?)?, unit: open.filter(|_| codec == Dolby::Eac3), bytes: left, + group, }); Ok(units) } -/// File the whole sync frame `frame` in `units`: as a new access unit presented at `next`, or -/// as a substream of the `open` one, joining it when it went out from this PES too. Returns -/// whether it began a unit. +/// File the whole sync frame `frame`, begun in MoQ group `group`, in `units`: as a new access +/// unit presented at `next`, or as a substream of the `open` one, joining it when it went out +/// from this PES too. Returns whether it began a unit. fn dolby_place( codec: Dolby, frame: Bytes, header: &crate::codec::legacy::Header, - next: Timestamp, + (next, group): (Timestamp, u64), open: &mut Option, - units: &mut Vec<(Timestamp, Bytes)>, + units: &mut Vec<(Timestamp, Bytes, u64)>, ) -> anyhow::Result { if codec.begins_unit(&frame) { *open = Some(DolbyUnit { at: next, samples: header.samples, rate: header.sample_rate, + group, }); - units.push((next, frame)); + units.push((next, frame, group)); return Ok(true); } let Some(unit) = *open else { @@ -3584,8 +4000,8 @@ fn dolby_place( unit.rate, ); match units.last_mut() { - Some((at, bytes)) if *at == unit.at => *bytes = [bytes.as_ref(), frame.as_ref()].concat().into(), - _ => units.push((unit.at, frame)), + Some((at, bytes, _)) if *at == unit.at => *bytes = [bytes.as_ref(), frame.as_ref()].concat().into(), + _ => units.push((unit.at, frame, unit.group)), } Ok(false) } @@ -3618,6 +4034,55 @@ fn ac3_sync_prefix(bytes: &[u8]) -> bool { bytes.iter().zip([0x0b, 0x77]).all(|(&byte, sync)| byte == sync) } +/// Build the elementary-stream payload for a frame of `kind`, repeating an AAC program config +/// element when `pce`. Video needs the resolved avcC/hvcC to rewrite length-prefixed NALs as +/// Annex-B. Section-framed verbatim streams carry no PES payload, so they have none. +fn es_payload(kind: &Kind, description: Option<&Bytes>, frame: &Frame, pce: bool) -> anyhow::Result>> { + Ok(match kind { + Kind::Video(stream_type) => Some(video_es_payload(*stream_type, description, frame)?), + Kind::Aac { config, .. } => { + let pce = match pce { + true => config.program_config.as_deref().unwrap_or_default(), + false => &[], + }; + let raw_len = pce.len() + frame.payload.len(); + let header = adts::write_header(config.object_type, config.sample_rate, config.channel_config, raw_len)?; + let mut framed = Vec::with_capacity(header.len() + raw_len); + framed.extend_from_slice(&header); + framed.extend_from_slice(pce); + framed.extend_from_slice(&frame.payload); + Some(framed) + } + // Each moq Opus frame is one packet; prefix the Opus-in-TS control header. + Kind::Opus { .. } => Some(opus_es_payload(&frame.payload)), + // Legacy audio frames were ingested whole (framing header included), so + // they pass through untouched. PES-framed verbatim payloads likewise. + Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 => Some(frame.payload.to_vec()), + Kind::Verbatim { + framing: catalog::Framing::Pes, + .. + } => Some(frame.payload.to_vec()), + Kind::Verbatim { + framing: catalog::Framing::Section, + .. + } => None, + // The frame is the PES already, written by [`Export::place`]. + Kind::PesPacket { .. } => None, + }) +} + +/// The PES stream_id a frame of `kind` goes out under, `None` to derive it from whether it is +/// video. Verbatim PES re-emits its original stream_id, falling back to private_stream_1 for an +/// undecoded stream with none recorded. +fn stream_id(kind: &Kind) -> Option { + match kind { + Kind::Verbatim { stream_id, .. } => Some(stream_id.unwrap_or(StreamId::PRIVATE_STREAM_1)), + // Opus is private-data PES, carried under private_stream_1 like ffmpeg. + Kind::Opus { .. } => Some(StreamId::PRIVATE_STREAM_1), + _ => None, + } +} + /// Build the Annex-B elementary-stream payload for one video frame: rewrite the /// length-prefixed NALs to start-code-delimited NALs, prepending the parameter /// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC on keyframes so a @@ -3911,6 +4376,7 @@ mod tests { cut: 0, arrived: now, read: now, + group: (pts.as_micros() as u64, true), }; track.queue("audio", pending, &mut jitter).unwrap(); } @@ -3939,13 +4405,13 @@ mod tests { let units = super::dolby_units( super::Dolby::Eac3, &mut None, - 0, + (0, 0), Timestamp::from_micros(0).unwrap(), &payload, ) .unwrap(); assert_eq!(units.len(), 30); - for (k, (at, _)) in units.iter().enumerate() { + for (k, (at, _, _)) in units.iter().enumerate() { assert_eq!(at.as_micros(), k as u128 * 1_536 * 1_000_000 / 44_100, "unit {k}"); } } @@ -3959,11 +4425,17 @@ mod tests { core.resize(128, 0); let unit = [core, eac3(0x40, 0x3f)].concat(); let payload = bytes::Bytes::from(unit.repeat(2)); - let units = - super::dolby_units(Dolby::Eac3, &mut None, 0, Timestamp::from_micros(0).unwrap(), &payload).unwrap(); + let units = super::dolby_units( + Dolby::Eac3, + &mut None, + (0, 0), + Timestamp::from_micros(0).unwrap(), + &payload, + ) + .unwrap(); let at: Vec<_> = units .iter() - .map(|(at, bytes)| (at.as_micros(), bytes.to_vec())) + .map(|(at, bytes, _)| (at.as_micros(), bytes.to_vec())) .collect(); assert_eq!(at, [(0, unit.clone()), (32_000, unit)]); } @@ -3977,7 +4449,7 @@ mod tests { let err = super::dolby_units( super::Dolby::Eac3, &mut None, - 0, + (0, 0), Timestamp::from_micros(0).unwrap(), &payload, ) diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 01282108118..0373bb72c9c 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -73,6 +73,10 @@ struct Live { scale: f64, /// The multiplex rate the receivers pad to. rate: u64, + /// The receivers render with [`Export::with_sync`]. + sync: bool, + /// The media time of the first frame, in µs. + origin: u64, /// How much later than its video the source sends audio of the same media time, in µs; /// negative sends the video later. A TS source sends video well ahead of its decode time /// and audio just in time, so its audio lags. @@ -138,6 +142,8 @@ impl Live { audio, scale, rate: MUX_RATE, + sync: false, + origin: 0, audio_lag, start: moq_clock::now(), tick: 0, @@ -183,11 +189,14 @@ impl Live { /// A receiver joining now. async fn join(&self, every: Duration) -> Leg { - let export = super::export_test::open_export(self.source.clone()) + let mut export = super::export_test::open_export(self.source.clone()) .await .unwrap() .with_delay(DELAY) .with_mux_rate(self.rate); + if self.sync { + export = export.with_sync(); + } Leg { export, every, @@ -236,7 +245,7 @@ impl Live { .write( group, &[Frame { - timestamp: Timestamp::from_micros(video).unwrap(), + timestamp: Timestamp::from_micros(self.origin + video).unwrap(), duration: None, payload: length_prefixed(&[&slice]), keyframe, @@ -257,7 +266,7 @@ impl Live { let index = self.audio_index; self.audio .write(Frame { - timestamp: Timestamp::from_micros(index * AUDIO_US).unwrap(), + timestamp: Timestamp::from_micros(self.origin + index * AUDIO_US).unwrap(), duration: None, payload: Bytes::from_iter((0..180u16).map(|i| (i ^ index as u16) as u8)), keyframe: index.is_multiple_of(AUDIO_GROUP), @@ -826,16 +835,243 @@ async fn two_legs_pad_the_same_slots_at_a_fractional_rate() { assert_same_packets(&a, &b, from); } -/// A slow group reaches both legs. Both have to skip it the same way. +/// A slow group reaches both legs, and each skips it when its own reads find the next group +/// far enough ahead, so each may render a different part of it. What both render is the same, +/// continuity counters included, and what one lacks is stuffing. #[tokio::test(start_paused = true)] -#[ignore = "1+1 (ST 2022-7) packet identity across a skip is deferred to an m2 follow-up"] async fn two_legs_render_a_skip_the_same_way() { for lag in LAGS { - let (a, b, from) = pair(1.0, lag, 12 * GOP, |tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; - assert_same_packets(&a, &b, from); + let (a, b, from) = synced(lag, JOIN, 12 * GOP, |tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; + assert_same_synced(&a, &b, from); } } +// Under `--sync` every byte a leg renders is a function of the objects it holds, their media +// time, the catalog and the rate, so the continuity counters match too, and content one leg +// lacks leaves stuffing in its positions rather than moving what follows. + +/// Two `--sync` legs of one broadcast on `lag`: one there from the start and reading after +/// every frame, the other joining at video tick `join` and reading every 100 ms. Returns them +/// with the slot time 2 s after the second joined, from which they must agree. +async fn synced(lag: i64, join: u64, ticks: u64, stall: impl Fn(u64) -> bool + Copy) -> (Leg, Leg, u64) { + synced_on(1.0, 0, lag, join, ticks, stall).await +} + +async fn synced_on( + scale: f64, + origin: u64, + lag: i64, + join: u64, + ticks: u64, + stall: impl Fn(u64) -> bool + Copy, +) -> (Leg, Leg, u64) { + let mut live = Live::new(scale, lag).await; + live.sync = true; + live.origin = origin; + let mut a = live.join(Duration::ZERO).await; + live.run(join, &mut [&mut a], stall).await; + let mut b = live.join(Duration::from_millis(100)).await; + live.run(ticks, &mut [&mut a, &mut b], stall).await; + live.finish(&mut [&mut a, &mut b]).await; + let joined = b.out.first().expect("the joiner rendered").1.timestamp.as_micros() as u64; + (a, b, joined + 2_000_000) +} + +/// Each slot a leg rendered from media time `from` (µs), keyed by that time. +fn slots(leg: &Leg, from: u64) -> std::collections::BTreeMap { + leg.out + .iter() + .map(|(_, frame)| (frame.timestamp.as_micros() as u64, frame.payload.clone())) + .filter(|(at, _)| *at >= from) + .collect() +} + +fn pid(packet: &[u8]) -> u16 { + u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]) +} + +/// Assert two `--sync` legs rendered the same packets, continuity counters included, in every +/// slot from media time `from` (µs), compared by the slot's media time: a position may hold +/// stuffing in one leg where the other has content, and nothing else may differ. Returns how +/// many positions held content in one leg alone. +fn assert_same_synced(a: &Leg, b: &Leg, from: u64) -> usize { + for leg in [a, b] { + let stats = leg.export.stats(); + assert_eq!((stats.overruns, stats.late), (0, 0), "a lane fell behind"); + } + let (a, b) = (slots(a, from), slots(b, from)); + let common: Vec = a.keys().filter(|at| b.contains_key(at)).copied().collect(); + assert!( + common.len() > 40, + "not enough overlap to be worth comparing: {}", + common.len() + ); + assert_eq!( + common.len(), + a.len().min(b.len()), + "a slot one leg rendered is missing from the other" + ); + let mut alone = 0; + for at in common { + let (x, y) = (&a[&at], &b[&at]); + assert_eq!(x.len(), y.len(), "slot at {at} µs rendered to a different size"); + for (n, (p, q)) in x.chunks(188).zip(y.chunks(188)).enumerate() { + if p == q { + continue; + } + let offset = p.iter().zip(q).position(|(x, y)| x != y).unwrap_or(0); + assert!( + pid(p) == 0x1fff || pid(q) == 0x1fff, + "slot at {at} µs differs at packet {n}, byte {offset}: PID {:#x} vs {:#x}, {:#04x} vs {:#04x}", + pid(p), + pid(q), + p[offset], + q[offset], + ); + alone += 1; + } + } + alone +} + +/// Whether a leg sent the PAT before any media, wherever its first slot fell on the grid. +fn leads_with_the_tables(leg: &Leg) -> bool { + let mut pids = leg.out.iter().flat_map(|(_, frame)| frame.payload.chunks(188).map(pid)); + pids.find(|&pid| pid == 0 || pid == VIDEO_PID || pid == AUDIO_PID) == Some(0) +} + +/// Legs that joined mid-group and a group later, read on different cadences with each track +/// lagging in turn, render the same packets, continuity counters included, and the joiner +/// starts on the tables. +#[tokio::test(start_paused = true)] +async fn synced_legs_render_the_same_packets() { + for lag in LAGS { + for join in [JOIN, JOIN + GOP, 4 * GOP] { + let (a, b, from) = synced(lag, join, 12 * GOP, |_| false).await; + let alone = assert_same_synced(&a, &b, from); + assert_eq!( + alone, 0, + "lag {lag}, join {join}: clean legs differ in {alone} positions" + ); + assert!(leads_with_the_tables(&b), "lag {lag}, join {join}"); + } + } +} + +/// Each leg loses groups the other receives. Each renders the other's packets wherever it +/// holds them, so one leg's stuffing is the other's content, and the two merged are a clean +/// leg's output. +#[tokio::test(start_paused = true)] +async fn complementary_losses_stay_complementary() { + let lost_a = |tick: u64| (5 * GOP + 3..6 * GOP).contains(&tick); + let lost_b = |tick: u64| (7 * GOP + 5..8 * GOP).contains(&tick); + for lag in LAGS { + let mut paths = Vec::new(); + for _ in 0..3 { + let mut live = Live::new(1.0, lag).await; + live.sync = true; + paths.push(live); + } + let [mut clean, mut path_a, mut path_b] = <[Live; 3]>::try_from(paths).ok().unwrap(); + assert!(clean.start == path_a.start && clean.start == path_b.start); + let mut c = clean.join(Duration::ZERO).await; + let mut a = path_a.join(Duration::ZERO).await; + let mut b = path_b.join(Duration::from_millis(100)).await; + for tick in 1..=12 * GOP { + clean.run(tick, &mut [&mut c], |_| false).await; + path_a.run(tick, &mut [&mut a], lost_a).await; + path_b.run(tick, &mut [&mut b], lost_b).await; + } + clean.finish(&mut [&mut c]).await; + path_a.finish(&mut [&mut a]).await; + path_b.finish(&mut [&mut b]).await; + + let from = 2 * GOP * VIDEO_US; + assert!(assert_same_synced(&a, &c, from) > 0, "lag {lag}: leg A lost nothing"); + assert!(assert_same_synced(&b, &c, from) > 0, "lag {lag}: leg B lost nothing"); + assert!(assert_same_synced(&a, &b, from) > 0); + let (a, b, c) = (slots(&a, from), slots(&b, from), slots(&c, from)); + for (at, clean) in &c { + let (Some(a), Some(b)) = (a.get(at), b.get(at)) else { + continue; + }; + let merged: Vec = a + .chunks(188) + .zip(b.chunks(188)) + .flat_map(|(a, b)| if pid(a) == 0x1fff { b } else { a }) + .copied() + .collect(); + assert_eq!( + &merged[..], + &clean[..], + "lag {lag}: the merged legs differ from a clean leg at {at} µs" + ); + } + } +} + +/// A broadcast timeline crossing the 33-bit PTS and PCR wrap while both legs run, with the +/// second joining just after it, renders the same packets. +#[tokio::test(start_paused = true)] +async fn synced_legs_render_the_same_packets_across_the_wrap() { + // 2^33 ticks of 90 kHz, in µs: the wrap falls 3.7 s in, and the joiner's first group is + // the first to start after it. + let wrap = (1u64 << 33) * 100 / 9; + let origin = (wrap - 3_700_000) / VIDEO_US * VIDEO_US; + let join = (wrap - origin).div_ceil(VIDEO_US * GOP) * GOP + 1; + let first = |leg: &Leg| leg.units().iter().find(|unit| unit.pid == VIDEO_PID).unwrap().decode; + for lag in LAGS { + let (a, b, from) = synced_on(1.0, origin, lag, join, join + 12 * GOP, |_| false).await; + assert!( + first(&a) > (1 << 33) - 10 * 90_000, + "the first leg started after the wrap" + ); + assert!(first(&b) < 10 * 90_000, "the joiner started before the wrap"); + assert_eq!(assert_same_synced(&a, &b, from), 0); + assert!(leads_with_the_tables(&a) && leads_with_the_tables(&b)); + } +} + +/// A source clock 30 ppm off the receivers' moves when slots go out, never what they carry. +#[tokio::test(start_paused = true)] +async fn synced_legs_render_a_drifting_source_the_same_way() { + for scale in [1.0 - 30e-6, 1.0 + 30e-6] { + let (a, b, from) = synced_on(scale, 0, 0, JOIN, 120_000_000 / VIDEO_US, |_| false).await; + assert_eq!(assert_same_synced(&a, &b, from), 0, "scale {scale}"); + } +} + +/// A rate the catalog does not record and no override names is measured, which would give +/// each leg a grid of its own, so `--sync` refuses it. +#[tokio::test(start_paused = true)] +#[should_panic(expected = "--sync needs a constant multiplex rate")] +async fn sync_refuses_an_output_without_a_constant_rate() { + let mut live = Live::new(1.0, 0).await; + let export = super::export_test::open_export(live.source.clone()) + .await + .unwrap() + .with_delay(DELAY) + .with_sync(); + let mut leg = Leg { + export, + every: Duration::ZERO, + next_poll: Instant::now(), + out: Vec::new(), + }; + live.run(2 * GOP, &mut [&mut leg], |_| false).await; +} + +/// A rate too low for the tracks' lanes is refused at the program tables. +#[tokio::test(start_paused = true)] +#[should_panic(expected = "cannot carry its lanes")] +async fn sync_refuses_a_rate_its_lanes_do_not_fit() { + let mut live = Live::new(1.0, 0).await; + live.sync = true; + live.rate = 14 * 188 * 8 * 40; + let mut leg = live.join(Duration::ZERO).await; + live.run(2 * GOP, &mut [&mut leg], |_| false).await; +} + /// Whatever follows the source's clock has to follow it identically on both legs, though they /// joined at different points on it. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/ts/lane.rs b/rs/moq-mux/src/container/ts/lane.rs new file mode 100644 index 00000000000..10754303d33 --- /dev/null +++ b/rs/moq-mux/src/container/ts/lane.rs @@ -0,0 +1,774 @@ +//! Fixed lanes on a constant-rate multiplex, so every exporter of a broadcast writes the same +//! packet at the same position ([`Export::with_sync`](super::Export::with_sync)). +//! +//! At a constant rate, slot `index` carries the timeline's packets from +//! `packets_before(index)` up to `packets_before(index + 1)`. Each PID owns a fixed share of +//! them that is a function of the slot alone: the slot's PCR its first position, the PAT and +//! PMT theirs in every slot that repeats them, and each elementary stream a lane of so many +//! packets a second, spread evenly over the rest. The positions no lane owns carry the SI, +//! else stuffing. +//! +//! A unit takes the next positions of its PID's lane from where its MoQ group was anchored: +//! the first slot the group's first unit may go out in. Where it lands depends on its group's +//! earlier units and nothing else, so a leg that lacks a group, or joined after it, lays the +//! next one where every other leg does. A group still unfinished in its lane when the next +//! one is due pushes the next one back, which only a leg holding both does: a lane needs the +//! headroom to carry each group within its own duration. + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::sync::Arc; + +use mpeg2ts::ts::TsPacket; + +use super::export::{PCR_INTERVAL, PSI_INTERVAL}; +use super::schedule::{PACKET, packets_before}; + +/// Grid slots per second. +const SLOTS: u64 = (1_000_000_000 / PCR_INTERVAL.as_nanos()) as u64; +/// How many slots apart the PAT and PMT repeat. +pub(super) const PSI_SLOTS: u128 = PSI_INTERVAL.as_nanos() / PCR_INTERVAL.as_nanos(); +/// How fast a system PID's transport buffer drains (Rxsys, ISO 13818-1 2.4.2.3). +const RX_SYSTEM: u64 = 1_000_000; +/// How far a restarted grid's first packet may sit behind the last slot laid, or ahead of it, +/// and still be on the same timeline, so the grid runs on rather than jumps: two seconds back, +/// a minute on. +const BEHIND: u128 = 2 * SLOTS as u128; +const AHEAD: u128 = 60 * SLOTS as u128; + +/// What a position of a slot carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Owner { + /// The slot's PCR, on its own PID. + Clock, + /// The `n`th packet of the PAT and PMT, sent in a slot that repeats them and in the first a + /// leg lays, stuffing in the rest. + Table(usize), + /// A packet of the `n`th lane. + Lane(usize), + /// The SI, else stuffing. + Free, +} + +/// The share of the multiplex a PID's lane takes. +#[derive(Clone, Copy, Debug)] +pub(super) enum Need { + /// So many packets a second. + Fixed(u64), + /// What the fixed lanes leave, split evenly with the other remainders, at most so many + /// packets a second. + Rest(u64), +} + +/// The positions every PID owns, slot by slot, at one multiplex rate. +#[derive(Debug)] +pub(super) struct Layout { + rate: u64, + /// Each lane's PID and packets a second. + lanes: Vec<(u16, u64)>, + /// Where the PAT and the PMT's packets sit in every slot. + tables: Vec, + /// How many positions apart a system PID's packets go, so they arrive no faster than its + /// transport buffer drains. + spacing: usize, +} + +impl Layout { + /// Lay `lanes` out at `rate` bits per second beside `tables` packets of PAT and PMT, keeping + /// `reserve` packets a second free for the SI. + pub fn new(rate: u64, tables: usize, reserve: u64, lanes: &[(u16, Need)]) -> anyhow::Result { + let spacing = rate.div_ceil(RX_SYSTEM).max(1) as usize; + // The fewest packets any slot carries at the rate. + let least = rate / PACKET; + anyhow::ensure!( + (1 + tables as u64) * spacing as u64 <= least, + "MPEG-TS --sync at {rate} b/s has too few packets a slot to pace its {tables} table packets; raise --mux-rate" + ); + let per_slot = |packets: u64| packets.div_ceil(SLOTS); + let fixed: u64 = lanes + .iter() + .map(|(_, need)| match need { + Need::Fixed(packets) => per_slot(*packets), + Need::Rest(_) => 0, + }) + .sum(); + let taken = 1 + tables as u64 + fixed + per_slot(reserve); + let rests = lanes.iter().filter(|(_, need)| matches!(need, Need::Rest(_))).count() as u64; + let left = least.checked_sub(taken).filter(|left| rests == 0 || *left >= rests); + let Some(left) = left else { + anyhow::bail!( + "MPEG-TS --sync at {rate} b/s cannot carry its lanes: the PCR, the tables and each non-video stream at its codec's \ + most take {} b/s, leaving nothing for video; raise --mux-rate", + taken * PACKET + ); + }; + let lanes = lanes + .iter() + .map(|&(pid, need)| { + let packets = match need { + Need::Fixed(packets) => packets, + Need::Rest(most) => (left / rests * SLOTS).min(most), + }; + (pid, packets) + }) + .collect(); + let tables = (1..=tables).map(|n| n * spacing).collect(); + Ok(Self { + rate, + lanes, + tables, + spacing, + }) + } + + pub fn rate(&self) -> u64 { + self.rate + } + + /// The lane `pid` owns, if any. + pub fn lane(&self, pid: u16) -> Option { + self.lanes.iter().position(|&(lane, _)| lane == pid) + } + + /// How fast lane `lane` carries elementary bytes, in bits per second. + pub fn payload_rate(&self, lane: usize) -> u64 { + self.lanes[lane].1 * (TsPacket::SIZE as u64 - 4) * 8 + } + + /// The timeline's first packet in slot `index`. + pub fn first(&self, index: u128) -> u128 { + packets_before(index, self.rate) + } + + /// The timeline's first packet whose bytes begin at or after `time` nanoseconds of media + /// time, in the slot that time falls in. + pub fn at(&self, time: u128) -> u128 { + (time * u128::from(self.rate)).div_ceil(TsPacket::SIZE as u128 * 8 * 1_000_000_000) + } + + /// How many packets slot `index` carries. + pub fn count(&self, index: u128) -> usize { + (self.first(index + 1) - self.first(index)) as usize + } + + /// The slot that carries the timeline's packet `position`. + pub fn slot_of(&self, position: u128) -> u128 { + ((position + 1) * u128::from(PACKET)).div_ceil(u128::from(self.rate)) - 1 + } + + /// When the timeline's packet `position` begins to arrive, in nanoseconds of media time: its + /// byte position at the rate, one slot behind like every PCR. + pub fn time(&self, position: u128) -> u128 { + (position * TsPacket::SIZE as u128 * 8 * 1_000_000_000 / u128::from(self.rate)) + .saturating_sub(PCR_INTERVAL.as_nanos()) + } + + /// How many packets lane `lane` takes in slot `index`, counted from the start of the + /// timeline so no slot rounds on its own. + fn lane_count(&self, lane: usize, index: u128) -> usize { + let packets = u128::from(self.lanes[lane].1); + let before = |index: u128| index * packets / u128::from(SLOTS); + (before(index + 1) - before(index)) as usize + } + + /// Whether slot `index` repeats the PAT and PMT, on the absolute grid every exporter shares. + pub fn repeats_tables(index: u128) -> bool { + index.is_multiple_of(PSI_SLOTS) + } + + /// What each position of slot `index` carries. The clock takes the first, the tables + /// theirs, and the lanes share the rest by smooth weighted round-robin with the free + /// positions: each takes its count as evenly as it can, in PID order where two tie. + pub fn owners(&self, index: u128) -> Vec { + let count = self.count(index); + let mut owners = vec![Owner::Free; count]; + owners[0] = Owner::Clock; + for (n, &position) in self.tables.iter().enumerate() { + owners[position] = Owner::Table(n); + } + let open: Vec = (1..count).filter(|&p| owners[p] == Owner::Free).collect(); + let mut weights: Vec = (0..self.lanes.len()) + .map(|lane| self.lane_count(lane, index) as i64) + .collect(); + let used: i64 = weights.iter().sum(); + weights.push(open.len() as i64 - used); + debug_assert!( + weights.last().is_some_and(|free| *free >= 0), + "lanes overflow slot {index}" + ); + let total = open.len() as i64; + let mut credit = vec![0i64; weights.len()]; + for position in open { + let mut pick = 0; + for lane in 0..credit.len() { + credit[lane] += weights[lane]; + if credit[lane] > credit[pick] { + pick = lane; + } + } + credit[pick] -= total; + if pick < self.lanes.len() { + owners[position] = Owner::Lane(pick); + } + } + owners + } + + /// The next `count` positions of lane `lane` at or after the timeline's packet `from`. + pub fn take(&self, lane: usize, from: u128, count: usize) -> Vec { + let mut out = Vec::with_capacity(count); + let mut index = self.slot_of(from); + while out.len() < count { + let first = self.first(index); + for (position, owner) in self.owners(index).into_iter().enumerate() { + let at = first + position as u128; + if at >= from && owner == Owner::Lane(lane) { + out.push(at); + if out.len() == count { + break; + } + } + } + index += 1; + } + out + } +} + +/// What [`Lane::plan`] gave a unit. +pub(super) struct Placed { + /// The timeline position of each of its packets. + pub positions: Vec, + /// The continuity counter of its first packet: how many packets its group laid before it. + pub counter: u8, +} + +/// Why [`Lane::plan`] placed nothing. +#[derive(Debug, PartialEq, Eq)] +pub(super) enum Unplaced { + /// This leg did not hold the unit's group from its first unit, so it cannot know where + /// the unit goes. + Partial, + /// The lane cannot carry the unit by its deadline. + Late, +} + +/// A PID's place in its lane. +pub(super) struct Lane { + lane: usize, + /// The group being laid. + group: Option, + /// One past the last position any group took. + end: u128, + /// Groups that began behind the one before them. + overruns: u64, +} + +struct Group { + sequence: u64, + /// The next position it may take. + next: u128, + /// How many packets it has laid. + sent: usize, + /// This leg holds it from its first unit. + whole: bool, +} + +impl Lane { + pub fn new(lane: usize) -> Self { + Self { + lane, + group: None, + end: 0, + overruns: 0, + } + } + + /// Which lane of its [`Layout`] it is. + pub fn index(&self) -> usize { + self.lane + } + + /// Groups that began behind the one before them, on a lane short of headroom. + pub fn overruns(&self) -> u64 { + self.overruns + } + + /// Forget every group, for a timeline that starts again. + pub fn reset(&mut self) { + self.group = None; + self.end = 0; + } + + /// Whether a unit of group `sequence`, its first when `opens`, would be the first its group + /// places, so its first packet flags the break in the continuity counter. + pub fn opening(&self, (sequence, opens): (u64, bool)) -> bool { + match &self.group { + Some(group) if group.sequence == sequence => group.whole && group.sent == 0, + _ => opens, + } + } + + /// Place a unit of `count` packets from group `sequence`, which it `opens` when it is the + /// group's first unit, that may go out from `release` and must have arrived by `by`, + /// both in nanoseconds of media time. + /// + /// A group opens at its lane's first position from its first unit's release, and each unit + /// after it takes the next positions from there, never before its own release. A unit that + /// misses its deadline takes nothing, so the units after it keep theirs. + pub fn plan( + &mut self, + layout: &Layout, + (sequence, opens): (u64, bool), + release: u128, + by: u128, + count: usize, + ) -> Result { + let earliest = layout.at(release); + let from = match &mut self.group { + Some(group) if group.sequence == sequence => match group.whole { + true => group.next.max(earliest), + false => return Err(Unplaced::Partial), + }, + _ if !opens => { + self.group = Some(Group { + sequence, + next: 0, + sent: 0, + whole: false, + }); + return Err(Unplaced::Partial); + } + _ => { + if self.end > earliest { + self.overruns += 1; + } + self.group = Some(Group { + sequence, + next: self.end.max(earliest), + sent: 0, + whole: true, + }); + self.end.max(earliest) + } + }; + let positions = layout.take(self.lane, from, count.max(1)); + let last = *positions.last().expect("at least one position"); + if layout.time(last + 1) > by { + return Err(Unplaced::Late); + } + let group = self.group.as_mut().expect("a group was just set"); + let counter = (group.sent & 0x0f) as u8; + group.next = last + 1; + group.sent += count; + self.end = self.end.max(last + 1); + Ok(Placed { positions, counter }) + } +} + +/// Number `packets` on from `counter`, every one carrying a payload. +pub(super) fn number(packets: &mut [u8], counter: u8) { + for (n, packet) in packets.as_chunks_mut::<{ TsPacket::SIZE }>().0.iter_mut().enumerate() { + packet[3] = (packet[3] & 0xf0) | ((counter as usize + n) & 0x0f) as u8; + } +} + +/// The PID of a TS packet. +fn pid(packet: &[u8]) -> u16 { + u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]) +} + +/// One slot as [`Grid::lay`] fills it: its packets at their positions in it, the clock's left +/// to the caller. +pub(super) struct Laid { + pub packets: Vec, + pub positions: Vec, + pub count: usize, + pub keyframe: bool, + pub units: Vec, + /// It does not follow the slot laid before it. + pub jump: bool, +} + +/// The packets placed on a [`Layout`], laid out slot by slot. +pub(super) struct Grid { + layout: Arc, + /// Each placed packet, by its timeline position. + placed: BTreeMap, + /// Where each unit's first packet sits: its PID and whether it opens a keyframe. + starts: BTreeMap, + /// The PAT then the PMT, numbered as they go out. + tables: Vec<[u8; TsPacket::SIZE]>, + /// SI packets still to go out, in order, each no earlier than its position. + si: VecDeque<(u128, [u8; TsPacket::SIZE])>, + /// Where each SI PID's last packet went. + si_last: HashMap, + /// The next slot to lay. + next: Option, + /// The last slot laid, kept through a restart. + last: Option, + /// Packets that reached the grid after their position went out. + late: u64, +} + +impl Grid { + pub fn new(layout: Arc, tables: Vec<[u8; TsPacket::SIZE]>) -> Self { + Self { + layout, + placed: BTreeMap::new(), + starts: BTreeMap::new(), + tables, + si: VecDeque::new(), + si_last: HashMap::new(), + next: None, + last: None, + late: 0, + } + } + + pub fn layout(&self) -> &Arc { + &self.layout + } + + /// Packets that reached the grid after their position went out. + pub fn late(&self) -> u64 { + self.late + } + + /// Put a unit's `packets` at their `positions`, dropping any whose position went out. + pub fn place(&mut self, packets: &[u8], positions: &[u128], keyframe: bool) { + let gone = self.next.map_or(0, |next| self.layout.first(next)); + for (n, (packet, &at)) in packets + .as_chunks::<{ TsPacket::SIZE }>() + .0 + .iter() + .zip(positions) + .enumerate() + { + if at < gone { + self.late += 1; + continue; + } + if n == 0 { + self.starts.insert(at, (pid(packet), keyframe)); + } + self.placed.insert(at, *packet); + } + } + + /// Send `packets` of SI from slot `index` on, in the positions no lane owns. + pub fn push_si(&mut self, index: u128, packets: &[u8]) { + let from = self.layout.first(index); + for packet in packets.as_chunks::<{ TsPacket::SIZE }>().0 { + self.si.push_back((from, *packet)); + } + } + + /// Whether anything placed is still to go out. + pub fn is_empty(&self) -> bool { + self.placed.is_empty() && self.si.is_empty() + } + + /// The slot [`Self::lay`] lays next: on from the last, else the one before the first holding + /// a packet, so the tables a timeline starts with come before anything of it, unless a + /// restarted grid's timeline runs on from the slot laid last. + pub fn index(&self) -> Option { + self.next.or_else(|| { + let first = self.layout.slot_of(*self.placed.keys().next()?); + Some(match self.last { + Some(last) if first + BEHIND > last && first <= last + AHEAD => last + 1, + _ => first.saturating_sub(1), + }) + }) + } + + /// Drop everything placed. The grid runs on from where it was: its positions are the + /// slot's alone. + pub fn clear(&mut self) { + self.placed.clear(); + self.starts.clear(); + self.si.clear(); + } + + /// Start the grid afresh at the next slot holding a packet, for a timeline that restarted. + pub fn restart(&mut self) { + self.clear(); + self.next = None; + self.si_last.clear(); + } + + /// Lay slot `index`: every placed packet at its position, the tables at theirs where the + /// slot repeats them or starts a timeline, the SI in order in the free positions, paced + /// like the tables. + /// + /// Tables that start a timeline number as the last repetition before them did, so a + /// receiver tuning in on them counts on into the next. + pub fn lay(&mut self, index: u128) -> Laid { + self.next = Some(index + 1); + let jump = self.last.is_some_and(|last| index != last + 1); + let tables = Layout::repeats_tables(index) || jump || self.last.is_none(); + self.last = Some(index); + let first = self.layout.first(index); + let owners = self.layout.owners(index); + let count = owners.len(); + let mut out = Laid { + packets: Vec::new(), + positions: Vec::new(), + count, + keyframe: false, + units: Vec::new(), + jump, + }; + let emission = index / PSI_SLOTS; + let spacing = self.layout.spacing as u128; + for (position, owner) in owners.into_iter().enumerate() { + let at = first + position as u128; + let packet = match owner { + Owner::Clock => None, + Owner::Table(n) => self.tables.get(n).filter(|_| tables).map(|table| { + let mut packet = *table; + // The PAT is one packet; the PMT's packets follow it, numbered on from the + // last emission's on the absolute grid. + let counter = match n { + 0 => emission, + n => emission * (self.tables.len() as u128 - 1) + n as u128 - 1, + }; + packet[3] = (packet[3] & 0xf0) | (counter & 0x0f) as u8; + packet + }), + Owner::Lane(_) => self.placed.remove(&at), + Owner::Free => { + let ready = self.si.front().is_some_and(|(from, packet)| { + *from <= at && self.si_last.get(&pid(packet)).is_none_or(|last| *last + spacing <= at) + }); + ready.then(|| { + let (_, packet) = self.si.pop_front().expect("just peeked"); + self.si_last.insert(pid(&packet), at); + packet + }) + } + }; + if let Some((pid, keyframe)) = self.starts.remove(&at) { + out.keyframe |= keyframe; + out.units.push(pid); + } + if let Some(packet) = packet { + out.positions.push(position); + out.packets.extend_from_slice(&packet); + } + } + // A packet left behind its position (one placed into the slot just laid by a lane that + // no longer owns it) never goes out. + let end = first + count as u128; + self.placed = self.placed.split_off(&end); + self.starts = self.starts.split_off(&end); + out + } +} + +/// When a unit due by `by` nanoseconds of media time may first go out, `ahead` slots before. +pub(super) fn release(by: u128, ahead: u128) -> u128 { + by.saturating_sub(ahead * PCR_INTERVAL.as_nanos()) +} + +#[cfg(test)] +mod test { + use super::*; + + const RATE: u64 = 34 * 188 * 8 * 40; + + fn layout() -> Layout { + Layout::new( + RATE, + 2, + 0, + &[ + (0x100, Need::Rest(u64::MAX)), + (0x101, Need::Fixed(422)), + (0x102, Need::Fixed(40)), + ], + ) + .unwrap() + } + + #[test] + fn every_slot_gives_each_lane_its_share() { + let layout = layout(); + for index in 0..400u128 { + let owners = layout.owners(index); + assert_eq!(owners.len(), 34); + assert_eq!(owners[0], Owner::Clock); + for lane in 0..3 { + let got = owners.iter().filter(|owner| **owner == Owner::Lane(lane)).count(); + assert_eq!(got, layout.lane_count(lane, index), "lane {lane} in slot {index}"); + } + } + let total: usize = (0..40).map(|index| layout.lane_count(1, index)).sum(); + assert_eq!(total, 422, "a second carries its packets"); + } + + fn tables() -> Vec<[u8; TsPacket::SIZE]> { + let mut pat = [0xffu8; TsPacket::SIZE]; + pat[..4].copy_from_slice(&[0x47, 0x40, 0x00, 0x10]); + let mut pmt = pat; + pmt[1..3].copy_from_slice(&[0x50, 0x00]); + vec![pat, pmt] + } + + #[test] + fn the_tables_repeat_on_the_absolute_grid_and_lead_a_timeline() { + let layout = layout(); + for index in [0u128, 20, 21, 40, 1_000_000_020] { + let owners = layout.owners(index); + assert_eq!(owners[layout.tables[0]], Owner::Table(0)); + assert_eq!(owners[layout.tables[1]], Owner::Table(1)); + } + let mut grid = Grid::new(Arc::new(layout), tables()); + let mut sent = Vec::new(); + for index in 21..=41 { + let laid = grid.lay(index); + for packet in laid.packets.as_chunks::<{ TsPacket::SIZE }>().0 { + sent.push((index, pid(packet), packet[3] & 0x0f)); + } + } + // The first slot laid leads with them, numbered as slot 20's were, so slot 40's count on. + assert_eq!( + sent, + [(21, 0x0000, 1), (21, 0x1000, 1), (40, 0x0000, 2), (40, 0x1000, 2)] + ); + } + + #[test] + fn a_rate_too_low_for_the_lanes_is_refused() { + let err = Layout::new( + RATE, + 2, + 0, + &[(0x100, Need::Rest(u64::MAX)), (0x101, Need::Fixed(40 * 40))], + ) + .unwrap_err(); + assert!(err.to_string().contains("raise --mux-rate"), "{err}"); + } + + #[test] + fn positions_map_back_to_their_slots() { + let layout = Layout::new(10_000_000, 2, 0, &[(0x100, Need::Rest(u64::MAX))]).unwrap(); + for index in [0u128, 1, 7, 39, 40, 123_456_789] { + let first = layout.first(index); + assert_eq!(layout.slot_of(first), index); + assert_eq!(layout.slot_of(first + layout.count(index) as u128 - 1), index); + } + } + + /// The start of slot `index`, in nanoseconds of media time. + fn start(index: u128) -> u128 { + index * PCR_INTERVAL.as_nanos() + } + + #[test] + fn a_group_lays_from_its_own_anchor_whatever_came_before() { + let layout = layout(); + let by = 10_000_000_000; + let mut whole = Lane::new(0); + let mut joiner = Lane::new(0); + // Group 1, which the joiner lacks. + whole.plan(&layout, (1, true), start(100), by, 50).unwrap(); + whole.plan(&layout, (1, false), start(101), by, 5).unwrap(); + let a = whole.plan(&layout, (2, true), start(110), by, 20).unwrap(); + let b = joiner.plan(&layout, (2, true), start(110), by, 20).unwrap(); + assert_eq!(a.positions, b.positions); + assert_eq!((a.counter, b.counter), (0, 0)); + let a = whole.plan(&layout, (2, false), start(110), by, 3).unwrap(); + let b = joiner.plan(&layout, (2, false), start(110), by, 3).unwrap(); + assert_eq!(a.positions, b.positions); + assert_eq!((a.counter, b.counter), (4, 4)); + assert_eq!((whole.overruns(), joiner.overruns()), (0, 0)); + } + + #[test] + fn groups_released_in_one_slot_keep_their_own_anchors() { + let layout = layout(); + // 48 kHz AAC: frames 21.3 ms apart, two to a slot now and then. + let frame = 1_000_000_000 * 1024 / 48_000; + let mut whole = Lane::new(1); + let mut joiner = Lane::new(1); + let last = start(100) + 2_000_000; + whole.plan(&layout, (1, true), last - frame, u128::MAX, 2).unwrap(); + whole.plan(&layout, (1, false), last, u128::MAX, 2).unwrap(); + assert_eq!(layout.slot_of(layout.at(last)), layout.slot_of(layout.at(last + frame))); + let a = whole.plan(&layout, (2, true), last + frame, u128::MAX, 2).unwrap(); + let b = joiner.plan(&layout, (2, true), last + frame, u128::MAX, 2).unwrap(); + assert_eq!(a.positions, b.positions); + assert_eq!(whole.overruns(), 0); + } + + #[test] + fn a_group_held_from_its_middle_is_not_placed() { + let layout = layout(); + let mut lane = Lane::new(0); + assert_eq!( + lane.plan(&layout, (1, false), start(10), u128::MAX, 2).err(), + Some(Unplaced::Partial) + ); + assert_eq!( + lane.plan(&layout, (1, false), start(10), u128::MAX, 2).err(), + Some(Unplaced::Partial) + ); + assert!(lane.plan(&layout, (2, true), start(30), u128::MAX, 2).is_ok()); + } + + #[test] + fn a_late_unit_leaves_its_positions_to_the_next() { + let layout = layout(); + let mut lane = Lane::new(1); + let at = layout.time(layout.first(11)); + assert_eq!( + lane.plan(&layout, (1, true), start(10), at, 30).err(), + Some(Unplaced::Late) + ); + let placed = lane.plan(&layout, (1, false), start(10), u128::MAX, 2).unwrap(); + assert_eq!(placed.counter, 0); + assert!(placed.positions[0] >= layout.first(10)); + } + + #[test] + fn the_grid_lays_tables_and_si_beside_the_lanes() { + let layout = Layout::new(RATE, 2, 80, &[(0x100, Need::Rest(u64::MAX)), (0x101, Need::Fixed(422))]).unwrap(); + let mut grid = Grid::new(Arc::new(layout), tables()); + let mut si = [0xffu8; TsPacket::SIZE]; + si[..4].copy_from_slice(&[0x47, 0x40, 0x11, 0x10]); + grid.push_si(40, &[si, si].concat()); + let mut seen = Vec::new(); + for index in 40..44 { + let laid = grid.lay(index); + assert_eq!(laid.count, 34); + let first = grid.layout().first(index); + for (packet, position) in laid + .packets + .as_chunks::<{ TsPacket::SIZE }>() + .0 + .iter() + .zip(&laid.positions) + { + seen.push((pid(packet), first + *position as u128, packet[3] & 0x0f)); + } + } + let first = grid.layout().first(40); + let on = |pid: u16| -> Vec<(u128, u8)> { + seen.iter() + .filter(|seen| seen.0 == pid) + .map(|&(_, at, counter)| (at - first, counter)) + .collect() + }; + // The PAT and PMT sit at their positions, numbered on from the emissions before them on + // the grid. + let tables = &grid.layout().tables; + assert_eq!(on(0x0000), [(tables[0] as u128, 2)]); + assert_eq!(on(0x1000), [(tables[1] as u128, 2)]); + let si = on(0x0011); + assert_eq!(si.len(), 2); + assert!( + si[1].0 >= si[0].0 + grid.layout().spacing as u128, + "SI paced like the tables" + ); + } +} diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index ebcfd9ae97b..0353909a323 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -29,6 +29,7 @@ mod export; mod follower; mod health; mod import; +mod lane; mod mux_rate; mod programs; mod psi; diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index b5ce0daf017..448613606f9 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -7,6 +7,7 @@ use std::time::Duration; use mpeg2ts::ts::TsPacket; use super::export::PCR_INTERVAL; +use super::lane::Grid; /// Grid slots per second, so a multiplex rate in bits per second is also the per-slot /// allowance in [`PACKET`] units. @@ -19,7 +20,7 @@ const _: () = assert!( /// One packet in bits per second of slots: `rate` bits per second fills slot `index` up to /// `index * rate / PACKET` packets since the start of the timeline, so no slot rounds on /// its own. -const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; +pub(super) const PACKET: u64 = TsPacket::SIZE as u64 * 8 * SLOTS_PER_SECOND; /// The most of a packet a decoder buffer can receive: everything after its 4-byte header. const PAYLOAD: usize = TsPacket::SIZE - 4; /// How many empty slots to lay at most before the next packets: one second's worth. Past @@ -62,7 +63,7 @@ fn grid_pcr(index: u128) -> u128 { } /// How many packets `rate` bits per second carries before slot `index` on the media timeline. -fn packets_before(index: u128, rate: u64) -> u128 { +pub(super) fn packets_before(index: u128, rate: u64) -> u128 { index * u128::from(rate) / u128::from(PACKET) } @@ -392,6 +393,8 @@ pub(super) struct Slot { /// The padded slot opens with a packet of the clock's PID carrying its PCR, so it needs /// no clock packet of its own. pub clocked: bool, + /// The slot does not follow the one laid before it, so its PCR flags the break. + pub jump: bool, } impl Slot { @@ -552,11 +555,14 @@ pub(super) struct Schedule { /// Every source has ended, so a unit that misses its deadline is the tail going out late /// rather than an output that cannot keep up. ended: bool, + /// The fixed lanes every slot is laid on instead, under [`Export::with_sync`](super::Export::with_sync). + sync: Option, } impl Schedule { pub fn new(window: Duration) -> Self { Self { + sync: None, units: VecDeque::new(), window: slot(window.as_nanos()), rate: None, @@ -585,6 +591,21 @@ impl Schedule { self.rate = rate; } + /// Lay every slot on `grid`'s fixed lanes from now on, whatever the units pushed: their + /// packets go straight onto it ([`Self::sync`]). + pub fn set_sync(&mut self, grid: Grid) { + self.sync = Some(grid); + } + + /// The fixed lanes slots are laid on, under [`Self::set_sync`]. + pub fn sync(&mut self) -> Option<&mut Grid> { + self.sync.as_mut() + } + + pub fn grid(&self) -> Option<&Grid> { + self.sync.as_ref() + } + /// Hold the packets on `pid` to `buffer`. pub fn set_buffer(&mut self, pid: u16, buffer: Buffer) { let before = self.transport_buffer(pid); @@ -784,6 +805,9 @@ impl Schedule { /// Whether nothing is left to lay out: every unit sent, and the clock past its decode time. pub fn is_empty(&self) -> bool { + if let Some(grid) = &self.sync { + return grid.is_empty(); + } self.units.is_empty() && self .horizon @@ -792,6 +816,9 @@ impl Schedule { /// Drop everything queued and start the grid afresh. pub fn clear(&mut self) { + if let Some(grid) = &mut self.sync { + grid.restart(); + } self.units.clear(); self.decoding.clear(); self.transport.clear(); @@ -842,7 +869,10 @@ impl Schedule { /// The next slot to lay out, and the first slot a unit still to be pushed could go out in /// once every unit that may ride it is known: the `known` that settles it. pub fn upcoming(&self) -> Option<(u128, u128)> { - let index = self.index(None)?; + let index = match &self.sync { + Some(grid) => grid.index()?, + None => self.index(None)?, + }; Some((index, index + 1)) } @@ -855,6 +885,23 @@ impl Schedule { if known.is_none() && self.is_empty() { return Ok(None); } + if let Some(grid) = &mut self.sync { + let Some(index) = grid.index().filter(|&index| known.is_none_or(|known| index < known)) else { + return Ok(None); + }; + let laid = grid.lay(index); + return Ok(Some(Slot { + index, + pcr: rate_pcr(index, grid.layout().rate()) as u64, + nulls: laid.count - 1 - laid.positions.len(), + packets: laid.packets, + keyframe: laid.keyframe, + units: laid.units, + positions: Some(laid.positions), + clocked: false, + jump: laid.jump, + })); + } let Some(index) = self.index(known) else { return Ok(None); }; @@ -909,6 +956,7 @@ impl Schedule { units, positions: None, clocked: false, + jump: false, })) } @@ -930,6 +978,7 @@ impl Schedule { units: Vec::new(), positions: Some(Vec::new()), clocked: false, + jump: false, }; let clock = self.clock.filter(|pid| self.buffers.contains_key(pid)); // A new multiplex rate changes Rsys from this slot on, and Bsys keeps the bytes it holds. @@ -2573,6 +2622,7 @@ mod tests { units: vec![], positions: None, clocked: false, + jump: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -2611,6 +2661,7 @@ mod tests { units: vec![1, 3], positions: None, clocked: false, + jump: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index 92f26c2e166..e95c5eb6a2d 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -195,6 +195,13 @@ pub struct Export { /// How many measurements found the source's clock further off ours than the 30 ppm an /// MPEG-TS system clock may be (ISO/IEC 13818-1 2.4.2.1), which the output cannot follow. pub out_of_tolerance: u64, + /// Groups that `--sync` began behind the end of the group before them on their PID, the + /// lane too narrow to absorb that group: a leg missing the earlier group's tail places + /// them elsewhere, so the legs differ until the lane catches up. + pub overruns: u64, + /// Packets that `--sync` dropped for reaching their slot after it went out, leaving + /// stuffing there. + pub late: u64, } /// An export's rows with every stream-wide counter zero, so one [`Log`] reads either edge. From 0035732cc5052e18d53710d94ee402b432264e16 Mon Sep 17 00:00:00 2001 From: t0ms Date: Sat, 17 Oct 2026 11:46:49 +0100 Subject: [PATCH 02/13] fix(moq-mux): settle the TS --sync decode clock by count Under --sync, a video track whose SPS declares max_num_reorder_frames settles each frame's decode slot once more frames are held than a picture can follow in output after preceding them in decode (doubled plus one for a field-capable H.264 SPS), at the SPS's fixed delay, with the DTS capped at the PTS. The timing rule's settling and its delay that only grows depend on everything a leg read before, so a leg that lost a group's tail mid-pyramid raised its delay for good and laid every later group elsewhere. A cut drops the frames the clock still holds from the cut group: they wait on frames that will not come, so the slots they would take are a guess. Without a declared depth the export warns and keeps the timing rule. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- rs/moq-mux/src/codec/h264/mod.rs | 26 +++- rs/moq-mux/src/codec/h265/mod.rs | 1 + rs/moq-mux/src/codec/video.rs | 14 ++ rs/moq-mux/src/container/decode.rs | 114 +++++++++++++++- rs/moq-mux/src/container/ts/export.rs | 32 ++++- .../src/container/ts/export_timing_test.rs | 125 ++++++++++++++++-- 7 files changed, 293 insertions(+), 22 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 5a579bc7375..943e0474693 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -444,7 +444,8 @@ constant rate (`--mux-rate`, or the catalog's), costing headroom over the source's (a 10 Mb/s channel needs about 12), and takes neither `--stitch` nor `--passthrough`. A group that starts behind the one before it, on a lane too narrow to absorb that one, is logged and counted in `overruns`: the legs can -differ there. +differ there. So can reordered video whose SPS declares no reorder depth +(`max_num_reorder_frames`), which is logged. A stdout export ends with the broadcast. `export ts --linger 10s` waits that long for the same publisher instance to come back, and carries on with the same diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 9fd98fc234f..a82ec7352ce 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -154,6 +154,7 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { Some(crate::codec::video::Reorder { depth: vui.depth?, period: vui.period, + fields: !vui.frame_only, }) } @@ -847,29 +848,39 @@ mod tests { sps_reorder(fixtures::SPS_IPB), Some(Reorder { depth: 1, - period: fixed + period: fixed, + fields: false }) ); assert_eq!( sps_reorder(fixtures::SPS_PYRAMID), Some(Reorder { depth: 2, - period: fixed + period: fixed, + fields: false }) ); assert_eq!( sps_reorder(fixtures::SPS_IPB_VARIABLE), - Some(Reorder { depth: 1, period: None }) + Some(Reorder { + depth: 1, + period: None, + fields: false + }) ); // The field-capable SPS still declares its depth and tick; only the frame rate needs - // `pic_struct` to read it. + // `pic_struct` to read it. A field may follow the other field of its own frame. + let field = sps_reorder(fixtures::SPS_FIELD); assert_eq!( - sps_reorder(fixtures::SPS_FIELD), + field, Some(Reorder { depth: 0, - period: Some((2, 50)) + period: Some((2, 50)), + fields: true }) ); + assert_eq!(field.map(Reorder::pictures), Some(1)); + assert_eq!(sps_reorder(fixtures::SPS_PYRAMID).map(Reorder::pictures), Some(2)); // Baseline with no VUI declares nothing. assert_eq!(sps_reorder(&[0x67, 0x42, 0xc0, 0x1f, 0xde]), None); // A VUI cut short is no declaration rather than a wrong one. @@ -902,7 +913,8 @@ mod tests { sps_reorder(fixtures::SPS_HRD), Some(Reorder { depth: 1, - period: Some((2, 50)) + period: Some((2, 50)), + fields: false }) ); assert_eq!(sps_hrd(fixtures::SPS_IPB), None); diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index dd56cbb0995..252199415b6 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -275,6 +275,7 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { Some(crate::codec::video::Reorder { depth: u32::try_from(depth).ok()?, period: sps_period(&sps.rbsp), + fields: false, }) } diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 118f78317b5..41664ec4f3d 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -61,9 +61,23 @@ pub(crate) struct Reorder { /// One picture's duration as `(units, scale)`, i.e. `units / scale` seconds, when the header /// declares a fixed picture rate. pub period: Option<(u64, u64)>, + /// Whether a picture may be a single field (H.264 `frame_mbs_only_flag` 0), while + /// [`Self::depth`] counts frames and field pairs. + pub fields: bool, } impl Reorder { + /// The most frames, each one picture, that can precede one in decode order and follow it in + /// output order: the depth, or with fields both fields of each pair it counts and the other + /// field of the frame's own pair. + pub(crate) fn pictures(self) -> usize { + let depth = usize::try_from(self.depth).unwrap_or(usize::MAX); + match self.fields { + true => depth.saturating_mul(2).saturating_add(1), + false => depth, + } + } + /// The minimum decode delay and lookahead in 90 kHz ticks. pub(crate) fn timing(self, framerate: Option) -> (u64, u64) { let period = match self.period { diff --git a/rs/moq-mux/src/container/decode.rs b/rs/moq-mux/src/container/decode.rs index d0a88d7e921..68ad5d67e0d 100644 --- a/rs/moq-mux/src/container/decode.rs +++ b/rs/moq-mux/src/container/decode.rs @@ -23,6 +23,10 @@ pub(crate) const MAX_REORDER: u64 = 2 * 90_000; /// (reordering deeper than all of those) is nudged one tick past the last DTS, so the clock /// stays strictly increasing. /// +/// That settling, and a delay that only grows, depend on everything read before, so two +/// exporters that joined or lost frames at different points can clock the same frame apart. +/// [`Self::pop_counted`] depends on the frames held alone, for exporters that must agree. +/// /// Ticks are continuous (unwrapped) 90 kHz, so the clock never wraps mid-stream; the 33-bit /// wire wrap happens once at emission. pub(crate) struct DecodeClock { @@ -89,11 +93,119 @@ impl DecodeClock { let slot = self.slots.remove(0); let (item, _) = self.held.pop_front()?; let dts = slot.saturating_sub(self.delay); + Some((item, self.after_last(dts))) + } + + /// The next frame and its DTS, once more than `depth` frames are held, or at once when + /// `flush`. `depth` is the most frames the sequence header lets precede one in decode order + /// and follow it in output order, so no frame still to come is presented before all of + /// them. The delay is `delay` throughout, and a DTS never passes its PTS. + pub(crate) fn pop_counted(&mut self, depth: usize, delay: u64, flush: bool) -> Option<(T, u64)> { + if !flush && self.held.len() <= depth { + return None; + } + let (item, pts) = self.held.pop_front()?; + let slot = self.slots.remove(0); + let dts = slot.saturating_sub(delay).min(pts); + Some((item, self.after_last(dts))) + } + + /// Forget every held frame, as if it had never been read. + pub(crate) fn discard(&mut self) { + self.held.clear(); + self.slots.clear(); + } + + fn after_last(&mut self, dts: u64) -> u64 { let dts = match self.last { Some(last) if dts <= last => last + 1, _ => dts, }; self.last = Some(dts); - Some((item, dts)) + dts + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A B-pyramid (`max_num_reorder_frames` 2) in decode order, in 40 ms frames. + const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7]; + const FRAME: u64 = 3_600; + + /// Every frame of `read` with its DTS, popped as each is read: counted at a depth of 2, or + /// by the timing rule with the lookahead that depth implies. + fn clocked(read: &[u64], counted: bool) -> Vec<(u64, u64)> { + let mut clock = DecodeClock::default(); + let mut out = Vec::new(); + let pop = |clock: &mut DecodeClock, flush| match counted { + true => clock.pop_counted(2, 2 * FRAME, flush), + false => clock.pop(4 * FRAME, 2 * FRAME, flush), + }; + for &pts in read { + clock.push(pts, pts); + out.extend(std::iter::from_fn(|| pop(&mut clock, false))); + } + out.extend(std::iter::from_fn(|| pop(&mut clock, true))); + out + } + + #[test] + fn counted_slots_are_the_encoders() { + let out = clocked(&PYRAMID.map(|i| (i + 2) * FRAME), true); + let dts: Vec = out.iter().map(|&(_, dts)| dts / FRAME).collect(); + assert_eq!(dts, (0..9).collect::>()); + } + + /// Two references spaced wider than the lookahead a depth of 2 implies, then the frames + /// presented between them: a conforming depth of 2, which the timing rule settles before + /// those frames arrive, and the count waits for. + #[test] + fn references_far_apart_keep_their_slots() { + let read = [0, 6, 10, 1, 2, 3, 4, 5, 7, 8, 9].map(|i| (i + 2) * FRAME); + let out = clocked(&read, true); + let dts: Vec = out.iter().map(|&(_, dts)| dts / FRAME).collect(); + assert_eq!(dts, (0..11).collect::>()); + assert_ne!(clocked(&read, false), out); + } + + /// A group cut short after a frame presented below one it never received: its held frames + /// take slots from an incomplete set, but the next group keeps the DTS it would have had, + /// where the timing rule raises its delay for good. + #[test] + fn a_cut_group_leaves_the_next_alone() { + let first = PYRAMID.map(|i| (i + 2) * FRAME); + let next = PYRAMID.map(|i| (i + 20) * FRAME); + let full: Vec = first.into_iter().chain(next).collect(); + let cut: Vec = first[..8].iter().copied().chain(next).collect(); + let tail = |out: Vec<(u64, u64)>| out[out.len() - next.len()..].to_vec(); + assert_eq!(tail(clocked(&full, true)), tail(clocked(&cut, true))); + assert_ne!(tail(clocked(&full, false)), tail(clocked(&cut, false))); + } + + /// Frames discarded at a cut leave every frame the clock gives out at the DTS it has with + /// nothing lost. + #[test] + fn a_discard_keeps_the_frames_given_out() { + let first = PYRAMID.map(|i| (i + 2) * FRAME); + let next = PYRAMID.map(|i| (i + 20) * FRAME); + let full: Vec = first.into_iter().chain(next).collect(); + let full = clocked(&full, true); + + let mut clock = DecodeClock::default(); + let mut out = Vec::new(); + for &pts in &first[..8] { + clock.push(pts, pts); + out.extend(std::iter::from_fn(|| clock.pop_counted(2, 2 * FRAME, false))); + } + clock.discard(); + for pts in next { + clock.push(pts, pts); + out.extend(std::iter::from_fn(|| clock.pop_counted(2, 2 * FRAME, false))); + } + out.extend(std::iter::from_fn(|| clock.pop_counted(2, 2 * FRAME, true))); + assert!(out.iter().all(|frame| full.contains(frame)), "{out:?}"); + assert_eq!(out.len(), 6 + next.len()); } } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 944342a04a8..6d246ba5a80 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -396,9 +396,15 @@ impl Track { jitter.lose(&self.pid, ahead); return Ok(()); } + // Frames still held from a group cut short wait on frames that will not come, so under + // sync they go, as on a leg that never read them. + if self.sync.is_some() && self.held.is_some_and(|held| held != pending.cut) { + self.clock.discard(); + } self.held = Some(pending.cut); if pending.frame.keyframe { - self.timing.describe(stream_type, self.source.description(), name); + self.timing + .describe(stream_type, self.source.description(), name, self.sync.is_some()); } let pts = to_ticks(pending.frame.timestamp); self.clock.push((pending, self.source.description().cloned()), pts); @@ -809,10 +815,17 @@ impl Track { /// silence: the jitter buffer drops them, where their own arrival would queue them behind a /// schedule that has already gone past them. Nothing settles the frames still held when the /// track ends, so they keep their own arrival and go out late if they must. + /// + /// Under sync, a stream whose SPS declares its depth is settled by counting frames at the + /// SPS's delay, so a frame's DTS never depends on what the leg read before it. fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) -> anyhow::Result<()> { let (delay, lookahead) = self.timing.reorder(); let lookahead = lookahead.max(self.timing.jitter); - while let Some(((mut pending, description), dts)) = self.clock.pop(lookahead, delay, flush) { + let counted = self.sync.as_ref().and(self.timing.depth()); + while let Some(((mut pending, description), dts)) = match counted { + Some(depth) => self.clock.pop_counted(depth, delay, flush), + None => self.clock.pop(lookahead, delay, flush), + } { if !self.finished { pending.arrived = pending.arrived.max(self.empty); } @@ -925,7 +938,7 @@ impl Timing { } /// Read the reorder depth from the codec config a keyframe is carried with. - fn describe(&mut self, stream_type: StreamType, description: Option<&Bytes>, name: &str) { + fn describe(&mut self, stream_type: StreamType, description: Option<&Bytes>, name: &str, synced: bool) { if self.description.as_ref() == description { return; } @@ -933,6 +946,15 @@ impl Timing { self.declared = description.and_then(|d| declared_reorder(stream_type, d)); self.hrd = description.and_then(|d| declared_hrd(stream_type, d)); tracing::debug!(track = %name, reorder = ?self.reorder(), "video reordering declared"); + if synced && self.declared.is_none() { + tracing::warn!(track = %name, "no reorder depth declared: if it reorders, --sync legs may give it different DTS"); + } + } + + /// The frames a picture can follow in output order after preceding them in decode order, + /// when the SPS declares it ([`Reorder::pictures`]). + fn depth(&self) -> Option { + self.declared.map(Reorder::pictures) } /// The receiver's buffers (H.222.0 2.14.3.1, 2.17.2): EB is the HRD's CPB, else the @@ -2304,7 +2326,9 @@ where // The video lane's reach is its first keyframe's HRD, which every leg reads alike. for (name, track) in self.tracks.iter_mut() { if let Kind::Video(stream_type) = track.kind { - track.timing.describe(stream_type, track.source.description(), name); + track + .timing + .describe(stream_type, track.source.description(), name, true); } } } diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 0373bb72c9c..953f0c4acdc 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -31,6 +31,11 @@ const VIDEO_US: u64 = 40_000; const AUDIO_US: u64 = 21_333; /// Video frames per group: 600 ms. const GOP: u64 = 15; +/// A B-pyramid group in decode order (`max_num_reorder_frames` 2): each frame's place in +/// presentation order. +const PYRAMID: [u64; GOP as usize] = [0, 4, 2, 1, 3, 8, 6, 5, 7, 12, 10, 9, 11, 14, 13]; +/// How many frames after its tick a B-pyramid frame is presented: its decode delay. +const PYRAMID_DELAY: u64 = 2; /// Audio frames per group, roughly matching the video group duration. const AUDIO_GROUP: u64 = 28; @@ -75,6 +80,8 @@ struct Live { rate: u64, /// The receivers render with [`Export::with_sync`]. sync: bool, + /// Video is a B-pyramid: see [`Self::reorder`]. + reorder: bool, /// The media time of the first frame, in µs. origin: u64, /// How much later than its video the source sends audio of the same media time, in µs; @@ -143,6 +150,7 @@ impl Live { scale, rate: MUX_RATE, sync: false, + reorder: false, origin: 0, audio_lag, start: moq_clock::now(), @@ -176,6 +184,30 @@ impl Live { } } + /// Send video as a B-pyramid in decode order, each frame [`PYRAMID_DELAY`] frames after its + /// tick, under an SPS that declares the depth. + fn reorder(&mut self) { + use crate::codec::h264::fixtures::{PPS as PYRAMID_PPS, SPS_PYRAMID}; + + let description = + crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_PYRAMID)], &[Bytes::from_static(PYRAMID_PPS)]) + .unwrap(); + let mut catalog = self.catalog.modify().unwrap(); + for config in catalog.video.renditions.values_mut() { + config.description = Some(description.clone()); + } + self.reorder = true; + } + + /// Abort the open video group, as a relay does to a subscriber left too far behind: the + /// subscriber reads it as evicted, and loses the rest of it. + fn evict(&mut self) { + if let Some(group) = self.group.take() { + group.abort(moq_net::Error::Old).unwrap(); + } + self.stalling = true; + } + /// Send nothing for `gap`, as a publisher that died, then resume at the live edge on the /// same timeline from the next group, as its replacement or the other half of a 1+1 pair /// would. @@ -223,10 +255,13 @@ impl Live { } else { let tick = self.tick; let keyframe = tick.is_multiple_of(GOP); - if keyframe && let Some(group) = self.group.take() { - match std::mem::take(&mut self.stalling) { - true => self.stalled.push(group), - false => group.finish().unwrap(), + if keyframe { + let stalled = std::mem::take(&mut self.stalling); + if let Some(group) = self.group.take() { + match stalled { + true => self.stalled.push(group), + false => group.finish().unwrap(), + } } } if keyframe { @@ -240,12 +275,16 @@ impl Live { } else { vec![0x41u8; 400] }; + let presented = match self.reorder { + true => (tick - tick % GOP + PYRAMID[(tick % GOP) as usize] + PYRAMID_DELAY) * VIDEO_US, + false => video, + }; let group = self.group.as_mut().expect("the first tick is a keyframe"); HangContainer::Legacy(crate::container::Kind::Video) .write( group, &[Frame { - timestamp: Timestamp::from_micros(self.origin + video).unwrap(), + timestamp: Timestamp::from_micros(self.origin + presented).unwrap(), duration: None, payload: length_prefixed(&[&slice]), keyframe, @@ -866,8 +905,12 @@ async fn synced_on( stall: impl Fn(u64) -> bool + Copy, ) -> (Leg, Leg, u64) { let mut live = Live::new(scale, lag).await; - live.sync = true; live.origin = origin; + synced_with(live, join, ticks, stall).await +} + +async fn synced_with(mut live: Live, join: u64, ticks: u64, stall: impl Fn(u64) -> bool + Copy) -> (Leg, Leg, u64) { + live.sync = true; let mut a = live.join(Duration::ZERO).await; live.run(join, &mut [&mut a], stall).await; let mut b = live.join(Duration::from_millis(100)).await; @@ -958,18 +1001,64 @@ async fn synced_legs_render_the_same_packets() { } } +/// Legs of a B-pyramid render the same packets when the first lost a group's tail before the +/// other joined, cut after a frame presented below one it never received. +#[tokio::test(start_paused = true)] +async fn synced_legs_render_reordered_video_the_same_way() { + let stall = |tick: u64| (2 * GOP + 8..3 * GOP).contains(&tick); + for lag in LAGS { + for join in [JOIN, JOIN + GOP, 4 * GOP + 2] { + let mut live = Live::new(1.0, lag).await; + live.reorder(); + let (a, b, from) = synced_with(live, join, 12 * GOP, stall).await; + let alone = assert_same_synced(&a, &b, from); + assert_eq!( + alone, 0, + "lag {lag}, join {join}: clean legs differ in {alone} positions" + ); + } + } +} + /// Each leg loses groups the other receives. Each renders the other's packets wherever it /// holds them, so one leg's stuffing is the other's content, and the two merged are a clean /// leg's output. #[tokio::test(start_paused = true)] async fn complementary_losses_stay_complementary() { - let lost_a = |tick: u64| (5 * GOP + 3..6 * GOP).contains(&tick); + complementary_losses(false, false).await; +} + +/// As [`complementary_losses_stay_complementary`] on a B-pyramid, with leg A's loss cutting +/// it between a reference and a frame presented before it. +#[tokio::test(start_paused = true)] +async fn complementary_losses_of_reordered_video_stay_complementary() { + complementary_losses(true, false).await; +} + +/// Legs of a B-pyramid stopped in turn, as receiver processes can be: each falls behind, the +/// relay evicts the group it was reading, and what its decode clock still held from that +/// group goes. The legs stay complementary. +#[tokio::test(start_paused = true)] +async fn legs_stopped_in_turn_stay_complementary() { + complementary_losses(true, true).await; +} + +/// Each leg loses part of the broadcast the other receives: with `stop`, by reading nothing +/// for over a second while the relay evicts its group, else by the source stalling a group +/// for that leg alone. +async fn complementary_losses(reorder: bool, stop: bool) { + let lost_a = |tick: u64| (5 * GOP + 8..6 * GOP).contains(&tick); let lost_b = |tick: u64| (7 * GOP + 5..8 * GOP).contains(&tick); + let stopped_a = |tick: u64| (5 * GOP + 8..7 * GOP).contains(&tick); + let stopped_b = |tick: u64| (8 * GOP + 5..10 * GOP).contains(&tick); for lag in LAGS { let mut paths = Vec::new(); for _ in 0..3 { let mut live = Live::new(1.0, lag).await; live.sync = true; + if reorder { + live.reorder(); + } paths.push(live); } let [mut clean, mut path_a, mut path_b] = <[Live; 3]>::try_from(paths).ok().unwrap(); @@ -979,8 +1068,26 @@ async fn complementary_losses_stay_complementary() { let mut b = path_b.join(Duration::from_millis(100)).await; for tick in 1..=12 * GOP { clean.run(tick, &mut [&mut c], |_| false).await; - path_a.run(tick, &mut [&mut a], lost_a).await; - path_b.run(tick, &mut [&mut b], lost_b).await; + match (stop, stopped_a(tick)) { + (false, _) => path_a.run(tick, &mut [&mut a], lost_a).await, + (true, false) => path_a.run(tick, &mut [&mut a], |_| false).await, + (true, true) => { + if !stopped_a(tick - 1) { + path_a.evict(); + } + path_a.run(tick, &mut [], |_| false).await + } + } + match (stop, stopped_b(tick)) { + (false, _) => path_b.run(tick, &mut [&mut b], lost_b).await, + (true, false) => path_b.run(tick, &mut [&mut b], |_| false).await, + (true, true) => { + if !stopped_b(tick - 1) { + path_b.evict(); + } + path_b.run(tick, &mut [], |_| false).await + } + } } clean.finish(&mut [&mut c]).await; path_a.finish(&mut [&mut a]).await; From a0bfd9eda70cfc36ad7ef50b545a1210281fc5cd Mon Sep 17 00:00:00 2001 From: t0ms Date: Mon, 19 Oct 2026 13:17:07 +0100 Subject: [PATCH 03/13] fix(moq-mux): open a TS --sync group once its HRD has the one before A unit is released as early as a full CPB would need it, so a group that fills its lane late can still be on it when the next group's first unit is released: a leg holding it lays the next group behind, where a leg that lost it does not (an overrun), and the two diverge. An H.264 picture opening a buffering period says how long its HRD holds it before decoding (initial_cpb_removal_delay), so how much of the CPB is empty then. Release that picture as much later as its lane takes to carry that room, never so late that it misses its deadline. The HRD received every earlier picture before its first bit (H.264 C.3), so a lane faster than the HRD's rate has carried the group before on any leg that holds it. The SPS walk keeps the NAL HRD's schedule count and delay length, and a reader takes the delay from the access unit's SEI. A stream without a NAL HRD or a buffering period, and H.265, keep today's release. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/codec/h264/mod.rs | 157 +++++++++++++++++- rs/moq-mux/src/container/ts/export.rs | 39 +++++ .../src/container/ts/export_timing_test.rs | 75 ++++++++- rs/moq-mux/src/container/ts/lane.rs | 29 +++- 4 files changed, 286 insertions(+), 14 deletions(-) diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index a82ec7352ce..2502c40a035 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -168,6 +168,55 @@ pub(crate) fn sps_color(nal: &[u8]) -> Option { sps_vui(nal)?.color } +/// How a buffering period SEI lays out the delays of the NAL HRD an SPS NAL unit declares. +pub(crate) fn sps_buffering(nal: &[u8]) -> Option { + sps_vui(nal)?.buffering +} + +/// How a buffering period SEI lays out the delays of an SPS's NAL HRD (ITU-T H.264 D.1.2, E.1.2). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Buffering { + /// `cpb_cnt_minus1 + 1`: the schedules it gives a delay and an offset for. + schedules: u32, + /// `initial_cpb_removal_delay_length_minus1 + 1`: the bits each takes. + bits: u32, +} + +/// The `initial_cpb_removal_delay`, in 90 kHz ticks, that a buffering period SEI in an access +/// unit of `length_size`-prefixed NAL units gives the NAL HRD's last schedule, the one +/// [`sps_hrd`] reads: how long the HRD holds the unit's first bit before decoding it. +pub(crate) fn initial_cpb_removal_delay(au: &[u8], length_size: usize, buffering: Buffering) -> Option { + use h264_parser::bitreader::BitReader; + use h264_parser::eg::read_ue; + + for nal in crate::codec::annexb::length_prefixed_nals(au, length_size) + .ok()? + .map_while(std::result::Result::ok) + { + match nal.first().map(|header| header & 0x1f) { + Some(6) => {} + // SEI precedes the first slice of its access unit (7.4.1.2.3). + Some(1..=5) => return None, + _ => continue, + } + let rbsp = h264_parser::nal::ebsp_to_rbsp(&nal[1..]); + for message in h264_parser::sei::SeiMessage::parse(&rbsp).ok()? { + let h264_parser::sei::SeiPayload::Unknown(0, payload) = message.payload else { + continue; + }; + let mut r = BitReader::new(&payload); + read_ue(&mut r).ok()?; // seq_parameter_set_id + let mut delay = None; + for _ in 0..buffering.schedules { + delay = Some(r.read_bits(buffering.bits).ok()?); + r.skip_bits(buffering.bits).ok()?; // initial_cpb_removal_delay_offset + } + return delay; + } + } + None +} + /// What an SPS VUI declares about picture timing, sample shape, and colour. struct Vui { /// The sample aspect, when the VUI specifies one. @@ -182,6 +231,8 @@ struct Vui { depth: Option, /// The NAL HRD, when present. hrd: Option, + /// How a buffering period SEI lays out the NAL HRD's delays. + buffering: Option, } impl Vui { @@ -328,9 +379,12 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } } let nal_hrd = r.read_flag()?; - let hrd = match nal_hrd { - true => Some(read_hrd(&mut r)?), - false => None, + let (hrd, buffering) = match nal_hrd { + true => { + let (hrd, buffering) = read_hrd(&mut r)?; + (Some(hrd), Some(buffering)) + } + false => (None, None), }; let vcl_hrd = r.read_flag()?; if vcl_hrd { @@ -348,6 +402,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { period, depth: None, hrd, + buffering, })); } r.skip_bits(1)?; // motion_vectors_over_pic_boundaries_flag @@ -362,11 +417,13 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { period, depth: Some(depth), hrd, + buffering, })) } -/// Read `hrd_parameters()` (ITU-T H.264 E.1.2) down to its last schedule (E.2.2). -fn read_hrd(r: &mut h264_parser::bitreader::BitReader) -> h264_parser::Result { +/// Read `hrd_parameters()` (ITU-T H.264 E.1.2) down to its last schedule (E.2.2), and how a +/// buffering period SEI lays out its delays. +fn read_hrd(r: &mut h264_parser::bitreader::BitReader) -> h264_parser::Result<(crate::codec::video::Hrd, Buffering)> { use h264_parser::eg::read_ue; let cpb_cnt_minus1 = read_ue(r)?; @@ -388,8 +445,15 @@ fn read_hrd(r: &mut h264_parser::bitreader::BitReader) -> h264_parser::Result Vec { + let mut fields = vec![(1, 1)]; // seq_parameter_set_id 0 + for &delay in delays { + fields.extend([(delay, bits), (1, bits)]); + } + fields.push((1, 1)); // bit_equal_to_one, then zeros to the byte + let (mut payload, mut acc, mut held) = (Vec::new(), 0u64, 0); + for (value, width) in fields { + acc = acc << width | u64::from(value); + held += width; + while held >= 8 { + held -= 8; + payload.push((acc >> held) as u8); + } + acc &= (1 << held) - 1; + } + if held > 0 { + payload.push((acc << (8 - held)) as u8); + } + let mut nal = vec![0x06, 0x00, payload.len() as u8]; + nal.extend(payload); + nal.push(0x80); + nal + } } #[cfg(test)] @@ -920,6 +1020,49 @@ mod tests { assert_eq!(sps_hrd(fixtures::SPS_IPB), None); } + fn length_prefixed(nals: &[&[u8]]) -> Vec { + nals.iter() + .flat_map(|nal| (nal.len() as u32).to_be_bytes().into_iter().chain(nal.iter().copied())) + .collect() + } + + #[test] + fn initial_cpb_removal_delay_reads_the_last_schedule() { + use crate::codec::video::Hrd; + use fixtures::buffering_period; + + assert_eq!( + sps_buffering(fixtures::SPS_HRD), + Some(Buffering { schedules: 1, bits: 19 }) + ); + assert_eq!(sps_buffering(fixtures::SPS_IPB), None); + let short = sps_buffering(fixtures::SPS_HRD_SHORT).expect("a NAL HRD"); + assert_eq!(short, Buffering { schedules: 1, bits: 18 }); + assert_eq!( + sps_hrd(fixtures::SPS_HRD_SHORT), + Some(Hrd { + bit_rate: 800_000, + cpb_size: 320_000 + }) + ); + let aud: &[u8] = &[0x09, 0xf0]; + let slice: &[u8] = &[0x65, 0x88, 0x84]; + let au = length_prefixed(&[aud, fixtures::SEI_BUFFERING_SHORT, slice]); + assert_eq!(initial_cpb_removal_delay(&au, 4, short), Some(32_399)); + let two = Buffering { schedules: 2, bits: 18 }; + let au = length_prefixed(&[&buffering_period(&[90_000, 36_000], 18), slice]); + assert_eq!(initial_cpb_removal_delay(&au, 4, two), Some(36_000)); + // No buffering period before the first slice, none at all, and one cut short. + let au = length_prefixed(&[slice, fixtures::SEI_BUFFERING_SHORT]); + assert_eq!(initial_cpb_removal_delay(&au, 4, short), None); + assert_eq!( + initial_cpb_removal_delay(&length_prefixed(&[aud, slice]), 4, short), + None + ); + let au = length_prefixed(&[fixtures::SEI_BUFFERING_SHORT]); + assert_eq!(initial_cpb_removal_delay(&au[..au.len() - 3], 4, short), None); + } + fn annexb_frame(nals: &[&[u8]]) -> Bytes { let mut buf = BytesMut::new(); for nal in nals { diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 6d246ba5a80..a6bdb452f51 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -708,6 +708,10 @@ impl Track { let by = decode.as_nanos().saturating_sub(self.drain().as_nanos()); let release = lane::release(by, self.ahead_slots(delay)); let video = matches!(self.kind, Kind::Video(_)); + let room = match video && group.1 { + true => self.timing.room(frame), + false => None, + }; let Some(sync) = &mut self.sync else { return Ok(None); }; @@ -742,6 +746,16 @@ impl Track { } } let count = packets.len() / TsPacket::SIZE; + // A unit is released as early as a full CPB would need it. A group's first picture whose + // buffering period says how much room its encoder's HRD has left when it decodes it goes + // out as much later as its lane takes to carry that room. The HRD received every earlier + // picture before that picture's first bit (ITU-T H.264 C.3), so a lane faster than the + // HRD's rate has by then carried the group before on any leg that holds it. It never goes + // out so late that it would miss its own deadline. + let release = match room { + Some(bits) => sync.layout.later(sync.lane.index(), release, bits, by, count), + None => release, + }; let overruns = sync.lane.overruns(); let plan = sync.lane.plan(&sync.layout, group, release, by, count); if sync.lane.overruns() > overruns { @@ -926,6 +940,8 @@ struct Timing { level: Option, /// The HRD its SPS declares. hrd: Option, + /// The NAL unit length and buffering period layout of an H.264 rendition with a NAL HRD. + buffering: Option<(usize, crate::codec::h264::Buffering)>, } impl Timing { @@ -945,6 +961,7 @@ impl Timing { self.description = description.cloned(); self.declared = description.and_then(|d| declared_reorder(stream_type, d)); self.hrd = description.and_then(|d| declared_hrd(stream_type, d)); + self.buffering = description.and_then(|d| declared_buffering(stream_type, d)); tracing::debug!(track = %name, reorder = ?self.reorder(), "video reordering declared"); if synced && self.declared.is_none() { tracing::warn!(track = %name, "no reorder depth declared: if it reorders, --sync legs may give it different DTS"); @@ -982,6 +999,18 @@ impl Timing { self.hrd.map(|hrd| fill(hrd.cpb_size, hrd.bit_rate)) } + /// The room, in bits, the HRD's CPB has left when it decodes `frame`, the first picture of a + /// buffering period: its size less what reaches it at its BitRate over the + /// `initial_cpb_removal_delay` the period's SEI gives. + fn room(&self, frame: &Frame) -> Option { + let ((length_size, buffering), hrd) = (self.buffering?, self.hrd?); + let delay = crate::codec::h264::initial_cpb_removal_delay(&frame.payload, length_size, buffering)?; + Some( + hrd.cpb_size + .saturating_sub(hrd.bit_rate.saturating_mul(u64::from(delay)) / 90_000), + ) + } + /// The least reorder delay and lookahead the SPS declares, in ticks, so two exporters agree /// on them from their first keyframe whatever reordering each has seen. /// @@ -4329,6 +4358,16 @@ fn declared_hrd(stream_type: StreamType, description: &[u8]) -> Option { } } +/// The NAL unit length and buffering period layout of an H.264 rendition whose first SPS +/// declares a NAL HRD. +fn declared_buffering(stream_type: StreamType, description: &[u8]) -> Option<(usize, crate::codec::h264::Buffering)> { + if !matches!(stream_type, StreamType::H264) { + return None; + } + let avcc = crate::codec::h264::Avcc::parse(description).ok()?; + Some((avcc.length_size, crate::codec::h264::sps_buffering(avcc.sps.first()?)?)) +} + /// The reordering declared by the first SPS in a video rendition's avcC/hvcC. fn declared_reorder(stream_type: StreamType, description: &[u8]) -> Option { match stream_type { diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 953f0c4acdc..a5cc431d61c 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -82,6 +82,11 @@ struct Live { sync: bool, /// Video is a B-pyramid: see [`Self::reorder`]. reorder: bool, + /// The `initial_cpb_removal_delay` each group's first picture opens a buffering period with: + /// see [`Self::buffering`]. + periods: Option, + /// A video tick whose frame is four times the size of a keyframe. + burst: Option, /// The media time of the first frame, in µs. origin: u64, /// How much later than its video the source sends audio of the same media time, in µs; @@ -151,6 +156,8 @@ impl Live { rate: MUX_RATE, sync: false, reorder: false, + periods: None, + burst: None, origin: 0, audio_lag, start: moq_clock::now(), @@ -199,6 +206,20 @@ impl Live { self.reorder = true; } + /// Declare an 800 kb/s CBR NAL HRD with 0.4 s of buffer delay, and open each group with a + /// buffering period SEI giving its first picture `delay` ticks of `initial_cpb_removal_delay`. + fn buffering(&mut self, delay: u32) { + use crate::codec::h264::fixtures::SPS_HRD_SHORT; + + let description = + crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_HRD_SHORT)], &[Bytes::from_static(PPS)]).unwrap(); + let mut catalog = self.catalog.modify().unwrap(); + for config in catalog.video.renditions.values_mut() { + config.description = Some(description.clone()); + } + self.periods = Some(delay); + } + /// Abort the open video group, as a relay does to a subscriber left too far behind: the /// subscriber reads it as evicted, and loses the rest of it. fn evict(&mut self) { @@ -270,10 +291,15 @@ impl Live { if stall(tick) { self.stalling = true; } else if !self.stalling { - let slice = if keyframe { - vec![0x65u8; 3_000] - } else { - vec![0x41u8; 400] + let slice = match (keyframe, self.burst == Some(tick)) { + (true, _) => vec![0x65u8; 3_000], + (false, true) => vec![0x41u8; 12_000], + (false, false) => vec![0x41u8; 400], + }; + // The delay field is as wide as `SPS_HRD_SHORT` declares. + let sei = match self.periods { + Some(delay) if keyframe => crate::codec::h264::fixtures::buffering_period(&[delay], 18), + _ => Vec::new(), }; let presented = match self.reorder { true => (tick - tick % GOP + PYRAMID[(tick % GOP) as usize] + PYRAMID_DELAY) * VIDEO_US, @@ -286,7 +312,10 @@ impl Live { &[Frame { timestamp: Timestamp::from_micros(self.origin + presented).unwrap(), duration: None, - payload: length_prefixed(&[&slice]), + payload: match sei.is_empty() { + true => length_prefixed(&[&slice]), + false => length_prefixed(&[&sei, &slice]), + }, keyframe, }], ) @@ -1117,6 +1146,42 @@ async fn complementary_losses(reorder: bool, stop: bool) { } } +/// A group ending in a burst still fills its lane at the next group's release, so a leg holding +/// it would lay the next group behind where a leg that lost it does. When the next group's +/// first picture opens a buffering period, it goes out late enough for neither leg's lane to be +/// busy, and both lay it alike. +#[tokio::test(start_paused = true)] +async fn a_leg_that_lost_a_burst_lays_the_next_buffering_period_alike() { + let lost = |tick: u64| (5 * GOP + 8..6 * GOP).contains(&tick); + for lag in LAGS { + let mut paths = Vec::new(); + for _ in 0..2 { + let mut live = Live::new(1.0, lag).await; + live.sync = true; + // A lane the HRD's Rx narrows cannot carry the first keyframe by a decode time of zero. + live.origin = 10_000_000; + // Half the HRD's buffer delay: a group's first picture waits 0.2 s before it decodes. + live.buffering(18_000); + live.burst = Some(6 * GOP - 1); + paths.push(live); + } + let [mut whole, mut lossy] = <[Live; 2]>::try_from(paths).ok().unwrap(); + assert!(whole.start == lossy.start); + let mut a = whole.join(Duration::ZERO).await; + let mut b = lossy.join(Duration::from_millis(100)).await; + for tick in 1..=10 * GOP { + whole.run(tick, &mut [&mut a], |_| false).await; + lossy.run(tick, &mut [&mut b], lost).await; + } + whole.finish(&mut [&mut a]).await; + lossy.finish(&mut [&mut b]).await; + assert!( + assert_same_synced(&a, &b, 2 * GOP * VIDEO_US) > 0, + "lag {lag}: the lossy leg lost nothing" + ); + } +} + /// A broadcast timeline crossing the 33-bit PTS and PCR wrap while both legs run, with the /// second joining just after it, renders the same packets. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/ts/lane.rs b/rs/moq-mux/src/container/ts/lane.rs index 10754303d33..9e68fad510f 100644 --- a/rs/moq-mux/src/container/ts/lane.rs +++ b/rs/moq-mux/src/container/ts/lane.rs @@ -12,8 +12,10 @@ //! the first slot the group's first unit may go out in. Where it lands depends on its group's //! earlier units and nothing else, so a leg that lacks a group, or joined after it, lays the //! next one where every other leg does. A group still unfinished in its lane when the next -//! one is due pushes the next one back, which only a leg holding both does: a lane needs the -//! headroom to carry each group within its own duration. +//! one is due pushes the next one back, which only a leg holding both does. A lane needs the +//! headroom to carry each group within its own duration, unless the next group's first picture +//! opens an H.264 buffering period: the export then releases it late enough for its HRD to rule +//! that out. use std::collections::{BTreeMap, HashMap, VecDeque}; use std::sync::Arc; @@ -213,6 +215,18 @@ impl Layout { owners } + /// `release` moved later by as long as lane `lane` takes to carry `bits` of payload, unless a + /// unit of `count` packets laid from there would arrive after `by`. + pub fn later(&self, lane: usize, release: u128, bits: u64, by: u128, count: usize) -> u128 { + let later = release + u128::from(bits) * 1_000_000_000 / u128::from(self.payload_rate(lane).max(1)); + let positions = self.take(lane, self.at(later), count.max(1)); + let last = *positions.last().expect("at least one position"); + match self.time(last + 1) > by { + true => release, + false => later, + } + } + /// The next `count` positions of lane `lane` at or after the timeline's packet `from`. pub fn take(&self, lane: usize, from: u128, count: usize) -> Vec { let mut out = Vec::with_capacity(count); @@ -730,6 +744,17 @@ mod test { assert!(placed.positions[0] >= layout.first(10)); } + #[test] + fn a_release_moves_later_only_while_its_unit_meets_its_deadline() { + let layout = layout(); + // Half a second of lane 0's payload. + let bits = layout.payload_rate(0) / 2; + let later = layout.later(0, start(100), bits, start(200), 2); + assert!((start(119)..=start(120)).contains(&later), "{later}"); + assert_eq!(layout.later(0, start(100), bits, start(110), 2), start(100)); + assert_eq!(layout.later(0, start(100), 0, start(110), 2), start(100)); + } + #[test] fn the_grid_lays_tables_and_si_beside_the_lanes() { let layout = Layout::new(RATE, 2, 80, &[(0x100, Need::Rest(u64::MAX)), (0x101, Need::Fixed(422))]).unwrap(); From 692aa6bc01d2f68c24ee6819f22d5c45999e5bd0 Mon Sep 17 00:00:00 2001 From: t0ms Date: Mon, 19 Oct 2026 16:45:11 +0100 Subject: [PATCH 04/13] fix(moq-mux): run the TS --sync clock on to the last decode time Under --sync the export places packets straight on the grid, so the schedule never recorded a decode-time horizon and stopped the clock once the grid drained: the last PCR came 335 ms before the last decode time. mux_sync now records each placed unit's decode time, and is_empty holds the clock until the grid's next slot passes it. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 3 +- .../src/container/ts/export_timing_test.rs | 45 ++++++++++++++++++- rs/moq-mux/src/container/ts/schedule.rs | 11 ++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index a6bdb452f51..f0699ff1ac3 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1419,7 +1419,7 @@ where /// Render every packet as a function of the broadcast alone, so two exports of it that /// joined at different moments and received it with different skew emit identical TS for - /// the content both hold, continuity counters included: a 1+1 (SMPTE ST 2022-7) pair. + /// the content both hold, continuity counters included, as the legs of a 1+1 pair must. /// /// Each PID gets a fixed lane of the constant multiplex rate, a function of the slot and /// the catalog, and each MoQ group lays its units from a position its first unit's @@ -2720,6 +2720,7 @@ where let Some(planned) = queued.planned else { return Ok(()); }; + self.schedule.decodes_at(queued.decode.as_nanos()); let grid = self.schedule.sync().context("MPEG-TS --sync lanes not laid out")?; grid.place(&planned.packets, &planned.positions, queued.frame.keyframe); Ok(()) diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index a5cc431d61c..304958a1520 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -1,7 +1,7 @@ //! Timing of a delayed [`Export`] on a live clock: what a receiver renders when it joins a //! broadcast already running, when one track loses frames, when the source falls silent and //! resumes, when the source's clock runs off the receiver's, and when two receivers render one -//! broadcast for a 1+1 (SMPTE ST 2022-7) pair. +//! broadcast for a 1+1 pair. //! //! Every case runs on the paused clock, with each frame written at the instant a live source //! would send it. A receiver's view is measured from its own output: the lead of each PES (its @@ -752,6 +752,49 @@ async fn the_video_held_at_the_end_goes_out() { } } +/// The last PCR a leg carried, 90 kHz. +fn last_pcr(out: &[(Instant, Frame)]) -> u64 { + let bytes: Vec = out + .iter() + .flat_map(|(_, frame)| frame.payload.iter().copied()) + .collect(); + let mut reader = TsPacketReader::new(Cursor::new(bytes)); + let mut pcr = None; + while let Some(packet) = reader.read_ts_packet().unwrap() { + if let Some(clock) = packet.adaptation_field.as_ref().and_then(|af| af.pcr) { + pcr = Some(clock.as_u64() / 300); + } + } + pcr.expect("a PCR") +} + +/// The media goes out up to the delay ahead of its decode time, so at the end of the broadcast +/// the clock runs on alone until the last unit decodes, with or without `--sync`: a receiver +/// removes a unit from its buffer only once the clock reaches its decode time. +#[tokio::test(start_paused = true)] +async fn the_clock_runs_on_to_the_last_decode_time() { + for sync in [false, true] { + let mut live = Live::new(1.0, LAGS[0]).await; + live.sync = sync; + live.origin = 10_000_000; + let mut leg = live.join(Duration::ZERO).await; + live.run(4 * GOP, &mut [&mut leg], |_| false).await; + live.finish(&mut [&mut leg]).await; + + let last = leg + .units() + .iter() + .map(|unit| unit.decode) + .max() + .expect("units went out"); + let short = diff90(last, last_pcr(&leg.out)) as f64 / 90.0; + assert!( + short <= SLOT_MS, + "sync {sync}: the clock stops {short} ms before the last decode time" + ); + } +} + // A source's clock is never the receiver's. A transport stream's 27 MHz may be off by 30 ppm, // which walks a fixed anchor 108 ms an hour: a source running slow makes every frame late in // turn once the walk passes the delay, and one running fast makes the buffer grow without bound. diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 448613606f9..bb5dd3b1d65 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -606,6 +606,12 @@ impl Schedule { self.sync.as_ref() } + /// Under [`Self::set_sync`], keep the clock running until `decode`, nanoseconds on the + /// media timeline, as a unit queued here would: the grid holds packets, not decode times. + pub fn decodes_at(&mut self, decode: u128) { + self.horizon = self.horizon.max(Some(decode.div_ceil(PCR_INTERVAL.as_nanos()))); + } + /// Hold the packets on `pid` to `buffer`. pub fn set_buffer(&mut self, pid: u16, buffer: Buffer) { let before = self.transport_buffer(pid); @@ -806,7 +812,10 @@ impl Schedule { /// Whether nothing is left to lay out: every unit sent, and the clock past its decode time. pub fn is_empty(&self) -> bool { if let Some(grid) = &self.sync { - return grid.is_empty(); + return grid.is_empty() + && self + .horizon + .is_none_or(|horizon| grid.index().is_none_or(|next| next > horizon)); } self.units.is_empty() && self From fb1e1790e4bce7ddf8b10af0117392048c37dc84 Mon Sep 17 00:00:00 2001 From: t0ms Date: Mon, 19 Oct 2026 16:45:11 +0100 Subject: [PATCH 05/13] docs(moq-cli): qualify --sync identity; no ST 2022-7 claim Name what a 1+1 pair's legs can still differ in (SI across a revision or a table the catalog adds or drops), and say an ST 2022-7 pair also needs identical RTP framing, which export ts does not write. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 9 ++++++--- rs/moq-cli/src/args.rs | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 943e0474693..21b4eedcba7 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -435,8 +435,9 @@ continuity error, and the rest carry on. It takes neither `--linger`, `export ts --sync` renders every packet from the broadcast alone, so two exports of it, joined at different moments and fed with different skew, emit identical -TS for the content both receive, continuity counters included: the two legs of -an SMPTE ST 2022-7 pair. Each PID gets a fixed lane of the constant rate, sized +TS for the content both receive, continuity counters included, as the legs of a +1+1 pair must; an SMPTE ST 2022-7 pair also needs identical RTP framing around +it, which this command does not write. Each PID gets a fixed lane of the constant rate, sized for its codec's most, with video taking the rest; content one leg lacks leaves its packets as stuffing there. The PCR rides a PID of its own, and each PID's continuity counter restarts at every group with the break flagged. It needs a @@ -445,7 +446,9 @@ source's (a 10 Mb/s channel needs about 12), and takes neither `--stitch` nor `--passthrough`. A group that starts behind the one before it, on a lane too narrow to absorb that one, is logged and counted in `overruns`: the legs can differ there. So can reordered video whose SPS declares no reorder depth -(`max_num_reorder_frames`), which is logged. +(`max_num_reorder_frames`), which is logged, and SI while one leg holds an older +revision than the other or around an SI table the catalog adds or drops, which +each leg takes up when its catalog update arrives. A stdout export ends with the broadcast. `export ts --linger 10s` waits that long for the same publisher instance to come back, and carries on with the same diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 881a010d3fd..ed3baec395e 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -1210,8 +1210,8 @@ pub struct Transport { pub passthrough: bool, /// Render every packet from the broadcast alone, on a fixed lane per PID, so two exports - /// of it emit identical TS for the content both receive, continuity counters included: - /// a 1+1 (SMPTE ST 2022-7) pair. Needs a constant rate (`--mux-rate`, or the catalog's). + /// of it emit identical TS for the content both receive, continuity counters included, + /// as the legs of a 1+1 pair must. Needs a constant rate (`--mux-rate`, or the catalog's). #[usage(long, conflicts("--stitch", "--passthrough"))] pub sync: bool, From 018fe439710ad0c3c05444f196a487d10bb20f4d Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 10:29:19 +0100 Subject: [PATCH 06/13] fix(mux): run a sync grid on after a stall that resumes behind it A --sync leg that stalls keeps laying the slots it owes, and on current main its subscription can resume at the group it was in, as far behind those slots as the stall lasted. The two-second window then took the resumed content for a new timeline: the grid jumped back and laid the slots again under a PCR that stepped back. A minute either way keeps it on the same timeline, the missed content left as stuffing. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/lane.rs | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/lane.rs b/rs/moq-mux/src/container/ts/lane.rs index 9e68fad510f..93d172c2599 100644 --- a/rs/moq-mux/src/container/ts/lane.rs +++ b/rs/moq-mux/src/container/ts/lane.rs @@ -32,9 +32,10 @@ pub(super) const PSI_SLOTS: u128 = PSI_INTERVAL.as_nanos() / PCR_INTERVAL.as_nan /// How fast a system PID's transport buffer drains (Rxsys, ISO 13818-1 2.4.2.3). const RX_SYSTEM: u64 = 1_000_000; /// How far a restarted grid's first packet may sit behind the last slot laid, or ahead of it, -/// and still be on the same timeline, so the grid runs on rather than jumps: two seconds back, -/// a minute on. -const BEHIND: u128 = 2 * SLOTS as u128; +/// and still be on the same timeline, so the grid runs on rather than jumps: a minute either +/// way. A leg that stalled has laid the slots it owed, and its subscription may resume as far +/// behind them as the relay kept its groups. +const BEHIND: u128 = 60 * SLOTS as u128; const AHEAD: u128 = 60 * SLOTS as u128; /// What a position of a slot carries. @@ -651,6 +652,29 @@ mod test { ); } + /// A leg that stalled has laid the slots it owed, and its subscription may resume as far + /// behind them as the relay kept its groups. Its grid runs on from the last slot, leaving what + /// it missed as stuffing, rather than lay those slots again under a PCR that steps back. A + /// timeline further off starts afresh. + #[test] + fn a_grid_restarted_behind_its_last_slot_runs_on() { + let layout = Arc::new(layout()); + let mut packet = [0xaau8; TsPacket::SIZE]; + packet[..4].copy_from_slice(&[0x47, 0x41, 0x00, 0x10]); + let last = 10_000u128; + for (behind, next) in [(3 * u128::from(SLOTS), last + 1), (BEHIND + 1, last - BEHIND - 2)] { + let mut grid = Grid::new(layout.clone(), tables()); + grid.lay(last); + grid.restart(); + grid.place(&packet, &[layout.first(last - behind)], false); + assert_eq!(grid.index(), Some(next), "{behind} slots behind"); + if next == last + 1 { + assert!(!grid.lay(next).jump); + assert!(grid.is_empty(), "what it missed never goes out"); + } + } + } + #[test] fn a_rate_too_low_for_the_lanes_is_refused() { let err = Layout::new( From 59d39cd9c54d0a87ad0b3763b5e3aedee0b5bc1f Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 10:55:55 +0100 Subject: [PATCH 07/13] fix(mux): count what a sync grid runs on past as late A grid that runs on after a restart drops what it holds behind the slot it lays. It counted those packets late only when they reached it after it had resumed, so a stopped leg's loss showed in one run and not the next. Count them as they go, and let the stopped-legs test hold a stopped leg's late packets while the clean leg must have none. Co-authored-by: Claude Opus 5.5 --- .../src/container/ts/export_timing_test.rs | 17 ++++++++++++++--- rs/moq-mux/src/container/ts/lane.rs | 4 ++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 7a31cc30b0a..cd2be14c699 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -1014,6 +1014,11 @@ fn assert_same_synced(a: &Leg, b: &Leg, from: u64) -> usize { let stats = leg.export.stats(); assert_eq!((stats.overruns, stats.late), (0, 0), "a lane fell behind"); } + same_synced(a, b, from) +} + +/// As [`assert_same_synced`], whatever the legs counted. +fn same_synced(a: &Leg, b: &Leg, from: u64) -> usize { let (a, b) = (slots(a, from), slots(b, from)); let common: Vec = a.keys().filter(|at| b.contains_key(at)).copied().collect(); assert!( @@ -1166,9 +1171,15 @@ async fn complementary_losses(reorder: bool, stop: bool) { path_b.finish(&mut [&mut b]).await; let from = 2 * GOP * VIDEO_US; - assert!(assert_same_synced(&a, &c, from) > 0, "lag {lag}: leg A lost nothing"); - assert!(assert_same_synced(&b, &c, from) > 0, "lag {lag}: leg B lost nothing"); - assert!(assert_same_synced(&a, &b, from) > 0); + // A stopped leg may resume behind the slots it laid, and drop what it reads for them. + for (leg, stopped) in [(&a, stop), (&b, stop), (&c, false)] { + let stats = leg.export.stats(); + assert_eq!(stats.overruns, 0, "lag {lag}: a lane fell behind"); + assert!(stopped || stats.late == 0, "lag {lag}: {} packets late", stats.late); + } + assert!(same_synced(&a, &c, from) > 0, "lag {lag}: leg A lost nothing"); + assert!(same_synced(&b, &c, from) > 0, "lag {lag}: leg B lost nothing"); + assert!(same_synced(&a, &b, from) > 0); let (a, b, c) = (slots(&a, from), slots(&b, from), slots(&c, from)); for (at, clean) in &c { let (Some(a), Some(b)) = (a.get(at), b.get(at)) else { diff --git a/rs/moq-mux/src/container/ts/lane.rs b/rs/moq-mux/src/container/ts/lane.rs index 93d172c2599..8661ad6a79b 100644 --- a/rs/moq-mux/src/container/ts/lane.rs +++ b/rs/moq-mux/src/container/ts/lane.rs @@ -523,6 +523,9 @@ impl Grid { let tables = Layout::repeats_tables(index) || jump || self.last.is_none(); self.last = Some(index); let first = self.layout.first(index); + // What a restarted grid holds behind the slot it runs on from went out as stuffing. + let ahead = self.placed.split_off(&first); + self.late += std::mem::replace(&mut self.placed, ahead).len() as u64; let owners = self.layout.owners(index); let count = owners.len(); let mut out = Laid { @@ -671,6 +674,7 @@ mod test { if next == last + 1 { assert!(!grid.lay(next).jump); assert!(grid.is_empty(), "what it missed never goes out"); + assert_eq!(grid.late(), 1, "and reached the grid after its position went out"); } } } From 1a91e9e698b194bdc0cf21c35fabb41631596e46 Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 12:21:14 +0100 Subject: [PATCH 08/13] fix(mux): keep a later SI entry off the sync clock PID Under --sync the PCR rides a PID of its own, picked when the tables are built clear of the SI present then. An SI entry a later catalog update adds was checked against the track PIDs alone, so it could land on the clock PID, sharing its continuity counter and flagging a time-base break at each emission. Check it against the PCR PID too. The --sync help and the CLI page now say every leg needs the same rate. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- rs/moq-cli/src/args.rs | 3 +- rs/moq-mux/src/container/ts/export.rs | 8 +++- rs/moq-mux/src/container/ts/export_test.rs | 44 ++++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index c7ddf6db656..c33109aad73 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -438,7 +438,8 @@ it, which this command does not write. Each PID gets a fixed lane of the constan for its codec's most, with video taking the rest; content one leg lacks leaves its packets as stuffing there. The PCR rides a PID of its own, and each PID's continuity counter restarts at every group with the break flagged. It needs a -constant rate (`--mux-rate`, or the catalog's), costing headroom over the +constant rate, the same on every leg since the lanes follow from it +(`--mux-rate`, or the catalog's), costing headroom over the source's (a 10 Mb/s channel needs about 12), and takes neither `--stitch` nor `--passthrough`. A group that starts behind the one before it, on a lane too narrow to absorb that one, is logged and counted in `overruns`: the legs can diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 01b7f801582..e5c55aa66df 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -1228,7 +1228,8 @@ pub struct Transport { /// Render every packet from the broadcast alone, on a fixed lane per PID, so two exports /// of it emit identical TS for the content both receive, continuity counters included, - /// as the legs of a 1+1 pair must. Needs a constant rate (`--mux-rate`, or the catalog's). + /// as the legs of a 1+1 pair must. Needs a constant rate, the same on every leg + /// (`--mux-rate`, or the catalog's). #[usage(long, conflicts("--stitch", "--passthrough"))] pub sync: bool, diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 8f7711dbbda..5168a920c2f 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1836,7 +1836,13 @@ where if !self.stale.is_empty() { self.resubscribe(&catalog, &mpegts)?; } - let es_pids: Vec = self.tracks.values().map(|t| t.pid).collect(); + // Under `--sync` the PCR rides a PID of its own, which SI must not take either. + let es_pids: Vec = self + .tracks + .values() + .map(|t| t.pid) + .chain(self.psi.as_ref().map(|psi| psi.pcr_pid)) + .collect(); reject_colliding_si_pids(&mpegts, self.pmt_pid(), &es_pids)?; // The layout is locked but the timing is not: a `jitter` or `framerate` published // after the tables still sizes the decode clock. diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 1cd1973b2f9..f960dc6eae3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -6060,6 +6060,50 @@ impl SiCadenceRig { } } +/// Under `--sync` the PCR rides a PID of its own, picked when the tables are built: an SI +/// entry the catalog adds after them must not land on it. +#[tokio::test(start_paused = true)] +async fn sync_refuses_a_later_si_entry_on_the_clock_pid() { + let mut rig = si_cadence_rig(0x0011, 0x42, Duration::from_secs(2)).await; + rig.exporter = rig.exporter.with_mux_rate(2_000_000).with_sync(); + let mut frames = Vec::new(); + for millis in (0..1_000).step_by(40) { + frames.extend(rig.media_frames(millis).await); + } + let clock = frames + .iter() + .flat_map(|frame| frame.payload.chunks(188)) + .find(|packet| packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0) + .map(|packet| u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2])) + .expect("the tables and the clock went out"); + rig.catalog + .modify() + .unwrap() + .ext + .mpegts + .si + .entry(clock) + .or_default() + .insert( + 0x70, + tscat::SiEntry { + track: "late.si".to_string(), + interval: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let err = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Err(err) = rig.exporter.next().await { + break err; + } + } + }) + .await + .expect("the export reads the catalog update"); + assert!(err.to_string().contains("mpegts.si PID"), "{err}"); +} + /// #2934: a revised SI snapshot goes out on the revision floor instead of waiting /// out the repetition interval (here the floor has long elapsed, so it rides the /// very next frame). For a clock table (TDT/TOT) the old interval-grid hold From 56f45f0784d225b9c0186ffd593e7d17e10d2a35 Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 12:33:34 +0100 Subject: [PATCH 09/13] fix(mux): read a reorder depth past the DPB as undeclared Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/codec/h264/mod.rs | 73 ++++++++++++++++++++++++++++++-- rs/moq-mux/src/codec/h265/mod.rs | 9 ++-- rs/moq-mux/src/codec/video.rs | 4 ++ 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 2502c40a035..58697637f6a 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -148,11 +148,14 @@ impl Sps { /// The reordering an SPS NAL unit declares in its VUI: `max_num_reorder_frames` from the /// bitstream restriction, and the frame period when `fixed_frame_rate_flag` is set. /// -/// `None` when the SPS carries no VUI or no bitstream restriction, or fails to parse. +/// `None` when the SPS carries no VUI or no bitstream restriction, declares more than +/// [`Reorder::MAX_DEPTH`](crate::codec::video::Reorder::MAX_DEPTH), or fails to parse. pub(crate) fn sps_reorder(nal: &[u8]) -> Option { + use crate::codec::video::Reorder; + let vui = sps_vui(nal)?; - Some(crate::codec::video::Reorder { - depth: vui.depth?, + Some(Reorder { + depth: vui.depth.filter(|depth| *depth <= Reorder::MAX_DEPTH)?, period: vui.period, fields: !vui.frame_only, }) @@ -939,6 +942,70 @@ mod tests { const SC4: &[u8] = &[0, 0, 0, 1]; + /// An RBSP written bit by bit. + #[derive(Default)] + struct Bits(Vec); + + impl Bits { + fn put(&mut self, value: u64, width: u32) { + self.0.extend((0..width).rev().map(|bit| value >> bit & 1 == 1)); + } + + fn ue(&mut self, value: u32) { + let code = u64::from(value) + 1; + let width = 64 - code.leading_zeros(); + self.put(0, width - 1); + self.put(code, width); + } + + /// The SPS NAL unit, stop bit and emulation prevention added. + fn sps(mut self) -> Vec { + self.0.push(true); + self.0.resize(self.0.len().next_multiple_of(8), false); + let mut nal = vec![0x67]; + let mut zeros = 0; + for byte in self.0.chunks(8) { + let byte = byte.iter().fold(0u8, |byte, &bit| byte << 1 | u8::from(bit)); + if zeros == 2 && byte <= 3 { + nal.push(3); + zeros = 0; + } + zeros = if byte == 0 { zeros + 1 } else { 0 }; + nal.push(byte); + } + nal + } + } + + /// A 64x48 baseline SPS whose VUI carries only a bitstream restriction declaring `depth`. + fn sps_declaring(depth: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(66, 8); // profile_idc + bits.put(30, 16); // constraint flags, reserved bits, level_idc + for value in [0, 0, 2, 1] { + bits.ue(value); // seq_parameter_set_id .. max_num_ref_frames + } + bits.put(0, 1); // gaps_in_frame_num_value_allowed_flag + bits.ue(3); // pic_width_in_mbs_minus1 + bits.ue(2); // pic_height_in_map_units_minus1 + bits.put(0b1101, 4); // frame_mbs_only, direct_8x8_inference, frame_cropping, vui_parameters_present + bits.put(0, 8); // the VUI's aspect ratio .. pic_struct flags + bits.put(0b11, 2); // bitstream_restriction_flag, motion_vectors_over_pic_boundaries_flag + for value in [0, 0, 16, 16, depth, depth] { + bits.ue(value); // max_bytes_per_pic_denom .. max_dec_frame_buffering + } + bits.sps() + } + + /// No decoded picture buffer holds more than 16 frames (A.3.1), so a deeper declaration is no + /// declaration rather than a count to hold that many frames for. + #[test] + fn an_impossible_reorder_depth_is_undeclared() { + assert_eq!(sps_reorder(&sps_declaring(2)).map(|reorder| reorder.depth), Some(2)); + assert_eq!(sps_reorder(&sps_declaring(16)).map(|reorder| reorder.depth), Some(16)); + assert_eq!(sps_reorder(&sps_declaring(1 << 20)), None); + } + #[test] fn sps_reorder_reads_the_vui() { use crate::codec::video::Reorder; diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 252199415b6..1e2d1bf6b48 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -266,14 +266,17 @@ pub(crate) fn config_from_sps(sps_nal: &[u8], in_band: bool) -> Result Option { + use crate::codec::video::Reorder; + let sps = SpsNALUnit::parse(&mut &nal[..]).ok()?; let highest = sps.rbsp.sps_max_sub_layers_minus1 as usize; let reorder = &sps.rbsp.sub_layer_ordering_info.sps_max_num_reorder_pics; let depth = *reorder.get(highest).or(reorder.last())?; - Some(crate::codec::video::Reorder { - depth: u32::try_from(depth).ok()?, + Some(Reorder { + depth: u32::try_from(depth).ok().filter(|depth| *depth <= Reorder::MAX_DEPTH)?, period: sps_period(&sps.rbsp), fields: false, }) diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index eacddcbba7c..66cb1c1f192 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -88,6 +88,10 @@ pub(crate) struct Reorder { } impl Reorder { + /// The deepest reordering a conforming sequence header declares: no decoded picture buffer + /// holds more than 16 frames (H.264 A.3.1, HEVC A.4.2). + pub(crate) const MAX_DEPTH: u32 = 16; + /// The most frames, each one picture, that can precede one in decode order and follow it in /// output order: the depth, or with fields both fields of each pair it counts and the other /// field of the frame's own pair. From db0ba7d374353cf98895cb1d61b2eb7f6ebc6dee Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 12:33:34 +0100 Subject: [PATCH 10/13] fix(mux): refuse an empty sync lane, skip SI still queued, flag tables on a jump Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 5 +- rs/moq-mux/src/container/ts/lane.rs | 80 ++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 5168a920c2f..b9f49b6a5e9 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -2454,9 +2454,10 @@ where /// Give each track its lane of `rate` under [`Self::with_sync`], beside the tables in /// `psi` and room for the SI, and lay every slot on them from now on. fn lay_lanes(&mut self, rate: u64, psi: &Psi) -> anyhow::Result<()> { + // Flagged, so each emission has the adaptation field a jump sets the flag in ([`Grid::lay`]). let mut tables = Vec::new(); - section_packets(&mut tables, Pid::PAT, &psi.pat, false)?; - section_packets(&mut tables, psi.pmt_pid, &psi.pmt, false)?; + section_packets(&mut tables, Pid::PAT, &psi.pat, true)?; + section_packets(&mut tables, psi.pmt_pid, &psi.pmt, true)?; let tables = tables.as_chunks::<{ TsPacket::SIZE }>().0.to_vec(); let mut needs: Vec<(u16, Need)> = self .tracks diff --git a/rs/moq-mux/src/container/ts/lane.rs b/rs/moq-mux/src/container/ts/lane.rs index 8661ad6a79b..d87c92b3464 100644 --- a/rs/moq-mux/src/container/ts/lane.rs +++ b/rs/moq-mux/src/container/ts/lane.rs @@ -17,7 +17,7 @@ //! opens an H.264 buffering period: the export then releases it late enough for its HRD to rule //! that out. -use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::Arc; use mpeg2ts::ts::TsPacket; @@ -104,7 +104,7 @@ impl Layout { taken * PACKET ); }; - let lanes = lanes + let lanes: Vec<(u16, u64)> = lanes .iter() .map(|&(pid, need)| { let packets = match need { @@ -114,6 +114,11 @@ impl Layout { (pid, packets) }) .collect(); + if let Some((pid, _)) = lanes.iter().find(|(_, packets)| *packets == 0) { + anyhow::bail!( + "MPEG-TS --sync cannot carry PID {pid:#x}: the rate its stream declares is less than a packet a second" + ); + } let tables = (1..=tables).map(|n| n * spacing).collect(); Ok(Self { rate, @@ -411,7 +416,8 @@ pub(super) struct Grid { placed: BTreeMap, /// Where each unit's first packet sits: its PID and whether it opens a keyframe. starts: BTreeMap, - /// The PAT then the PMT, numbered as they go out. + /// The PAT then the PMT, numbered as they go out. Each section's first packet carries an + /// adaptation field, whose `discontinuity_indicator` is set where the grid jumps. tables: Vec<[u8; TsPacket::SIZE]>, /// SI packets still to go out, in order, each no earlier than its position. si: VecDeque<(u128, [u8; TsPacket::SIZE])>, @@ -470,11 +476,17 @@ impl Grid { } } - /// Send `packets` of SI from slot `index` on, in the positions no lane owns. + /// Send `packets` of SI from slot `index` on, in the positions no lane owns. A PID whose + /// last emission is still queued sends none of this one, so SI the free positions cannot + /// carry skips emissions rather than piling up; each emission numbers from zero and flags + /// its break, so a skipped one leaves no gap. pub fn push_si(&mut self, index: u128, packets: &[u8]) { let from = self.layout.first(index); + let queued: HashSet = self.si.iter().map(|(_, packet)| pid(packet)).collect(); for packet in packets.as_chunks::<{ TsPacket::SIZE }>().0 { - self.si.push_back((from, *packet)); + if !queued.contains(&pid(packet)) { + self.si.push_back((from, *packet)); + } } } @@ -516,7 +528,8 @@ impl Grid { /// like the tables. /// /// Tables that start a timeline number as the last repetition before them did, so a - /// receiver tuning in on them counts on into the next. + /// receiver tuning in on them counts on into the next; after a jump they flag the break + /// from the repetition the stream carried last. pub fn lay(&mut self, index: u128) -> Laid { self.next = Some(index + 1); let jump = self.last.is_some_and(|last| index != last + 1); @@ -551,6 +564,9 @@ impl Grid { n => emission * (self.tables.len() as u128 - 1) + n as u128 - 1, }; packet[3] = (packet[3] & 0xf0) | (counter & 0x0f) as u8; + if packet[1] & 0x40 != 0 && packet[3] & 0x20 != 0 && packet[4] > 0 { + packet[5] = (packet[5] & !0x80) | u8::from(jump) << 7; + } packet }), Owner::Lane(_) => self.placed.remove(&at), @@ -626,12 +642,32 @@ mod test { fn tables() -> Vec<[u8; TsPacket::SIZE]> { let mut pat = [0xffu8; TsPacket::SIZE]; - pat[..4].copy_from_slice(&[0x47, 0x40, 0x00, 0x10]); + pat[..6].copy_from_slice(&[0x47, 0x40, 0x00, 0x30, 0x01, 0x80]); let mut pmt = pat; pmt[1..3].copy_from_slice(&[0x50, 0x00]); vec![pat, pmt] } + /// A jump numbers the tables on from a repetition the stream never carried, so they flag + /// the break in their counters as the PCR does; elsewhere they flag nothing. + #[test] + fn the_tables_flag_a_jump() { + let mut grid = Grid::new(Arc::new(layout()), tables()); + let flagged = |laid: Laid| -> Vec { + laid.packets + .chunks(TsPacket::SIZE) + .filter(|packet| matches!(pid(packet), 0x0000 | 0x1000)) + .map(|packet| packet[5] & 0x80 != 0) + .collect() + }; + assert_eq!(flagged(grid.lay(PSI_SLOTS + 3)), [false, false], "a timeline's start"); + assert_eq!(flagged(grid.lay(5 * PSI_SLOTS + 7)), [true, true], "a jump"); + for index in 5 * PSI_SLOTS + 8..6 * PSI_SLOTS { + assert!(flagged(grid.lay(index)).is_empty(), "slot {index} repeats nothing"); + } + assert_eq!(flagged(grid.lay(6 * PSI_SLOTS)), [false, false], "a repetition"); + } + #[test] fn the_tables_repeat_on_the_absolute_grid_and_lead_a_timeline() { let layout = layout(); @@ -691,6 +727,15 @@ mod test { assert!(err.to_string().contains("raise --mux-rate"), "{err}"); } + /// A lane of no packets would never own the positions a frame's packets wait for. + #[test] + fn a_lane_of_no_packets_is_refused() { + for need in [Need::Fixed(0), Need::Rest(0)] { + let err = Layout::new(RATE, 2, 80, &[(0x100, Need::Rest(u64::MAX)), (0x101, need)]).unwrap_err(); + assert!(err.to_string().contains("PID 0x101"), "{need:?}: {err}"); + } + } + #[test] fn positions_map_back_to_their_slots() { let layout = Layout::new(10_000_000, 2, 0, &[(0x100, Need::Rest(u64::MAX))]).unwrap(); @@ -824,4 +869,25 @@ mod test { "SI paced like the tables" ); } + + #[test] + fn si_the_free_positions_cannot_carry_does_not_pile_up() { + let layout = Layout::new(RATE, 2, 80, &[(0x100, Need::Rest(u64::MAX)), (0x101, Need::Fixed(422))]).unwrap(); + let mut grid = Grid::new(Arc::new(layout), tables()); + let mut si = [0xffu8; TsPacket::SIZE]; + si[..4].copy_from_slice(&[0x47, 0x40, 0x11, 0x10]); + let emission = si.repeat(40); + let mut sent = 0; + for index in 0..400 { + grid.push_si(index, &emission); + assert!(grid.si.len() <= 40, "slot {index}: {} SI packets queued", grid.si.len()); + let laid = grid.lay(index); + sent += laid + .packets + .chunks(TsPacket::SIZE) + .filter(|packet| pid(packet) == 0x11) + .count(); + } + assert!(sent >= 400, "the SI still fills the free positions: {sent} packets"); + } } From 79ae7c9f23605e63728bc8822daec69183c8ee70 Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 15:18:55 +0100 Subject: [PATCH 11/13] fix(mux): stop admitting SI once the last flush's media is out Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 4 +- rs/moq-mux/src/container/ts/export_test.rs | 67 ++++++++++++++++++++++ rs/moq-mux/src/container/ts/lane.rs | 7 ++- rs/moq-mux/src/container/ts/schedule.rs | 16 ++++-- 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index b9f49b6a5e9..c7f7cfd2b87 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -2953,9 +2953,11 @@ where { self.flush_packs(|_, due| due <= index + lead)?; } + // With nothing more coming, SI joins only slots the media still runs to: an entry due + // in every slot would otherwise keep the grid from emptying. if self.sync && let Some((index, _)) = self.schedule.upcoming() - && known.is_none_or(|known| index < known) + && known.map_or(!self.schedule.media_done(), |known| index < known) { self.queue_si(index)?; } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index f960dc6eae3..3f180f66dd3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -9775,6 +9775,8 @@ struct Placed { mux_rate: Option, /// Export with [`Export::with_sync`], which needs `mux_rate`. sync: bool, + /// Also carry an SDT from a snapshot track, re-emitted at this interval. + sdt: Option, } impl Placed { @@ -9788,6 +9790,7 @@ impl Placed { rendition: false, mux_rate: None, sync: false, + sdt: None, } } } @@ -9861,6 +9864,33 @@ async fn export_placed(placed: Placed) -> crate::Result> { } } let mut data = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + // Held open: dropping the producer ends the track before export subscribes. + let _sdt = placed.sdt.map(|interval| { + let mut track = broadcast.create_track("0x0011-0x42.si", timed()).unwrap(); + track + .write_frame( + Timestamp::ZERO, + Bytes::from(make_long_section(0x42, 1, 0, 0, 0, &[0xaa; 8])), + ) + .unwrap(); + catalog + .modify() + .unwrap() + .ext + .mpegts + .si + .entry(0x0011) + .or_default() + .insert( + 0x42, + tscat::SiEntry { + track: "0x0011-0x42.si".to_string(), + interval: Some(interval), + ..Default::default() + }, + ); + track + }); for (k, &pts) in placed.video.iter().enumerate() { let slice = vec![if k == 0 { 0x65u8 } else { 0x41 }; 400]; @@ -10144,6 +10174,43 @@ async fn placed_pes_goes_out_whole_at_its_placement_under_sync() { } } +/// Under `--sync`, an SI entry due in every slot goes out while the media does and stops with +/// it, so the flush at the end returns. +#[test] +fn si_due_every_slot_ends_with_the_media_under_sync() { + let (sent, received) = std::sync::mpsc::channel(); + // A thread of its own: a flush that never returns blocks the runtime that would time it out. + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .unwrap(); + let _ = sent.send(runtime.block_on(export_placed(Placed { + mux_rate: Some(2_000_000), + sync: true, + sdt: Some(Duration::ZERO), + ..Placed::new(vec![(1_000, pes_bare(0xbf, b"private_stream_2"))]) + }))); + }); + let slices = received + .recv_timeout(Duration::from_secs(5)) + .expect("the flush at the end never returned") + .unwrap(); + let sdt = slices + .iter() + .filter(|slice| { + slice + .payload + .as_chunks::<188>() + .0 + .iter() + .any(|packet| (u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2])) == 0x0011) + }) + .count(); + assert!(sdt > 1, "the SDT repeats with the media: {sdt} slices"); +} + /// Tune-in drops a PES placed before the keyframe departs, and keeps one placed beside it, /// after it departs but before its PTS, as well as one placed after it among the media, /// each in the slot of its placement. Comparing placement with the keyframe's PTS would drop diff --git a/rs/moq-mux/src/container/ts/lane.rs b/rs/moq-mux/src/container/ts/lane.rs index d87c92b3464..a70ca46f61d 100644 --- a/rs/moq-mux/src/container/ts/lane.rs +++ b/rs/moq-mux/src/container/ts/lane.rs @@ -492,7 +492,12 @@ impl Grid { /// Whether anything placed is still to go out. pub fn is_empty(&self) -> bool { - self.placed.is_empty() && self.si.is_empty() + !self.holds_media() && self.si.is_empty() + } + + /// Whether media placed is still to go out, whatever SI is queued. + pub fn holds_media(&self) -> bool { + !self.placed.is_empty() } /// The slot [`Self::lay`] lays next: on from the last, else the one before the first holding diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index eab6202241f..d9f28b5d0fa 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -814,10 +814,7 @@ impl Schedule { /// Whether nothing is left to lay out: every unit sent, and the clock past its decode time. pub fn is_empty(&self) -> bool { if let Some(grid) = &self.sync { - return grid.is_empty() - && self - .horizon - .is_none_or(|horizon| grid.index().is_none_or(|next| next > horizon)); + return grid.is_empty() && self.media_done(); } self.units.is_empty() && self @@ -825,6 +822,17 @@ impl Schedule { .is_none_or(|horizon| self.next.is_some_and(|next| next > horizon)) } + /// Under [`Self::sync`], whether every media packet placed has gone out and the clock is past + /// its decode time, whatever SI is still queued. + pub fn media_done(&self) -> bool { + self.sync.as_ref().is_some_and(|grid| { + !grid.holds_media() + && self + .horizon + .is_none_or(|horizon| grid.index().is_none_or(|next| next > horizon)) + }) + } + /// Drop everything queued and start the grid afresh. pub fn clear(&mut self) { if let Some(grid) = &mut self.sync { From 75d2e322f82a4363624b9bccde7764e91775c89d Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 16:43:27 +0100 Subject: [PATCH 12/13] test(mux): require the SI flush test's export to end Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export_test.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 3f180f66dd3..5490c10ce3b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -9777,6 +9777,8 @@ struct Placed { sync: bool, /// Also carry an SDT from a snapshot track, re-emitted at this interval. sdt: Option, + /// Finish the catalog once everything is written, so the export must end rather than stall. + ends: bool, } impl Placed { @@ -9791,6 +9793,7 @@ impl Placed { mux_rate: None, sync: false, sdt: None, + ends: false, } } } @@ -9938,6 +9941,9 @@ async fn export_placed(placed: Placed) -> crate::Result> { } audio.finish().unwrap(); data.finish().unwrap(); + if placed.ends { + catalog.finish().unwrap(); + } let mut exporter = open_ts( crate::source::announced(&consumer).await, @@ -9953,13 +9959,16 @@ async fn export_placed(placed: Placed) -> crate::Result> { exporter = exporter.with_sync(); } let mut out = Vec::new(); - while let Ok(next) = tokio::time::timeout(DRAIN, exporter.next()).await { + loop { + let Ok(next) = tokio::time::timeout(DRAIN, exporter.next()).await else { + assert!(!placed.ends, "the export stalled instead of ending"); + return Ok(out); + }; match next? { Some(frame) => out.push(frame), - None => break, + None => return Ok(out), } } - Ok(out) } /// Where a PES went out: the slice it began in, its first byte's offset into the slice, and @@ -10175,11 +10184,12 @@ async fn placed_pes_goes_out_whole_at_its_placement_under_sync() { } /// Under `--sync`, an SI entry due in every slot goes out while the media does and stops with -/// it, so the flush at the end returns. +/// it, so the export ends. #[test] fn si_due_every_slot_ends_with_the_media_under_sync() { let (sent, received) = std::sync::mpsc::channel(); // A thread of its own: a flush that never returns blocks the runtime that would time it out. + // A stall panics the thread instead, which drops `sent`. std::thread::spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -10190,12 +10200,13 @@ fn si_due_every_slot_ends_with_the_media_under_sync() { mux_rate: Some(2_000_000), sync: true, sdt: Some(Duration::ZERO), + ends: true, ..Placed::new(vec![(1_000, pes_bare(0xbf, b"private_stream_2"))]) }))); }); let slices = received .recv_timeout(Duration::from_secs(5)) - .expect("the flush at the end never returned") + .expect("the export never ended") .unwrap(); let sdt = slices .iter() From 87d7e7d81a8d9aa67978bede80b69383033e6d9d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 23:02:43 -0700 Subject: [PATCH 13/13] fix(mux): find the sync PCR a PID below the streams when none is above Under --sync the dedicated PCR PID was searched only above the highest elementary-stream PID, so a program with a stream on 0x1FFE failed to export though lower PIDs were free. The search now wraps to FIRST_ES_PID, still deterministic across legs. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 4 +++- rs/moq-mux/src/container/ts/export_test.rs | 24 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 26539fee884..8778f8e5cf7 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -2452,7 +2452,8 @@ where } /// The PID the PCR rides alone under [`Self::with_sync`]: the first above the program's - /// elementary streams that nothing else in it uses, so every leg picks the same one. + /// elementary streams that nothing else in it uses, else the first below them, so every + /// leg picks the same one. fn clock_pid(&self) -> anyhow::Result { let used: HashSet = self .tracks @@ -2468,6 +2469,7 @@ where .max() .unwrap_or(FIRST_ES_PID); (top + 1..0x1FFF) + .chain(FIRST_ES_PID..top) .find(|pid| !used.contains(pid)) .context("MPEG-TS --sync found no PID left for the PCR") } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index c61e5b38e80..895d4b725ce 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -6177,6 +6177,30 @@ async fn sync_refuses_a_later_si_entry_on_the_clock_pid() { assert!(err.to_string().contains("mpegts.si PID"), "{err}"); } +/// Under `--sync` a program whose streams take the top PID still finds a free one below +/// them for the PCR. +#[tokio::test(start_paused = true)] +async fn sync_clock_pid_falls_below_the_streams() { + let mut rig = si_cadence_rig(0x0011, 0x42, Duration::from_secs(2)).await; + { + let mut guard = rig.catalog.modify().unwrap(); + let name = guard.video.renditions.keys().next().unwrap().clone(); + guard.ext.mpegts.tracks.insert(name, tscat::Track::new(0x1FFE)); + } + rig.exporter = rig.exporter.with_mux_rate(2_000_000).with_sync(); + let mut frames = Vec::new(); + for millis in (0..1_000).step_by(40) { + frames.extend(rig.media_frames(millis).await); + } + let clock = frames + .iter() + .flat_map(|frame| frame.payload.chunks(188)) + .find(|packet| packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0) + .map(|packet| u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2])) + .expect("the tables and the clock went out"); + assert_eq!(clock, 0x1001); +} + /// #2934: a revised SI snapshot goes out on the revision floor instead of waiting /// out the repetition interval (here the floor has long elapsed, so it rides the /// very next frame). For a clock table (TDT/TOT) the old interval-grid hold