diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 5ae868e04f0..5eeb015d1ce 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -72,6 +72,15 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} + # A KLV stream joins the program mid-stream and leaves it: the export announces each + # layout at the next PMT version, and carries the stream only while a PMT lists it. + - name: TS PMT revision + if: ${{ !cancelled() }} + run: nix develop --command just test ts --revision --strict + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + - name: TS audio-only SCTE-35 if: ${{ !cancelled() }} run: nix develop --command just test ts --clockless audio-scte35 diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 773422db2b9..0f38d8c3f04 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -115,8 +115,10 @@ supplies a PES boundary or clock fails rather than retaining bytes indefinitely. It takes one program: a multi-program stream is refused until `--program` picks one, or `--program all` publishes each as its own broadcast (`event.hang` becomes `event/1.hang`, -`event/2.hang`). A damaged packet is dropped on its own PID, so video freezes -until its next keyframe while the other tracks carry on. The importer also +`event/2.hang`), following the PAT: a program it adds publishes, one it drops +ends, and one that returns publishes as a new instance. A damaged packet is +dropped on its own PID, so video freezes until its next keyframe while the +other tracks carry on. The importer also logs feed health (stalled streams, TR 101 290 errors) without changing what it publishes. `--passthrough` publishes the multiplex whole instead, every packet in order on one [`m2ts`](/concept/hang#transport-streams) track with every diff --git a/doc/bin/srt.md b/doc/bin/srt.md index 1e0069edfc4..8de4d1518ec 100644 --- a/doc/bin/srt.md +++ b/doc/bin/srt.md @@ -37,7 +37,8 @@ With the default `--linger 0s`, that catalog must already be available. A multi-program feed is refused unless `--program` picks one: `--program 2` imports program 2 alone, and `--program all` publishes each program as its own broadcast (`event.hang` becomes -`event/1.hang`, `event/2.hang`, and so on). +`event/1.hang`, `event/2.hang`, and so on), following the PAT as programs +join and leave. ```bash moq --connect https://relay.example.com/anon --broadcast event.hang import srt --listen '[::]:9000' --program all diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index c91e3683676..94dbced6884 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -162,6 +162,7 @@ They are the only service-layer fields this document parses; everything else abo A publisher MUST include `program` when the broadcast came from a transport stream, and omit it otherwise. A publisher MUST NOT change `transportStreamId` or `programNumber` in place: the carried `si` describes that service, so a source whose PAT names another ends the broadcast. +A PAT that moves the PMT to another PID changes `pmtPid` in place. ## si {#field-si} The standalone service information tables, keyed by the PID they ride on and then by `table_id`: @@ -372,6 +373,7 @@ A consumer rebuilding a transport stream: - MUST place each track on its recorded `pid`, and MUST assign an unused PID to a track with none. - MUST put the PMT on `program.pmtPid` and build the PAT and PMT from `program`. - MUST choose a PCR PID and emit PCRs on it at the program clock's repetition interval. With no media stream it MUST use an unused PID with adaptation-only PCR packets and no PMT elementary-stream entry. It MUST NOT refuse a program for lacking video. +- MUST follow the catalog as it changes: when a track joins or leaves `tracks` or `programDescriptors` changes, it MUST emit a PMT at the next `version_number` (modulo 32) before any packet of a stream it adds, and MUST NOT emit a stream's packets after a PMT that no longer lists it. Each version it emits MUST differ from the one before, and no older version may follow a newer one. The PCR PID MUST NOT move until the stream it rides ends. When `program.pmtPid` changes, it MUST emit a PAT at the next `version_number` naming the new PID, and MUST NOT emit the PMT on the new PID before that PAT or on the old one after it. - MUST re-emit each track's `descriptors` as its ES-level descriptors, and `programDescriptors` as the PMT's `program_info`. - MUST re-emit each `si` entry's sections byte-for-byte on that entry's PID, reading them from its track ({{si-track}}), at least as often as its `interval` when declared. - MUST packetize each `verbatim` frame as one section on its track's PID. @@ -457,6 +459,8 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. - An `m2ts` track unwraps PCRs from its history, clamps extrapolated object times at a forward discontinuity, and cuts a fallback group after one second. `randomAccess` covers every published group; `muxRate` follows the measured source rate. - A `tracks` entry is never changed in place: a stream the PMT revises continues under a new name, and one it drops leaves the section. `programDescriptors` follows the PMT; `program`'s `transportStreamId` and `programNumber` do not change. +- A consumer rebuilding a stream follows the catalog's revisions at the next PMT `version_number`, and carries a stream only while a PMT lists it. +- `pmtPid` follows a PAT that moves the PMT, and a consumer rebuilding a stream moves the PMT behind a PAT at the next `version_number`. ## draft-lcurley-moq-mpegts-00 {:numbered="false"} diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 525abf4dd6f..2992a761385 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -73,8 +73,8 @@ refused means an error or a counted drop. | CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted when the import reads every program. Today dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | | DVB SI on 0x0010, 0x0011, 0x0012, 0x0014 | Preserved: snapshot track per PID and `table_id` | landed | | Program identity, program and ES descriptors | Preserved: descriptors follow each PMT revision; an identity change refuses | landed | -| PMT add, remove, retype; PID reuse | Normalized without a new epoch: a changed or reused PID ends its track and continues under a fresh name, and the PMT is rebuilt at the next `version_number`. Import landed; today refusal or stale PMT on export | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | -| PAT changes: moved PMT PID, programs added or removed | Normalized: followed; a removed program retires, and returns as a new epoch. Today ignored, or the whole multiplex stops | [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) | +| PMT add, remove, retype; PID reuse | Normalized without a new epoch: a changed or reused PID ends its track and continues under a fresh name, and the export rebuilds the PMT at the next `version_number`. Today `export ts --sync` ends at an added track and announces no other revision | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | +| PAT changes: moved PMT PID, programs added or removed | Normalized: a moved PMT is followed, and export moves it behind a PAT at the next `version_number`; under `--program all` an added program publishes, a removed one retires, and one returning publishes a new epoch. A listed program whose PMT never arrives waits, unannounced | landed | | Programs without video | Preserved when the PMT names a PCR PID; refused without one | landed | | PCR | Normalized: rebased, carried on the first video PID, else audio, else a dedicated PCR-only PID | landed | | Mux rate, null stuffing | Normalized: re-padded to `muxRate` while the source is CBR | landed | @@ -88,8 +88,7 @@ remain with their current quests. ## Required -- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - a TS export announces each PMT revision at the next `version_number` -- [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program +- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - synced TS legs revise their PMT identically - [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted - [Release clock false alarms](/quest/m1/ts-reliability/release-clock-false-alarms.md) - no false jitter leap at a 500 ms delay and no false drift alarm diff --git a/quest/m1/ts-reliability/pat-lifecycle.md b/quest/m1/ts-reliability/pat-lifecycle.md deleted file mode 100644 index 3998e01bae2..00000000000 --- a/quest/m1/ts-reliability/pat-lifecycle.md +++ /dev/null @@ -1,48 +0,0 @@ -# [M] Live PAT changes keep programs honest - -## Goal - -A live TS source whose PAT changes mid-stream keeps every broadcast meaning -one program. A moved PMT PID, a program added, and a program removed or -returning each have one observable outcome: followed, published, retired, -or published as a new epoch. None is silently ignored. - -## Plan - -What the code does today: - -- A PAT naming another PMT PID for the program adds a reassembler but never - drops the old one, and `program.pmtPid` keeps the first. -- `Programs` ignores a program a later PAT adds, without a warning. -- Under `--program all`, a later PAT that removes one program reaches every - per-program `Import`; the removed program's `handle_pat` errors, - `Programs::feed` propagates it, and the whole multiplex ingest stops, - including unrelated programs. -- A PAT naming another `transport_stream_id`, or a single-program import's - PAT naming another program, already refuses the import. - -Decided 2026-10-20 by the maintainer: - -- A new PMT PID for the same program moves the reassembler and updates - `program.pmtPid`, which the export follows with a new PAT. `Programs` - publishes a program a later PAT adds as its own broadcast, as it does for - the first PAT, rather than ignoring it. -- A later PAT that removes a program retires that program's broadcast: its - tracks finish and it is unannounced, while the other programs carry on. - If the program returns, it publishes a new epoch of its broadcast, never - the retired instance's names for new content. A single-program import - whose selected program leaves the PAT ends, as today. -- Open: a PAT that names a program whose PMT never arrives. Decide whether - the import refuses it or waits, and which clock bounds the wait (found in - [moq-dev/moq#6122](https://github.com/moq-dev/moq/pull/6122)). - -Rust: `rs/moq-mux` TS `Import::handle_pat` and `Programs`. - -Tests use the generated `fixture::Mux` builder: a PMT PID move, and a -`--program all` MPTS whose PAT adds a program, drops one and later restores -it: the other program runs uninterrupted, and the returning one publishes a -new epoch. - -## Related - -- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - the export follows a moved PMT PID with a new PAT diff --git a/quest/m1/ts-reliability/pmt-export.md b/quest/m1/ts-reliability/pmt-export.md index 3b44a2a8949..2b635d3ca1b 100644 --- a/quest/m1/ts-reliability/pmt-export.md +++ b/quest/m1/ts-reliability/pmt-export.md @@ -1,54 +1,41 @@ -# [M] TS export follows a PMT revision +# [M] Synced TS legs revise their PMT identically ## Goal -A TS export of a broadcast whose layout changes mid-stream announces the -revised layout as an ordinary PMT revision. A track added, retired, or -replaced under a fresh name changes the PMT at the next `version_number`; -nothing else blips. +Two `moq export ts --sync` legs of a broadcast whose layout changes +mid-stream revise their PMT identically: the same `version_number`, in the +same slot, listing the same streams, wherever both legs hold the content +around it. A track added no longer ends a synced export. ## Plan -The import already keeps track names honest: a changed `stream_type` or ES -descriptor loop finishes the old track, drops its catalog entry, and opens -a new uniquely named one; a removed stream finishes its track and leaves -the catalog; `programDescriptors` updates in place; a program identity -change refuses. +Outside `--sync`, the export follows a PMT revision: it rebuilds the PMT at +the next `version_number` when the track set or an entry changes, drops a +retired track's PID once its track ends, and keeps the PCR PID unless its +track retires. -What the export does today: +What `--sync` does today: -- `Export` builds the PAT/PMT once. A track added afterward fails the export - ("TS track layout changed after PAT/PMT was emitted"), so every revision - the import publishes ends the export. -- A track that leaves the catalog keeps its PID in the PMT forever, and - `programDescriptors` is read once. -- A switch to another instance already rebuilds the tables at the next - `version_number`. +- The tables are laid once, from the catalog the leg opened on. A track + added afterward fails the export ("… under --sync"). +- A track that leaves the catalog keeps its PID and its lane, and an entry + or `programDescriptors` change is not announced. +- So legs that open on either side of a catalog change lay different PMTs, + and a pair does not survive an added stream. -Decided 2026-10-20 by the maintainer: +Decided 2026-10-22 by the maintainer: under `moq export ts --sync`, the +revision is a function of the catalog alone, so identical legs revise +their PMT identically. -- Export rebuilds the PMT at the next `version_number` when the track set - or an entry changes, drops a retired track's PID once its track ends, and - keeps the PCR PID unless its track retires. A TS receiver sees an - ordinary PMT revision and a Hang player sees a rendition switch. +Open: where a revision lands. Each leg reads the catalog update at its own +moment, so the slot that carries the revised PMT, and its `version_number`, +must follow from something both legs read alike, such as an added track's +first unit on its lane or the end of a retired track's last group. A +revision that adds a track also needs a lane the rate can carry. -A `programDescriptors` change is an entry change too: the PMT carries the -revised `program_info` at the next `version_number`. +Rust: `rs/moq-mux` TS export (`with_sync`). -Under `moq export ts --sync`, the revision is a function of the catalog -alone, so identical legs revise their PMT identically. - -Rust: `rs/moq-mux` TS export. Draft -([mpegts](/drafts/draft-lcurley-moq-mpegts.md)): the rebuild section says -how a consumer revises the PMT. - -Tests use the generated `fixture::Mux` builder through import and export: -PMT add, remove, retype, descriptor-only change, version rollover (31 to -0), PID reuse after removal, and a program descriptor change, each -asserting the exported PMT's streams and version sequence. Extend the -TSDuck round trip with a generated mid-stream PMT revision so -`compliance.py` checks the rebuilt version sequence in CI. - -## Related - -- [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a moved PMT PID reaches the export as a new PAT +Tests: two synced legs, joined apart, across a PMT add, remove, retype and +descriptor change, write the same packets from the revision on, PMTs +included; a leg that joins after a revision lays the revised PMT at the +same `version_number`. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 06163cb48a9..f86b4831d90 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -125,22 +125,26 @@ where /// Tracks [`Self::follow`] left on the ended broadcast, resubscribed as the returned /// catalog lists them. stale: HashSet, - /// The PAT and PMT `version_number`, advanced by each switch to another instance so a - /// demux that caches tables by version reads the new ones. + /// The PAT `version_number`, advanced by each switch to another instance so a demux that + /// caches tables by version reads the new ones. version: VersionNumber, + /// The PMT's, advanced by each switch too, and by each revision of the program ([`Self::revise`]). + pmt_version: VersionNumber, /// After a switch to another instance, the PIDs whose first packet is queued flagging the /// break. `None` for an export that never switched. flagged: Option>, /// How long after its decode time each frame goes out. delay: Duration, - /// Holds every track's frames until `delay` past their decode time, keyed by PID. - jitter: jitter::Buffer, + /// Holds every track's frames until `delay` past their decode time. + jitter: jitter::Buffer, /// A frame the jitter buffer let go, waiting for the tail of the generation before it to go out. - held: Option>, + held: Option>, /// Jitter-buffer generation of the program being muxed; a newer one rewinds it. generation: u64, tracks: HashMap, + /// How many tracks the export has taken on, which numbers the next ([`Key`]). + serials: u64, /// Tables muxed with frames still packed, which ride ahead of the next unit queued, so /// they go out in the order they were written. tables: Vec, @@ -276,8 +280,28 @@ struct Synced { /// the media timeline, and its [`Track::psi_slot`]. type PesStart = (u128, Option); +/// A track's key in the jitter buffer: its PID, then the order the export took it on in. A +/// track taking over the PID of one still going out to its end is a track of its own there, +/// and waits for that one to send everything ([`Export::take_over`]). +type Key = (u16, u64); + struct Track { source: ExportSource, + /// The order the export took it on in ([`Key`]). + serial: u64, + /// The PMT lists it: from the program tables for a track they were built with, else from + /// its first frame out, so a track taking over a PID is announced after the tail of the + /// one before it ([`Export::revise`]). + listed: bool, + /// The last catalog snapshot lists it. + cataloged: bool, + /// The program descriptors the PMT carried when it left the catalog while listed, which + /// the PMT carries beside the current ones while it drains ([`Export::table`]). + left_under: Vec, + /// It left the catalog while listed, and its stream beside the current ones outgrew a + /// section: the PMT drops it, nothing more of it goes out, and it retires at once + /// ([`Export::table`]). + cut: bool, /// The first frame, held until the program tables are built. pending: Option, /// When the source was subscribed. Nothing it delivers reached the export before then, @@ -333,7 +357,7 @@ struct Track { struct Input<'a> { waiting_for_header: bool, video_start: Option, - jitter: &'a mut jitter::Buffer, + jitter: &'a mut jitter::Buffer, /// Counts the frames dropped ahead of the tune-in point or their codec config. trimmed: &'a mut u64, } @@ -343,6 +367,11 @@ impl Track { fn new(source: ExportSource, kind: Kind) -> Self { Self { source, + serial: 0, + listed: false, + cataloged: true, + left_under: Vec::new(), + cut: false, pending: None, subscribed: moq_clock::now(), latest: moq_clock::Instant::ZERO, @@ -367,12 +396,16 @@ impl Track { } } + fn key(&self) -> Key { + (self.pid, self.serial) + } + /// Queue a frame read from the source in the jitter buffer, once its decode time is known. /// /// The decode clock runs here, as frames are read, because the deadline is on decode time: /// reordered video arrives in decode order with PTS 0, 120, 40, 80, so a PTS deadline would /// strand a B-frame behind its reference or send it first. - fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) -> anyhow::Result<()> { + fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) -> anyhow::Result<()> { if pending.restart != self.restart { // A marker declared a break in the timeline, so the decode clock restarts too. self.release(name, jitter, true)?; @@ -401,7 +434,7 @@ impl Track { && self.sync.is_none() && !radl(stream_type, self.source.description(), &pending.frame) { - jitter.lose(&self.pid, ahead); + jitter.lose(&self.key(), ahead); return Ok(()); } // Frames still held from a group cut short wait on frames that will not come, so under @@ -428,7 +461,7 @@ impl Track { // A placed frame is judged at release, against the keyframe's departure ([`Export::leads_video`]). let aligned = !is_video && !self.kind.placed(); loop { - if !input.waiting_for_header && input.jitter.full(&self.pid) { + if !input.waiting_for_header && input.jitter.full(&self.key()) { break; } match self.source.poll_event(waiter)? { @@ -442,7 +475,8 @@ impl Track { // A group arrives from its first frame, so the first read from it opens it. let sequence = self.source.group().unwrap_or_default(); let opens = self.group.replace(sequence) != Some(sequence); - if input.waiting_for_header && !self.source.header_ready() { + // A track taken on after the tables waits for its config as the first ones did. + if (input.waiting_for_header || !self.listed) && !self.source.header_ready() { *input.trimmed += 1; continue; } @@ -794,7 +828,7 @@ impl Track { /// /// Under sync, a stream whose SPS declares its depth is settled by counting frames at the /// SPS's delay, so a frame's DTS never depends on what the leg read before it. - fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) -> anyhow::Result<()> { + fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) -> anyhow::Result<()> { let (delay, lookahead) = self.timing.reorder(); let lookahead = lookahead.max(self.timing.jitter); let counted = self.sync.as_ref().and(self.timing.depth()); @@ -817,7 +851,7 @@ impl Track { pending: Pending, dts: Option, description: Option, - jitter: &mut jitter::Buffer, + jitter: &mut jitter::Buffer, ) -> anyhow::Result<()> { let Pending { frame, @@ -879,7 +913,7 @@ impl Track { planned, }, }; - if jitter.push(self.pid, arrival)? == Push::Late { + if jitter.push(self.key(), arrival)? == Push::Late { tracing::warn!( track = %name, dropped = jitter.dropped(), @@ -1057,8 +1091,18 @@ impl Kind { /// The program tables plus the resolved PID layout. struct Psi { pat: Bytes, + /// The PAT last laid into a slot, by version and the PMT PID it names; `None` before the first. + pat_laid: Option<(VersionNumber, u16)>, pmt: Bytes, + /// What `pmt` encodes, which a revision is compared with. + table: super::psi::Pmt, + /// The PMT last laid into a slot and its version, `None` before the first. + laid: Option<(VersionNumber, super::psi::Pmt)>, + /// The PID each slot's PCR goes out on. pcr_pid: u16, + /// The PID a revision moves the PCR to, until a slot lays a PMT naming it: the PCR stays on + /// `pcr_pid` until then, where the PMT a receiver holds places it. + announced: Option, /// PID the PMT rides on: the source's original (preserved from the service /// record) or the synthesized [`PMT_PID`] for a media-only source. pmt_pid: u16, @@ -1349,6 +1393,7 @@ where instance, stale: HashSet::new(), version: VersionNumber::default(), + pmt_version: VersionNumber::default(), flagged: None, tables: Vec::new(), delay: Duration::ZERO, @@ -1356,6 +1401,7 @@ where held: None, generation: 0, tracks: HashMap::new(), + serials: 0, counters: HashMap::new(), program_descriptors: Vec::new(), program: None, @@ -1581,9 +1627,11 @@ where // 4. Mux each frame the jitter buffer lets go (the first carries the buffered // PAT/PMT), then lay out every grid slot whose time has come ([`Self::lay_due`]). loop { + self.retire()?; if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); } + self.take_over(); let ready = match self.held.take() { Some(ready) => ready, None => match self.jitter.poll_next(waiter) { @@ -1609,9 +1657,9 @@ where let (name, kind) = self .tracks .iter() - .find(|(_, t)| t.pid == ready.track) + .find(|(_, t)| t.key() == ready.track) .map(|(name, t)| (name.clone(), t.kind.clone())) - .context("frame for an unknown PID")?; + .context("frame for an unknown track")?; if self.leads_video(&kind, &ready.item) { self.jitter.discard(&ready); } else { @@ -1655,6 +1703,7 @@ where self.flush_packs(|_, _| true)?; } self.lay_due(waiter)?; + self.retire()?; if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); } @@ -1770,7 +1819,7 @@ where reject_ambiguous_tracks(&catalog, &mpegts)?; self.source.retain_selectable(&mut catalog); - self.program_descriptors = mpegts.program_descriptors.clone(); + let descriptors = std::mem::replace(&mut self.program_descriptors, mpegts.program_descriptors.clone()); self.program = mpegts.program.clone(); // An explicit override wins even when it is refused (leaving the output // unpadded), so a bad flag cannot silently fall back to the catalog rate. @@ -1847,35 +1896,47 @@ where } } - // The program tables are written once; reject a track added afterwards. + // Once the program tables are out, a track added is taken on and announced by a PMT + // revision ([`Self::revise`]), except under sync, whose tables are laid once. // - // A track that leaves the catalog is not a layout change: it keeps its PID and is - // read to its own end. A publisher retires a rendition as its track finishes or - // drops, and the catalog update races that end on another stream, so the leaving - // cannot say which it was. The track's end does: a finish ends it cleanly, and a - // drop is the error the export reports. + // A track that leaves the catalog is read to its own end, keeping its PID until then + // ([`Self::retire`]). A publisher retires a rendition as its track finishes or drops, + // and the catalog update races that end on another stream, so the leaving cannot say + // which it was. The track's end does: a finish ends it cleanly, and a drop is the + // error the export reports. if self.psi.is_some() { - for name in active.keys() { - anyhow::ensure!( - self.tracks.contains_key(name), - "TS track layout changed after PAT/PMT was emitted: '{name}' added" - ); + for (name, track) in self.tracks.iter_mut() { + let cataloged = active.contains_key(name); + if track.listed && track.cataloged && !cataloged { + track.left_under = descriptors.clone(); + } + track.cataloged = cataloged; + } + if self.sync { + for name in active.keys() { + anyhow::ensure!( + self.tracks.contains_key(name), + "TS track layout changed after PAT/PMT was emitted under --sync: '{name}' added" + ); + } } if !self.stale.is_empty() { self.resubscribe(&catalog, &mpegts)?; } + if !self.sync { + self.admit(&catalog, &mpegts, active.keys())?; + } // The PCR may ride a PID of its own, which SI must not take either. let mut es_pids: Vec = self.tracks.values().map(|t| t.pid).collect(); es_pids.push(self.psi.as_ref().unwrap().pcr_pid); reject_colliding_si_pids(&mpegts, self.pmt_pid(), &es_pids)?; - // The layout is locked but the timing is not: a `jitter` or `framerate` published - // after the tables still sizes the decode clock. + // A `jitter` or `framerate` published after the tables still sizes the decode clock. for (name, config) in catalog.video.renditions.iter() { if let Some(track) = self.tracks.get_mut(name) { track.timing.configure(config, name); } } - return Ok(()); + return self.revise(); } // Assign a PID to every desired track: prefer the original recorded in the @@ -1909,37 +1970,54 @@ where // Reuse each track's existing source (and any pending frame) by name; refresh // its PID, kind, and descriptors from this snapshot. Drop tracks no longer present. let mut old = std::mem::take(&mut self.tracks); - for (name, config) in catalog.video.renditions.iter() { + for name in active.keys() { + if let Some(mut track) = self.configure(name, old.remove(name), &catalog, &mpegts)? { + track.pid = pids[name]; + self.tracks.insert(name.clone(), track); + } + } + self.expect_continuous(); + Ok(()) + } + + /// `name`'s track, configured from this snapshot: `track`, else a new one reading the + /// track from the live edge. `None` for a rendition [`ExportSource`] cannot read. + fn configure( + &mut self, + name: &str, + track: Option, + catalog: &Catalog, + mpegts: &catalog::Mpegts, + ) -> anyhow::Result> { + let budget = self.budget(); + let mut track = if let Some(config) = catalog.video.renditions.get(name) { let kind = video_kind(config, name)?; - let mut track = match old.remove(name) { + let mut track = match track { Some(track) => track, - None => match ExportSource::for_video(&self.source, name, config, self.budget())? - .map(|s| at_edge(s, self.replay)) - { - Some(source) => Track::new(source, kind.clone()), - None => continue, + None => match ExportSource::for_video(&self.source, name, config, budget)? { + Some(source) => Track::new(at_edge(source, self.replay), kind.clone()), + None => return Ok(None), }, }; track.kind = kind; track.timing.configure(config, name); - self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); - } - for (name, config) in catalog.audio.renditions.iter() { + track + } else if let Some(config) = catalog.audio.renditions.get(name) { let kind = audio_kind(config, name)?; - let mut track = match old.remove(name) { + let mut track = match track { Some(track) => track, - None => match ExportSource::for_audio(&self.source, name, config, self.budget())? - .map(|s| at_edge(s, self.replay)) - { - Some(source) => Track::new(source, kind.clone()), - None => continue, + None => match ExportSource::for_audio(&self.source, name, config, budget)? { + Some(source) => Track::new(at_edge(source, self.replay), kind.clone()), + None => return Ok(None), }, }; track.buffer = Some(audio_buffer(config, &kind)); track.kind = kind; - self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); - } - for (name, entry) in mpegts.tracks.iter() { + track + } else { + let Some(entry) = mpegts.tracks.get(name) else { + return Ok(None); + }; let kind = match (&entry.verbatim, &entry.pes_packet) { (Some(verbatim), _) => Kind::Verbatim { stream_type: verbatim.stream_type, @@ -1947,37 +2025,104 @@ where (None, Some(pes)) => Kind::PesPacket { stream_type: pes.stream_type, }, - (None, None) => continue, + (None, None) => return Ok(None), }; - let mut track = match old.remove(name) { + let mut track = match track { Some(track) => track, - None => Track::new( - ExportSource::for_stream(&self.source, name, self.budget())?, - kind.clone(), - ), + None => Track::new(ExportSource::for_stream(&self.source, name, budget)?, kind.clone()), }; track.buffer = verbatim_buffer(&kind, &entry.descriptors); track.kind = kind; - self.refresh(name, track, pids[name], entry.descriptors.clone()); + track + }; + if track.serial == 0 { + self.serials += 1; + track.serial = self.serials; + } + track.descriptors = track_descriptors(mpegts, name); + Ok(Some(track)) + } + + /// Take on each of `names` the export does not carry yet, and refresh each it carries + /// that the PMT does not list yet, as before the tables. A listed track keeps its PID, kind + /// and descriptors, which its packets went out under: a revised stream comes under a new + /// name, as the import publishes it, and takes the PID over. The PMT lists one from its + /// first frame out ([`Self::mux`]). + fn admit<'a>( + &mut self, + catalog: &Catalog, + mpegts: &catalog::Mpegts, + names: impl Iterator, + ) -> anyhow::Result<()> { + for name in names { + let track = match self.tracks.remove(name) { + Some(track) if track.listed => Some(track), + track => { + let current = track.as_ref().map(Track::key); + match self.configure(name, track, catalog, mpegts)? { + Some(mut track) => { + track.pid = self.pid_for(mpegts.tracks.get(name).map(|entry| entry.pid), current)?; + if let Some(current) = current.filter(|¤t| current != track.key()) { + self.jitter.rekey(¤t, track.key()); + if let Some(held) = self.held.as_mut().filter(|held| held.track == current) { + held.track = track.key(); + } + } + Some(track) + } + None => None, + } + } + }; + if let Some(track) = track { + self.tracks.insert(name.clone(), track); + } } - // The clock anchors on whichever track a source sends latest against its decode time - // (audio just in time while video runs most of a second ahead), so it waits to hear - // from each continuous one: audio, video, and the passthrough streams the schedule - // models (DVB AC-3, teletext). Not from sparse ones such as SCTE-35, subtitles or ID3, - // which would hold every join for the full bound. + self.expect_continuous(); + Ok(()) + } + + /// The PID for a track the PMT does not list yet, which has `current` if it was taken on + /// before: the one its entry `recorded`, unless the tables or a track still cataloged + /// hold it, else the one it has, else the first one nothing in the program holds. A + /// track that left the catalog holds its PID only for itself, so one replacing it on + /// the PID takes the PID over ([`Self::retire`]). + fn pid_for(&self, recorded: Option, current: Option) -> anyhow::Result { + let pmt_pid = self.pmt_pid(); + let pcr_pid = self.psi.as_ref().map(|psi| psi.pcr_pid); + let others = || self.tracks.values().filter(move |track| Some(track.key()) != current); + let reserved = |pid: u16| pid == 0 || pid == pmt_pid || pid >= 0x1FFF; + if let Some(pid) = + recorded.filter(|&pid| !reserved(pid) && !others().any(|track| track.cataloged && track.pid == pid)) + { + return Ok(pid); + } + if let Some((pid, _)) = current { + return Ok(pid); + } + (FIRST_ES_PID..0x1FFF) + .find(|&pid| { + Some(pid) != pcr_pid + && pid != pmt_pid + && !others().any(|track| track.pid == pid) + && self.si.keys().all(|(si, _)| *si != pid) + }) + .context("TS export has no unused PID for a track") + } + + /// Tell the jitter buffer which tracks a clock being acquired waits to hear from. + /// + /// The clock anchors on whichever track a source sends latest against its decode time + /// (audio just in time while video runs most of a second ahead), so it waits to hear + /// from each continuous one: audio, video, and the passthrough streams the schedule + /// models (DVB AC-3, teletext). Not from sparse ones such as SCTE-35, subtitles or ID3, + /// which would hold every join for the full bound. + fn expect_continuous(&mut self) { let continuous = self .tracks .values() .filter(|track| track.kind.class() != super::stats::Class::Data || track.buffer.is_some()); - self.jitter.expect(continuous.map(|track| track.pid)); - Ok(()) - } - - /// Keep `track` under `name` with this snapshot's PID and descriptors. - fn refresh(&mut self, name: &str, mut track: Track, pid: u16, descriptors: Vec) { - track.pid = pid; - track.descriptors = descriptors; - self.tracks.insert(name.to_string(), track); + self.jitter.expect(continuous.map(Track::key)); } /// Point each stale track this snapshot lists at the returned broadcast. @@ -2057,7 +2202,10 @@ where if self.psi.is_none() { return stats; } - for track in self.tracks.values() { + // A PID taken over goes to the track the PMT lists on it, the one taken on last. + let mut tracks: Vec<&Track> = self.tracks.values().filter(|track| track.listed).collect(); + tracks.sort_by_key(|track| track.serial); + for track in tracks { let (units, quiet) = self.liveness.stream(track.pid); let row = super::stats::Stream { track: track.kind.suffix().to_string(), @@ -2077,6 +2225,40 @@ where self.jitter.len() } + /// How many tracks the export carries, listed or not. + #[cfg(test)] + pub(super) fn tracks(&self) -> usize { + self.tracks.len() + } + + /// `pid`'s access units and how long it has been quiet, as [`Self::stats`] reads them. + #[cfg(test)] + pub(super) fn liveness(&self, pid: u16) -> (u64, Option) { + self.liveness.stream(pid) + } + + /// The track the PMT lists on `pid`: of those listed there, the one taken on last. + #[cfg(test)] + pub(super) fn listed_on(&self, pid: u16) -> Option<&str> { + self.tracks + .iter() + .filter(|(_, track)| track.listed && track.pid == pid) + .max_by_key(|(_, track)| track.serial) + .map(|(name, _)| name.as_str()) + } + + /// Take the next frame due out of the jitter buffer and hold it, as a generation boundary + /// holds the new generation's first frame behind the old one's tail. Returns its PID. + #[cfg(test)] + pub(super) fn hold_next(&mut self) -> Option { + let Poll::Ready(ready) = self.jitter.poll_next(&kio::Waiter::noop()) else { + return None; + }; + let pid = ready.track.0; + self.held = Some(ready); + Some(pid) + } + /// When the next queued frame is due, or the next grid slot while media is queued or /// every source has finished and the tail runs out. #[cfg(test)] @@ -2109,7 +2291,8 @@ where /// /// The same publisher instance (an equal epoch) continues the stream: the program and /// its PSI stay as announced, and its catalog resubscribes each track as it lists it, so a - /// gap is a skip like any other. One that adds a track fails as a layout change. + /// gap is a skip like any other. A track it adds is announced by a PMT revision, as any + /// catalog update's is, or fails as a layout change under [`Self::with_sync`]. /// /// Another instance (another epoch, or an epochless route) is a full program switch: a /// fresh export of `broadcast` with a new PMT from its catalog, and nothing carried from @@ -2154,13 +2337,24 @@ where // program that went out advances them again. let emitted = self.psi.is_some(); if (!same && emitted) || self.flagged.is_some() { - next.version = self.version; + // A revision no slot laid reached no receiver: the next follows the last laid. + next.version = self + .psi + .as_ref() + .and_then(|psi| psi.pat_laid) + .map_or(self.version, |(version, _)| version); + next.pmt_version = self + .psi + .as_ref() + .and_then(|psi| psi.laid.as_ref()) + .map_or(self.pmt_version, |(version, _)| *version); next.flagged = Some(HashSet::new()); next.pcr_discontinuity = true; next.epoch = self.epoch; next.emitted_epoch = self.emitted_epoch; if !same && emitted { next.version.increment(); + next.pmt_version.increment(); next.epoch += 1; } } @@ -2220,7 +2414,7 @@ where .tracks .values() .filter(|track| matches!(track.kind, Kind::Video(_))) - .filter_map(|track| self.jitter.peek(&track.pid)) + .filter_map(|track| self.jitter.peek(&track.key())) .map(|next| match placed { true => next.key, false => next.frame.timestamp.max(start), @@ -2298,9 +2492,18 @@ where } } - /// PID the PMT rides on: the source's original (preserved in the service record), - /// or the synthesized [`PMT_PID`] for a media-only source or an invalid (zero) value. + /// PID the PMT rides on: once the tables are built, the one they lay it on, which only + /// [`Self::revise`] moves; before, the catalog's ([`Self::cataloged_pmt_pid`]). fn pmt_pid(&self) -> u16 { + match &self.psi { + Some(psi) => psi.pmt_pid, + None => self.cataloged_pmt_pid(), + } + } + + /// PID the catalog gives the PMT: the source's original (preserved in the service record), + /// or the synthesized [`PMT_PID`] for a media-only source or an invalid (zero) value. + fn cataloged_pmt_pid(&self) -> u16 { self.program .as_ref() .map(|s| s.pmt_pid) @@ -2328,39 +2531,130 @@ where } } - // Order tracks by PID for a stable layout; first video track carries the PCR. - let mut tracks: Vec<&Track> = self.tracks.values().collect(); - tracks.sort_by_key(|t| t.pid); - + for track in self.tracks.values_mut() { + track.listed = true; + } let pcr_pid = match sync { // A media PID flags the break at every group, which on the PCR's PID would declare a // break in the time base, so under sync the PCR rides a PID of its own. Some(_) => self.clock_pid()?, - None => tracks - .iter() - .find(|t| matches!(t.kind, Kind::Video(_))) - .or_else(|| { - tracks.iter().find(|t| { - matches!( - t.kind, - Kind::Aac { .. } | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 - ) - }) - }) - .map(|t| t.pid) + None => self + .media_clock() // Sparse data cannot carry the clock's cadence: reserve a PID used only by PCR. .or_else(|| { (FIRST_ES_PID..0x1fff).find(|pid| { *pid != self.pmt_pid() - && tracks.iter().all(|track| track.pid != *pid) + && self.tracks.values().all(|track| track.pid != *pid) && self.si.keys().all(|(si, _)| si != pid) }) }) .context("TS export has no unused PID for the PCR")?, }; - let streams = tracks + // Preserve the source's program identity so the rebuilt PAT/PMT stay consistent + // with the carried SI; synthesize a minimal identity otherwise. + let pmt_pid = self.pmt_pid(); + let program_number = self + .program + .as_ref() + .map(|s| s.program_number) + .filter(|&id| id != 0) + .unwrap_or(1); + + let pat = self.pat(program_number, pmt_pid)?; + let table = self.table(program_number, pcr_pid)?; + + self.schedule.set_tables(pmt_pid); + self.schedule.set_clock(pcr_pid); + for track in self.tracks.values() { + self.liveness.register(track.pid); + } + let psi = Psi { + pat, + pat_laid: None, + pmt: super::psi::encode_pmt(&table, self.pmt_version)?, + table, + laid: None, + pcr_pid, + announced: None, + pmt_pid, + }; + if let Some(rate) = sync { + self.lay_lanes(rate, &psi)?; + } + self.psi = Some(psi); + Ok(()) + } + + /// The PID the PCR rides on without sync: the program's first video track, by PID, else + /// its first audio track. `None` for a program of sparse data alone. + fn media_clock(&self) -> Option { + let mut tracks: Vec<&Track> = self + .tracks + .values() + .filter(|track| track.listed && !track.cut) + .collect(); + tracks.sort_by_key(|track| track.pid); + tracks .iter() + .find(|t| matches!(t.kind, Kind::Video(_))) + .or_else(|| { + tracks.iter().find(|t| { + matches!( + t.kind, + Kind::Aac { .. } | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 + ) + }) + }) + .map(|t| t.pid) + } + + /// [`Self::listing`], unless what tracks draining since they left the catalog add outgrows + /// a section beside the rest. Then those tracks drain under the current program descriptors + /// for good, as they would once ended; and if their streams still do not fit, those end + /// there ([`Track::cut`]): the PMT lists the program the catalog does. + fn table(&mut self, program_number: u16, pcr_pid: u16) -> anyhow::Result { + let fits = |table: &super::psi::Pmt| super::psi::encode_pmt(table, VersionNumber::default()).is_ok(); + let mut table = self.listing(program_number, pcr_pid)?; + if !fits(&table) && self.tracks.values().any(|track| !track.left_under.is_empty()) { + tracing::warn!( + "TS export drops the program descriptors draining tracks left under: with them the PMT outgrows a section" + ); + for track in self.tracks.values_mut() { + track.left_under.clear(); + } + table = self.listing(program_number, pcr_pid)?; + } + let stale = &self.stale; + let mut draining = self + .tracks + .iter_mut() + .filter(|(name, track)| track.listed && !track.cut && !track.cataloged && !stale.contains(*name)) + .peekable(); + if fits(&table) || draining.peek().is_none() { + return Ok(table); + } + tracing::warn!( + "TS export ends the streams of tracks that left the catalog: with them the PMT outgrows a section" + ); + for (_, track) in draining { + track.cut = true; + } + self.listing(program_number, pcr_pid) + } + + /// The PMT of `program_number` with the clock on `pcr_pid`, listing each listed track in + /// PID order: on a PID taken over, the track taken on last. + fn listing(&self, program_number: u16, pcr_pid: u16) -> anyhow::Result { + let mut listed: BTreeMap = BTreeMap::new(); + for track in self.tracks.values().filter(|track| track.listed && !track.cut) { + let entry = listed.entry(track.pid).or_insert(track); + if track.serial > entry.serial { + *entry = track; + } + } + let streams = listed + .values() .map(|t| { let stream_type = match &t.kind { Kind::Video(stream_type) => *stream_type as u8, @@ -2398,11 +2692,27 @@ where }) .collect::>>()?; - // Re-emit the captured program-level descriptors. With none (a non-TS source), - // derive the SCTE-35 'CUEI' registration when a 0x86 verbatim stream is present. - let program_info = if !self.program_descriptors.is_empty() { - self.program_descriptors.clone() - } else if tracks.iter().any(|t| { + // Re-emit the captured program-level descriptors, and while tracks drain those each left + // the catalog under, in the order they were taken on: the streams of both revisions are + // listed until they end, so are theirs. A stale track ([`Self::followed`]) sends no tail. + let mut draining: Vec<&Track> = self + .tracks + .iter() + .filter(|(name, track)| track.listed && !track.cut && !track.cataloged && !self.stale.contains(*name)) + .map(|(_, track)| track) + .collect(); + draining.sort_by_key(|track| track.serial); + let mut program_info = self.program_descriptors.clone(); + for descriptor in draining.iter().flat_map(|track| &track.left_under) { + if !program_info.contains(descriptor) { + program_info.push(descriptor.clone()); + } + } + // With none (a non-TS source), derive the SCTE-35 'CUEI' registration when a 0x86 + // verbatim stream is present. + let program_info = if !program_info.is_empty() { + program_info + } else if listed.values().any(|t| { // Only derive CUEI for section-framed 0x86 (SCTE-35); a whole-PES 0x86 // (e.g. DTS audio) must not advertise SCTE-35 section signaling. matches!(&t.kind, Kind::Verbatim { stream_type: 0x86 }) @@ -2411,49 +2721,228 @@ where } else { Vec::new() }; + Ok(super::psi::Pmt { + program_number, + pcr_pid: Some(Pid::new(pcr_pid)?), + program_info, + streams, + }) + } - // Preserve the source's program identity so the rebuilt PAT/PMT stay consistent - // with the carried SI; synthesize a minimal identity otherwise. - let pmt_pid = self.pmt_pid(); - let transport_stream_id = self.program.as_ref().map(|s| s.transport_stream_id).unwrap_or(1); - let program_number = self - .program - .as_ref() - .map(|s| s.program_number) - .filter(|&id| id != 0) - .unwrap_or(1); + /// Rebuild the PMT from the tracks it lists, at the next `version_number` if that changes + /// it, to go out with the next frame: a receiver sees an ordinary PMT revision. A PMT the + /// catalog's `program.pmtPid` moves goes to its new PID behind a PAT at the next + /// `version_number` naming it. Under sync the tables stay as laid. + /// + /// Tables go out with the frame they were muxed with, and frames on different PIDs leave in + /// deadline order, not mux order, so every copy queued that has not started out takes the + /// revision too: none older can follow it. A revision before any slot laid the one before + /// replaces it, so each version that goes out follows the one before by one, and differs + /// from it. + fn revise(&mut self) -> anyhow::Result<()> { + let Some(psi) = self.psi.as_ref().filter(|_| !self.sync) else { + return Ok(()); + }; + let (program_number, pcr_pid) = (psi.table.program_number, psi.announced.unwrap_or(psi.pcr_pid)); + let table = self.table(program_number, pcr_pid)?; + let psi = self.psi.as_ref().expect("the tables are built"); + if let Some(pmt_pid) = self.moved_pmt(psi) { + match psi.pat_laid { + Some((version, laid)) if laid == pmt_pid => self.version = version, + Some((version, _)) if version == self.version => self.version.increment(), + _ => {} + } + let pat = self.pat(program_number, pmt_pid)?; + let psi = self.psi.as_mut().expect("the tables are built"); + let from = std::mem::replace(&mut psi.pmt_pid, pmt_pid); + let pmt = psi.pmt.clone(); + psi.pat = pat.clone(); + self.schedule.set_tables(pmt_pid); + self.last_psi = None; + // A PAT and the PMT it names go out together: a copy whose PAT has started out + // keeps its PMT where that PAT names it. + let mut moved = |packets: &mut Vec, sent: usize| { + let at = pat_from(packets, sent); + retable(packets, at, Pid::PAT, Pid::PAT, &pat)?; + retable(packets, at, from, pmt_pid, &pmt) + }; + moved(&mut self.tables, 0)?; + self.schedule.rewrite(&mut moved)?; + } + let psi = self.psi.as_mut().expect("the tables are built"); + if table == psi.table { + return Ok(()); + } + match &psi.laid { + Some((version, laid)) if *laid == table => self.pmt_version = *version, + Some((version, _)) if *version == self.pmt_version => self.pmt_version.increment(), + _ => {} + } + psi.pmt = super::psi::encode_pmt(&table, self.pmt_version)?; + psi.table = table; + self.last_psi = None; + let (pid, pmt) = (psi.pmt_pid, psi.pmt.clone()); + retable(&mut self.tables, 0, pid, pid, &pmt)?; + self.schedule + .rewrite(|packets, sent| retable(packets, sent, pid, pid, &pmt)) + } - let pat = Pat { + /// The PID the catalog moved the PMT to from the one `psi` lays it on, unless something + /// the export lays already rides it, when the PMT stays put. + fn moved_pmt(&self, psi: &Psi) -> Option { + let pmt_pid = self.cataloged_pmt_pid(); + if pmt_pid == psi.pmt_pid { + return None; + } + let taken = pmt_pid == psi.pcr_pid + || self.tracks.values().any(|track| track.pid == pmt_pid) + || self.si.keys().any(|(pid, _)| *pid == pmt_pid); + if taken { + tracing::warn!( + from = psi.pmt_pid, + to = pmt_pid, + "TS export keeps the PMT on its PID: the catalog moves it to one in use" + ); + return None; + } + Some(pmt_pid) + } + + /// The PAT naming `pmt_pid` for `program_number`, at the current `version_number`. + fn pat(&self, program_number: u16, pmt_pid: u16) -> anyhow::Result { + let transport_stream_id = self.program.as_ref().map(|s| s.transport_stream_id).unwrap_or(1); + super::psi::encode_pat(&Pat { transport_stream_id, version_number: self.version, table: vec![ProgramAssociation { program_num: program_number, program_map_pid: Pid::new(pmt_pid)?, }], + }) + } + + /// Drop each track that left the catalog once everything it sent has gone out, and with + /// it its PID from the PMT, unless a track listed since holds the PID. The PCR stays on its + /// PID unless the track it rode on is dropped with the PID, when it moves to the PID + /// [`Self::media_clock`] gives once a PMT naming that one is laid ([`Self::note_tables`]), + /// staying on a PID of its own without one. + /// + /// One that left before the PMT listed it goes at once if it holds nothing, or, waiting to + /// take a PID over ([`Self::take_over`]) from a track that has not ended, with what it holds + /// when a later track takes its place there: such tracks would otherwise pile up behind + /// sources that never end. One whose stream the PMT dropped ([`Track::cut`]) goes at once. + /// + /// A stale track ([`Self::followed`]) is not dropped while its catalog may list it again on + /// its PID: only once a later track listed there holds it. + fn retire(&mut self) -> anyhow::Result<()> { + if self.sync || self.psi.is_none() { + return Ok(()); + } + let held = self.held.as_ref().map(|ready| ready.track); + let taken = |track: &Track| { + self.tracks + .values() + .any(|other| other.listed && other.pid == track.pid && other.serial > track.serial) }; - let pmt = super::psi::Pmt { - program_number, - pcr_pid: Some(Pid::new(pcr_pid)?), - program_info, - streams, + let replaced = |track: &Track| { + self.tracks + .values() + .any(|other| other.pid == track.pid && other.serial > track.serial) }; - - self.schedule.set_tables(pmt_pid); - self.schedule.set_clock(pcr_pid); - for track in self.tracks.values() { - self.liveness.register(track.pid); + let behind_unended = |track: &Track| { + self.tracks + .values() + .any(|other| other.pid == track.pid && other.serial < track.serial && !other.finished) + }; + let done: Vec = self + .tracks + .iter() + .filter(|(name, track)| { + let sent = || { + track.finished + && track.pack.is_none() + && held != Some(track.key()) + && !self.jitter.holds(&track.key()) + && (taken(track) || !self.schedule.holds(track.pid)) + }; + let superseded = || !track.listed && behind_unended(track) && replaced(track); + // Nothing of one never listed went out, and with nothing held none will. + let idle = || { + !track.listed + && track.pack.is_none() + && held != Some(track.key()) + && !self.jitter.holds(&track.key()) + }; + !track.cataloged + && (!self.stale.contains(*name) || taken(track)) + && (track.cut || sent() || superseded() || idle()) + }) + .map(|(name, _)| name.clone()) + .collect(); + if done.is_empty() { + return Ok(()); } - let psi = Psi { - pat: super::psi::encode_pat(&pat)?, - pmt: super::psi::encode_pmt(&pmt, self.version)?, - pcr_pid, - pmt_pid, + let psi = self.psi.as_ref().expect("the tables are built"); + let pcr_pid = psi.announced.unwrap_or(psi.pcr_pid); + let mut cleared = false; + for name in done { + let track = self.tracks.remove(&name).expect("a track retiring"); + self.stale.remove(&name); + if let Some(ready) = self.held.take_if(|ready| ready.track == track.key()) { + self.jitter.discard(&ready); + } + self.jitter.forget(&track.key()); + cleared |= track.pid == pcr_pid; + // A stream listed on the PID later counts its silence from its own listing. + if track.listed && !self.tracks.values().any(|other| other.listed && other.pid == track.pid) { + self.liveness.retire(track.pid); + } + } + if cleared + && !self.tracks.values().any(|track| track.pid == pcr_pid) + && let Some(pid) = self.media_clock() + { + let psi = self.psi.as_mut().expect("the tables are built"); + psi.announced = (pid != psi.pcr_pid).then_some(pid); + } + self.expect_continuous(); + self.revise() + } + + /// Hold each track taking over a PID in the jitter buffer until every track before it + /// there has ended and its last packet has gone out, so none of theirs goes out under the + /// PMT revised for it. Release times alone would interleave them: the new track's first + /// frame can be due before the tail of the one it replaces. A frame of one already out of + /// the jitter buffer, held at a generation boundary as [`Self::admit`] moved its track onto + /// the PID, drops: held on, it would keep that tail from going out. + fn take_over(&mut self) { + let held = self.held.as_ref().map(|ready| ready.track); + let sending = |track: &Track| { + !track.finished + || track.pack.is_some() + || held == Some(track.key()) + || self.jitter.holds(&track.key()) + || self.schedule.holds(track.pid) }; - if let Some(rate) = sync { - self.lay_lanes(rate, &psi)?; + let waits: Vec<(Key, bool)> = self + .tracks + .values() + .map(|track| { + let behind = !track.listed + && self + .tracks + .values() + .any(|other| other.pid == track.pid && other.serial < track.serial && sending(other)); + (track.key(), behind) + }) + .collect(); + for (key, wait) in waits { + if wait && held == Some(key) { + let ready = self.held.take().expect("the held frame"); + self.jitter.discard(&ready); + } + self.jitter.wait(&key, wait); } - self.psi = Some(psi); - Ok(()) } /// The PID the PCR rides alone under [`Self::with_sync`]: the first above the program's @@ -2520,7 +3009,30 @@ where pes, .. } = queued; - let is_video = matches!(self.tracks.get(name).context("missing track")?.kind, Kind::Video(_)); + let track = self.tracks.get_mut(name).context("missing track")?; + if track.cut { + return Ok(()); + } + let is_video = matches!(track.kind, Kind::Video(_)); + // A track taken on after the tables is announced with its first frame, which goes out as + // a PES of its own behind the revised tables, on its PID after whatever the track it + // takes the PID over from sent. + let first = !std::mem::replace(&mut track.listed, true); + if first { + let (pid, serial) = (track.pid, track.serial); + // A stream taking the PID over counts its silence from its own listing, even from a + // stale track there ([`Self::followed`]) that retires only once this one is listed. + if self + .tracks + .values() + .any(|other| other.listed && other.pid == pid && other.serial < serial) + { + self.liveness.retire(pid); + } + self.liveness.register(pid); + self.flush_packs(|track, _| track.pid == pid)?; + self.revise()?; + } // Refresh PSI at keyframes or after the interval lapses. let psi = (is_video && frame.keyframe) || due(frame.timestamp, self.last_psi, PSI_INTERVAL); if psi { @@ -2620,7 +3132,7 @@ where }; // Only a delay leaves the frames time to wait for the ones after them. let track = self.tracks.get(name).context("missing track")?; - if let Some((samples, rate)) = track.samples(&es_payload).filter(|_| !self.delay.is_zero()) { + if let Some((samples, rate)) = track.samples(&es_payload).filter(|_| !self.delay.is_zero() && !first) { return self.pack(name, unit, decode, out, es_payload, (samples, rate, pes)); } self.flush(name)?; @@ -2673,8 +3185,9 @@ where 1 } }; - if let Some(buffer) = buffer { - self.schedule.set_buffer(pid, buffer); + match buffer { + Some(buffer) => self.schedule.set_buffer(pid, buffer), + None => self.schedule.clear_buffer(pid), } self.schedule.push_placed( pid, @@ -2843,8 +3356,9 @@ where } }; let track = self.tracks.get(name).context("missing track")?; - if let Some(buffer) = track.buffer() { - self.schedule.set_buffer(pid, buffer); + match track.buffer() { + Some(buffer) => self.schedule.set_buffer(pid, buffer), + None => self.schedule.clear_buffer(pid), } // A padded output cannot burst a picture late, so a CPB that the source may fill to // its declared buffer delay needs that much send-ahead, all of it out of the delay. @@ -2969,7 +3483,7 @@ where let Some(psi) = self.psi.as_ref() else { return Ok(()); }; - let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); + let pmt_pid = psi.pmt_pid; if known.is_none() { self.flush_packs(|_, _| true)?; } @@ -2993,6 +3507,7 @@ where let Some(slot) = self.schedule.next(known)? else { break; }; + let pcr_pid = self.psi.as_ref().expect("the tables are built").pcr_pid; let clock = pcr_packet(pcr_pid, slot.pcr)?; let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); // The slot opens with its PCR, in the clock packet or the clock PID's packet that @@ -3003,6 +3518,7 @@ where } // Under sync every packet was numbered where it was planned. if !self.sync { + self.note_tables(&payload, pmt_pid); self.number(&mut payload); } let frame = Frame { @@ -3020,6 +3536,40 @@ where Ok(()) } + /// Note the PAT and PMT a slot lays, which are the current ones: a revision rewrites every + /// copy that has not started out ([`Self::revise`]). The PCR moves to the PID the PMT names + /// from the next slot on. + fn note_tables(&mut self, packets: &[u8], pmt_pid: u16) { + let Some(psi) = self.psi.as_mut() else { + return; + }; + let packets = packets.as_chunks::<{ TsPacket::SIZE }>().0; + let starts = |pid: u16| { + packets + .iter() + .any(|packet| packet_pid(packet) == pid && packet[1] & 0x40 != 0) + }; + if starts(Pid::PAT) && psi.pat_laid.is_none_or(|(version, _)| version != self.version) { + psi.pat_laid = Some((self.version, psi.pmt_pid)); + } + if starts(pmt_pid) + && psi + .laid + .as_ref() + .is_none_or(|(version, _)| *version != self.pmt_version) + { + psi.laid = Some((self.pmt_version, psi.table.clone())); + } + if starts(pmt_pid) + && let Some(pid) = psi.announced + && psi.table.pcr_pid.is_some_and(|named| named.as_u16() == pid) + { + psi.pcr_pid = pid; + psi.announced = None; + self.schedule.set_clock(pid); + } + } + /// Number the continuity counters of packets about to go out. A packet without a /// payload (a clock packet) repeats the counter before it (ISO 13818-1 2.4.3.3); before /// anything has gone out on its PID, any value starts a valid run. @@ -3065,6 +3615,43 @@ where } } +/// The PID of `packet`. +fn packet_pid(packet: &[u8]) -> u16 { + u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]) +} + +/// The first packet at or after `from` of `packets` that starts a PAT section, or the end. +fn pat_from(packets: &[u8], from: usize) -> usize { + let packets = packets.as_chunks::<{ TsPacket::SIZE }>().0; + (from..packets.len()) + .find(|&at| packet_pid(&packets[at]) == Pid::PAT && packets[at][1] & 0x40 != 0) + .unwrap_or(packets.len()) +} + +/// Put `section`, on `to`, in place of each section on `pid` that starts at or after packet +/// `from` of `packets`, flagging a break where the one it replaces did. +fn retable(packets: &mut Vec, from: usize, pid: u16, to: u16, section: &[u8]) -> anyhow::Result<()> { + let old = std::mem::take(packets); + let old = old.as_chunks::<{ TsPacket::SIZE }>().0; + let mut at = 0; + while let Some(packet) = old.get(at) { + at += 1; + if at <= from || packet_pid(packet) != pid || packet[1] & 0x40 == 0 { + packets.extend_from_slice(packet); + continue; + } + let flag = packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x80 != 0; + section_packets(packets, to, section, flag)?; + while old + .get(at) + .is_some_and(|next| packet_pid(next) == pid && next[1] & 0x40 == 0) + { + at += 1; + } + } + Ok(()) +} + /// Packetize a PES payload into 188-byte TS packets, flagging a break in the first when `flag`. fn pes_packets(out: &mut Vec, unit: &PesUnit, payload: &[u8], flag: bool) -> anyhow::Result<()> { let pts = to_ts_timestamp(unit.timestamp)?; @@ -4081,6 +4668,7 @@ mod tests { let source = super::ExportSource::for_stream(&source, name, Duration::from_secs(2)).unwrap(); let mut track = super::Track::new(source, kind); track.pid = pid; + track.listed = true; tracks.push(track); } let frame = |at| super::Frame { @@ -4091,7 +4679,7 @@ mod tests { }; let start = moq_clock::now(); let mut jitter = super::jitter::Buffer::new(Duration::from_millis(500)); - jitter.expect([1, 2]); + jitter.expect([(1, 0), (2, 0)]); let mut trimmed = 0; for publisher in &mut publishers { publisher.write(frame(0)).unwrap(); @@ -4151,7 +4739,7 @@ mod tests { std::task::Poll::Ready(ready) => Some(ready), std::task::Poll::Pending => None, }) - .find(|ready| ready.track == 1) + .find(|ready| ready.track == (1, 0)) .expect("a video start"); assert_eq!(video.item.frame.timestamp.as_micros(), 600_000, "{order:?}"); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 895d4b725ce..5514f5ab583 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -8139,6 +8139,297 @@ async fn the_same_instance_continues_after_a_drop() { same_instance_returns(false).await; } +/// A track the returned catalog leaves out stays listed, silent on its PID, with no tail to +/// send, so the PMT carries only the returned catalog's program descriptors, not also those the +/// track was listed under. +#[tokio::test(start_paused = true)] +async fn a_track_left_out_of_a_return_keeps_no_program_descriptors() { + let origin = crate::source::produce_origin(); + let epoch = moq_net::Epoch::mint(); + let route = || moq_net::origin::Route::default().with_epoch(epoch.clone()); + let describe = |catalog: &mut crate::catalog::Producer, tag: u8| { + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![tscat::Descriptor { + tag, + data: Bytes::from_static(&[0xaa; 4]), + }]; + }; + + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + describe(&mut catalog, 0x80); + let mut kept = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + let mut left = aac_rendition(&mut broadcast, &mut catalog, "b.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + let start = tokio::time::Instant::now(); + for ms in (0..200).step_by(20) { + write_aac(&mut kept, ms); + write_aac(&mut left, ms); + } + let mut frames = drain_frames(&mut export).await; + drop((broadcast, catalog, kept, left)); + frames.extend(drain_to_end(&mut export).await.0); + + tokio::time::sleep(Duration::from_secs(1)).await; + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + describe(&mut catalog, 0x81); + let back = origin.consume().routed_broadcast("live").await.unwrap(); + let stream = crate::catalog::Consumer::new(&back, Default::default()).await.unwrap(); + let mut export = export.follow(back, stream).unwrap(); + frames.extend(drain_frames(&mut export).await); + let mut kept = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + let resumed = start.elapsed().as_millis() as u64; + for ms in (resumed..resumed + 200).step_by(20) { + write_aac(&mut kept, ms); + } + frames.extend(drain_frames(&mut export).await); + describe(&mut catalog, 0x82); + for ms in (resumed + 200..resumed + 400).step_by(20) { + write_aac(&mut kept, ms); + } + frames.extend(drain_frames(&mut export).await); + kept.finish().unwrap(); + catalog.finish().unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + end.unwrap(); + + let bytes: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + let mut reader = TsPacketReader::new(Cursor::new(bytes)); + let mut infos: Vec> = Vec::new(); + while let Some(packet) = reader.read_ts_packet().unwrap() { + if let Some(TsPayload::Pmt(pmt)) = packet.payload { + let tags: Vec = pmt.program_info.iter().map(|d| d.tag).collect(); + if infos.last() != Some(&tags) { + infos.push(tags); + } + } + } + assert_eq!(infos, [vec![0x80], vec![0x81], vec![0x82]]); +} + +/// Returns of one instance whose catalogs rename the track on its PID retire each one left +/// out once the next holds the PID: the export keeps one track however many returns come. +#[tokio::test(start_paused = true)] +async fn a_track_renamed_on_its_pid_across_returns_retires_the_one_before() { + let origin = crate::source::produce_origin(); + let epoch = moq_net::Epoch::mint(); + let route = || moq_net::origin::Route::default().with_epoch(epoch.clone()); + let rendition = |broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + name: &str| { + let track = aac_rendition(broadcast, catalog, name); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name.to_string(), tscat::Track::new(0x101)); + track + }; + + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let mut track = rendition(&mut broadcast, &mut catalog, "0.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + let start = tokio::time::Instant::now(); + for ms in (0..200).step_by(20) { + write_aac(&mut track, ms); + } + drain_frames(&mut export).await; + drop((broadcast, catalog, track)); + let _ = drain_to_end(&mut export).await; + + for round in 1..=4 { + tokio::time::sleep(Duration::from_secs(1)).await; + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let back = origin.consume().routed_broadcast("live").await.unwrap(); + let stream = crate::catalog::Consumer::new(&back, Default::default()).await.unwrap(); + export = export.follow(back, stream).unwrap(); + drain_frames(&mut export).await; + let mut track = rendition(&mut broadcast, &mut catalog, &format!("{round}.aac")); + let resumed = start.elapsed().as_millis() as u64; + for ms in (resumed..resumed + 200).step_by(20) { + write_aac(&mut track, ms); + } + drain_frames(&mut export).await; + drop((broadcast, catalog, track)); + let _ = drain_to_end(&mut export).await; + } + assert_eq!(export.tracks(), 1); +} + +/// A track a same-instance return renames on its PID counts its silence from its own listing, +/// not from the last unit of the stale one before it, which retires only once it is listed. +#[tokio::test(start_paused = true)] +async fn a_track_renamed_on_its_pid_across_a_return_is_quiet_from_its_listing() { + let origin = crate::source::produce_origin(); + let epoch = moq_net::Epoch::mint(); + let route = || moq_net::origin::Route::default().with_epoch(epoch.clone()); + let rendition = |broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + name: &str| { + let track = aac_rendition(broadcast, catalog, name); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name.to_string(), tscat::Track::new(0x101)); + track + }; + + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let mut track = rendition(&mut broadcast, &mut catalog, "0.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(Duration::from_secs(1)) + .with_replay(); + let start = tokio::time::Instant::now(); + for ms in (0..200).step_by(20) { + write_aac(&mut track, ms); + } + drain_frames(&mut export).await; + drop((broadcast, catalog, track)); + let _ = drain_to_end(&mut export).await; + let (before, _) = export.liveness(0x101); + + tokio::time::sleep(Duration::from_secs(10)).await; + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let back = origin.consume().routed_broadcast("live").await.unwrap(); + let stream = crate::catalog::Consumer::new(&back, Default::default()).await.unwrap(); + let mut export = export.follow(back, stream).unwrap(); + drain_frames(&mut export).await; + let mut track = rendition(&mut broadcast, &mut catalog, "1.aac"); + let resumed = start.elapsed().as_millis() as u64; + let mut quiet = Vec::new(); + for ms in (resumed..resumed + 4000).step_by(20) { + write_aac(&mut track, ms); + poll_frames(&mut export); + let (units, since) = export.liveness(0x101); + if export.listed_on(0x101) == Some("1.aac") && units == before { + quiet.push(since); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!(!quiet.is_empty(), "the track was listed with none of its units out"); + assert!( + quiet + .iter() + .all(|since| since.is_some_and(|since| since < Duration::from_secs(1))), + "{quiet:?}" + ); + drop((broadcast, catalog, track)); +} + +/// A PMT revision no slot laid before a switch reached no receiver: the replacement's PMT +/// follows the last one out by one version. +#[tokio::test(start_paused = true)] +async fn a_switch_follows_the_last_pmt_out() { + let origin = crate::source::produce_origin(); + let route = || moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + + let (mut old, mut old_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut old, &mut old_catalog, "a.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + let mut frames = drain_frames(&mut export).await; + old_catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![tscat::Descriptor { + tag: 0x80, + data: Bytes::from_static(&[0xaa; 4]), + }]; + frames.extend(drain_frames(&mut export).await); + + let (mut new, mut new_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut new, &mut new_catalog, "b.aac"); + let replacement = origin.consume().request_broadcast("live", None).await.unwrap(); + let stream = crate::catalog::Consumer::new(&replacement, Default::default()) + .await + .unwrap(); + let mut export = export.follow(replacement, stream).unwrap(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + frames.extend(drain_frames(&mut export).await); + audio.finish().unwrap(); + new_catalog.finish().unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + end.unwrap(); + + let mut versions: Vec = pmts(&frames).iter().map(|(version, _)| *version).collect(); + versions.dedup(); + assert_eq!(versions, [0, 1]); + drop((old, old_catalog)); +} + +/// A PAT revision no slot laid before a switch reached no receiver: the replacement's PAT +/// follows the last one out by one version. +#[tokio::test(start_paused = true)] +async fn a_switch_follows_the_last_pat_out() { + let origin = crate::source::produce_origin(); + let route = || moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + + let (mut old, mut old_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut old, &mut old_catalog, "a.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + let mut frames = drain_frames(&mut export).await; + old_catalog.modify().unwrap().ext.mpegts.program = Some(tscat::Program { + transport_stream_id: 1, + program_number: 1, + pmt_pid: 0x30, + }); + frames.extend(drain_frames(&mut export).await); + + let (mut new, mut new_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut new, &mut new_catalog, "b.aac"); + let replacement = origin.consume().request_broadcast("live", None).await.unwrap(); + let stream = crate::catalog::Consumer::new(&replacement, Default::default()) + .await + .unwrap(); + let mut export = export.follow(replacement, stream).unwrap(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + frames.extend(drain_frames(&mut export).await); + audio.finish().unwrap(); + new_catalog.finish().unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + end.unwrap(); + + let mut versions: Vec = pats(&frames).iter().map(|&(version, _)| version).collect(); + versions.dedup(); + assert_eq!(versions, [0, 1]); + drop((old, old_catalog)); +} + /// A returning route serves the same publisher's catalog and monotonically numbered groups. #[tokio::test(start_paused = true)] async fn a_return_keeps_catalog_and_group_history() { diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 1308354e8ac..3f13de21c96 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -113,9 +113,9 @@ pub struct Import { /// The PAT, reassembled off PID 0. pat: PatReader, - /// The PMT PIDs PATs have named for the imported program, each with its reassembler. + /// The PMT PID the latest PAT names for the imported program, with its reassembler. /// Several programs' PMTs may share one. - pmt_sections: HashMap, + pmt: Option<(Pid, SectionReassembler)>, /// PAT and PMT sections dropped for a CRC mismatch, keyed by PID. crc_errors: BTreeMap, /// Per elementary-stream-PID codec routing. @@ -242,7 +242,7 @@ impl Import { held: std::time::Duration::ZERO, held_edge: None, pat: PatReader::default(), - pmt_sections: HashMap::new(), + pmt: None, crc_errors: BTreeMap::new(), streams: HashMap::new(), retired_stats: BTreeMap::new(), @@ -461,7 +461,7 @@ impl Import { }; if pid.as_u16() == Pid::PAT { self.pat_packet(&pkt)?; - } else if self.pmt_sections.contains_key(&pid) { + } else if self.pmt.as_ref().is_some_and(|(pmt, _)| *pmt == pid) { self.pmt_packet(pid, &pkt)?; } else { // PIDs we don't decode and don't carry (`Stream::Ignored`: a base catalog's @@ -497,7 +497,9 @@ impl Import { /// Feed one packet on a PMT PID to its reassembler, applying every good PMT section. fn pmt_packet(&mut self, pid: Pid, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { let mut sections = Vec::new(); - self.pmt_sections.entry(pid).or_default().push(pkt, &mut sections); + if let Some((_, reassembler)) = &mut self.pmt { + reassembler.push(pkt, &mut sections); + } for section in sections { // Other tables may share the PID; only a PMT is this path's to check. if section.first() != Some(&psi::PMT_TABLE_ID) { @@ -1482,7 +1484,8 @@ impl Import { } /// Pick the imported program from a whole PAT and learn its PMT PID, so `decode` - /// reassembles that PMT and no other. + /// reassembles that PMT and no other. A PAT that moves the PMT to another PID moves the + /// reassembler, and the catalog's `program.pmtPid`, with it. /// /// Every PAT is checked, so a program added mid-stream ends an unselected import, a /// selected program that disappears ends a selected one, and a PAT naming the program under @@ -1515,13 +1518,22 @@ impl Import { identity.1 ); } - self.pmt_sections.entry(entry.pmt_pid).or_default(); - self.health.pmt_pids(&[entry.pmt_pid.as_u16()], self.liveness.now()); - if self.identity.is_none() { - self.identity = Some(identity); - self.record_program_identity(pat.transport_stream_id, entry)?; + self.identity = Some(identity); + if self.pmt.as_ref().is_some_and(|(pid, _)| *pid == entry.pmt_pid) { + return Ok(()); } - Ok(()) + // A section begun on the old PID no longer speaks for the program. + if let Some((from, _)) = &self.pmt { + tracing::info!( + program = entry.program_number, + from = from.as_u16(), + to = entry.pmt_pid.as_u16(), + "TS PMT moved to another PID" + ); + } + self.pmt = Some((entry.pmt_pid, SectionReassembler::default())); + self.health.pmt_pids(&[entry.pmt_pid.as_u16()], self.liveness.now()); + self.record_program_identity(pat.transport_stream_id, entry) } /// Capture the transport/service identity (TSID, service number, PMT PID) from the @@ -7050,7 +7062,7 @@ pub(super) mod test { /// One PES on `pid` carrying two whole MP2 frames: enough for the second to confirm the /// first, which is what the legacy path publishes on. - fn mp2_pes(pid: u16, cc: u8, pts: u64, fills: [u8; 2]) -> Vec { + pub(in crate::container::ts) fn mp2_pes(pid: u16, cc: u8, pts: u64, fills: [u8; 2]) -> Vec { let mut payload = mp2_frame(fills[0]); payload.extend_from_slice(&mp2_frame(fills[1])); audio_pes_packet(pid, cc, pts, &payload) @@ -10126,6 +10138,50 @@ pub(super) mod test { } } + /// A PAT that moves the PMT to another PID moves the import with it: the catalog records + /// the new PID, the PMT there revises the program, and the old PID no longer speaks for it. + #[test] + fn a_moved_pmt_is_followed() { + use super::super::fixture::Mux; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pmt_pid = || { + catalog + .snapshot() + .ext + .mpegts + .program + .as_ref() + .map(|program| program.pmt_pid) + }; + + let mut mux = Mux::default(); + mux.pat(&[(1, 0x100)]).pmt(0x100, 1, 0x61, &[(MP2, 0x61)], 0); + mux.out.extend(mp2_pes(0x61, 0, 90_000, [0xAA, 0xBB])); + import.decode(&mux.bytes()).unwrap(); + assert_eq!(pmt_pid(), Some(0x100)); + + let mut mux = Mux::default(); + mux.pat(&[(1, 0x200)]) + .pmt(0x200, 1, 0x61, &[(MP2, 0x61), (MP2, 0x62)], 0) + .pmt(0x100, 1, 0x61, &[], 0); + mux.out.extend(mp2_pes(0x61, 1, 108_000, [0xAA, 0xBB])); + mux.out.extend(mp2_pes(0x62, 0, 108_000, [0xCC, 0xDD])); + import.decode(&mux.bytes()).unwrap(); + import.finish().unwrap(); + assert_eq!(pmt_pid(), Some(0x200)); + assert_eq!( + catalog.snapshot().audio.renditions.len(), + 2, + "the PMT on the new PID adds a stream, and the one on the old PID removes none" + ); + } + /// A section-framed PID the PMT drops keeps its stats row, so what it delivered stays counted. #[test] fn a_dropped_section_route_keeps_its_stats_row() { diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 94a424abac5..7d9786ec916 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -71,3 +71,5 @@ mod pes_packet_test; mod recovery_test; #[cfg(test)] mod release_test; +#[cfg(test)] +mod revision_test; diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index e48dd5780ce..ef157f8f517 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -9,23 +9,35 @@ use super::import::{Framer, PatReader}; use super::{Ext, Import, psi, stats}; use crate::catalog; -/// Imports every program the first PAT lists, each as its own broadcast on `origin`, with its -/// own clock and catalog. +/// Imports every program the PAT lists, each as its own broadcast on `origin`, with its own +/// clock and catalog. /// /// Program `n` of `event.hang` publishes on `event/n.hang`, keeping the catalog suffix last so -/// format detection still sees it; a name without one gains a plain `/n`. Input ahead of that -/// PAT is dropped, as a single [`Import`] drops media ahead of its PSI. A program a later PAT -/// adds is not published; one it removes ends the import. +/// format detection still sees it; a name without one gains a plain `/n`. Input ahead of the +/// first PAT listing a program is dropped, as a single [`Import`] drops media ahead of its PSI. +/// +/// The PAT is followed as it changes. A program a later PAT adds publishes as the first +/// PAT's do, from the packet after that PAT. One it removes is retired: its tracks finish and +/// its broadcast is withdrawn, while the other programs carry on. A program that returns +/// publishes a new instance, under a new epoch. A program whose PMT never arrives waits, +/// unannounced: it publishes nothing until its PMT declares a track. pub struct Programs { origin: moq_net::origin::Producer, name: moq_net::PathOwned, config: catalog::Config, - /// Input not yet read for the PAT: a trailing partial packet, or what follows the PAT. + /// Input not yet framed: before the first PAT, all of it; after, a trailing partial packet + /// the programs already have. pending: Vec, - /// Reads the PAT out of the input. + /// Reads every PAT out of the input. scan: PatScan, - /// Empty until the PAT arrives. + /// Empty until the PAT arrives, and again once a PAT removes every program. programs: Vec, + /// Whether a PAT has listed a program yet. + listed: bool, + /// Whether any program has announced, retired ones included. + announced: bool, + /// What retired programs counted, so the multiplex's counters keep it. + retired: Retired, } /// One program's broadcast. @@ -39,6 +51,18 @@ struct Program { /// The route and catalog track until the catalog first names a servable track: announced any /// sooner, a relay would ask for tracks it cannot serve yet. pending: Option<(moq_net::origin::Route, moq_net::track::Consumer)>, + /// Where in [`Programs::pending`] this program's input starts, for the one call that adds + /// it: just after the PAT that did. + start: usize, +} + +/// The counters of programs a later PAT removed. +#[derive(Default)] +struct Retired { + /// Each PID's row as its last program left it, until a program carrying it again replaces it. + streams: BTreeMap, + crc_errors: BTreeMap, + errors: super::health::Errors, } impl Program { @@ -71,68 +95,167 @@ impl Programs { pending: Vec::new(), scan: PatScan::default(), programs: Vec::new(), + listed: false, + announced: false, + retired: Retired::default(), } } /// Demux a chunk of the multiplex. A trailing partial packet is retained for the next call. + /// + /// The input is cut at each PAT that changes the programs: what comes before it reaches + /// the programs the PAT before it listed, so a removed program never reads the PAT that + /// removed it, and an added one starts just after it. pub fn decode(&mut self, data: &[u8]) -> anyhow::Result<()> { - if !self.programs.is_empty() { - return self.feed(data); - } - self.pending.extend_from_slice(data); - let Some(pat) = self.scan.scan(&mut self.pending) else { - return Ok(()); + // What the programs already have: the partial packet the last call kept. + let mut fed = if self.programs.is_empty() { + 0 + } else { + self.pending.len() }; - for program in pat.program_numbers() { - let name = program_broadcast(self.name.as_str(), program); - let mut broadcast = self - .origin - .create_broadcast(&name) - .with_context(|| format!("failed to create broadcast {name}"))?; - // TS carries undecoded elementary streams verbatim, so each catalog carries the - // `mpegts` extension, and owns its broadcast's catalog tracks. - let config = self - .config - .clone() - .with_catalog(catalog::hang::Catalog::::default()); - let catalog = catalog::Producer::new(&mut broadcast, config)?; - let mut import = Import::new(broadcast.clone(), catalog.reserve()).with_program(program); - // Seeded with the scanner's PAT rather than replaying its packets, so every section - // reaches the importer and nothing the scanner counted is counted again. - import.handle_pat(&pat)?; - // Each program is its own publisher instance for this run, so a restart - // replaces it rather than resuming into it. - let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); - let listed = broadcast.consume().track(hang::Catalog::DEFAULT_NAME)?; - self.programs.push(Program { - number: program, - broadcast, - import, - catalog, - pending: Some((route, listed)), - }); + self.pending.extend_from_slice(data); + let mut off = 0; + while let Some(at) = self.scan.framer.next(&self.pending, &mut off) { + let pkt: &[u8; TsPacket::SIZE] = self.pending[at..at + TsPacket::SIZE].try_into().unwrap(); + if (u16::from(pkt[1] & 0x1f) << 8 | u16::from(pkt[2])) != Pid::PAT { + continue; + } + // The importers count what they read themselves. + let mut read = 0; + let crc_error = match self.programs.is_empty() { + true => &mut self.scan.crc_error, + false => &mut read, + }; + let Some(pat) = self.scan.pat.push(pkt, crc_error) else { + continue; + }; + let numbers = pat.program_numbers(); + let current = |number: &u16| self.programs.iter().any(|program| program.number == *number); + if numbers.iter().all(current) && self.programs.iter().all(|program| numbers.contains(&program.number)) { + continue; + } + let cut = at.max(fed); + self.feed(fed, cut)?; + fed = cut; + self.follow(&pat, at + TsPacket::SIZE)?; } - let pending = std::mem::take(&mut self.pending); - self.feed(&pending) + self.feed(fed, self.pending.len())?; + for program in &mut self.programs { + program.start = 0; + } + // Before the first PAT, what precedes the partial packet is dropped; after it, the + // programs have it all. + self.pending.drain(..off); + Ok(()) } - fn feed(&mut self, data: &[u8]) -> anyhow::Result<()> { + /// Hand `pending[from..to]` to every program, each from its own start. + fn feed(&mut self, from: usize, to: usize) -> anyhow::Result<()> { for program in &mut self.programs { - program.import.decode(data)?; - program.announce_if_listed()?; + let from = from.max(program.start); + if from < to { + program.import.decode(&self.pending[from..to])?; + } + self.announced |= program.announce_if_listed()?; } Ok(()) } + /// Bring the programs in line with `pat`: retire those it no longer lists, keeping the + /// order of the rest, and start those it adds on the input from `start`. + fn follow(&mut self, pat: &psi::Pat, start: usize) -> anyhow::Result<()> { + let numbers = pat.program_numbers(); + let (kept, gone) = std::mem::take(&mut self.programs) + .into_iter() + .partition(|program| numbers.contains(&program.number)); + self.programs = kept; + for program in gone { + self.retire(program); + } + for number in numbers { + if self.programs.iter().any(|program| program.number == number) { + continue; + } + if self.listed { + tracing::info!(program = number, "TS program joined the PAT"); + } + self.start(number, pat, start)?; + } + self.listed |= !self.programs.is_empty(); + Ok(()) + } + + /// Publish program `number` as its own broadcast, its importer seeded with `pat`. + fn start(&mut self, number: u16, pat: &psi::Pat, start: usize) -> anyhow::Result<()> { + let name = program_broadcast(self.name.as_str(), number); + let mut broadcast = self + .origin + .create_broadcast(&name) + .with_context(|| format!("failed to create broadcast {name}"))?; + // TS carries undecoded elementary streams verbatim, so each catalog carries the + // `mpegts` extension, and owns its broadcast's catalog tracks. + let config = self + .config + .clone() + .with_catalog(catalog::hang::Catalog::::default()); + let catalog = catalog::Producer::new(&mut broadcast, config)?; + let mut import = Import::new(broadcast.clone(), catalog.reserve()).with_program(number); + // Seeded with the PAT rather than replaying its packets, so every section reaches the + // importer and nothing already counted is counted again. + import.handle_pat(pat)?; + // Each program, and each return of one, is its own publisher instance, so a restart + // replaces it rather than resuming into it. + let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + let listed = broadcast.consume().track(hang::Catalog::DEFAULT_NAME)?; + self.programs.push(Program { + number, + broadcast, + import, + catalog, + pending: Some((route, listed)), + start, + }); + Ok(()) + } + + /// End a program a PAT no longer lists: finish its tracks, then its catalog, and close its + /// broadcast, so it is withdrawn now rather than lingering. Subscribers already reading a + /// track read it to its end. The rest of the multiplex carries on whatever this program + /// left unfinished. + fn retire(&mut self, mut program: Program) { + tracing::info!( + program = program.number, + "TS program left the PAT; retiring its broadcast" + ); + let finished = program + .import + .finish() + .and_then(|()| program.catalog.finish().map_err(anyhow::Error::from)); + if let Err(err) = finished { + tracing::warn!(program = program.number, error = %err, "TS program retired unfinished"); + } + program.broadcast.close(); + self.retired.streams.extend(program.import.stats().streams); + for (&pid, &count) in program.import.crc_errors() { + let merged = self.retired.crc_errors.entry(pid).or_default(); + *merged = (*merged).max(count); + } + self.retired.errors.max(program.import.errors()); + } + /// Every program's counters in one map: PIDs are unique across a multiplex. /// /// Every importer reads the same PAT, and programs sharing a PMT PID read the same PMT /// sections, so a dropped section counts once however many read it. Every importer also - /// grades every packet of the multiplex, so each TR 101 290 error counts once too. + /// grades every packet of the multiplex, so each TR 101 290 error counts once too. A + /// retired program's rows stay until a program carrying the PID again replaces them. pub fn stats(&self) -> stats::Snapshot { - let mut stats = stats::Snapshot::default(); - let mut crc_errors = BTreeMap::::new(); - let mut errors = super::health::Errors::default(); + let mut stats = stats::Snapshot { + streams: self.retired.streams.clone(), + ..Default::default() + }; + let mut crc_errors = self.retired.crc_errors.clone(); + let mut errors = self.retired.errors.clone(); for program in &self.programs { stats.streams.extend(program.import.stats().streams); for (&pid, &count) in program.import.crc_errors() { @@ -151,11 +274,12 @@ impl Programs { /// Finish each program's tracks, then its catalog while it still lists them. /// /// Returns each program's [`Finish`](moq_net::broadcast::Finish), which keeps it servable - /// until its media is stale; dropping one closes that program. + /// until its media is stale; dropping one closes that program. A program retired earlier + /// has none. /// Fails when the input ended before its PAT listed a program or any program declared a track. pub fn finish(&mut self) -> anyhow::Result> { - anyhow::ensure!(!self.programs.is_empty(), "input ended before its PAT listed a program"); - let mut announced = false; + anyhow::ensure!(self.listed, "input ended before its PAT listed a program"); + let mut announced = self.announced; let mut finished = Vec::with_capacity(self.programs.len()); for program in &mut self.programs { program.import.finish()?; @@ -187,39 +311,15 @@ fn program_broadcast(name: &str, program: u16) -> String { } } -/// Finds the first whole PAT listing a program, before any importer exists to read one. +/// Reads every whole PAT out of the multiplex, ahead of the importers it starts. #[derive(Default)] struct PatScan { framer: Framer, pat: PatReader, - /// PAT sections dropped for a bad CRC before the first whole PAT. + /// PAT sections dropped for a bad CRC while no importer read them. crc_error: u64, } -impl PatScan { - /// Read `pending` up to the first whole PAT listing a program, returning it with `pending` - /// trimmed to start just after the packet that completed it. Otherwise only a trailing - /// partial packet is kept: the reader holds a partial PAT itself. - fn scan(&mut self, pending: &mut Vec) -> Option { - let mut off = 0; - let mut found = None; - while let Some(at) = self.framer.next(pending, &mut off) { - let pkt: &[u8; TsPacket::SIZE] = pending[at..at + TsPacket::SIZE].try_into().unwrap(); - if (u16::from(pkt[1] & 0x1f) << 8 | u16::from(pkt[2])) != Pid::PAT { - continue; - } - if let Some(pat) = self.pat.push(pkt, &mut self.crc_error) - && !pat.program_numbers().is_empty() - { - found = Some(pat); - break; - } - } - pending.drain(..off); - found - } -} - #[cfg(test)] mod test { use std::time::Duration; @@ -227,10 +327,132 @@ mod test { use super::*; use crate::catalog::hang::Container; use crate::container::Consumer; + use crate::container::ts::fixture::Mux; use crate::container::ts::import::test::{ - corrupt, fifty_programs, pat_section, psi_packets, two_programs, two_section_pat, + corrupt, fifty_programs, mp2_pes, pat_section, psi_packets, two_programs, two_section_pat, }; + const MP2: u8 = 0x03; + + /// The epoch each program publishes under, announced yet or not. + fn epochs(programs: &Programs) -> Vec<(u16, moq_net::Epoch)> { + programs + .programs + .iter() + .map(|program| { + let route = program.broadcast.route(); + let route = route.as_ref().or(program.pending.as_ref().map(|(route, _)| route)); + ( + program.number, + route.and_then(|route| route.epoch.clone()).expect("an epoch"), + ) + }) + .collect() + } + + /// One step of a two-program multiplex: a PAT listing `listed`, both programs' PMTs, and an + /// MP2 PES on each program's audio whether or not the PAT lists it. + struct Steps { + cc: [u8; 2], + pts: u64, + } + + impl Steps { + fn step(&mut self, programs: &mut Programs, listed: &[(u16, u16)]) { + let mut mux = Mux::default(); + mux.pat(listed) + .pmt(0x100, 1, 0x61, &[(MP2, 0x61)], 0) + .pmt(0x200, 2, 0x71, &[(MP2, 0x71)], 0); + for (cc, pid) in self.cc.iter_mut().zip([0x61, 0x71]) { + mux.out.extend(mp2_pes(pid, *cc, self.pts, [pid as u8; 2])); + *cc = (*cc + 1) & 0x0f; + } + // Past both 72 ms frames. + self.pts += 18_000; + programs.decode(&mux.bytes()).unwrap(); + } + } + + /// A program a later PAT adds publishes, one it drops retires alone, and one it restores + /// publishes a new instance, while the program no PAT touched runs throughout. + #[test] + fn programs_follow_the_pat() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + let mut steps = Steps { + cc: [0; 2], + pts: 90_000, + }; + + steps.step(&mut programs, &[(1, 0x100)]); + let one = epochs(&programs); + assert_eq!(one.len(), 1); + + steps.step(&mut programs, &[(1, 0x100), (2, 0x200)]); + let both = epochs(&programs); + assert_eq!(both[0], one[0], "program 1 carries on"); + assert_eq!(both[1].0, 2, "the added program publishes"); + let two = programs.programs[1].broadcast.consume(); + + steps.step(&mut programs, &[(1, 0x100)]); + assert_eq!(epochs(&programs), one, "program 1 carries on"); + assert!(two.is_closed(), "the dropped program's broadcast is withdrawn"); + + steps.step(&mut programs, &[(1, 0x100), (2, 0x200)]); + let back = epochs(&programs); + assert_eq!(back[0], one[0], "program 1 carries on"); + assert_eq!(back[1].0, 2); + assert_ne!(back[1].1, both[1].1, "the returning program is a new instance"); + + let _finished = programs.finish().unwrap(); + let stats = programs.stats(); + assert_eq!(stats.streams[&0x61].units, 8, "program 1 delivered every frame"); + assert_eq!(stats.streams[&0x71].units, 2, "the new instance counts its own frames"); + } + + /// A PAT that drops a program reaches no importer of it, so the multiplex carries on + /// rather than ending on that program's refusal. + #[test] + fn a_dropped_program_does_not_end_the_multiplex() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + let mut steps = Steps { + cc: [0; 2], + pts: 90_000, + }; + steps.step(&mut programs, &[(1, 0x100), (2, 0x200)]); + // Fed a packet at a time, so the PAT that drops program 2 completes in a call of its own. + let mut mux = Mux::default(); + mux.pat(&[(1, 0x100)]); + mux.out.extend(mp2_pes(0x61, 1, 108_000, [0x61; 2])); + for packet in mux.bytes().chunks(TsPacket::SIZE) { + programs.decode(packet).unwrap(); + } + assert_eq!(epochs(&programs).len(), 1); + let _finished = programs.finish().unwrap(); + } + + /// A program whose PMT never arrives waits, unannounced, and holds up neither the others + /// nor the end of the input. + #[test] + fn a_program_without_a_pmt_waits_unannounced() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + let mut steps = Steps { + cc: [0; 2], + pts: 90_000, + }; + steps.step(&mut programs, &[(1, 0x100), (3, 0x300)]); + steps.step(&mut programs, &[(1, 0x100), (3, 0x300)]); + let _finished = programs.finish().unwrap(); + let routes: Vec<_> = programs + .programs + .iter() + .map(|program| (program.number, program.broadcast.route().is_some())) + .collect(); + assert_eq!(routes, vec![(1, true), (3, false)]); + } + #[tokio::test(start_paused = true)] async fn internal_program_abort_preserves_the_catalog_error() { let origin = crate::source::produce_origin(); @@ -258,9 +480,16 @@ mod test { assert_eq!(program_broadcast("event", 1), "event/1"); } - /// The PAT in a whole buffer, as one [`PatScan`] reads it. + /// The programs the first whole PAT in a buffer starts. fn pat_programs(data: &[u8]) -> Option> { - Some(PatScan::default().scan(&mut data.to_vec())?.program_numbers()) + let mut programs = Programs::new( + crate::source::produce_origin(), + "event.hang", + catalog::Config::default(), + ); + programs.decode(data).unwrap(); + let numbers: Vec = programs.programs.iter().map(|program| program.number).collect(); + (!numbers.is_empty()).then_some(numbers) } #[test] diff --git a/rs/moq-mux/src/container/ts/revision_test.rs b/rs/moq-mux/src/container/ts/revision_test.rs new file mode 100644 index 00000000000..a480c72a1f8 --- /dev/null +++ b/rs/moq-mux/src/container/ts/revision_test.rs @@ -0,0 +1,1686 @@ +//! A program whose PMT revises mid-stream, imported and exported again: the export announces +//! each revision with a PMT at the next `version_number`, its PAT unchanged. + +use std::time::Duration; + +use super::catalog::Ext; +use super::fixture::{Flags, MS, Mux}; +use crate::container::Frame; + +const PMT_PID: u16 = 0x20; +/// The PMT PID the export gives a program that recorded none. +const OWN_PMT_PID: u16 = 0x1000; +const CLOCK_PID: u16 = 0x1ff; +const AUDIO_PID: u16 = 0x101; +const DATA_PID: u16 = 0x102; +const OTHER_AUDIO_PID: u16 = 0x103; + +const AUDIO: u8 = 0x0f; +const PRIVATE: u8 = 0x06; +const METADATA: u8 = 0x15; + +/// ISO 639 language descriptors for English and French. +const ENG: &[u8] = &[0x0a, 0x04, b'e', b'n', b'g', 0x00]; +const FRA: &[u8] = &[0x0a, 0x04, b'f', b'r', b'a', 0x00]; +/// `metadata_descriptor` for asynchronous KLV (ISO 13818-1 2.6.60). +const KLV: &[u8] = &[0x26, 0x09, 0x01, 0x00, 0xff, b'K', b'L', b'V', b'A', 0x0f, 0x00]; + +/// How long the export holds each frame. +const DELAY: Duration = Duration::from_secs(2); + +/// A PMT the export wrote: its `version_number`, PCR PID, program descriptors, and each +/// stream as `(stream_type, PID, descriptor bytes)`. +#[derive(Debug, PartialEq)] +struct Table { + version: u8, + pcr_pid: u16, + info: Vec, + streams: Vec<(u8, u16, Vec)>, +} + +/// One span of the source: the streams its PMT lists. +struct Span<'a> { + streams: &'a [(u8, u16, &'a [u8])], +} + +/// Import `spans` one by one, each opened by a PAT and a PMT at the next version and running +/// for `intervals` PCRs 40 ms apart (the first for three seconds), with an AAC frame on each +/// audio stream and a PES on each private one every interval. The export polls after each, +/// as a live one keeps up with the catalog, and runs to its end after the last. Returns +/// every PMT it wrote, each run of repeats once. +async fn reexport(spans: &[Span<'_>], intervals: u64) -> Vec { + let spans: Vec<_> = spans.iter().map(|span| (PMT_PID, span)).collect(); + tables(&reexport_frames(&spans, intervals).await, PMT_PID) +} + +/// [`reexport`], each span's PAT naming its PMT on the PID beside it. Returns every frame +/// the export wrote. +async fn reexport_frames(spans: &[(u16, &Span<'_>)], intervals: u64) -> Vec { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + + let mut ticks = 1_000 * MS; + let mut export = None; + let mut frames = Vec::new(); + for (version, &(pmt_pid, span)) in spans.iter().enumerate() { + let mut mux = Mux::default(); + mux.pat(&[(1, pmt_pid)]) + .pmt_at(version as u8 % 32, pmt_pid, 1, CLOCK_PID, span.streams, 0); + // The import publishes its catalog once it has measured the multiplex rate. + let intervals = if version == 0 { 75 } else { intervals }; + for _ in 0..intervals { + mux.packet(CLOCK_PID, Flags::pcr(ticks)); + for &(stream_type, pid, _) in span.streams { + let payload = match stream_type { + AUDIO => aac(ticks / 300 + 9_000, pid as u8), + PRIVATE => pes(0xbd, pid as u8), + _ => pes(0xfc, pid as u8), + }; + mux.whole(pid, &payload); + } + ticks += 40 * MS; + } + import.decode(&mux.bytes()).unwrap(); + let export = match &mut export { + Some(export) => export, + None => { + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let export = export.insert(opened.with_delay(DELAY).with_replay()); + // The first frame out carries the tables built from this span alone. + let first = tokio::time::timeout(DELAY * 2, export.next()).await; + frames.push(first.expect("a first frame").unwrap().unwrap()); + export + } + }; + settle(export, &mut frames).await; + } + let mut mux = Mux::default(); + mux.packet(CLOCK_PID, Flags::pcr(ticks)); + import.decode(&mux.bytes()).unwrap(); + import.finish().unwrap(); + catalog.finish().unwrap(); + + drain(&mut export.unwrap(), &mut frames).await; + frames +} + +/// Run `export` until it waits on its source, as a live one keeps up with each catalog +/// snapshot as it comes. +async fn settle(export: &mut super::Export, frames: &mut Vec) +where + S::Ext: super::catalog::Catalog, +{ + while let Ok(frame) = tokio::time::timeout(Duration::from_millis(1), export.next()).await { + frames.push(frame.unwrap().expect("the export runs on")); + } +} + +/// Run `export` to its end. +async fn drain(export: &mut super::Export, frames: &mut Vec) +where + S::Ext: super::catalog::Catalog, +{ + loop { + match tokio::time::timeout(DELAY * 10, export.next()) + .await + .expect("the export ends") + { + Ok(Some(frame)) => frames.push(frame), + Ok(None) => break, + Err(err) => panic!("the export failed: {err}"), + } + } +} + +/// An ADTS AAC frame presenting at `pts`, its payload filled with `fill`. +fn aac(pts: u64, fill: u8) -> Vec { + let mut frame = super::adts::write_header(2, 48_000, 2, 64).unwrap().to_vec(); + frame.resize(7 + 64, fill); + let field = [ + 0x21 | ((pts >> 29) & 0x0e) as u8, + (pts >> 22) as u8, + 1 | ((pts >> 14) & 0xfe) as u8, + (pts >> 7) as u8, + 1 | ((pts << 1) & 0xfe) as u8, + ]; + let mut pes = vec![0, 0, 1, 0xc0]; + pes.extend_from_slice(&((3 + field.len() + frame.len()) as u16).to_be_bytes()); + pes.extend_from_slice(&[0x80, 0x80, field.len() as u8]); + pes.extend_from_slice(&field); + pes.extend_from_slice(&frame); + pes +} + +/// A PES of `stream_id` without a PTS, its payload filled with `fill`. +fn pes(stream_id: u8, fill: u8) -> Vec { + let payload = [fill; 24]; + let mut pes = vec![0, 0, 1, stream_id]; + pes.extend_from_slice(&((3 + payload.len()) as u16).to_be_bytes()); + pes.extend_from_slice(&[0x84, 0x00, 0x00]); + pes.extend_from_slice(&payload); + pes +} + +/// Every packet in `frames`. +fn packets(frames: &[Frame]) -> impl Iterator { + frames + .iter() + .flat_map(|frame| frame.payload.as_chunks::<188>().0.iter()) +} + +/// The PID of `packet`. +fn pid(packet: &[u8; 188]) -> u16 { + u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]) +} + +/// Every PMT on `pmt_pid` in `frames`, each run of repeats once. +fn tables(frames: &[Frame], pmt_pid: u16) -> Vec
{ + let mut out: Vec
= Vec::new(); + let mut section = Vec::new(); + for packet in packets(frames) { + if pid(packet) != pmt_pid { + continue; + } + let mut at = 4; + if packet[3] & 0x20 != 0 { + at += 1 + usize::from(packet[4]); + } + if packet[1] & 0x40 != 0 { + section = packet[at + 1 + usize::from(packet[at])..].to_vec(); + } else if !section.is_empty() { + section.extend_from_slice(&packet[at..]); + } else { + continue; + } + let len = 3 + (usize::from(section[1] & 0x0f) << 8 | usize::from(section[2])); + if section.len() < len { + continue; + } + let whole = std::mem::take(&mut section); + let section = &whole[..len]; + let pmt = super::psi::Pmt::parse(section).expect("a whole PMT"); + let bytes = |descriptors: &[super::catalog::Descriptor]| { + descriptors + .iter() + .flat_map(|d| [vec![d.tag, d.data.len() as u8], d.data.to_vec()].concat()) + .collect::>() + }; + let table = Table { + version: (section[5] >> 1) & 0x1f, + pcr_pid: pmt.pcr_pid.map_or(0x1fff, |pid| pid.as_u16()), + info: bytes(&pmt.program_info), + streams: pmt + .streams + .iter() + .map(|es| (es.stream_type, es.pid.as_u16(), bytes(&es.descriptors))) + .collect(), + }; + if out.last() != Some(&table) { + out.push(table); + } + } + out +} + +/// Every PAT in `frames` as its `version_number` and the PMT PID it names, each run of +/// repeats once. +fn pats(frames: &[Frame]) -> Vec<(u8, u16)> { + let mut out = Vec::new(); + for pat in packets(frames).filter_map(pat) { + if out.last() != Some(&pat) { + out.push(pat); + } + } + out +} + +/// The `version_number` and PMT PID of the PAT `packet` starts, if it starts one. +fn pat(packet: &[u8; 188]) -> Option<(u8, u16)> { + if pid(packet) != 0 || packet[1] & 0x40 == 0 { + return None; + } + let mut at = 4; + if packet[3] & 0x20 != 0 { + at += 1 + usize::from(packet[4]); + } + let section = &packet[at + 1 + usize::from(packet[at])..]; + Some(( + (section[5] >> 1) & 0x1f, + u16::from(section[10] & 0x1f) << 8 | u16::from(section[11]), + )) +} + +/// Each table's version and streams as `(stream_type, PID)`. +fn layouts(tables: &[Table]) -> Vec<(u8, Vec<(u8, u16)>)> { + tables + .iter() + .map(|table| { + let streams = table + .streams + .iter() + .map(|&(stream_type, pid, _)| (stream_type, pid)) + .collect(); + (table.version, streams) + }) + .collect() +} + +#[tokio::test(start_paused = true)] +async fn a_stream_added_mid_stream_revises_the_pmt() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID)]), + (1, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + ] + ); + assert!( + tables.iter().all(|table| table.pcr_pid == AUDIO_PID), + "the clock stays put" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_stream_removed_mid_stream_leaves_the_pmt_once_it_ends() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + (1, vec![(AUDIO, AUDIO_PID)]), + ] + ); +} + +/// A changed `stream_type` keeps the PID, and the PMT names the new type at the next version. +#[tokio::test(start_paused = true)] +async fn a_retyped_stream_revises_its_entry_in_place() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (METADATA, DATA_PID, KLV)], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + (1, vec![(AUDIO, AUDIO_PID), (METADATA, DATA_PID)]), + ] + ); + assert_eq!(tables[1].streams[1].2, KLV); +} + +/// An audio language retag reopens the rendition, which takes the PID and the clock over. +#[tokio::test(start_paused = true)] +async fn a_descriptor_change_revises_the_entry_and_keeps_the_clock() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, ENG)], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, FRA)], + }, + ], + 25, + ) + .await; + assert_eq!( + tables, + vec![ + Table { + version: 0, + pcr_pid: AUDIO_PID, + info: Vec::new(), + streams: vec![(AUDIO, AUDIO_PID, ENG.to_vec())], + }, + Table { + version: 1, + pcr_pid: AUDIO_PID, + info: Vec::new(), + streams: vec![(AUDIO, AUDIO_PID, FRA.to_vec())], + }, + ] + ); +} + +/// A listed track keeps the descriptors its packets went out under: a catalog revising them +/// under the same name revises nothing, as a revised stream comes under a new name. +#[tokio::test(start_paused = true)] +async fn a_listed_track_keeps_its_descriptors() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let language = |language: &[u8]| super::catalog::Descriptor { + tag: language[0], + data: bytes::Bytes::copy_from_slice(&language[2..]), + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .get_mut(&name) + .unwrap() + .descriptors = vec![language(ENG)]; + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .get_mut(&name) + .unwrap() + .descriptors = vec![language(FRA)]; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + audio.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tables = tables(&frames, OWN_PMT_PID); + let streams: Vec<_> = tables.iter().map(|table| table.streams.clone()).collect(); + assert_eq!(streams, [vec![(AUDIO, AUDIO_PID, ENG.to_vec())]]); +} + +/// The PCR leaves a PID only with the track it rode on, for the next audio track. +#[tokio::test(start_paused = true)] +async fn the_clock_moves_only_when_its_track_retires() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, ENG), (AUDIO, OTHER_AUDIO_PID, FRA)], + }, + Span { + streams: &[(AUDIO, OTHER_AUDIO_PID, FRA)], + }, + ], + 25, + ) + .await; + let clocks: Vec<(u8, u16)> = tables.iter().map(|table| (table.version, table.pcr_pid)).collect(); + assert_eq!(clocks, vec![(0, AUDIO_PID), (1, OTHER_AUDIO_PID)]); + assert_eq!(tables[1].streams, vec![(AUDIO, OTHER_AUDIO_PID, FRA.to_vec())]); +} + +/// The PCR leaves the PID of a track that retires only behind a PMT naming the PID it moves +/// to: a receiver holding the PMT before finds the clock on the PID that one names. +#[tokio::test(start_paused = true)] +async fn the_clock_moves_behind_the_pmt_naming_its_pid() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut clock) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut clock, 0x21), (&mut other, 0x21)], + 0, + 3_000, + ) + .await; + remove(&mut catalog, &name); + clock.finish().unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut other, 0x21)], 3_000, 6_000).await; + other.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let mut named = None; + let mut clocks = Vec::new(); + for frame in &frames { + for packet in packets(std::slice::from_ref(frame)) { + if packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0 { + if let Some(named) = named { + assert_eq!(pid(packet), named, "a PCR on a PID the PMT out does not name"); + } + clocks.push(pid(packet)); + } + } + if let Some(table) = tables(std::slice::from_ref(frame), OWN_PMT_PID).last() { + named = Some(table.pcr_pid); + } + } + clocks.dedup(); + assert_eq!(clocks, [AUDIO_PID, OTHER_AUDIO_PID]); +} + +/// A PID the PMT dropped can come back as another stream. +#[tokio::test(start_paused = true)] +async fn a_pid_removed_can_return_as_another_stream() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (METADATA, DATA_PID, KLV)], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + (1, vec![(AUDIO, AUDIO_PID)]), + (2, vec![(AUDIO, AUDIO_PID), (METADATA, DATA_PID)]), + ] + ); +} + +/// A changed `program_info` is a revision of its own. +#[tokio::test(start_paused = true)] +async fn a_program_descriptor_change_revises_the_pmt() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from_static(&[0xaa; 6]), + }]; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + audio.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let versions: Vec<(u8, Vec)> = tables(&frames, OWN_PMT_PID) + .into_iter() + .map(|table| (table.version, table.info)) + .collect(); + assert_eq!( + versions, + vec![(0, Vec::new()), (1, vec![0x80, 6, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa])] + ); +} + +/// Layouts that change faster than the export lags behind its source still reach the wire +/// as a run of versions, each one up from the last and differing from it: a table copied into +/// a frame queued behind a revision never goes out after it, and a revision overtaken before +/// any frame carried it takes no version. +#[tokio::test(start_paused = true)] +async fn each_version_out_follows_the_last_by_one_change() { + let with: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])]; + let without: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[])]; + let spans: Vec = (0..12) + .map(|i| Span { + streams: if i % 2 == 0 { with } else { without }, + }) + .collect(); + let tables = reexport(&spans, 10).await; + assert!(tables.len() > 2, "{tables:?}"); + for pair in tables.windows(2) { + assert_eq!(pair[1].version, (pair[0].version + 1) % 32, "{tables:?}"); + assert_ne!(pair[1].streams, pair[0].streams, "{tables:?}"); + } +} + +/// The `version_number` runs on from 31 to 0. +#[tokio::test(start_paused = true)] +async fn the_version_wraps_after_31() { + let with: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])]; + let without: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[])]; + let spans: Vec = (0..34) + .map(|i| Span { + streams: if i % 2 == 0 { with } else { without }, + }) + .collect(); + // Each span outlasts the export's lag behind its source: a PID dropped for less is + // listed again before the table dropping it goes out. + let tables = reexport(&spans, 40).await; + let versions: Vec = tables.iter().map(|table| table.version).collect(); + let expected: Vec = (0..34).map(|i| i % 32).collect(); + assert_eq!(versions, expected); + for (i, table) in tables.iter().enumerate() { + assert_eq!(table.streams.len(), if i % 2 == 0 { 2 } else { 1 }, "revision {i}"); + } +} + +/// A track's writer, its frames in the legacy container. +type Writer = crate::container::Producer; + +/// Open an AAC track on `broadcast` and list it in `catalog`, with `pid` recorded if given. +fn aac_track( + broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + pid: Option, +) -> (String, Writer) { + let track = broadcast + .create_track( + broadcast.unique_name(".aac"), + hang::container::track_info(hang::catalog::PRIORITY.audio), + ) + .unwrap(); + let name = track.name().to_string(); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AAC { profile: 2 }, 48_000, 2); + config.container = hang::catalog::Container::Legacy; + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.insert(name.clone(), config); + if let Some(pid) = pid { + guard + .ext + .mpegts + .tracks + .insert(name.clone(), super::catalog::Track::new(pid)); + } + drop(guard); + let container = crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio); + (name, Writer::new(track, container)) +} + +/// Write an AAC frame at `millis` to each of `tracks`. +fn write_aac(tracks: &mut [&mut Writer], millis: u64) { + for track in tracks { + write_marked(track, millis, 0x21); + } +} + +/// Write an AAC frame at `millis` to `track`, its payload filled with `mark`. +fn write_marked(track: &mut Writer, millis: u64, mark: u8) { + let mut payload = vec![mark; 16]; + payload[0] = 0x21; + track + .write(Frame { + timestamp: moq_net::Timestamp::from_micros(millis * 1_000).unwrap(), + duration: None, + payload: bytes::Bytes::from(payload), + keyframe: true, + }) + .unwrap(); + track.cut(None).unwrap(); +} + +/// The marks of the frames ([`write_marked`]) that went out on `pid`, in order. +fn marks(frames: &[Frame], pid: u16, marks: &[u8]) -> Vec { + let mut payload = Vec::new(); + for packet in packets(frames).filter(|packet| self::pid(packet) == pid && packet[3] & 0x10 != 0) { + let at = if packet[3] & 0x20 != 0 { + 5 + usize::from(packet[4]) + } else { + 4 + }; + payload.extend_from_slice(&packet[at..]); + } + let mut out = Vec::new(); + for (at, run) in payload.windows(15).enumerate() { + let mark = run[0]; + if marks.contains(&mark) && run.iter().all(|&byte| byte == mark) && (at == 0 || payload[at - 1] != mark) { + out.push(mark); + } + } + out +} + +/// The export of `consumer`. +async fn export_of(consumer: &moq_net::broadcast::Consumer) -> super::Export> { + let source = crate::source::announced(consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + opened.with_delay(DELAY).with_replay() +} + +/// Open the export of `consumer` once its first frame is out. +async fn open( + consumer: &moq_net::broadcast::Consumer, + frames: &mut Vec, +) -> super::Export> { + let mut export = export_of(consumer).await; + let out = tokio::time::timeout(DELAY * 2, export.next()).await; + frames.push(out.expect("the tables went out").unwrap().unwrap()); + settle(&mut export, frames).await; + export +} + +/// Take `name` out of the catalog. +fn remove(catalog: &mut crate::catalog::Producer, name: &str) { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.remove(name); + guard.ext.mpegts.tracks.remove(name); +} + +/// Under sync the tables stay as laid, so a track added after them fails the export. +#[tokio::test(start_paused = true)] +async fn a_track_added_under_sync_fails_the_export() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let mut export = opened.with_delay(DELAY).with_mux_rate(2_000_000).with_sync(); + let out = tokio::time::timeout(DELAY * 2, export.next()).await; + out.expect("the tables went out").unwrap().unwrap(); + + let _second = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let err = tokio::time::timeout(DELAY * 2, async { + loop { + if let Err(err) = export.next().await { + break err; + } + } + }) + .await + .expect("the export reads the catalog update"); + assert!(err.to_string().contains("under --sync"), "{err}"); +} + +/// A track the catalog lists after the tables went out, but before its PID is recorded, +/// takes the frames it holds along to the recorded PID when that arrives. +#[tokio::test(start_paused = true)] +async fn a_pid_recorded_after_its_track_takes_the_held_frames_along() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let mut export = opened.with_delay(DELAY).with_replay(); + let out = tokio::time::timeout(DELAY * 2, export.next()).await; + let mut frames = vec![out.expect("the tables went out").unwrap().unwrap()]; + settle(&mut export, &mut frames).await; + + let (name, mut second) = aac_track(&mut broadcast, &mut catalog, None); + settle(&mut export, &mut frames).await; + for millis in (3_000..3_500).step_by(20) { + write_aac(&mut [&mut first, &mut second], millis); + } + settle(&mut export, &mut frames).await; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(OTHER_AUDIO_PID)); + settle(&mut export, &mut frames).await; + for millis in (3_500..6_000).step_by(20) { + write_aac(&mut [&mut first, &mut second], millis); + } + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tables = tables(&frames, OWN_PMT_PID); + assert_eq!( + tables.last().expect("a PMT").streams, + vec![(AUDIO, AUDIO_PID, Vec::new()), (AUDIO, OTHER_AUDIO_PID, Vec::new())] + ); + let carried: std::collections::BTreeSet = packets(&frames) + .filter(|packet| packet[1] & 0x40 != 0 && packet[3] & 0x10 != 0) + .filter(|packet| { + let at = if packet[3] & 0x20 != 0 { + 5 + usize::from(packet[4]) + } else { + 4 + }; + packet[at..].starts_with(&[0, 0, 1]) + }) + .map(pid) + .collect(); + assert_eq!( + carried, + [AUDIO_PID, OTHER_AUDIO_PID].into(), + "no PES left on another PID" + ); +} + +/// The next frame `export` writes. +async fn next(export: &mut super::Export) -> Frame +where + S::Ext: super::catalog::Catalog, +{ + let out = tokio::time::timeout(DELAY * 4, export.next()).await; + out.expect("a frame").unwrap().expect("the export runs on") +} + +/// A track not listed yet, whose first frame is held behind the old generation's tail, takes +/// that frame along to the PID recorded meanwhile. +#[tokio::test(start_paused = true)] +async fn a_pid_recorded_while_its_frame_is_held_takes_the_frame_along() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let mut frames = Vec::new(); + let mut export = open(&consumer, &mut frames).await; + while export.buffered() > 0 { + frames.push(next(&mut export).await); + } + let (name, mut second) = aac_track(&mut broadcast, &mut catalog, None); + settle(&mut export, &mut frames).await; + write_aac(&mut [&mut second], 3_000); + for millis in (3_020..3_500).step_by(20) { + write_aac(&mut [&mut first, &mut second], millis); + } + settle(&mut export, &mut frames).await; + tokio::time::advance(DELAY * 2).await; + let held = export.hold_next().expect("a frame due"); + assert_ne!(held, AUDIO_PID, "the second track's frame is held"); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(OTHER_AUDIO_PID)); + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert_eq!( + tables(&frames, OWN_PMT_PID).last().expect("a PMT").streams, + vec![(AUDIO, AUDIO_PID, Vec::new()), (AUDIO, OTHER_AUDIO_PID, Vec::new())] + ); +} + +/// A track taking a PID over goes out behind everything the track before it there sent, even a +/// tail sent further ahead of its decode time than the new track's first frame: none of the +/// old stream goes out under the PMT revised for the new one. +#[tokio::test(start_paused = true)] +async fn a_track_taking_a_pid_over_goes_out_behind_the_tail_before_it() { + const OLD: u8 = 0xa5; + const NEW: u8 = 0x5a; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut old, OLD), (&mut other, 0x21)], + 0, + 3_000, + ) + .await; + + // The old stream's tail goes out ahead of its time, running on past where the new one starts. + for millis in (3_000..3_800).step_by(20) { + write_marked(&mut old, millis, OLD); + } + old.finish().unwrap(); + remove(&mut catalog, &name); + settle(&mut export, &mut frames).await; + let (_, mut new) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + settle(&mut export, &mut frames).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut new, NEW), (&mut other, 0x21)], + 3_000, + 5_000, + ) + .await; + new.finish().unwrap(); + other.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let marks = marks(&frames, AUDIO_PID, &[OLD, NEW]); + let first = marks + .iter() + .position(|&mark| mark == NEW) + .expect("the new stream went out"); + assert!(marks[..first].contains(&OLD), "the old stream went out first"); + assert!( + marks[first..].iter().all(|&mark| mark == NEW), + "the old stream's tail went out after the new one began: {marks:x?}" + ); +} + +/// A PID the PMT stops listing is gone rather than quiet, so a stream returning on it counts +/// its silence from its own listing, not from the last unit of the stream before it. +#[tokio::test(start_paused = true)] +async fn a_pid_that_leaves_the_pmt_is_gone_until_it_returns() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut old, 0x21), (&mut other, 0x21)], + 0, + 2_000, + ) + .await; + old.finish().unwrap(); + remove(&mut catalog, &name); + live(&mut export, &mut frames, start, &mut [(&mut other, 0x21)], 2_000, 5_000).await; + assert!(!export.stats().streams.contains_key(&AUDIO_PID), "the PID left the PMT"); + assert_eq!( + export.liveness(AUDIO_PID).1, + None, + "a PID the PMT left is gone, not quiet" + ); + + let (_, mut new) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + settle(&mut export, &mut frames).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut new, 0x21), (&mut other, 0x21)], + 5_000, + 7_500, + ) + .await; + let (_, quiet) = export.liveness(AUDIO_PID); + assert!( + quiet.is_some_and(|quiet| quiet < Duration::from_secs(1)), + "the returned stream is quiet for {quiet:?}" + ); +} + +/// A track whose first frame is held behind a generation boundary, then moved onto a PID taken +/// over, goes out behind the tail there all the same: the held frame drops. +#[tokio::test(start_paused = true)] +async fn a_held_frame_moved_onto_a_pid_taken_over_waits_for_the_tail() { + const OLD: u8 = 0xa5; + const NEW: u8 = 0x5a; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (old_name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_marked(&mut old, millis, OLD); + } + let mut frames = Vec::new(); + let mut export = open(&consumer, &mut frames).await; + while export.buffered() > 0 { + frames.push(next(&mut export).await); + } + let (name, mut new) = aac_track(&mut broadcast, &mut catalog, None); + settle(&mut export, &mut frames).await; + write_marked(&mut new, 3_000, NEW); + for millis in (3_020..3_500).step_by(20) { + write_marked(&mut old, millis, OLD); + write_marked(&mut new, millis, NEW); + } + settle(&mut export, &mut frames).await; + tokio::time::advance(DELAY * 2).await; + let held = export.hold_next().expect("a frame due"); + assert_ne!(held, AUDIO_PID, "the new track's frame is held"); + { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.remove(&old_name); + guard.ext.mpegts.tracks.remove(&old_name); + guard + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(AUDIO_PID)); + } + for millis in (3_500..4_000).step_by(20) { + write_marked(&mut old, millis, OLD); + write_marked(&mut new, millis, NEW); + } + old.finish().unwrap(); + new.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let marks = marks(&frames, AUDIO_PID, &[OLD, NEW]); + let first = marks + .iter() + .position(|&mark| mark == NEW) + .expect("the new stream went out"); + assert!( + marks[first..].iter().all(|&mark| mark == NEW), + "the old stream's tail went out after the new one began: {marks:x?}" + ); +} + +/// A track that left the catalog while waiting to take a PID over goes once a later one takes +/// its place there, so such tracks do not pile up behind a stream that never ends. +#[tokio::test(start_paused = true)] +async fn a_track_waiting_to_take_a_pid_over_goes_once_superseded() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut old, 0x21)], 0, 2_000).await; + remove(&mut catalog, &name); + let mut unended = Vec::new(); + for round in 0..4 { + let (name, mut successor) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let from = 2_000 + round * 500; + live( + &mut export, + &mut frames, + start, + &mut [(&mut old, 0x21), (&mut successor, 0x21)], + from, + from + 500, + ) + .await; + remove(&mut catalog, &name); + unended.push(successor); + } + settle(&mut export, &mut frames).await; + assert_eq!( + export.tracks(), + 2, + "the stream that never ends, and the last track waiting behind it" + ); +} + +/// A track that leaves the catalog drains under the program descriptors it was listed with: the +/// PMT carries them beside the revision's own until it ends. +#[tokio::test(start_paused = true)] +async fn a_track_leaving_drains_under_the_program_descriptors_it_was_listed_with() { + let cuei = super::catalog::Descriptor { + tag: 0x05, + data: bytes::Bytes::from_static(b"CUEI"), + }; + let other = super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from_static(&[0xaa, 0xaa]), + }; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![cuei]; + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (name, mut leaving) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut leaving, 0x21)], + 0, + 3_000, + ) + .await; + { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.remove(&name); + guard.ext.mpegts.tracks.remove(&name); + guard.ext.mpegts.program_descriptors = vec![other]; + } + leaving.finish().unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 3_000, 6_000).await; + first.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let versions: Vec<(u8, Vec, Vec)> = tables(&frames, OWN_PMT_PID) + .into_iter() + .map(|table| (table.version, table.info, table.streams.iter().map(|es| es.1).collect())) + .collect(); + let (cuei, other) = (b"\x05\x04CUEI".to_vec(), vec![0x80, 2, 0xaa, 0xaa]); + assert_eq!( + versions, + vec![ + (0, cuei.clone(), vec![AUDIO_PID, OTHER_AUDIO_PID]), + (1, [other.clone(), cuei].concat(), vec![AUDIO_PID, OTHER_AUDIO_PID]), + (2, other, vec![AUDIO_PID]), + ] + ); +} + +/// A track draining its tail keeps only the program descriptors it left the catalog under +/// beside the current ones, however often those change meanwhile: carrying every set between +/// would outgrow a PMT section. +#[tokio::test(start_paused = true)] +async fn a_track_draining_keeps_only_the_program_descriptors_it_left_under() { + let descriptor = |tag: u8| super::catalog::Descriptor { + tag, + data: bytes::Bytes::from(vec![tag; 4]), + }; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![descriptor(0x80)]; + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (name, mut leaving) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + let mut tracks = [(&mut first, 0x21), (&mut leaving, 0x21)]; + live(&mut export, &mut frames, start, &mut tracks, 0, 3_000).await; + remove(&mut catalog, &name); + for (at, tag) in (3_000..).step_by(1_000).zip(0x81..=0x85) { + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![descriptor(tag)]; + live(&mut export, &mut frames, start, &mut tracks, at, at + 1_000).await; + } + leaving.finish().unwrap(); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21)], + 8_000, + 11_000, + ) + .await; + first.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tags = |info: &[u8]| { + let mut tags = Vec::new(); + let mut at = 0; + while at < info.len() { + tags.push(info[at]); + at += 2 + usize::from(info[at + 1]); + } + tags + }; + let tables = tables(&frames, OWN_PMT_PID); + let infos: Vec> = tables.iter().map(|table| tags(&table.info)).collect(); + assert_eq!( + infos, + [ + vec![0x80], + vec![0x81, 0x80], + vec![0x82, 0x80], + vec![0x83, 0x80], + vec![0x84, 0x80], + vec![0x85, 0x80], + vec![0x85], + ] + ); +} + +/// A track the catalog records on the PMT's PID as it moves the PMT takes another: the PMT +/// still rides that PID when the track is taken on, and moves behind a new PAT after. +#[tokio::test(start_paused = true)] +async fn a_track_recorded_on_the_pid_the_pmt_moves_from_takes_another() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program = Some(program(PMT_PID)); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 3_000).await; + catalog.modify().unwrap().ext.mpegts.program = Some(program(OWN_PMT_PID)); + let (_, mut added) = aac_track(&mut broadcast, &mut catalog, Some(PMT_PID)); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut added, 0x21)], + 3_000, + 6_000, + ) + .await; + first.finish().unwrap(); + added.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert_eq!(pats(&frames), [(0, PMT_PID), (1, OWN_PMT_PID)]); + let tables = tables(&frames, OWN_PMT_PID); + let pids: Vec = tables.last().unwrap().streams.iter().map(|es| es.1).collect(); + assert_eq!(pids.len(), 2, "{tables:?}"); + assert!(!pids.contains(&PMT_PID) && !pids.contains(&OWN_PMT_PID), "{tables:?}"); +} + +/// An SCTE-35 `splice_null` section. +const CUE: &[u8] = &[ + 0xfc, 0x30, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xf0, 0x0a, 0x05, 0x00, 0x00, 0x2b, 0xb4, 0x7f, + 0xdf, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xad, 0x25, 0xe8, 0x39, +]; + +/// Tables go out with the frame they were muxed with, and a placed frame waits for its +/// placement, so a revision muxed after another can go out first. The copies still queued +/// then carry the later table, under the version the earlier one took: the wire still +/// steps by one. +#[tokio::test(start_paused = true)] +async fn a_revision_out_before_the_one_it_follows_takes_its_version() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 3_000).await; + + // A cue and an AAC track start together, the cue muxed first, on the lower PID: its + // revision waits with it for its placement, the AAC track's goes out now. + let track = broadcast + .unique_track(".scte35", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let entry = super::catalog::Track { + pid: DATA_PID, + descriptors: Vec::new(), + verbatim: Some(super::catalog::Verbatim::new(0x86, super::catalog::Framing::Section)), + pes_packet: None, + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(track.name().to_string(), entry); + let mut cue = Writer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + let (_, mut added) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + settle(&mut export, &mut frames).await; + tokio::time::sleep_until(start + Duration::from_millis(3_000)).await; + cue.write(Frame { + timestamp: moq_net::Timestamp::from_millis(3_000).unwrap(), + duration: None, + payload: bytes::Bytes::from_static(CUE), + keyframe: true, + }) + .unwrap(); + cue.cut(None).unwrap(); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut added, 0x21)], + 3_000, + 6_000, + ) + .await; + for track in [&mut first, &mut added, &mut cue] { + track.finish().unwrap(); + } + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tables = tables(&frames, OWN_PMT_PID); + for pair in tables.windows(2) { + assert_eq!(pair[1].version, (pair[0].version + 1) % 32, "{tables:?}"); + } + let pids: Vec = tables.last().unwrap().streams.iter().map(|es| es.1).collect(); + assert_eq!(pids, [AUDIO_PID, DATA_PID, OTHER_AUDIO_PID], "{tables:?}"); +} + +/// A stream taking a PID over from a buffered one goes out on its own model, not the one +/// before it: a private section larger than the AAC buffer before it still goes out. +#[tokio::test(start_paused = true)] +async fn a_stream_taking_a_pid_over_leaves_the_buffer_before_it() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (name, mut before) = aac_track(&mut broadcast, &mut catalog, Some(DATA_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut before, 0x21)], + 0, + 3_000, + ) + .await; + remove(&mut catalog, &name); + before.finish().unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 3_000, 6_000).await; + + let track = broadcast + .unique_track(".private", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let entry = super::catalog::Track { + pid: DATA_PID, + descriptors: Vec::new(), + verbatim: Some(super::catalog::Verbatim::new(0x05, super::catalog::Framing::Section)), + pes_packet: None, + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(track.name().to_string(), entry); + let mut private = Writer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + settle(&mut export, &mut frames).await; + tokio::time::sleep_until(start + Duration::from_millis(6_000)).await; + let mut section = vec![0x80, 0x7f, 0xfd]; + section.resize(4_096, 0xab); + private + .write(Frame { + timestamp: moq_net::Timestamp::from_millis(6_000).unwrap(), + duration: None, + payload: bytes::Bytes::from(section), + keyframe: true, + }) + .unwrap(); + private.cut(None).unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 6_000, 9_000).await; + for track in [&mut first, &mut private] { + track.finish().unwrap(); + } + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let layout = layouts(&tables(&frames, OWN_PMT_PID)); + assert_eq!(layout.last().unwrap().1, [(AUDIO, AUDIO_PID), (0x05, DATA_PID)]); + let sent = packets(&frames).filter(|packet| pid(packet) == DATA_PID && packet[4..].iter().all(|&b| b == 0xab)); + assert_eq!(sent.count(), 4_096 / 184 - 1, "the section went out whole"); +} + +/// Tracks leaving one after another each drain under the program descriptors they left +/// under, until those outgrow a section beside the rest: then all drain under the current +/// set, and none comes back. +#[tokio::test(start_paused = true)] +async fn draining_descriptor_sets_that_outgrow_a_section_give_way_to_the_current_one() { + let descriptor = |tag: u8| super::catalog::Descriptor { + tag, + data: bytes::Bytes::from(vec![tag; 250]), + }; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![descriptor(0x80)]; + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (names, mut leaving): (Vec, Vec) = (0x103..=0x105) + .map(|pid| aac_track(&mut broadcast, &mut catalog, Some(pid))) + .unzip(); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + let mut at = 0; + for (index, tag) in (0..=names.len()).zip(0x81..) { + let until = if index == 0 { 3_000 } else { at + 1_000 }; + let mut tracks: Vec<(&mut Writer, u8)> = vec![(&mut first, 0x21)]; + tracks.extend(leaving.iter_mut().map(|track| (track, 0x21))); + live(&mut export, &mut frames, start, &mut tracks, at, until).await; + at = until; + if let Some(name) = names.get(index) { + remove(&mut catalog, name); + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![descriptor(tag)]; + } + } + for track in &mut leaving { + track.finish().unwrap(); + } + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21)], + at, + at + 3_000, + ) + .await; + first.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tags = |info: &[u8]| { + let mut tags = Vec::new(); + let mut at = 0; + while at < info.len() { + tags.push(info[at]); + at += 2 + usize::from(info[at + 1]); + } + tags + }; + let tables = tables(&frames, OWN_PMT_PID); + let infos: Vec> = tables.iter().map(|table| tags(&table.info)).collect(); + let mut runs = infos.clone(); + runs.dedup(); + assert_eq!(runs, [vec![0x80], vec![0x81, 0x80], vec![0x82, 0x80, 0x81], vec![0x83]]); +} + +/// A stream added while one that left drains, the two too many for a section beside the +/// rest, ends the drain there: the PMT lists the program the catalog does. +#[tokio::test(start_paused = true)] +async fn a_draining_stream_too_many_for_a_section_gives_way() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + // A stream with 255 bytes of descriptors: three fit a section, four do not. + fn track( + broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + pid: u16, + ) -> (String, Writer) { + let (name, writer) = aac_track(broadcast, catalog, Some(pid)); + let descriptor = super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from(vec![pid as u8; 250]), + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .get_mut(&name) + .unwrap() + .descriptors = vec![descriptor]; + (name, writer) + } + let (_, mut first) = track(&mut broadcast, &mut catalog, AUDIO_PID); + let (_, mut second) = track(&mut broadcast, &mut catalog, OTHER_AUDIO_PID); + let (name, mut leaving) = track(&mut broadcast, &mut catalog, 0x105); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + let mut tracks = [(&mut first, 0x21), (&mut second, 0x21), (&mut leaving, 0x21)]; + live(&mut export, &mut frames, start, &mut tracks, 0, 3_000).await; + remove(&mut catalog, &name); + let (_, mut added) = track(&mut broadcast, &mut catalog, 0x106); + let mut tracks = [ + (&mut first, 0x21), + (&mut second, 0x21), + (&mut leaving, 0x21), + (&mut added, 0x21), + ]; + live(&mut export, &mut frames, start, &mut tracks, 3_000, 4_000).await; + leaving.finish().unwrap(); + let mut tracks = [(&mut first, 0x21), (&mut second, 0x21), (&mut added, 0x21)]; + live(&mut export, &mut frames, start, &mut tracks, 4_000, 7_000).await; + for track in [&mut first, &mut second, &mut added] { + track.finish().unwrap(); + } + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let pids: Vec> = layouts(&tables(&frames, OWN_PMT_PID)) + .into_iter() + .map(|(_, streams)| streams.into_iter().map(|(_, pid)| pid).collect()) + .collect(); + assert_eq!( + pids, + [ + vec![AUDIO_PID, OTHER_AUDIO_PID, 0x105], + vec![AUDIO_PID, OTHER_AUDIO_PID, 0x106] + ] + ); +} + +/// Tracks the catalog takes back before sending anything, their sources left open, go with +/// it, each on a PID of its own as they are. +#[tokio::test(start_paused = true)] +async fn tracks_withdrawn_before_sending_anything_go() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 3_000).await; + let mut open = Vec::new(); + for at in (3_000..8_000).step_by(1_000) { + let (name, withdrawn) = aac_track(&mut broadcast, &mut catalog, None); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], at, at + 500).await; + remove(&mut catalog, &name); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21)], + at + 500, + at + 1_000, + ) + .await; + open.push(withdrawn); + } + assert_eq!(export.tracks(), 1); + first.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + drop(open); +} + +/// Write a frame stamped `millis`, filled with its mark, to each of `tracks` for each 20 ms +/// from `from` to `to`, `millis` after `start`, as a live source does, the export keeping up. +async fn live( + export: &mut super::Export, + frames: &mut Vec, + start: tokio::time::Instant, + tracks: &mut [(&mut Writer, u8)], + from: u64, + to: u64, +) where + S::Ext: super::catalog::Catalog, +{ + for millis in (from..to).step_by(20) { + tokio::time::sleep_until(start + Duration::from_millis(millis)).await; + for (track, mark) in tracks.iter_mut() { + write_marked(track, millis, *mark); + } + settle(export, frames).await; + } +} + +/// A PAT that moves the PMT before any table has gone out takes every copy queued with it: +/// the PAT names the new PID at the version the first had, and nothing goes out on the old. +#[tokio::test(start_paused = true)] +async fn a_pmt_moved_before_its_tables_go_out_moves_every_copy() { + const MOVED_PMT_PID: u16 = 0x30; + let span = Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }; + let frames = reexport_frames(&[(PMT_PID, &span), (MOVED_PMT_PID, &span)], 25).await; + assert_eq!(pats(&frames), vec![(0, MOVED_PMT_PID)]); + assert_eq!( + layouts(&tables(&frames, MOVED_PMT_PID)), + vec![(0, vec![(AUDIO, AUDIO_PID)])] + ); + assert!(packets(&frames).all(|packet| pid(packet) != PMT_PID)); +} + +/// The program a catalog records, its PMT on `pmt_pid`. +fn program(pmt_pid: u16) -> super::catalog::Program { + super::catalog::Program { + transport_stream_id: 1, + program_number: 1, + pmt_pid, + } +} + +/// A catalog that moves the PMT to another PID is followed by a PAT at the next version +/// naming it, and from then on the PMT rides the new PID alone. +#[tokio::test(start_paused = true)] +async fn a_moved_pmt_moves_behind_a_new_pat() { + const MOVED_PMT_PID: u16 = 0x30; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program = Some(program(PMT_PID)); + let (_, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + catalog.modify().unwrap().ext.mpegts.program = Some(program(MOVED_PMT_PID)); + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + audio.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert_eq!(pats(&frames), vec![(0, PMT_PID), (1, MOVED_PMT_PID)]); + assert_eq!( + layouts(&tables(&frames, MOVED_PMT_PID)), + vec![(0, vec![(AUDIO, AUDIO_PID)])] + ); + let moved = packets(&frames) + .position(|packet| pat(packet) == Some((1, MOVED_PMT_PID))) + .expect("the PAT naming the new PID"); + assert!( + packets(&frames).take(moved).all(|packet| pid(packet) != MOVED_PMT_PID), + "no PMT on the new PID before the PAT naming it" + ); + assert!( + packets(&frames).skip(moved).all(|packet| pid(packet) != PMT_PID), + "no PMT on the old PID after the PAT that moved it" + ); +} diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index d9f28b5d0fa..dcd7b01c42e 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -621,6 +621,16 @@ impl Schedule { } } + /// Hold the packets on `pid` to no buffer, for a stream with no T-STD model: one taking a + /// PID over must not inherit its predecessor's. + pub fn clear_buffer(&mut self, pid: u16) { + let before = self.transport_buffer(pid); + self.buffers.remove(&pid); + if self.transport_buffer(pid) != before { + self.remap(); + } + } + /// Hold the program tables, the PAT and the PMT on `pmt_pid`, to the one TBsys they share. pub fn set_tables(&mut self, pmt_pid: u16) { let held = self.pmt == Some(pmt_pid) @@ -674,6 +684,19 @@ impl Schedule { runs } + /// Rewrite each queued unit's packets with `rewrite`, given how many have gone out, and find + /// its TBsys runs again. + pub fn rewrite( + &mut self, + mut rewrite: impl FnMut(&mut Vec, usize) -> anyhow::Result<()>, + ) -> anyhow::Result<()> { + for unit in &mut self.units { + rewrite(&mut unit.packets, unit.sent)?; + } + self.remap(); + Ok(()) + } + /// Find each queued unit's TBsys runs again once the PIDs held to TBsys change. fn remap(&mut self) { let mut units = std::mem::take(&mut self.units); @@ -811,6 +834,11 @@ impl Schedule { !self.units.is_empty() } + /// Whether a unit on `pid` is still to go out. + pub fn holds(&self, pid: u16) -> bool { + self.units.iter().any(|unit| unit.pid == pid) + } + /// Whether nothing is left to lay out: every unit sent, and the clock past its decode time. pub fn is_empty(&self) -> bool { if let Some(grid) = &self.sync { diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index b3d6bbc27a3..d47e3282ccd 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -143,6 +143,8 @@ pub(crate) struct Buffer { /// The next generation's frames, held while its clock is acquired. acquire: Option>, tracks: BTreeMap>, + /// The tracks whose frames stay queued past their deadlines ([`Buffer::wait`]). + waiting: BTreeSet, /// The latest deadline given out. horizon: Option, /// The tracks an acquisition waits to hear from, bounded, so that it anchors on whichever @@ -353,6 +355,7 @@ impl Buffer { previous: None, acquire: None, tracks: BTreeMap::new(), + waiting: BTreeSet::new(), horizon: None, expect: BTreeSet::new(), released: false, @@ -806,9 +809,12 @@ impl Buffer { /// in decode order, ties by track. Not in deadline order: two frames of one decode time /// queued either side of a steering step can be due nanoseconds apart, which would make /// the order depend on when each was read rather than on the media. + /// + /// A waiting track ([`Self::wait`]) is passed over. fn front(&self) -> Option<(Instant, &K)> { self.tracks .iter() + .filter(|(key, _)| !self.waiting.contains(*key)) .filter_map(|(key, track)| { let (generation, decode, deadline, _) = track.queue.front()?; Some(((*generation, decode.as_nanos(), key), *deadline)) @@ -817,6 +823,39 @@ impl Buffer { .map(|((_, _, key), deadline)| (deadline, key)) } + /// Drop what each waiting track ([`Self::wait`]) holds of a generation older than every + /// other track's next frame. The generations go out in turn, so it would go out ahead of + /// the frames the track waits behind: it is late. As for any loss before a track's first + /// frame out, a track that has sent nothing loses all it holds and starts at its next sync + /// frame. + fn lapse(&mut self) { + let Some(next) = self + .tracks + .iter() + .filter(|(key, _)| !self.waiting.contains(*key)) + .filter_map(|(_, track)| track.queue.front().map(|(generation, ..)| *generation)) + .min() + else { + return; + }; + let older = |track: &Track| track.queue.front().is_some_and(|(generation, ..)| *generation < next); + for key in &self.waiting { + let Some(track) = self.tracks.get_mut(key).filter(|track| older(track)) else { + continue; + }; + if !track.started { + self.trimmed += track.queue.len() as u64; + track.queue.clear(); + track.waiting = true; + continue; + } + while older(track) { + track.queue.pop_front(); + self.dropped += 1; + } + } + } + /// When the acquisition under way ends, unless a track starts a new group first. fn acquired_by(&self) -> Option { self.acquire.as_ref().map(|acquire| { @@ -880,6 +919,7 @@ impl Buffer { self.acquired(); continue; } + self.lapse(); let front = self.front().map(|(deadline, key)| (deadline, key.clone())); if let Some((deadline, key)) = &front && (self.delay.is_zero() || now >= *deadline) @@ -941,6 +981,77 @@ impl Buffer { Some(item) } + /// Whether `key` has a frame waiting, queued or held for a clock being acquired. + pub fn holds(&self, key: &K) -> bool { + self.peek(key).is_some() + || self + .acquire + .as_ref() + .is_some_and(|acquire| acquire.tracks.contains_key(key)) + } + + /// Drop what is kept of `key`, a track that will send nothing more: an ended track's lead + /// would otherwise place every later floor, and its keys would pile up. The frames it still + /// holds, of a track leaving before its first went out, are trimmed. + pub fn forget(&mut self, key: &K) { + if let Some(track) = self.tracks.remove(key) { + self.trimmed += track.queue.len() as u64; + } + if let Some(acquire) = &mut self.acquire { + acquire.tracks.remove(key); + let held = acquire.frames.len(); + acquire.frames.retain(|(track, _)| track != key); + self.trimmed += (held - acquire.frames.len()) as u64; + if acquire.frames.is_empty() { + self.acquire = None; + } + } + self.waiting.remove(key); + self.leads.remove(key); + self.steer.tracks.remove(key); + if let Some(floor) = &mut self.steer.floor { + floor.tracks.remove(key); + } + self.expect.remove(key); + } + + /// Carry everything kept of `from` over to `to`, a key nothing uses yet. + pub fn rekey(&mut self, from: &K, to: K) { + fn carry(map: &mut BTreeMap, from: &K, to: &K) { + if let Some(value) = map.remove(from) { + map.insert(to.clone(), value); + } + } + carry(&mut self.tracks, from, &to); + carry(&mut self.leads, from, &to); + carry(&mut self.steer.tracks, from, &to); + if let Some(floor) = &mut self.steer.floor { + carry(&mut floor.tracks, from, &to); + } + if let Some(acquire) = &mut self.acquire { + carry(&mut acquire.tracks, from, &to); + for (key, _) in acquire.frames.iter_mut().filter(|(key, _)| key == from) { + *key = to.clone(); + } + } + if self.waiting.remove(from) { + self.waiting.insert(to.clone()); + } + if self.expect.remove(from) { + self.expect.insert(to); + } + } + + /// Keep `key`'s frames queued past their deadlines while `wait`: a track that must go out + /// behind another's frames, whatever their release times. What it holds of a generation + /// the others have left drops ([`Self::lapse`]). + pub fn wait(&mut self, key: &K, wait: bool) { + match wait { + true => self.waiting.insert(key.clone()), + false => self.waiting.remove(key), + }; + } + /// When the next queued frame is due, or the acquisition under way ends. #[cfg(test)] pub fn next_deadline(&self) -> Option { @@ -1578,6 +1689,37 @@ mod tests { ); } + /// A waiting track never goes out ahead of the others, not even with a generation they have + /// left: what it holds of that drops. + #[tokio::test(start_paused = true)] + async fn a_waiting_track_drops_what_it_holds_of_a_generation_left() { + let start = moq_clock::now(); + let mut buffer = Buffer::new(DELAY).replay(); + buffer.push(1, arrival(start, 0, "v0")).unwrap(); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v0"]); + + buffer.wait(&2, true); + let now = moq_clock::now(); + buffer.push(2, arrival(now, 20, "w20")).unwrap(); + buffer.push(2, arrival(now, 40, "w40")).unwrap(); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(due(&mut buffer).is_empty(), "a waiting track is passed over"); + + // The other track crosses a restart onto a new generation. + let now = moq_clock::now(); + buffer.push(1, after(1, arrival(now, 5_000, "v5000"))).unwrap(); + tokio::time::advance(DELAY).await; + assert_eq!(released(&mut buffer), [("v5000", 1)]); + assert_eq!(buffer.trimmed(), 2); + + buffer.wait(&2, false); + let now = moq_clock::now(); + buffer.push(2, arrival(now, 5_020, "w5020")).unwrap(); + tokio::time::advance(Duration::from_millis(20)).await; + assert_eq!(released(&mut buffer), [("w5020", 1)]); + } + /// What a run against a drifting source saw. struct Run { /// The error the run ended on, if it failed. diff --git a/test/justfile b/test/justfile index 133e50e2b91..01f2941b5b8 100644 --- a/test/justfile +++ b/test/justfile @@ -164,6 +164,7 @@ ts *args: python3 ./ts/holes.test.py python3 ./ts/passthrough.test.py python3 ./ts/klv.test.py + python3 ./ts/revision.test.py python3 ./ts/clockless.test.py ./ts/run.sh "$@" diff --git a/test/ts/README.md b/test/ts/README.md index 8e7f71e464e..a8dd2f5fee8 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -44,6 +44,7 @@ just test ts --leak # 1080p video with no HRD, filling its level's 2 just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --fragment-pes # legal headers split across packets, with FFmpeg/TSDuck grading just test ts --klv # KLV and DVB AC-3 carried as whole PES, graded against the source +just test ts --revision # the PMT revised mid-stream, a KLV stream joining and leaving just test ts --delay 2s # pass the exporter's --delay just test ts --hrd --rate 7000000 # congest the subscriber's leg below the media rate just test ts --loss 1 # drop 1% of the relay's datagrams to the subscriber @@ -68,6 +69,15 @@ source placement on the program clock, and none after its PTS; and FFmpeg readin each as KLV with the same PES and PTS. `holes.py` and the T-STD model grade the AC-3, whose buffer drains each sync frame at its own decode time. +`--revision` has `revision.py` add asynchronous KLV to the default clip's null +slots from a third of its program clock to two thirds, and revise the PMT at each +end: the next `version_number` lists the stream before its first packet, and the +one after drops it once its last packet is out. `revision.py grade` holds the +capture's PMTs to the source's layouts in order, each at the version after the one +before and the KLV entry byte-equal, every packet of the stream between the PMT +listing it and the one dropping it, and every PES of the stream in order and +byte-equal. + `--live` swaps the analyzer, not the rig: the same round-trip runs, but the subscriber's stdout goes straight into `pcr-timing.py` instead of a capture file. That is the only arm that can see release timing at all, and it is what nightly @@ -116,6 +126,7 @@ Severities: **hard** checks fail the run by default; **shape** checks report as | `packet-size` | hard | 188 (or 204) bytes per packet | | `sync` | hard | no invalid sync bytes / transport-error packets | | `pat` / `pmt` | hard | valid PAT mapping programs to a PMT that lists the elementary streams | +| `pmt-versions` | hard | each program's PMT changes only at a new `version_number`, and never returns to an older one | | `psi-crc` | hard | no section dropped for a bad CRC | | `continuity` | hard | no continuity-counter discontinuities | | `pcr-presence` | hard | every program's declared PCR PID carries PCR (0x1FFF declares none) | @@ -894,7 +905,7 @@ browser interop. It runs nightly on `main`, and release PRs touching a workspace crate `run.sh` builds (kept in sync by `sh/gh/ts-paths.py`), the harness, or the toolchain run it too: `just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, full, headroom, -open-GOP, KLV (strict), passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. +open-GOP, KLV (strict), PMT revision, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 829100951ef..083a7c465ee 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -14,9 +14,9 @@ in `pcr-timing.py`. Checks split into two severities: - - HARD (structural): fail the run by default. PAT/PMT, packet size, sync, - continuity counters, PSI CRC, PCR presence, PCR monotonicity, T-STD buffers, - system buffers. + - HARD (structural): fail the run by default. PAT/PMT, PMT versions, packet + size, sync, continuity counters, PSI CRC, PCR presence, PCR monotonicity, + T-STD buffers, system buffers. - SHAPE (broadcast profile): reported as WARN and only fail the run under `--strict`. Service descriptors (SDT). @@ -1527,6 +1527,64 @@ def take(self, start: float, end: float, first: bool, grade: Grade) -> None: grade.flag("Bsys overflow") +def pmt_tables(data: bytes, packet_size: int) -> dict[int, list[tuple[int, int, bytes]]]: + """Each program's PMTs in force (current_next_indicator set), by program_number: (index + of the packet completing the section, version_number, the section less its version and + CRC), on whichever PMT PID a PAT named, each run of repeats once.""" + pmt_pids = set() + for _, section in sections(packets_on(data, packet_size, {0}).get(0, [])): + if section[0] == 0x00 and section[5] & 0x01: + for at in range(8, len(section) - 4, 4): + if (section[at] << 8) | section[at + 1]: + pmt_pids.add(((section[at + 2] & 0x1F) << 8) | section[at + 3]) + found: dict[int, list[tuple[int, int, bytes]]] = {} + for packets in packets_on(data, packet_size, pmt_pids).values(): + for index, section in sections(packets): + if section[0] == 0x02 and section[5] & 0x01: + program = (section[3] << 8) | section[4] + body = section[:5] + section[6:-4] + found.setdefault(program, []).append((index, (section[5] >> 1) & 0x1F, body)) + tables: dict[int, list[tuple[int, int, bytes]]] = {} + for program, seen in found.items(): + runs = tables.setdefault(program, []) + for table in sorted(seen): + if not runs or runs[-1][1:] != table[1:]: + runs.append(table) + return tables + + +def check_pmt_versions(ts_path: str, packet_size: int) -> Check: + """Each program's PMT changes only at a new version_number (ISO 13818-1 2.4.4.9), and an + older version never follows a newer one. + + A demux keeps the table it has until the version changes, so a change made under the same + version never reaches it, and a copy of the table a revision replaced, arriving after it, + reverts the revision. A version one up to 15 behind the newest is older, modulo 32. So is + a table coming back under the version it went out at before the versions since have come + round to it, which modulo 32 alone reads as a skip forward. A revision that skips versions + is reported, not failed: what a receiver sees still changes at every new version.""" + with open(ts_path, "rb") as handle: + data = handle.read() + revisions = skipped = 0 + for program, runs in sorted(pmt_tables(data, packet_size).items()): + for at in range(1, len(runs)): + newest, (index, version, body) = runs[at - 1][1], runs[at] + behind = (newest - version) % 32 + last = max((n for n in range(at) if runs[n][1:] == (version, body)), default=None) + returned = last is not None and len({v for _, v, _ in runs[last + 1 : at]}) < 31 + if behind == 0: + detail = f"program {program}: the PMT changed at packet {index} under version {version}" + elif behind < 16 or returned: + detail = f"program {program}: version {version} at packet {index}, after version {newest}" + else: + revisions += 1 + skipped += (version - newest) % 32 - 1 + continue + return Check("pmt-versions", Severity.HARD, Status.FAIL, detail, {"revisions": revisions}) + detail = f"{revisions} revision(s), each at a new version" + (f", {skipped} version(s) skipped" if skipped else "") + return Check("pmt-versions", Severity.HARD, Status.PASS, detail, {"revisions": revisions, "skipped": skipped}) + + def check_system_buffers(ts_path: str, packet_size: int, scan: Scan) -> Check: """TBsys and Bsys (ISO 13818-1 2.4.2.3) for each program, on its PCR. @@ -1695,6 +1753,7 @@ def analyze(ts_path: str, reference_seconds: float | None = None) -> list[Check] pat, pmt = check_pat_pmt(analysis) checks.append(pat) checks.append(pmt) + checks.append(check_pmt_versions(ts_path, packet_size)) checks.append(check_psi_crc(analysis)) checks.append(check_continuity(analysis)) checks.append(check_pcr_presence(analysis, clock_by_pid)) diff --git a/test/ts/compliance.test.py b/test/ts/compliance.test.py index ee93a1f9a29..b2f2ade3115 100644 --- a/test/ts/compliance.test.py +++ b/test/ts/compliance.test.py @@ -1,10 +1,13 @@ import pathlib import sys +import tempfile import unittest sys.path.insert(0, str(pathlib.Path(__file__).parent)) import compliance # noqa: E402 +PMT_PID = 0x1000 + def eac3(stream: int) -> bytes: """A six-block 48 kHz E-AC-3 sync frame of 512 bytes; `stream` is byte 2's top bits.""" @@ -37,5 +40,66 @@ def test_reserved_reduced_rate_is_refused(self): compliance.ac3_frame(bytes(frame), 0) +def section(table_id: int, extension: int, version: int, body: bytes) -> bytes: + out = bytearray([table_id, 0xB0, 0, extension >> 8, extension & 0xFF, 0xC1 | version << 1, 0, 0]) + body + length = len(out) + 4 - 3 + out[1] |= length >> 8 + out[2] = length & 0xFF + return bytes(out) + compliance.crc32_mpeg(bytes(out)).to_bytes(4, "big") + + +def psi(pid: int, counter: int, table: bytes) -> bytes: + return bytes([0x47, 0x40 | pid >> 8, pid & 0xFF, 0x10 | counter & 15, 0]) + table + b"\xff" * (183 - len(table)) + + +def tables(*pmts: tuple[int, list[int]]) -> bytes: + """A PAT, then each PMT as (version, the PIDs of its AAC streams), on PMT_PID.""" + pat = section(0x00, 1, 0, bytes([0, 1, 0xE0 | PMT_PID >> 8, PMT_PID & 0xFF])) + out = [psi(0, 0, pat)] + for counter, (version, pids) in enumerate(pmts): + streams = b"".join(bytes([0x0F, 0xE0 | pid >> 8, pid & 0xFF, 0xF0, 0]) for pid in pids) + out.append(psi(PMT_PID, counter, section(0x02, 1, version, bytes([0xE1, 0x01, 0xF0, 0]) + streams))) + return b"".join(out) + + +class PmtVersions(unittest.TestCase): + def check(self, *pmts: tuple[int, list[int]]) -> compliance.Check: + with tempfile.NamedTemporaryFile(suffix=".ts") as handle: + handle.write(tables(*pmts)) + handle.flush() + return compliance.check_pmt_versions(handle.name, 188) + + def test_repeats_and_revisions_pass(self): + check = self.check((0, [0x101]), (0, [0x101]), (1, [0x101, 0x102]), (1, [0x101, 0x102]), (2, [0x101])) + self.assertEqual((check.status, check.metrics), (compliance.Status.PASS, {"revisions": 2, "skipped": 0})) + + def test_the_version_wraps(self): + check = self.check((31, [0x101]), (0, [0x101, 0x102])) + self.assertEqual((check.status, check.metrics["revisions"]), (compliance.Status.PASS, 1)) + + def test_a_skipped_version_is_reported(self): + check = self.check((0, [0x101]), (2, [0x101, 0x102])) + self.assertEqual((check.status, check.metrics["skipped"]), (compliance.Status.PASS, 1)) + + def test_a_change_under_the_same_version_fails(self): + self.assertEqual(self.check((0, [0x101]), (0, [0x101, 0x102])).status, compliance.Status.FAIL) + + def test_an_older_version_after_a_newer_fails(self): + check = self.check((0, [0x101]), (1, [0x101, 0x102]), (0, [0x101])) + self.assertEqual(check.status, compliance.Status.FAIL) + self.assertIn("after version 1", check.detail) + + def test_a_copy_past_half_the_versions_back_fails(self): + revisions = [(version, [0x101, 0x200 + version]) for version in range(18)] + check = self.check(*revisions, (0, [0x101, 0x200])) + self.assertEqual(check.status, compliance.Status.FAIL) + self.assertIn("after version 17", check.detail) + + def test_a_table_back_after_every_version_passes(self): + cycle = [(version % 32, [0x101, 0x200 + version % 32]) for version in range(33)] + check = self.check(*cycle) + self.assertEqual((check.status, check.metrics["revisions"]), (compliance.Status.PASS, 32)) + + if __name__ == "__main__": unittest.main() diff --git a/test/ts/revision.py b/test/ts/revision.py new file mode 100644 index 00000000000..15b7915628f --- /dev/null +++ b/test/ts/revision.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Revise a generated clip's PMT mid-stream, and grade a round-trip's capture of the revisions. + + ./revision.py inject source.ts out.ts [--report revision.json] + ./revision.py grade source.ts capture.ts [--report-json revision.json] + +`inject` adds asynchronous KLV (stream_type 0x06 with a `KLVA` registration, no PTS) in the +clip's null slots from a third of its program clock to two thirds, every 100 ms, and revises +the PMT at each end: it lists the stream at the next version_number before the stream's first +packet, and drops it at the version after that once its last packet is out. Every other packet +stays in place. + +`grade` holds the capture to what a TS export promises of a revision. Its PMT runs through the +source's layouts in order (the source's PIDs and stream types, the KLV entry byte-equal), each +at the version after the one before. Every packet of the stream falls between the PMT that lists +it and the one that drops it, and every PES of the stream arrives, in order and byte-equal. +compliance.py grades the rest of the capture, the version sequence included. + +Exit status is 0 when every check passes, 1 otherwise. +""" + +import argparse +import json +import sys +from pathlib import Path + +import compliance +import klv + +PID = 0x112 +STREAM = (0x06, PID, klv.KLVA) + + +class Failure(Exception): + """A named grading failure.""" + + def __init__(self, check, detail): + self.check = check + super().__init__(f"{check}: {detail}") + + +def revise(packet, version, streams): + """A PMT packet whose section is at `version` with `streams` appended, its CRC recomputed.""" + body = klv.payload(packet) + start = len(packet) - len(body) + 1 + body[0] + section = bytearray(klv.single_section(packet)) + section[5] = (section[5] & 0xC1) | version << 1 + entries = b"".join(bytes([kind, 0xE0 | route >> 8, route & 0xFF, 0xF0 | len(info) >> 8, len(info) & 0xFF]) + info for kind, route, info in streams) + section = section[:-4] + entries + length = len(section) + 4 - 3 + section[1] = (section[1] & 0xF0) | length >> 8 + section[2] = length & 0xFF + section += compliance.crc32_mpeg(bytes(section)).to_bytes(4, "big") + if start + len(section) > klv.SIZE: + raise ValueError("the PMT outgrows its packet") + out = bytearray(packet[:start]) + section + out += b"\xff" * (klv.SIZE - len(out)) + return bytes(out) + + +def clock_span(source, pcr_pid): + """The first and last PCR on `pcr_pid`, unwrapped.""" + values = [] + for packet in source: + if klv.pid(packet) == pcr_pid and (now := klv.pcr(packet)) is not None: + values.append(now if not values else values[-1] + (now - values[-1] % klv.PCR_WRAP) % klv.PCR_WRAP) + if len(values) < 2: + raise ValueError("fewer than two PCRs") + return values[0], values[-1] + + +def inject(data): + """`data` with the KLV stream and its two PMT revisions, and a report of what went in.""" + source = klv.packets(data) + prog = klv.program(data) + if PID in prog.streams: + raise ValueError(f"the clip already carries PID {PID:#x}") + first, last = clock_span(source, prog.pcr_pid) + start, end = first + (last - first) // 3, first + 2 * (last - first) // 3 + version = None + # 0 before the stream, 1 while it runs, 2 once it ended and its last packet is out. + phase = 0 + listed = False + queue = [] + counter = count = 0 + deadline = now = None + revised = [] + out = [] + for index, packet in enumerate(source): + route = klv.pid(packet) + if route == prog.pcr_pid and (value := klv.pcr(packet)) is not None: + now = value if now is None else now + (value - now % klv.PCR_WRAP) % klv.PCR_WRAP + if phase == 0 and now >= start: + phase, deadline = 1, now + while phase == 1 and deadline < end and now >= deadline: + pes = klv.async_klv(count) + queue.extend(klv.split(PID, pes, counter)) + counter += -(-len(pes) // 184) + count += 1 + deadline += klv.ASYNC_PERIOD + if phase == 1 and now >= end and not queue: + phase = 2 + if route == prog.pmt_pid and (klv.single_section(packet) or b"\0")[0] == 0x02: + version = (klv.single_section(packet)[5] >> 1) & 0x1F if version is None else version + if phase: + packet = revise(packet, (version + phase) % 32, [STREAM] if phase == 1 else []) + if len(revised) < phase: + revised.append(index) + listed = phase == 1 + if route == klv.NULL and queue and listed: + packet = queue.pop(0) + out.append(packet) + if phase != 2 or len(revised) != 2: + raise ValueError("the clip ended before the stream did and the PMT dropped it") + return b"".join(out), {"pes": count, "revised_at": revised} + + +def layouts(data): + """(index, version, {PID: (stream_type, descriptors)}) of each PMT of the first program, each + run of repeats once.""" + prog = klv.program(data) + found = [] + for index, packet in enumerate(klv.packets(data)): + section = klv.single_section(packet) + if klv.pid(packet) != prog.pmt_pid or section is None or section[0] != 0x02: + continue + table = ((section[5] >> 1) & 0x1F, klv.read_pmt(section)[1]) + if not found or found[-1][1:] != table: + found.append((index, *table)) + return found + + +def grade(source, capture): + """Grade `capture` against `source`, returning a report; raises Failure on the first miss.""" + types = lambda streams: {route: kind for route, (kind, _) in streams.items()} + src, cap = layouts(source), layouts(capture) + if [types(streams) for _, _, streams in cap] != [types(streams) for _, _, streams in src]: + raise Failure("layouts", f"the capture's PMTs list {[sorted(types(s).items()) for _, _, s in cap]}, the source's {[sorted(types(s).items()) for _, _, s in src]}") + for (_, newest, _), (index, version, _) in zip(cap, cap[1:]): + if version != (newest + 1) % 32: + raise Failure("version", f"version {version} at packet {index} follows version {newest}") + with_stream = [k for k, (_, _, streams) in enumerate(cap) if PID in streams] + if len(with_stream) != 1: + raise Failure("layouts", f"{len(with_stream)} of the capture's PMTs list PID {PID:#x}, not one") + listed = with_stream[0] + if cap[listed][2][PID] != (STREAM[0], STREAM[2]): + raise Failure("pmt", f"PID {PID:#x}: {cap[listed][2][PID]} in the capture, not {(STREAM[0], STREAM[2])}") + opened = cap[listed][0] + closed = cap[listed + 1][0] if listed + 1 < len(cap) else None + for index, packet in enumerate(klv.packets(capture)): + if klv.pid(packet) == PID and not (opened < index and (closed is None or index < closed)): + raise Failure("window", f"a packet on PID {PID:#x} at {index}, outside the PMTs listing it ({opened} to {closed})") + src_pes = [pes.data for pes in klv.private_pes(source, [PID])[PID]] + cap_pes = [pes.data for pes in klv.private_pes(capture, [PID])[PID]] + if cap_pes != src_pes: + raise Failure("pes", f"PID {PID:#x}: {len(cap_pes)} PES in the capture, not the source's {len(src_pes)} in order") + return {"versions": [version for _, version, _ in cap], "pes": len(cap_pes), "opened": opened, "closed": closed} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + add = sub.add_parser("inject") + add.add_argument("source", type=Path) + add.add_argument("output", type=Path) + add.add_argument("--report", type=Path) + check = sub.add_parser("grade") + check.add_argument("source", type=Path) + check.add_argument("capture", type=Path) + check.add_argument("--report-json", type=Path) + args = parser.parse_args() + + if args.command == "inject": + data, report = inject(args.source.read_bytes()) + args.output.write_bytes(data) + if args.report: + args.report.write_text(json.dumps(report, indent=2) + "\n") + print("revised the PMT:", report) + return 0 + + print("### PMT revisions (capture against the source)") + try: + report = grade(args.source.read_bytes(), args.capture.read_bytes()) + except Failure as err: + print(f" FAIL {err}") + return 1 + if args.report_json: + args.report_json.write_text(json.dumps(report, indent=2) + "\n") + print(f" PASS versions {report['versions']}; {report['pes']} PES on PID {PID:#x}, " + f"between the PMTs at packets {report['opened']} and {report['closed']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/revision.test.py b/test/ts/revision.test.py new file mode 100644 index 00000000000..7df4b037c48 --- /dev/null +++ b/test/ts/revision.test.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Controls for the PMT revision fixture and its grader: each miss it exists to catch fails it.""" + +import unittest + +import compliance +import klv +import revision + +VIDEO = 0x100 +PMT = 0x1000 + + +def section(table_id, extension, body): + out = bytearray([table_id, 0xB0, 0, extension >> 8, extension & 0xFF, 0xC1, 0, 0]) + body + length = len(out) + 4 - 3 + out[1] |= length >> 8 + out[2] = length & 0xFF + return bytes(out) + compliance.crc32_mpeg(bytes(out)).to_bytes(4, "big") + + +def psi(route, table): + return bytes([0x47, 0x40 | route >> 8, route & 0xFF, 0x10, 0]) + table + b"\xff" * (183 - len(table)) + + +def picture(counter, pcr): + """A video packet carrying `pcr` and the start of an unbounded PES.""" + base, ext = pcr // 300, pcr % 300 + field = bytes([7, 0x10]) + (base >> 1).to_bytes(4, "big") + bytes([(base & 1) << 7 | 0x7E | ext >> 8, ext & 0xFF]) + pes = bytes([0, 0, 1, 0xE0, 0, 0, 0x80, 0x80, 5]) + klv.clock_field(0x2, base % klv.PTS_WRAP) + body = pes + b"\x00" * (184 - len(field) - len(pes)) + return bytes([0x47, 0x40 | VIDEO >> 8, VIDEO & 0xFF, 0x30 | counter & 15]) + field + body + + +NULL = bytes([0x47, 0x1F, 0xFF, 0x10]) + b"\xff" * 184 +PAT = psi(0, section(0x00, 1, bytes([0, 1, 0xE0 | PMT >> 8, PMT & 0xFF]))) +TABLE = psi(PMT, section(0x02, 1, bytes([0xE0 | VIDEO >> 8, VIDEO & 0xFF, 0xF0, 0, 0x1B, 0xE0 | VIDEO >> 8, VIDEO & 0xFF, 0xF0, 0]))) + + +def media(pictures=150): + """A clip of `pictures` 40 ms apart, its tables every ten, each picture followed by null slots.""" + out = [] + for k in range(pictures): + if k % 10 == 0: + out += [PAT, TABLE] + out.append(picture(k, 27_000_000 + k * 40 * 27_000)) + out.extend([NULL] * 6) + return b"".join(out) + + +def replace(data, index, packet): + return data[: index * klv.SIZE] + packet + data[(index + 1) * klv.SIZE :] + + +class Revision(unittest.TestCase): + def setUp(self): + self.source, self.report = revision.inject(media()) + + def assertFails(self, check, capture): + with self.assertRaises(revision.Failure) as caught: + revision.grade(self.source, capture) + self.assertEqual(caught.exception.check, check, str(caught.exception)) + + def test_the_source_revises_twice_and_grades_itself(self): + tables = revision.layouts(self.source) + self.assertEqual([version for _, version, _ in tables], [0, 1, 2]) + self.assertEqual([revision.PID in streams for _, _, streams in tables], [False, True, False]) + self.assertEqual(revision.grade(self.source, self.source)["versions"], [0, 1, 2]) + self.assertGreater(self.report["pes"], 10) + + def test_only_null_slots_and_tables_change(self): + for old, new in zip(klv.packets(media()), klv.packets(self.source)): + if klv.pid(old) not in (klv.NULL, PMT): + self.assertEqual(old, new) + + def test_the_stream_runs_between_its_revisions(self): + opened, closed = self.report["revised_at"] + on_pid = [k for k, packet in enumerate(klv.packets(self.source)) if klv.pid(packet) == revision.PID] + self.assertLess(opened, on_pid[0]) + self.assertLess(on_pid[-1], closed) + + def test_a_skipped_version_fails(self): + _, closed = self.report["revised_at"] + capture = self.source + for k, packet in enumerate(klv.packets(self.source)): + if k >= closed and klv.pid(packet) == PMT: + capture = replace(capture, k, revision.revise(TABLE, 3, [])) + self.assertFails("version", capture) + + def test_a_stale_table_after_the_revision_fails(self): + packets = klv.packets(self.source) + _, closed = self.report["revised_at"] + listing = packets[self.report["revised_at"][0]] + after = next(k for k in range(closed + 1, len(packets)) if klv.pid(packets[k]) == klv.NULL) + self.assertFails("layouts", replace(self.source, after, listing)) + + def test_a_packet_after_the_stream_was_dropped_fails(self): + packets = klv.packets(self.source) + _, closed = self.report["revised_at"] + stream = next(packet for packet in packets if klv.pid(packet) == revision.PID) + after = next(k for k in range(closed + 1, len(packets)) if klv.pid(packets[k]) == klv.NULL) + self.assertFails("window", replace(self.source, after, stream)) + + def test_a_dropped_pes_fails(self): + packets = klv.packets(self.source) + index = [k for k, packet in enumerate(packets) if klv.pid(packet) == revision.PID][3] + self.assertFails("pes", replace(self.source, index, NULL)) + + def test_a_capture_that_never_revised_fails(self): + self.assertFails("layouts", media()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/ts/run.sh b/test/ts/run.sh index cbca53d1473..58ded8b8552 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -26,6 +26,7 @@ # ./run.sh --fragment-pes # split generated clear PES headers across packets # ./run.sh --clockless sections # sections, pes, or audio-scte35 without video # ./run.sh --klv # add KLV and DVB AC-3, carried as whole PES, and grade them +# ./run.sh --revision # revise the PMT mid-stream, adding then dropping a KLV stream # ./run.sh --delay 2s # pass the exporter's --delay # ./run.sh --rate 7000000 # bufferbloat: 7 Mb/s to the subscriber behind a deep queue (moq-shaper) # ./run.sh --loss 10 # drop 10% of the relay's datagrams to the subscriber (moq-shaper) @@ -87,6 +88,7 @@ FULL="" # the same with audio, in a multiplex the source fills with no n FRAGMENT_PES="" # use following null slots for fragmented PES headers CLOCKLESS="" # generated programs clocked by audio or a dedicated PCR-only PID KLV="" # inject asynchronous and synchronous KLV beside DVB AC-3, and grade every PES +REVISION="" # revise the PMT twice mid-stream, and grade the export's revisions LOSS="" # percent of relay -> subscriber datagrams dropped at random by moq-shaper RATE="" # bits/s moq-shaper carries from the relay to the subscriber, behind QUEUE QUEUE="" # how long a datagram may wait for RATE before the path tail-drops it @@ -184,6 +186,10 @@ while [[ $# -gt 0 ]]; do KLV=1 shift ;; + --revision) + REVISION=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -286,6 +292,10 @@ if [[ -n "$KLV" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAG echo "error: --klv generates its own clip and grades the demultiplexed export's PES" >&2 exit 1 fi +if [[ -n "$REVISION" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAGMENT_PES$PAIR$LIVE$PASSTHROUGH$MPTS$KLV$CLOCKLESS" ]]; then + echo "error: --revision revises the generated closed-GOP clip and grades the demultiplexed export's PMT" >&2 + exit 1 +fi if [[ -n "$CLOCKLESS" ]]; then case "$CLOCKLESS" in audio-scte35 | sections | pes) ;; *) @@ -567,6 +577,12 @@ else -c:a aac -b:a 128k \ -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" fi +# A KLV stream joins the program a third of the way in and leaves it two thirds in, each +# change announced by a PMT revision, in the clip's null slots. +if [[ -n "$REVISION" ]]; then + mv "$SRC_TS" "$HARNESS_RUN/media.ts" + python3 "$DIR/revision.py" inject "$HARNESS_RUN/media.ts" "$SRC_TS" --report "$HARNESS_RUN/revision-injected.json" +fi if [[ -n "$FRAGMENT_PES" ]]; then python3 "$DIR/fragment_pes.py" "$SRC_TS" "$HARNESS_RUN/source-fragmented.ts" \ @@ -970,6 +986,17 @@ if [[ -n "$KLV" ]]; then fi fi +# The export announces each layout the source's PMT revised to, at the next version. +if [[ -n "$REVISION" ]]; then + echo + if ! python3 "$DIR/revision.py" grade "$SRC_TS" "$SUB_TS" --report-json "$HARNESS_RUN/revision.json"; then + echo >&2 + echo "error: PMT revision analysis failed (see round-trip logs below)" >&2 + dump_logs + exit 1 + fi +fi + # The delay's lead over the source's hides packets the export spends beyond the source's # until it runs out, so they are counted outright. if [[ -n "$FULL" ]]; then