From 0631611f685b58e4579ee83d66b3615b00670caa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 10:30:57 -0700 Subject: [PATCH 01/49] quest: open the archive line Co-Authored-By: Claude Opus 5.5 From 3f42dce55e9cd63175fd8d8743ab9282c8c18a6b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 18:48:24 -0700 Subject: [PATCH 02/49] feat(moq-archive): record selected broadcast tracks with a Writer (#4043) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 5 + js/json/src/window/encoder.ts | 9 + js/json/src/window/producer.ts | 18 + js/json/src/window/window.test.ts | 22 + quest/m1/archive/README.md | 7 +- quest/m1/archive/browser.md | 2 - quest/m1/archive/cli.md | 47 ++ quest/m1/archive/dvr.md | 7 +- quest/m1/archive/proof.md | 2 +- quest/m1/archive/recovery.md | 26 + quest/m1/archive/writer.md | 80 -- rs/moq-archive/Cargo.toml | 8 +- rs/moq-archive/src/error.rs | 20 + rs/moq-archive/src/lib.rs | 5 +- rs/moq-archive/src/writer.rs | 1172 ++++++++++++++++++++++++++++ rs/moq-json/src/window/encoder.rs | 11 +- rs/moq-json/src/window/mod.rs | 40 +- rs/moq-json/src/window/producer.rs | 19 + rs/moq-mux/src/timeline.rs | 87 +++ 19 files changed, 1492 insertions(+), 95 deletions(-) create mode 100644 quest/m1/archive/cli.md create mode 100644 quest/m1/archive/recovery.md delete mode 100644 quest/m1/archive/writer.md create mode 100644 rs/moq-archive/src/writer.rs diff --git a/Cargo.lock b/Cargo.lock index ab053a33f4..8643d865b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4159,6 +4159,10 @@ dependencies = [ "async-trait", "bytes", "futures", + "hang", + "kio 0.6.0", + "moq-json", + "moq-mux", "moq-net", "object_store", "percent-encoding", @@ -4167,6 +4171,7 @@ dependencies = [ "tempfile", "thiserror 2.0.21", "tokio", + "tracing", ] [[package]] diff --git a/js/json/src/window/encoder.ts b/js/json/src/window/encoder.ts index e403ef919c..a5836a2507 100644 --- a/js/json/src/window/encoder.ts +++ b/js/json/src/window/encoder.ts @@ -129,6 +129,15 @@ export class Encoder { return this.#start + this.#window.length; } + /** + * Discard group-local state, so the next edit opens a new group with a header. + * + * Call this whenever the caller closes the current group behind the encoder's back. + */ + reset(): void { + this.#resyncGroup(); + } + /** Discard group-local state after an encoded frame did not reach the wire. */ #resyncGroup(): void { this.#flate = undefined; diff --git a/js/json/src/window/producer.ts b/js/json/src/window/producer.ts index b7051f60f7..20457f5e13 100644 --- a/js/json/src/window/producer.ts +++ b/js/json/src/window/producer.ts @@ -90,6 +90,24 @@ export class Producer { encoded.commit(); } + /** + * Finish the open group, leaving the next edit to open a replacement with a header. + * + * Idempotent: cutting when no group is open does nothing. A caller that stores complete groups + * cuts after its edits, so every edit so far sits in a group no later frame can extend. + */ + cut(): void { + if (!this.#group) return; + + // Reset first: the group closes either way below, and a throw must not leave the encoder + // appending ops to a group that is gone. + this.#encoder.reset(); + + const group = this.#group; + this.#group = undefined; + group.close(); + } + /** Finish the track, closing any open group. */ finish(): void { if (this.#finished) return; diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index 95d2f0bedd..f5bdb63cc5 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -75,6 +75,28 @@ test("push and pop round-trip", async () => { ]); }); +test("a cut is invisible to the consumer", async () => { + const live = new Live(); + await live.push(0); + live.producer.cut(); + live.producer.cut(); + await live.push(1); + + expect(await live.finish()).toEqual([ + { push: { index: 0, value: { n: 0 } } }, + { push: { index: 1, value: { n: 1 } } }, + ]); +}); + +test("reset makes the next edit a header", () => { + const encoder = new Encoder({}); + encoder.push({ n: 0 }).commit(); + expect(encoder.push({ n: 1 }).keyframe).toBe(false); + + encoder.reset(); + expect(encoder.push({ n: 2 }).keyframe).toBe(true); +}); + test("concurrent consumer reads are rejected", async () => { const track = new Track.Producer("test"); const consumer = new Consumer({ track: track.subscribe() }); diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index acff3c3023..f5ab5b2eb6 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -37,6 +37,10 @@ The segment engine is in `rs/moq-mux/src/timeline.rs`: `rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`: percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list/delete. +`moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) records enrolled tracks +through `Deferred`, omits failed tracks with `Pending::omit`, stores each +segment's timeline groups after `Producer::flush`, and expires DVR segments +with a deletion grace. It refuses a prefix that already holds a timeline. ### Format @@ -103,10 +107,11 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests -- [Recording writer](/quest/m1/archive/writer.md) - feed the segmenter from a `broadcast::Consumer`, store each segment, then commit its record +- [Archive endpoint](/quest/m1/archive/cli.md) - `moq ... export archive` records and `import archive` replays - [Recording reader](/quest/m1/archive/reader.md) - serve archived FETCH through a supplied `broadcast::Producer` - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [Offline archive HLS](/quest/m1/archive/hls.md) - render playlists from the archive timeline and fetch segment media lazily +- [Resume a recording](/quest/m1/archive/recovery.md) - recover the retained timeline on restart and clean up DVR orphans - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback - [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation diff --git a/quest/m1/archive/browser.md b/quest/m1/archive/browser.md index 520a5cb09e..78a0dd061c 100644 --- a/quest/m1/archive/browser.md +++ b/quest/m1/archive/browser.md @@ -34,5 +34,3 @@ remains downstream (moq.pro) work. ## Required - [JavaScript FETCH](/quest/m1/js-fetch.md) - generic on-demand group serving and IETF FETCH support - -- [Recording writer](/quest/m1/archive/writer.md) diff --git a/quest/m1/archive/cli.md b/quest/m1/archive/cli.md new file mode 100644 index 0000000000..84ebbf0e79 --- /dev/null +++ b/quest/m1/archive/cli.md @@ -0,0 +1,47 @@ +# [M] Archive endpoint in moq-cli + +## Goal + +`moq-cli` records and replays archives with no custom code, through an +`archive ` endpoint on the existing direction verbs: +`export archive` records a broadcast with `moq_archive::Writer`, and +`import archive` republishes a recording, serving its groups on demand +through `moq_archive::Reader`. Recording an ingest in one process is a +multi-stage line. + +```bash +moq --connect $RELAY --broadcast ev.hang export archive s3://rec/ev +moq --connect $RELAY \ + import --broadcast ev.hang srt --listen 0.0.0.0:9000 \ + -- export --broadcast ev.hang archive file:///rec/ev +moq --connect $RELAY --broadcast ev-replay.hang import archive s3://rec/ev +``` + +## Plan + +Parse the store URL with `object_store::parse_url`. Compile in every +`object_store` cloud backend (S3, GCS, Azure) behind features, with local +files always available. + +An archive stage carries exactly one broadcast; refuse a multi-broadcast +stage (an RTMP or SRT listener accepting many) rather than invent a prefix +scheme. + +Export reads its own catalog: video and audio renditions enroll as pacing +tracks, the catalog and any other track as non-pacing, and renditions added +later enroll as they appear. The writer stays catalog-agnostic. Expose the +writer's retention window and deletion grace as flags. + +Import hands the reader the stage's `broadcast::Producer`. The recording is +not live; decide with the reader whether import finishes the replayed +timeline or keeps following a growing archive. + +Update `doc/bin/cli.md` and every example invocation. + +## Required + +- [Recording reader](/quest/m1/archive/reader.md) - serves `import archive` + +## Closes + +- [#2281](https://github.com/moq-dev/moq/issues/2281) - close this issue when the quest finishes diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 930571650f..19c0b655ac 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -7,8 +7,9 @@ same timeline and group-range objects as an unbounded archive. ## Plan -The recording writer prerequisite owns retention, checkpoint recovery, deletion -grace, and restart cleanup. This quest consumes that contract and owns viewer +The recording writer owns retention and deletion grace, and +[Resume a recording](/quest/m1/archive/recovery.md) owns checkpoint recovery +and restart cleanup. This quest consumes that contract and owns viewer seek and return-to-live behavior, not a second writer implementation. The player reads the archive timeline, FETCHes old groups through the normal @@ -24,7 +25,7 @@ Use the writer/reader fixtures; a retention defect is fixed in its owning layer. ## Required -- [Recording writer](/quest/m1/archive/writer.md) +- [Resume a recording](/quest/m1/archive/recovery.md) - [Recording reader](/quest/m1/archive/reader.md) ## Closes diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md index 1f68bf7385..c28c05284b 100644 --- a/quest/m1/archive/proof.md +++ b/quest/m1/archive/proof.md @@ -63,5 +63,5 @@ object directly without any listing or separate index object. ## Required -- [Recording writer](/quest/m1/archive/writer.md) +- [Resume a recording](/quest/m1/archive/recovery.md) - [Offline archive HLS](/quest/m1/archive/hls.md) diff --git a/quest/m1/archive/recovery.md b/quest/m1/archive/recovery.md new file mode 100644 index 0000000000..70b302abd5 --- /dev/null +++ b/quest/m1/archive/recovery.md @@ -0,0 +1,26 @@ +# [M] Resume a recording + +## Goal + +A writer restarting on a prefix it exclusively owns recovers the retained +timeline and continues the same recording instead of refusing the prefix. + +## Plan + +`moq_archive::Writer::new` refuses any prefix that already holds a timeline. +Replace that refusal with recovery: list and replay the timeline's +`segments/` objects from a retained checkpoint, then resume the timeline +producer at the next segment ID with the recovered window. That needs a way +to seed `moq_mux::timeline::Producer` with a window and next segment, which it +lacks today. + +For a DVR, reconcile a complete `groups/` listing of every recorded track +against the recovered records before accepting input. Wait the deletion grace +period from successful recovery, then delete unreferenced group objects left +by interrupted expiration or uploads. Failed or incomplete recovery or listing +deletes nothing. Preserve `.info` and timeline checkpoint objects. + +An unbounded archive resumes the same way without deleting anything. Source +group sequences that restart below the recovered ranges must not produce +overlapping objects; decide whether that refuses the track or starts a new +recording. diff --git a/quest/m1/archive/writer.md b/quest/m1/archive/writer.md deleted file mode 100644 index 16eb74108d..0000000000 --- a/quest/m1/archive/writer.md +++ /dev/null @@ -1,80 +0,0 @@ -# [L] Recording writer - -## Goal - -A generic `moq-archive` writer consumes explicitly selected tracks from a -`broadcast::Consumer`, feeds their complete groups to the existing segmenter, -stores one object per track per segment, and commits the timeline record only -after the objects it names are durable. - -## Plan - -The application supplies the broadcast, object prefix, and arbitrary track -registrations, each pacing or non-pacing; the writer does not parse a catalog -or know which tracks are media. - -Enrollment creates the track's `.info` before accepting any group. If that -create fails, enrollment fails and no range for the track can enter the -timeline. On `AlreadyExists`, validate the immutable `.info` and accept only matching -parsed `version`, `priority`, and `timescale` values; a priority or timescale mismatch fails enrollment. - -Feed the existing segmenter (`rs/moq-mux/src/timeline.rs`): take -`Producer::deferred` (:917), enroll each registration through -`Deferred::track` or `pacing_track` (:625-631), and report every complete -group read from the consumer through its `Recorder` (:1096). The application -calls `cut(pts)` (:635) when it knows an aligned keyframe boundary. A segment -may hold many groups per track, especially one-group-per-frame audio. A pacing -track that stops without closing blocks segment completion on purpose; the -application applies its own deadline and calls `cut(pts)` or removes the -track. Storage does not invent a timeout. - -Accept new group IDs in strictly increasing order per track; refuse duplicate -or decreasing arrivals. Already accepted groups may finish in any order, so -buffer completion independently and encode in sequence order. Require -nonoverlapping object ranges across segments, including forced cuts; never -backfill a closed segment or move its stalled groups into an overlapping range. - -For each `Pending` record from `Deferred::next` (:658), encode and PUT one -object per participating track at `groups/.`, buffering -groups independently of relay retention. The bounds come from the exact -recorded ranges; tracks with no complete groups have no object. After all PUTs settle, drop the ranges of every track whose PUT -failed and commit with `Producer::push` (:948). That needs a per-track -omission on `Pending` beside `Pending::gap()` (:712-719), which clears every -track; add it here. Never publish a range first and hope the relay still has -it. A later segment continues normally after any omission. - -Catalog update applicability and cross-track configuration dependencies belong -to [Catalog track identity](/quest/m2/catalog-tracks.md). Recording a catalog -as an ordinary track does not bind its updates to media groups; this writer -does not introduce catalog-specific commit or retention prerequisites. - -The archive timeline uses the same object envelope but is not enrolled in its -own records. After pushing segment N and applying retention pops, close the -recording-owned Window group and store the complete groups under the timeline -track's `segments/N` key (19-digit padded N). Commit IDs consecutively, -including all-gap segments. Make it durable before committing N+1; a timeline PUT -failure stops the recording at its preceding durable timeline object. Never -cut a source group. On a clean source end, drain with -`Deferred::finish` (:650), commit the final partial segment, and call -`Producer::finish` (:1015). Do not write a completion marker. - -Retention is writer policy. During the next segment commit, use -`Producer::pop` (:983) for expired records before closing that segment's -timeline group. Make it durable, wait the configured grace period, then delete -the corresponding segment objects. No trimming occurs after the final segment -is committed, and timeline objects are never rewritten. - -Before a DVR writer resumes input, recover its full retained timeline and -reconcile a complete media-key listing under exclusive prefix ownership. -Wait the deletion grace period from successful recovery, then delete unreferenced `groups/` objects -left by interrupted expiration or uploads. Failed or incomplete recovery must -prevent deletion. Preserve `.info` and timeline checkpoint objects. - -Keep archive policy out of protocol libraries. As the native application that -owns its storage and track choices, `moq-cli` attaches the writer to every -import path and enrolls the resulting `broadcast::Consumer` tracks. Downstream -(moq.pro) gateways attach the same writer once it ships in a release. - -## Closes - -- [#2281](https://github.com/moq-dev/moq/issues/2281) - close this issue when the quest finishes diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index d95e48b1ce..238c1c00d3 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -15,15 +15,21 @@ categories = ["multimedia", "filesystem"] [dependencies] bytes = { workspace = true } futures = { workspace = true } +hang = { workspace = true } +kio = { workspace = true } +moq-mux = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "sync", "time"] } +tracing = { workspace = true } [dev-dependencies] async-trait = "0.1" +moq-json = { path = "../moq-json" } object_store = { workspace = true, features = ["fs"] } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index ad372a3209..0ccb85bc5e 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -73,6 +73,26 @@ pub enum Error { /// `.info` JSON is malformed. #[error("json: {0}")] Json(String), + + /// The prefix already holds a recording timeline; resuming is unsupported. + #[error("prefix already holds a recording: {0}")] + Occupied(String), + + /// The track was already enrolled, or is the recording's own timeline. + #[error("track already enrolled: {0}")] + Enrolled(String), + + /// The source broadcast or one of its tracks failed. + #[error("source: {0}")] + Source(String), + + /// The recording's timeline could not be segmented or published. + #[error("timeline: {0}")] + Timeline(String), + + /// The writer stopped accepting commands. + #[error("writer closed")] + Closed, } impl From for Error { diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 603284461c..8a06522e6a 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -1,7 +1,8 @@ //! Versioned hang recording objects on any [`object_store::ObjectStore`]. //! //! The crate owns the portable layout and codecs: percent-encoded track names, `.info` JSON, -//! the binary segment envelope, and put/get/list/delete. Callers that need runtime dispatch +//! the binary segment envelope, and put/get/list/delete. A [`Writer`] records selected tracks of +//! a broadcast into those objects. Callers that need runtime dispatch //! supply `Arc`; the archive API itself stays generic. //! //! Group bounds are finite inclusive ranges in first-to-last order: @@ -18,12 +19,14 @@ pub mod info; mod path; pub mod segment; pub mod store; +pub mod writer; pub use error::{Error, Result}; pub use info::Info; pub use path::Key; pub use segment::{Frame, Group, Object}; pub use store::Store; +pub use writer::Writer; /// Recording format version written into `.info` and the binary envelope. pub const VERSION: u64 = 1; diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs new file mode 100644 index 0000000000..13ac32294a --- /dev/null +++ b/rs/moq-archive/src/writer.rs @@ -0,0 +1,1172 @@ +//! Record explicitly selected tracks of a [`broadcast::Consumer`] into a [`Store`]. +//! +//! The application enrolls each track as pacing or non-pacing through a [`Control`]; the writer +//! never parses a catalog. Every complete group feeds the timeline segmenter. For each closed +//! segment the writer stores one object per participating track, omits any track whose object +//! failed, commits the record through its own timeline encoder, and stores that segment's timeline +//! groups before starting the next one. The timeline therefore only advertises durable objects. +//! +//! ```no_run +//! # async fn example(source: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { +//! use moq_archive::object_store::memory::InMemory; +//! +//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +//! let writer = moq_archive::Writer::new(store, source, Default::default()).await?; +//! let control = writer.control(); +//! control.pacing_track("video").await?; +//! control.pacing_track("audio").await?; +//! control.track("catalog.json").await?; +//! writer.run().await?; +//! # Ok(()) +//! # } +//! ``` + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{FutureExt, StreamExt}; +use hang::timeline::Record; +use moq_mux::timeline::{self, Deferred, DeferredDrain, Pending, Recorder}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStore; +use tokio::sync::{mpsc, watch}; +use tokio::time::Instant; + +use crate::segment::{Frame, Group, Object}; +use crate::store::list::Query; +use crate::{Error, Info, Key, Result, Store}; + +/// Subscribers ask for every cached group; the publisher clamps this to its own max age. +const REPLAY: Duration = Duration::from_secs(u32::MAX as u64); + +/// How a [`Writer`] segments and retains its recording. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct Config { + /// Segment pacing for the recording's timeline. + pub timeline: timeline::Config, + /// Expire old segments (a DVR), or keep everything when `None` (an archive). + pub retention: Option, +} + +impl Config { + /// Set [`timeline`](Self::timeline). + pub fn with_timeline(mut self, timeline: timeline::Config) -> Self { + self.timeline = timeline; + self + } + + /// Set [`retention`](Self::retention). + pub fn with_retention(mut self, retention: impl Into>) -> Self { + self.retention = retention.into(); + self + } +} + +/// A DVR retention policy. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Retention { + /// Keep at least this much content, measured from timeline records. + pub window: Duration, + /// How long expired objects outlive the timeline that stopped advertising them, so readers + /// holding an older timeline can finish their GETs. + pub grace: Duration, +} + +impl Retention { + /// Keep `window` of content and delete expired objects `grace` after they leave the timeline. + pub fn new(window: Duration, grace: Duration) -> Self { + Self { window, grace } + } +} + +/// Records enrolled tracks until the source broadcast closes. +/// +/// Enroll tracks through [`control`](Self::control), then drive the recording with +/// [`run`](Self::run). +pub struct Writer { + control: Control, + commands: mpsc::UnboundedReceiver, + committer: Committer, + grace: Option, + // Owns the recording's timeline track. + _timeline: broadcast::Producer, +} + +/// Enrolls, cuts, and removes tracks on a [`Writer`]. +pub struct Control { + shared: Arc>, + deferred: Deferred, + commands: mpsc::UnboundedSender, +} + +impl Clone for Control { + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + deferred: self.deferred.clone(), + commands: self.commands.clone(), + } + } +} + +struct Shared { + store: Store, + source: broadcast::Consumer, + /// The recording-owned timeline track, which no source track may shadow. + timeline: String, + /// Every name ever enrolled. A name is never reused, so its object ranges stay increasing. + enrolled: Mutex>, +} + +enum Command { + Enroll { + name: String, + subscriber: Box, + recorder: Recorder, + timescale: Timescale, + }, + Remove(String), + /// A cut may have completed a segment. + Poke, +} + +impl Writer { + /// Start a recording under `store`'s prefix, reading tracks from `source`. + /// + /// Refuses a prefix that already holds a timeline: resuming a recording is unsupported. + pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { + let broadcast = broadcast::Info::new().produce(); + let timeline = timeline::Producer::new(&broadcast, config.timeline); + let deferred = timeline.deferred().map_err(timeline_error)?; + let section = timeline.section(); + + let mut existing = store.list(&Query::segments(§ion.track)?); + if let Some(entry) = existing.next().await { + return Err(Error::Occupied(store.path(&entry?.key)?.to_string())); + } + drop(existing); + + let replay = track::Subscription::default().with_max_age(REPLAY); + let groups = broadcast + .consume() + .track(§ion.track) + .map_err(source_error)? + .subscribe(replay) + .await + .map_err(source_error)? + .ordered(); + let info = Info::new(groups.info().priority, groups.info().timescale.as_u64())?; + store.put_info(§ion.track, &info).await?; + + let shared = Arc::new(Shared { + store, + source, + timeline: section.track, + enrolled: Mutex::new(HashSet::new()), + }); + let (commands, receiver) = mpsc::unbounded_channel(); + let retention = config.retention; + Ok(Self { + control: Control { + shared: shared.clone(), + deferred, + commands, + }, + commands: receiver, + committer: Committer { + shared, + timeline, + groups, + timescale: section.timescale.into(), + retention: retention.as_ref().map(|r| r.window), + window: VecDeque::new(), + }, + grace: retention.map(|r| r.grace), + _timeline: broadcast, + }) + } + + /// A handle that enrolls, cuts, and removes tracks, before or during [`run`](Self::run). + pub fn control(&self) -> Control { + self.control.clone() + } + + /// Record until the source broadcast closes and every enrolled track ends. + /// + /// Then flush the final segment and finish the timeline. Fails when the timeline cannot be + /// committed or stored: the recording stops at its last durable timeline object. Returns the + /// source's error, after finishing, when the broadcast aborted. + pub async fn run(self) -> Result<()> { + let Self { + control, + mut commands, + committer, + grace, + _timeline, + } = self; + let shared = control.shared.clone(); + let source = shared.source.clone(); + let mut segments = Segments::Live(control.deferred.clone()); + drop(control); + + let mut tracks: HashMap = HashMap::new(); + // Reported groups waiting for the record that names them, kept past a track's removal. + let mut buffered: HashMap> = HashMap::new(); + let mut reads = FuturesUnordered::new(); + let mut committer = Some(committer); + let mut commit: Option> = None; + let mut deletions: VecDeque<(Instant, Vec)> = VecDeque::new(); + let mut closed = false; + // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. + let mut accepting = true; + + loop { + if commit.is_none() + && let Some(pending) = segments.next() + { + let objects = take_objects(&pending, &mut buffered); + let committer = committer.take().expect("idle committer"); + commit = Some(committer.commit(pending, objects).boxed()); + } + + if commit.is_none() && closed && tracks.is_empty() { + // Refuse late commands, so an enrollment racing the end fails instead of vanishing. + commands.close(); + if !accepting { + match segments { + Segments::Live(deferred) => { + segments = Segments::Drain(deferred.finish()); + continue; + } + Segments::Drain(_) => break, + } + } + } + + let deadline = deletions.front().map(|(deadline, _)| *deadline); + tokio::select! { + biased; + command = commands.recv(), if accepting => match command { + None => accepting = false, + Some(Command::Enroll { name, subscriber, recorder, timescale }) => { + let (cancel, cancelled) = watch::channel(()); + reads.push(guard(cancelled.clone(), recv(name.clone(), subscriber)).boxed()); + tracks.insert(name, TrackState { + recorder, + timescale, + largest: None, + accepted: BTreeMap::new(), + subscribed: true, + _cancel: cancel, + cancelled, + }); + } + Some(Command::Remove(name)) => { + tracks.remove(&name); + } + Some(Command::Poke) => {} + }, + Some(read) = reads.next(), if !reads.is_empty() => { + handle(read, &mut tracks, &mut buffered, &mut reads); + } + (done, result) = async { commit.as_mut().unwrap().await }, if commit.is_some() => { + commit = None; + committer = Some(done); + let expired = result?; + if let Some(grace) = grace && !expired.is_empty() { + deletions.push_back((Instant::now() + grace, expired)); + } + } + _ = source.closed(), if !closed => closed = true, + _ = tokio::time::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { + let (_, keys) = deletions.pop_front().unwrap(); + delete(&shared.store, keys).await; + } + } + } + + let Segments::Drain(drain) = segments else { + unreachable!("the loop only ends after draining"); + }; + drain.result().map_err(timeline_error)?; + let mut committer = committer.expect("idle committer"); + committer.timeline.finish().map_err(timeline_error)?; + + // Complete expirations already committed to the timeline; nothing new expires after the + // final segment. + for (deadline, keys) in deletions { + tokio::time::sleep_until(deadline).await; + delete(&shared.store, keys).await; + } + + if source.is_finished() { + Ok(()) + } else { + Err(source_error(source.closed().await)) + } + } +} + +impl Control { + /// Enroll `name` without letting it influence segmentation, such as a catalog or metadata track. + /// + /// Subscribes to the track and creates its `.info` before accepting any group. Fails when the + /// name was already enrolled, is the recording's timeline, or `.info` conflicts. + pub async fn track(&self, name: &str) -> Result<()> { + self.enroll(name, false).await + } + + /// Enroll the continuously publishing track `name` as a pacing track. + /// + /// It votes on segment boundaries and holds every segment back until its groups are known. A + /// pacing track that stalls without closing stalls the recording by design; apply a deadline + /// with [`cut`](Self::cut) or [`remove`](Self::remove). + pub async fn pacing_track(&self, name: &str) -> Result<()> { + self.enroll(name, true).await + } + + /// Declare a segment boundary at `pts`, overriding the configured minimum-duration pacing. + /// + /// The first cut takes over boundaries for good, as with [`timeline::Producer::cut`]. + pub fn cut(&self, pts: Timestamp) -> Result<()> { + self.deferred.cut(pts).map_err(timeline_error)?; + self.send(Command::Poke) + } + + /// Stop recording `name`, dropping its incomplete groups. The name cannot be enrolled again. + pub fn remove(&self, name: &str) -> Result<()> { + self.send(Command::Remove(name.to_string())) + } + + async fn enroll(&self, name: &str, pacing: bool) -> Result<()> { + if name == self.shared.timeline || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { + return Err(Error::Enrolled(name.to_string())); + } + let result = self.subscribe(name, pacing).await; + if result.is_err() { + self.shared.enrolled.lock().unwrap().remove(name); + } + result + } + + async fn subscribe(&self, name: &str, pacing: bool) -> Result<()> { + let replay = track::Subscription::default().with_max_age(REPLAY); + let subscriber = self + .shared + .source + .track(name) + .map_err(source_error)? + .subscribe(replay) + .await + .map_err(source_error)?; + let timescale = subscriber.info().timescale; + let info = Info::new(subscriber.info().priority, timescale.as_u64())?; + self.shared.store.put_info(name, &info).await?; + + let recorder = match pacing { + true => self.deferred.pacing_track(name), + false => self.deferred.track(name), + }; + self.send(Command::Enroll { + name: name.to_string(), + subscriber: Box::new(subscriber), + recorder, + timescale, + }) + } + + fn send(&self, command: Command) -> Result<()> { + self.commands.send(command).map_err(|_| Error::Closed) + } +} + +/// One enrolled track's accepted groups. +struct TrackState { + recorder: Recorder, + timescale: Timescale, + /// The newest group accepted; later arrivals must exceed it. + largest: Option, + /// Accepted groups in sequence order: `None` while reading, `Some` once complete. Reported in + /// order, so a complete group waits for every earlier accepted one. + accepted: BTreeMap>, + /// The subscription is still delivering groups. + subscribed: bool, + /// Dropping the sender cancels every read for this track. + _cancel: watch::Sender<()>, + cancelled: watch::Receiver<()>, +} + +impl TrackState { + /// Report every complete group at the front of the accepted queue. + fn report(&mut self, name: &str, buffered: &mut HashMap>) { + while let Some(entry) = self.accepted.first_entry() { + if entry.get().is_none() { + break; + } + let group = entry.remove().expect("checked above"); + let (Some(first), Some(last)) = (group.frames.first(), group.frames.last()) else { + continue; + }; + // Frame timestamps were validated while reading, so these conversions succeed. + let (Ok(first), Ok(last)) = ( + Timestamp::new(first.timestamp, self.timescale), + Timestamp::new(last.timestamp, self.timescale), + ) else { + continue; + }; + self.recorder.record(group.sequence, first, true); + self.recorder.end(last); + buffered + .entry(name.to_string()) + .or_default() + .insert(group.sequence, group); + } + } +} + +// Each read is already boxed as a future, so the variant sizes cost nothing extra. +#[allow(clippy::large_enum_variant)] +enum Read { + Group { + name: String, + subscriber: Box, + result: moq_net::Result>, + }, + Frames { + name: String, + sequence: u64, + result: moq_net::Result>, + }, + Cancelled, +} + +async fn recv(name: String, mut subscriber: Box) -> Read { + let result = subscriber.recv_group().await; + Read::Group { + name, + subscriber, + result, + } +} + +async fn frames(name: String, mut group: group::Consumer) -> Read { + let sequence = group.sequence; + let mut frames = Vec::new(); + let result = loop { + match group.read_frame().await { + Ok(Some(frame)) => frames.push(frame), + Ok(None) => break Ok(frames), + Err(err) => break Err(err), + } + }; + Read::Frames { name, sequence, result } +} + +/// Resolve to [`Read::Cancelled`] once the track's state drops. +async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { + tokio::select! { + read = read => read, + _ = cancelled.changed() => Read::Cancelled, + } +} + +fn handle( + read: Read, + tracks: &mut HashMap, + buffered: &mut HashMap>, + reads: &mut FuturesUnordered>, +) { + let name = match read { + Read::Cancelled => return, + Read::Group { + name, + subscriber, + result, + } => { + let Some(track) = tracks.get_mut(&name) else { + return; + }; + match result { + Ok(Some(group)) => { + let sequence = group.sequence; + if track.largest.is_some_and(|largest| sequence <= largest) { + tracing::debug!(track = %name, sequence, "refusing a duplicate or decreasing group"); + } else { + track.largest = Some(sequence); + track.accepted.insert(sequence, None); + reads.push(guard(track.cancelled.clone(), frames(name.clone(), group)).boxed()); + } + reads.push(guard(track.cancelled.clone(), recv(name.clone(), subscriber)).boxed()); + } + Ok(None) => track.subscribed = false, + Err(err) => { + tracing::warn!(track = %name, %err, "track ended"); + track.subscribed = false; + } + } + name + } + Read::Frames { name, sequence, result } => { + let Some(track) = tracks.get_mut(&name) else { + return; + }; + let group = result + .map_err(|err| err.to_string()) + .and_then(|frames| convert(sequence, frames, track.timescale).map_err(|err| err.to_string())); + match group { + Ok(group) if !group.frames.is_empty() => { + track.accepted.insert(sequence, Some(group)); + } + Ok(_) => { + track.accepted.remove(&sequence); + } + Err(err) => { + tracing::warn!(track = %name, sequence, %err, "dropping an incomplete group"); + track.accepted.remove(&sequence); + } + } + track.report(&name, buffered); + name + } + }; + + // A track whose subscription ended closes once its last accepted group settles. + if tracks + .get(&name) + .is_some_and(|track| !track.subscribed && track.accepted.is_empty()) + { + tracks.remove(&name); + } +} + +/// Convert a complete group's frames into the track's timescale. +fn convert(sequence: u64, frames: Vec, timescale: Timescale) -> Result { + let frames = frames + .into_iter() + .map(|frame| { + let timestamp = frame.timestamp.convert(timescale).map_err(|_| Error::Overflow)?.value(); + crate::path::check_id(timestamp)?; + Ok(Frame { + timestamp, + payload: frame.payload, + }) + }) + .collect::>()?; + crate::path::check_id(sequence)?; + Ok(Group { sequence, frames }) +} + +/// Take the buffered groups a pending record names, one object per track. +/// +/// A track missing any advertised group cannot produce an object whose table matches its ranges, +/// so it gets `None` and is omitted. +fn take_objects( + pending: &Pending, + buffered: &mut HashMap>, +) -> Vec<(String, Option)> { + pending + .tracks + .iter() + .map(|(name, ranges)| { + let track = buffered.entry(name.clone()).or_default(); + let mut groups = Vec::new(); + let mut complete = true; + for range in ranges { + for sequence in range.start..=range.end { + match track.remove(&sequence) { + Some(group) => groups.push(group), + None => complete = false, + } + } + } + (name.clone(), complete.then_some(Object { groups })) + }) + .collect() +} + +/// An in-flight segment commit, returning the committer and the expired objects to delete. +type Commit = BoxFuture<'static, (Committer, Result>)>; + +/// Deferred records to commit: live, then the terminal drain. +enum Segments { + Live(Deferred), + Drain(DeferredDrain), +} + +impl Segments { + fn next(&self) -> Option { + match self { + Self::Live(deferred) => deferred.next(), + Self::Drain(drain) => drain.next(), + } + } +} + +/// Commits one segment at a time: media objects, the record, retention, then the timeline object. +struct Committer { + shared: Arc>, + timeline: timeline::Producer, + /// The recording's own timeline track, read back to store its complete groups. + groups: track::Ordered, + /// Units per second of record `pts` and `duration`. + timescale: u64, + retention: Option, + /// Committed records still in the timeline window, oldest first. + window: VecDeque, +} + +impl Committer { + /// Commit `pending`, returning the committer and the expired objects to delete after the grace. + async fn commit(mut self, pending: Pending, objects: Vec<(String, Option)>) -> (Self, Result>) { + let result = self.commit_inner(pending, objects).await; + (self, result) + } + + async fn commit_inner(&mut self, mut pending: Pending, objects: Vec<(String, Option)>) -> Result> { + let shared = self.shared.clone(); + let store = &shared.store; + let puts = objects.iter().map(|(name, object)| async move { + let result = match object { + Some(object) => store.put_groups(name, object).await.map(|_| ()), + None => Err(Error::Timeline("advertised groups are not buffered".into())), + }; + (name, result) + }); + for (name, result) in futures::future::join_all(puts).await { + if let Err(err) = result { + tracing::warn!(track = %name, segment = pending.segment, %err, "omitting a track that was not stored"); + pending = pending.omit(name); + } + } + + let segment = pending.segment; + let record = (*pending).clone(); + self.timeline.push(pending).map_err(timeline_error)?; + self.window.push_back(record); + + let expired = self.expire(); + if !expired.is_empty() { + self.timeline.pop(expired.len() as u64).map_err(timeline_error)?; + } + self.timeline.flush().map_err(timeline_error)?; + + let object = self.read_timeline()?; + store.put_segments(&shared.timeline, segment, &object).await?; + + let mut keys = Vec::new(); + for record in expired { + for (name, ranges) in &record.tracks { + if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { + keys.push(Key::groups(name.clone(), first.start..=last.end)?); + } + } + } + Ok(keys) + } + + /// Pop the oldest records while the rest still cover the retention window. + fn expire(&mut self) -> Vec { + let Some(retention) = self.retention else { + return Vec::new(); + }; + let target = (retention.as_micros() * self.timescale as u128).div_ceil(1_000_000); + let mut total: u128 = self.window.iter().map(|record| record.duration as u128).sum(); + let mut expired = Vec::new(); + while self.window.len() > 1 { + let front = self.window.front().expect("non-empty").duration as u128; + if total - front < target { + break; + } + total -= front; + expired.push(self.window.pop_front().expect("non-empty")); + } + expired + } + + /// Collect the timeline groups completed since the last segment. + fn read_timeline(&mut self) -> Result { + let waiter = kio::Waiter::noop(); + let timescale = self.groups.info().timescale; + let mut groups = Vec::new(); + while let Poll::Ready(result) = self.groups.poll_next_group(&waiter) { + let Some(mut group) = result.map_err(timeline_error_net)? else { + break; + }; + let mut frames = Vec::new(); + loop { + match group.poll_read_frame(&waiter) { + Poll::Ready(Ok(Some(frame))) => frames.push(frame), + Poll::Ready(Ok(None)) => break, + Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), + // Flushing closed every group, so an open one is a bug. + Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), + } + } + groups.push(convert(group.sequence, frames, timescale)?); + } + Ok(Object { groups }) + } +} + +async fn delete(store: &Store, keys: Vec) { + let deletes = keys.iter().map(|key| async move { (key, store.delete(key).await) }); + for (key, result) in futures::future::join_all(deletes).await { + // An object that outlives its expiry advertises nothing; recovery cleans it up. + if let Err(err) = result { + tracing::warn!(?key, %err, "failed to delete an expired object"); + } + } +} + +fn timeline_error(err: moq_mux::Error) -> Error { + Error::Timeline(err.to_string()) +} + +fn timeline_error_net(err: moq_net::Error) -> Error { + Error::Timeline(err.to_string()) +} + +fn source_error(err: moq_net::Error) -> Error { + Error::Source(err.to_string()) +} + +#[cfg(test)] +mod tests { + + use futures::TryStreamExt; + use futures::stream::BoxStream; + use object_store::memory::InMemory; + use object_store::path::Path; + use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, PutMultipartOptions, PutOptions, + PutPayload, PutResult, + }; + + use super::*; + + /// An in-memory store whose group PUTs fail for one track. + #[derive(Debug, Clone)] + struct Failing { + inner: Arc, + track: &'static str, + } + + impl std::fmt::Display for Failing { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Failing") + } + } + + #[async_trait::async_trait] + impl ObjectStore for Failing { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + if location.as_ref().contains(&format!("/{}/groups/", self.track)) { + return Err(object_store::Error::NotImplemented { + operation: "put".into(), + implementer: "Failing".into(), + }); + } + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } + } + + const TIMELINE: &str = hang::timeline::DEFAULT_NAME; + + fn ms(v: u64) -> Timestamp { + Timestamp::from_millis(v).unwrap() + } + + /// A millisecond track that keeps an hour of history, so enrollment sees every group. + fn track(broadcast: &broadcast::Producer, name: &str) -> track::Producer { + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + broadcast.create_track(name, info).unwrap() + } + + /// Write one complete group with a frame at each timestamp. + fn group(track: &track::Producer, sequence: u64, timestamps: &[u64]) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for ×tamp in timestamps { + group + .write_frame(ms(timestamp), format!("{sequence}@{timestamp}")) + .unwrap(); + } + group.finish().unwrap(); + } + + /// Replay every stored timeline object, returning the records still in the window. + async fn window(store: &Store) -> Vec { + let entries: Vec<_> = store + .list(&Query::segments(TIMELINE).unwrap()) + .try_collect() + .await + .unwrap(); + let mut segments: Vec = entries + .into_iter() + .map(|entry| match entry.key { + Key::Segments { segment, .. } => segment, + key => panic!("unexpected {key:?}"), + }) + .collect(); + segments.sort(); + assert_eq!( + segments, + (0..segments.len() as u64).collect::>(), + "timeline objects are consecutive" + ); + + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + let mut records = BTreeMap::new(); + for segment in segments { + let object = store.get_segments(TIMELINE, segment).await.unwrap(); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } + } + while let Some(event) = decoder.next_event() { + match event { + moq_json::window::Event::Push { index, value } => { + records.insert(index, value); + } + moq_json::window::Event::Pop(range) | moq_json::window::Event::Skip(range) => { + for index in range { + records.remove(&index); + } + } + _ => unreachable!(), + } + } + } + records.into_values().collect() + } + + /// The groups each track contributes, from the records. + fn ranges(records: &[Record], name: &str) -> Vec<(u64, u64)> { + records + .iter() + .flat_map(|record| record.tracks.get(name).into_iter().flatten()) + .map(|range| (range.start, range.end)) + .collect() + } + + /// Every range in the records resolves to one stored object with exactly those groups. + async fn check_objects(store: &Store, records: &[Record]) { + for record in records { + for (name, ranges) in &record.tracks { + let first = ranges.first().unwrap().start; + let last = ranges.last().unwrap().end; + let object = store.get_groups(name, first..=last).await.unwrap(); + let advertised: Vec = ranges.iter().flat_map(|range| range.start..=range.end).collect(); + let stored: Vec = object.groups.iter().map(|group| group.sequence).collect(); + assert_eq!(stored, advertised, "{name} segment {}", record.segment); + } + } + } + + #[tokio::test] + async fn records_selected_tracks() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let catalog = track(&source, "catalog.json"); + let _ignored = track(&source, "ignored"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + control.track("catalog.json").await.unwrap(); + + group(&catalog, 0, &[0]); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + catalog.finish().unwrap(); + source.finish(); + + tokio::spawn(writer.run()).await.unwrap().unwrap(); + + let records = window(&store).await; + assert_eq!(records.len(), 6); + assert_eq!(ranges(&records, "video"), (0..6).map(|s| (s, s)).collect::>()); + assert_eq!(ranges(&records, "catalog.json"), vec![(0, 0)]); + assert!(records.iter().all(|record| !record.tracks.contains_key("ignored"))); + check_objects(&store, &records).await; + + let object = store.get_groups("video", 2..=2).await.unwrap(); + assert_eq!(object.groups[0].frames[1].timestamp, 2500); + assert_eq!(object.groups[0].frames[1].payload, "2@2500"); + assert_eq!(store.get_info("video").await.unwrap(), Info::new(0, 1000).unwrap()); + store.get_info(TIMELINE).await.unwrap(); + assert!(store.get_info("ignored").await.is_err()); + } + + #[tokio::test] + async fn a_failed_put_omits_only_that_track() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let failing = Failing { + inner: Arc::new(InMemory::new()), + track: "audio", + }; + let store = Store::new(failing, "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + control.pacing_track("audio").await.unwrap(); + + for sequence in 0..3 { + group(&video, sequence, &[sequence * 1000]); + group(&audio, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + audio.finish().unwrap(); + source.finish(); + + writer.run().await.unwrap(); + + let records = window(&store).await; + assert_eq!(records.len(), 3); + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2)]); + assert!(ranges(&records, "audio").is_empty()); + check_objects(&store, &records).await; + } + + #[tokio::test] + async fn retention_pops_then_deletes_expired_objects() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.finish(); + + writer.run().await.unwrap(); + + // Segments 3 and 4 hold two seconds; the partial final segment 5 is always kept. + let records = window(&store).await; + assert_eq!( + records.iter().map(|record| record.segment).collect::>(), + vec![3, 4, 5] + ); + check_objects(&store, &records).await; + + let stored: HashSet<_> = store + .list(&Query::groups("video").unwrap()) + .map_ok(|entry| entry.key) + .try_collect() + .await + .unwrap(); + let expected: HashSet<_> = (3..6).map(|s| Key::groups("video", s..=s).unwrap()).collect(); + assert_eq!(stored, expected); + } + + #[tokio::test] + async fn commands_are_refused_once_the_recording_ends() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + // A long grace keeps `run` waiting on deletions after the timeline finishes. + let config = + Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + source.finish(); + + let run = tokio::spawn(writer.run()); + while window(&store).await.last().map(|record| record.segment) != Some(5) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!run.is_finished()); + assert_eq!(control.remove("video"), Err(Error::Closed)); + run.abort(); + } + + #[tokio::test] + async fn decreasing_arrivals_are_refused() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + group(&video, 0, &[0]); + group(&video, 2, &[1000]); + group(&video, 1, &[500]); + group(&video, 3, &[2000]); + video.finish().unwrap(); + source.finish(); + + writer.run().await.unwrap(); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (2, 2), (3, 3)]); + check_objects(&store, &records).await; + } + + #[tokio::test] + async fn removing_a_stalled_pacing_track_releases_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + control.pacing_track("audio").await.unwrap(); + let run = tokio::spawn(writer.run()); + + group(&audio, 0, &[0]); + // The audio group never completes, so audio never reports past it. + let mut stalled = audio.create_group(group::Info { sequence: 1 }).unwrap(); + stalled.write_frame(ms(1000), "stalled").unwrap(); + for sequence in 0..4 { + group(&video, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + source.finish(); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!run.is_finished(), "a stalled pacing track holds the recording open"); + assert!(window(&store).await.is_empty(), "and holds every segment back"); + + control.remove("audio").unwrap(); + run.await.unwrap().unwrap(); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2), (3, 3)]); + assert_eq!( + ranges(&records, "audio"), + vec![(0, 0)], + "the incomplete group is dropped" + ); + check_objects(&store, &records).await; + drop(stalled); + } + + #[tokio::test] + async fn enrollment_is_refused_for_duplicates_and_the_timeline() { + let source = broadcast::Info::new().produce(); + let _video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + assert_eq!(control.track("video").await, Err(Error::Enrolled("video".into()))); + assert_eq!(control.track(TIMELINE).await, Err(Error::Enrolled(TIMELINE.into()))); + } + + #[tokio::test] + async fn a_conflicting_info_fails_enrollment() { + let source = broadcast::Info::new().produce(); + let _video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + store.put_info("video", &Info::new(7, 1000).unwrap()).await.unwrap(); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + assert_eq!( + control.pacing_track("video").await, + Err(Error::Priority { + existing: 7, + intended: 0 + }) + ); + } + + #[tokio::test] + async fn an_existing_recording_is_refused() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().pacing_track("video").await.unwrap(); + group(&video, 0, &[0]); + video.finish().unwrap(); + source.finish(); + writer.run().await.unwrap(); + + let again = broadcast::Info::new().produce(); + assert!(matches!( + Writer::new(store, again.consume(), Config::default()).await, + Err(Error::Occupied(_)) + )); + } +} diff --git a/rs/moq-json/src/window/encoder.rs b/rs/moq-json/src/window/encoder.rs index 58abe657c1..6af1f43a5d 100644 --- a/rs/moq-json/src/window/encoder.rs +++ b/rs/moq-json/src/window/encoder.rs @@ -138,7 +138,7 @@ impl Pending<'_, T> { impl Drop for Pending<'_, T> { fn drop(&mut self) { if self.edit.is_some() { - self.encoder.resync(); + self.encoder.reset(); } } } @@ -216,8 +216,11 @@ impl Encoder { self.offset..self.start + self.window.len() as u64 } - /// Discard group-local state after an encoded frame did not reach the wire. - fn resync(&mut self) { + /// Discard group-local state, so the next edit opens a new group with a header. + /// + /// Call this whenever the caller closes the current group behind the encoder's back. It is also + /// how the encoder recovers after an encoded frame did not reach the wire. + pub fn reset(&mut self) { self.flate = None; self.op_bytes = 0; self.header_len = 0; @@ -288,7 +291,7 @@ impl Encoder { let encoded = self.frame(bytes)?; let group_bytes = self.header_len.saturating_add(self.op_bytes); if group_bytes > moq_net::group::MAX_CACHE_BYTES { - self.resync(); + self.reset(); Ok(None) } else { Ok(Some(encoded)) diff --git a/rs/moq-json/src/window/mod.rs b/rs/moq-json/src/window/mod.rs index 6a319aa806..254f3d568e 100644 --- a/rs/moq-json/src/window/mod.rs +++ b/rs/moq-json/src/window/mod.rs @@ -26,8 +26,9 @@ //! //! The publisher rolls a group when the ops in it outgrow //! [`ProducerConfig::op_ratio`](ProducerConfig::op_ratio) times the header that opened it, exactly as -//! [`snapshot`](crate::snapshot) rolls on its delta budget. That is purely a compression decision: -//! there is no caller-driven cut and no age bound, and a [`Consumer`] never surfaces it. A header +//! [`snapshot`](crate::snapshot) rolls on its delta budget. That is purely a compression decision +//! with no age bound, and a [`Consumer`] never surfaces it. A caller that stores complete groups +//! can also end one with [`Producer::cut`]; the next edit opens a new group with a header. A header //! restating records a reader already has yields nothing, so however often the publisher rolls, the //! reader sees one continuous stream of [`Event`]s. [`ProducerConfig::checkpoint_records`] bounds //! the suffix repeated on each roll for a long-lived window. @@ -604,6 +605,41 @@ mod test { assert!(group.decode(br#"{"offset":0,"records":[]}"#).is_err()); } + #[test] + fn cut_finishes_the_group_and_the_next_edit_restates() { + let track = moq_net::broadcast::Info::new() + .produce() + .create_track("test", None) + .unwrap(); + let replay = moq_net::track::Subscription::default().with_max_age(std::time::Duration::from_secs(30)); + let mut groups = track.subscribe(replay); + let mut producer = Producer::::new(track, ProducerConfig::default()); + + producer.cut().unwrap(); + producer.push(&rec(0)).unwrap(); + producer.push(&rec(1)).unwrap(); + producer.cut().unwrap(); + producer.cut().unwrap(); + producer.push(&rec(2)).unwrap(); + + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(Some(mut first))) = groups.poll_recv_group(&waiter) else { + panic!("the first group is published"); + }; + assert_eq!(first.frame_count(), 2); + assert!(first.poll_finished(&waiter).is_ready(), "cut finishes the open group"); + + let Poll::Ready(Ok(Some(mut second))) = groups.poll_recv_group(&waiter) else { + panic!("the next edit opens a group"); + }; + let Poll::Ready(Ok(Some(header))) = second.poll_read_frame(&waiter) else { + panic!("the group starts with a header"); + }; + let header: Value = serde_json::from_slice(&header.payload).unwrap(); + assert_eq!(header, json!({ "offset": 0, "records": [rec(0), rec(1), rec(2)] })); + assert!(groups.poll_recv_group(&waiter).is_pending()); + } + #[test] fn rolling_is_invisible_to_the_consumer() { // The same edits, framed two ways: one group for everything, versus a roll per edit. diff --git a/rs/moq-json/src/window/producer.rs b/rs/moq-json/src/window/producer.rs index 1bdfba06e4..5967f80bb5 100644 --- a/rs/moq-json/src/window/producer.rs +++ b/rs/moq-json/src/window/producer.rs @@ -70,6 +70,14 @@ impl Producer { self.inner.lock().unwrap().pop(count) } + /// Finish the open group, leaving the next edit to open a replacement with a header. + /// + /// Idempotent: cutting when no group is open does nothing. A caller that stores complete groups + /// cuts after its edits, so every edit so far sits in a group no later frame can extend. + pub fn cut(&mut self) -> Result<()> { + self.inner.lock().unwrap().cut() + } + /// Finish the track, closing any open group. /// /// Borrows rather than consumes, matching snapshot and stream, so the handle stays @@ -115,6 +123,17 @@ impl Inner { Ok(()) } + fn cut(&mut self) -> Result<()> { + let Some(group) = self.track.group.take() else { + return Ok(()); + }; + // The group closes either way, so reset first: a `finish` error must not leave the encoder + // appending ops to a group that is gone. + self.encoder.reset(); + group.finish()?; + Ok(()) + } + fn finish(&mut self) -> Result<()> { if self.finished { return Ok(()); diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs index 4af4902d1e..ecc240c092 100644 --- a/rs/moq-mux/src/timeline.rs +++ b/rs/moq-mux/src/timeline.rs @@ -697,6 +697,19 @@ impl Pending { .clear(); self } + + /// Drop one track whose media could not be stored, keeping every other track's ranges. + /// + /// Like [`gap`](Self::gap) for a single track: the record stays committable without advertising + /// objects that are not durable. + pub fn omit(mut self, track: &str) -> Self { + self.record + .as_mut() + .expect("a pending record is present until commit") + .tracks + .remove(track); + self + } } impl Deref for Pending { @@ -759,6 +772,17 @@ impl Output { } } + fn flush(&mut self) -> crate::Result<()> { + let Some(sink) = self.sink.as_mut() else { + return Ok(()); + }; + match sink.cut() { + Ok(()) => Ok(()), + Err(moq_json::Error::Net(err)) => Err(err.into()), + Err(err) => Err(err.into()), + } + } + fn finish(&mut self) -> crate::Result<()> { self.closed = true; let Some(sink) = self.sink.as_mut() else { @@ -964,6 +988,14 @@ impl Producer { self.output.lock().unwrap().pop(count) } + /// Close the timeline track's open group, so every push and pop so far sits in a complete group. + /// + /// A recorder stores those groups before committing the next segment. The next edit opens a + /// new group restating the window. + pub fn flush(&self) -> crate::Result<()> { + self.output.lock().unwrap().flush() + } + fn publish_ready(&self) { // Serialize the dequeue and publish steps together. Recorders can report from different // tasks, and taking one record before locking the output would let a later record win the @@ -1345,6 +1377,61 @@ mod test { )); } + #[tokio::test] + async fn omitting_a_track_keeps_the_others() { + let (broadcast, mut timeline) = setup(); + let segmenter = timeline.deferred().unwrap(); + let mut video = segmenter.pacing_track("video0"); + let mut audio = segmenter.pacing_track("audio0"); + video.record(0, ms(0), true); + audio.record(0, ms(0), true); + video.record(1, ms(2_000), true); + audio.record(1, ms(2_000), true); + + let pending = segmenter.next().expect("the complete segment is ready for storage"); + timeline.push(pending.omit("audio0")).unwrap(); + drop((video, audio)); + let tail = segmenter.finish(); + while let Some(pending) = tail.next() { + timeline.push(pending).unwrap(); + } + timeline.finish().unwrap(); + + assert_eq!( + drain(&broadcast, &timeline).await, + vec![ + entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), + entry(1, 2_000, 0, &[("audio0", &[(1, 1)]), ("video0", &[(1, 1)])]), + ] + ); + } + + #[test] + fn flush_closes_the_timeline_group() { + let (broadcast, timeline) = setup(); + let segmenter = timeline.deferred().unwrap(); + let replay = moq_net::track::Subscription::default().with_max_age(Duration::from_secs(30)); + let groups = broadcast.consume().track(DEFAULT_NAME).unwrap().subscribe(replay); + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(mut groups)) = groups.poll_ok(&waiter) else { + panic!("the deferred timeline track exists"); + }; + + let mut video = segmenter.pacing_track("video0"); + video.record(0, ms(0), true); + video.record(1, ms(2_000), true); + timeline.push(segmenter.next().unwrap()).unwrap(); + timeline.pop(1).unwrap(); + timeline.flush().unwrap(); + + let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { + panic!("the pushed record opened a group"); + }; + assert_eq!(group.frame_count(), 2, "the push header and the pop share one group"); + assert!(group.poll_finished(&waiter).is_ready(), "flush closes it"); + assert!(groups.poll_recv_group(&waiter).is_pending()); + } + #[test] fn dropping_a_pending_record_requeues_it() { let (_broadcast, timeline) = setup(); From 62a48219a18f665439d1d09068ecbd2303733384 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 19:28:44 -0700 Subject: [PATCH 03/49] feat(archive): serve recordings through a supplied broadcast (#4044) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 13 + quest/m1/archive/README.md | 5 +- quest/m1/archive/cli.md | 4 - quest/m1/archive/dvr.md | 5 +- quest/m1/archive/hls.md | 4 - quest/m1/archive/reader.md | 49 --- rs/moq-archive/Cargo.toml | 3 +- rs/moq-archive/src/error.rs | 10 + rs/moq-archive/src/lib.rs | 5 +- rs/moq-archive/src/reader/index.rs | 175 +++++++++++ rs/moq-archive/src/reader/mod.rs | 390 ++++++++++++++++++++++++ rs/moq-archive/src/reader/tests.rs | 467 +++++++++++++++++++++++++++++ 12 files changed, 1069 insertions(+), 61 deletions(-) delete mode 100644 quest/m1/archive/reader.md create mode 100644 rs/moq-archive/src/reader/index.rs create mode 100644 rs/moq-archive/src/reader/mod.rs create mode 100644 rs/moq-archive/src/reader/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 8643d865b9..31aac7a2c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4166,6 +4166,7 @@ dependencies = [ "moq-net", "object_store", "percent-encoding", + "quick_cache", "serde", "serde_json", "tempfile", @@ -6869,6 +6870,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick_cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" +dependencies = [ + "equivalent", + "foldhash", + "hashbrown 0.17.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.12" diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index f5ab5b2eb6..80e59ac56c 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -41,6 +41,10 @@ percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list through `Deferred`, omits failed tracks with `Pending::omit`, stores each segment's timeline groups after `Producer::flush`, and expires DVR segments with a deletion grace. It refuses a prefix that already holds a timeline. +`moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays the timeline onto a +supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` with a byte-bounded +object LRU. `Reader::refresh` follows by listing timeline keys after its cursor, so gaps and +DVR expiry recover from the next checkpoint; `Reader::finish` applies out-of-band finality. ### Format @@ -108,7 +112,6 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests - [Archive endpoint](/quest/m1/archive/cli.md) - `moq ... export archive` records and `import archive` replays -- [Recording reader](/quest/m1/archive/reader.md) - serve archived FETCH through a supplied `broadcast::Producer` - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [Offline archive HLS](/quest/m1/archive/hls.md) - render playlists from the archive timeline and fetch segment media lazily - [Resume a recording](/quest/m1/archive/recovery.md) - recover the retained timeline on restart and clean up DVR orphans diff --git a/quest/m1/archive/cli.md b/quest/m1/archive/cli.md index 84ebbf0e79..cfa76925bf 100644 --- a/quest/m1/archive/cli.md +++ b/quest/m1/archive/cli.md @@ -38,10 +38,6 @@ timeline or keeps following a growing archive. Update `doc/bin/cli.md` and every example invocation. -## Required - -- [Recording reader](/quest/m1/archive/reader.md) - serves `import archive` - ## Closes - [#2281](https://github.com/moq-dev/moq/issues/2281) - close this issue when the quest finishes diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 19c0b655ac..685185fa65 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -23,10 +23,13 @@ Test seeks within the retained window, expiry during a seek, missing groups, restart recovery, and return to live without duplicated or rewound playback. Use the writer/reader fixtures; a retention defect is fixed in its owning layer. +The reader evicts popped spans from its object cache, but a group it already +served stays in `moq_net`'s track cache until the pool reclaims it. Decide +whether expiry during a seek needs a group eviction API in `moq-net`. + ## Required - [Resume a recording](/quest/m1/archive/recovery.md) -- [Recording reader](/quest/m1/archive/reader.md) ## Closes diff --git a/quest/m1/archive/hls.md b/quest/m1/archive/hls.md index a967808de4..f4a2966c22 100644 --- a/quest/m1/archive/hls.md +++ b/quest/m1/archive/hls.md @@ -36,7 +36,3 @@ explicit binding. Track immutability and update correlation belong to Prove aligned audio/video switching, missing track segments, discontinuities, caller-supplied finality, and bounded LRU reads. - -## Required - -- [Recording reader](/quest/m1/archive/reader.md) diff --git a/quest/m1/archive/reader.md b/quest/m1/archive/reader.md deleted file mode 100644 index f317970680..0000000000 --- a/quest/m1/archive/reader.md +++ /dev/null @@ -1,49 +0,0 @@ -# [L] Recording reader - -## Goal - -A reader takes an archive and a caller-provided `broadcast::Producer`, then -serves FETCH requests for every track and group the archive timeline -advertises. - -## Plan - -Bootstrap by listing the timeline track's `segments/` keys, sorting them, and -replaying from a retained Window checkpoint. Records derive each track's -`groups/.` key from its minimum and maximum advertised IDs. -Follow a growing archive with GET of timeline segment N+1; media listings do -not need refreshing. Not Found is not finality. A reader behind DVR retention -re-bootstraps from a retained checkpoint, and timeline pops evict cached ranges. -Alternatively enumerate timeline keys after the last replayed key with -`list_with_offset`, consuming and sorting the full result before replay. -Pagination tokens continue one enumeration, not future refreshes. - -Filename listings provide a cached group-range index without media GETs. A -backend with verified ordered offset listing may find a cold FETCH candidate -using `PaginatedListStore` with offset `groups/` (19 padded -digits, no dot) and `max_keys = 1`. Check the lower bound and committed timeline -membership; the table resolves internal gaps. Generic listings are unordered, -so never take their first entry as the nearest match. A stale listing cannot -veto a successful GET referenced by a newer durable timeline record. - -Use `track::Dynamic` (`rs/moq-net/src/model/track.rs:1652`, minted by -`Producer::dynamic` :1587) to accept requested tracks and groups on the -supplied producer; a cache-miss `Consumer::fetch_group` (:2352) parks on it. -Map `(track, group)` to its range-named object through the cached index, GET -that one object, validate the envelope version and every group/frame table -entry against the retrieved length, filename bounds, and exact committed -ranges, and place it in a byte-bounded LRU. -Adjacent group FETCHes reuse the same object. A request for one audio track or -rendition never downloads another track's object. - -Reproduce the original group sequence, frame timestamps, and payload bytes, -including a requested `frame_start`. A group absent from the timeline, a -missing object, an unknown envelope version, or a malformed table behaves -exactly like a group the source never delivered; siblings and later segments -remain usable. - -The reader does not interpret media, own routing, or expose storage paths. It -does not infer a terminal broadcast state: a caller such as a managed -recordings API may supply finality out of band, and the reader then finishes -the replayed timeline track. Otherwise an archive may be growing, crashed, or -missing its tail, and remains readable. diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 238c1c00d3..9f03c360b5 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -17,10 +17,12 @@ bytes = { workspace = true } futures = { workspace = true } hang = { workspace = true } kio = { workspace = true } +moq-json = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } +quick_cache = "0.7" serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } @@ -29,7 +31,6 @@ tracing = { workspace = true } [dev-dependencies] async-trait = "0.1" -moq-json = { path = "../moq-json" } object_store = { workspace = true, features = ["fs"] } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index 0ccb85bc5e..85010486b9 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -74,6 +74,10 @@ pub enum Error { #[error("json: {0}")] Json(String), + /// Publishing the replayed archive failed. + #[error("moq: {0}")] + Moq(String), + /// The prefix already holds a recording timeline; resuming is unsupported. #[error("prefix already holds a recording: {0}")] Occupied(String), @@ -104,6 +108,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: moq_net::Error) -> Self { + Self::Moq(err.to_string()) + } +} + impl From for Error { fn from(err: serde_json::Error) -> Self { Self::Json(err.to_string()) diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 8a06522e6a..b582fffbbd 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -2,7 +2,8 @@ //! //! The crate owns the portable layout and codecs: percent-encoded track names, `.info` JSON, //! the binary segment envelope, and put/get/list/delete. A [`Writer`] records selected tracks of -//! a broadcast into those objects. Callers that need runtime dispatch +//! a broadcast into those objects, and a [`Reader`] serves a recording back through a `moq_net` +//! broadcast. Callers that need runtime dispatch //! supply `Arc`; the archive API itself stays generic. //! //! Group bounds are finite inclusive ranges in first-to-last order: @@ -17,6 +18,7 @@ pub use object_store; mod error; pub mod info; mod path; +pub mod reader; pub mod segment; pub mod store; pub mod writer; @@ -24,6 +26,7 @@ pub mod writer; pub use error::{Error, Result}; pub use info::Info; pub use path::Key; +pub use reader::Reader; pub use segment::{Frame, Group, Object}; pub use store::Store; pub use writer::Writer; diff --git a/rs/moq-archive/src/reader/index.rs b/rs/moq-archive/src/reader/index.rs new file mode 100644 index 0000000000..bedbd7b9e2 --- /dev/null +++ b/rs/moq-archive/src/reader/index.rs @@ -0,0 +1,175 @@ +use std::collections::{BTreeMap, HashMap}; +use std::ops::{Range, RangeInclusive}; + +use hang::timeline::Record; + +use crate::path::check_range; + +/// One track's stored object for one record: its filename bounds and the exact runs it holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Span { + /// Inclusive first-to-last group sequence, the object's filename bounds. + pub bounds: RangeInclusive, + /// The advertised runs, ascending and nonoverlapping; groups between them never existed. + pub runs: Vec>, +} + +impl Span { + /// Build from a record's ranges, refusing empty, reversed, out-of-range, or unordered runs. + fn new(ranges: &[hang::timeline::Range]) -> Option { + let mut runs: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + let run = range.start..=range.end; + check_range(&run).ok()?; + if let Some(prev) = runs.last() + && run.start() <= prev.end() + { + return None; + } + runs.push(run); + } + let bounds = *runs.first()?.start()..=*runs.last()?.end(); + Some(Self { bounds, runs }) + } + + fn contains(&self, group: u64) -> bool { + self.runs.iter().any(|run| run.contains(&group)) + } +} + +/// The committed group ranges advertised by the replayed timeline window. +#[derive(Default)] +pub(super) struct Index { + /// Per track, each span keyed by its smallest group sequence. + tracks: HashMap>, + /// Per window index, the `(track, smallest)` spans that record added, so a pop can evict them. + records: BTreeMap>, +} + +impl Index { + /// Add a record's spans at window `index`. + /// + /// A track whose ranges are malformed or overlap another record's span is left out, so its + /// groups behave like ones the source never delivered; the record's other tracks stay usable. + pub fn push(&mut self, index: u64, record: &Record) { + let mut added = Vec::new(); + for (track, ranges) in &record.tracks { + let Some(span) = Span::new(ranges) else { + tracing::warn!(track, segment = record.segment, "ignoring malformed archive ranges"); + continue; + }; + + let spans = self.tracks.entry(track.clone()).or_default(); + let smallest = *span.bounds.start(); + let before = spans.range(..=span.bounds.end()).next_back(); + if before.is_some_and(|(_, prev)| prev.bounds.end() >= span.bounds.start()) { + tracing::warn!(track, segment = record.segment, "ignoring overlapping archive ranges"); + continue; + } + + spans.insert(smallest, span); + added.push((track.clone(), smallest)); + } + self.records.insert(index, added); + } + + /// Remove the records at these window indices, returning the evicted spans by track. + pub fn pop(&mut self, indices: Range) -> Vec<(String, Span)> { + let popped: Vec = self.records.range(indices).map(|(index, _)| *index).collect(); + let mut evicted = Vec::new(); + for index in popped { + for (track, smallest) in self.records.remove(&index).unwrap_or_default() { + let span = self.tracks.get_mut(&track).and_then(|spans| spans.remove(&smallest)); + if let Some(span) = span { + evicted.push((track, span)); + } + } + } + evicted + } + + /// Whether any replayed record named this track. + pub fn has_track(&self, track: &str) -> bool { + self.tracks.contains_key(track) + } + + /// The span advertising `group` on `track`, if the retained window commits it. + pub fn get(&self, track: &str, group: u64) -> Option { + let (_, span) = self.tracks.get(track)?.range(..=group).next_back()?; + span.contains(group).then(|| span.clone()) + } +} + +#[cfg(test)] +mod tests { + use hang::timeline::Range; + + use super::*; + use crate::ID_MAX; + + fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { + let mut record = Record::new(segment, segment * 1000, 1000); + for (name, ranges) in tracks { + let ranges = ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(); + record.tracks.insert(name.to_string(), ranges); + } + record + } + + #[test] + fn lookup_respects_bounds_and_internal_gaps() { + let mut index = Index::default(); + index.push(0, &record(0, &[("audio", &[(0, 2), (5, 6)]), ("video", &[(0, 0)])])); + index.push(1, &record(1, &[("audio", &[(7, 9)])])); + + assert_eq!(index.get("audio", 1).unwrap().bounds, 0..=6); + assert_eq!(index.get("audio", 6).unwrap().runs, vec![0..=2, 5..=6]); + assert!(index.get("audio", 3).is_none(), "internal gap"); + assert_eq!(index.get("audio", 8).unwrap().bounds, 7..=9); + assert!(index.get("audio", 10).is_none()); + assert!(index.get("video", 1).is_none()); + assert!(index.get("chat", 0).is_none()); + assert!(index.has_track("video")); + assert!(!index.has_track("chat")); + } + + #[test] + fn pop_evicts_only_the_popped_records() { + let mut index = Index::default(); + index.push(0, &record(0, &[("video", &[(0, 1)])])); + index.push(1, &record(1, &[("video", &[(2, 3)])])); + + let evicted = index.pop(0..1); + assert_eq!(evicted.len(), 1); + assert_eq!(evicted[0].1.bounds, 0..=1); + assert!(index.get("video", 0).is_none()); + assert!(index.get("video", 2).is_some()); + assert!(index.pop(0..1).is_empty(), "already popped"); + assert!(index.has_track("video"), "a popped track stays known"); + } + + #[test] + fn malformed_and_overlapping_tracks_are_ignored() { + let mut index = Index::default(); + index.push(0, &record(0, &[("video", &[(4, 6)]), ("audio", &[(3, 1)])])); + assert!(index.get("audio", 1).is_none(), "reversed run"); + + // Overlapping the earlier video span drops only the video entry. + index.push(1, &record(1, &[("video", &[(6, 8)]), ("audio", &[(10, 11)])])); + assert_eq!(index.get("video", 6).unwrap().bounds, 4..=6); + assert!(index.get("video", 7).is_none()); + assert!(index.get("audio", 10).is_some()); + + // Unordered runs and IDs past the recording range. + index.push( + 2, + &record(2, &[("chat", &[(5, 6), (1, 2)]), ("data", &[(ID_MAX, ID_MAX + 1)])]), + ); + assert!(index.get("chat", 5).is_none()); + assert!(index.get("data", ID_MAX).is_none()); + + // Popping a record whose track was ignored leaves the earlier span alone. + index.pop(1..2); + assert!(index.get("video", 4).is_some()); + } +} diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs new file mode 100644 index 0000000000..64b22c9fab --- /dev/null +++ b/rs/moq-archive/src/reader/mod.rs @@ -0,0 +1,390 @@ +//! Serve an archive's groups through a caller-supplied broadcast. +//! +//! A [`Reader`] replays the archive timeline onto its timeline track and answers FETCH for every +//! group that timeline commits. Each request maps to one range-named object, fetched with a single +//! GET, validated against the record that advertised it, and kept in a byte-bounded cache so +//! adjacent groups reuse it. A request for one track never downloads another track's object. +//! +//! The reader never infers that a recording ended: it keeps following until the caller, who may +//! know finality out of band, calls [`Reader::finish`]. +//! +//! ```no_run +//! # async fn example(store: moq_archive::Store) -> moq_archive::Result<()> { +//! use moq_archive::reader::{Config, Reader}; +//! +//! let broadcast = moq_net::broadcast::Info::new().produce(); +//! let mut reader = Reader::open(store, &broadcast, Config::new("timeline.z")).await?; +//! let serve = reader.serve(); +//! // Spawn `serve` on any executor, then call `reader.refresh()` to follow a growing archive. +//! # drop(serve); +//! # Ok(()) +//! # } +//! ``` + +mod index; + +use std::future::Future; +use std::ops::RangeInclusive; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use hang::timeline::Record; +use moq_json::window; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStore; + +use self::index::{Index, Span}; +use crate::store::list::Query; +use crate::{Error, Key, Object, Result, Store}; + +/// Configuration for [`Reader::open`]. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Config { + /// The timeline track name, from the catalog's `archive` entry. + pub timeline: String, + /// Upper bound on cached object bytes shared by every track. + pub cache: u64, +} + +impl Config { + /// Read the timeline named `timeline` with a 64 MiB object cache. + pub fn new(timeline: impl Into) -> Self { + Self { + timeline: timeline.into(), + cache: 64 * 1024 * 1024, + } + } + + /// Set [`cache`](Self::cache). + pub fn with_cache(mut self, bytes: u64) -> Self { + self.cache = bytes; + self + } +} + +/// Serves an archive through a [`broadcast::Producer`] and follows its timeline. +pub struct Reader { + shared: Arc>, + dynamic: broadcast::Dynamic, + timeline: track::Producer, + decoder: window::Decoder, + /// The last timeline segment replayed. + cursor: Option, +} + +struct Shared { + store: Store, + index: Mutex, + cache: quick_cache::sync::Cache, Weight>, +} + +impl Reader { + /// Create the timeline track on `broadcast`, then replay every retained timeline object. + /// + /// Fails if the timeline's `.info` is missing or invalid, the timeline track already exists, + /// or the store cannot list the timeline. + pub async fn open(store: Store, broadcast: &broadcast::Producer, config: Config) -> Result { + let info = track_info(&store.get_info(&config.timeline).await?)?; + let timeline = broadcast.create_track(config.timeline, info)?; + + let items = usize::try_from(config.cache / (1024 * 1024)) + .unwrap_or(usize::MAX) + .max(16); + let shared = Arc::new(Shared { + store, + index: Mutex::default(), + cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), + }); + + let mut reader = Self { + shared, + dynamic: broadcast.dynamic(), + timeline, + decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), + cursor: None, + }; + reader.refresh().await?; + Ok(reader) + } + + /// Replay timeline objects committed since the last refresh. + /// + /// Lists timeline keys after the last replayed segment, then GETs them in segment order. Each + /// stored group opens with a window checkpoint, so a segment that expired or is malformed is + /// skipped and the next one recovers the retained window, evicting whatever it popped. A + /// malformed newest object is retried on the next refresh. + pub async fn refresh(&mut self) -> Result<()> { + let timeline = self.timeline.name().to_string(); + let mut query = Query::segments(&timeline)?; + if let Some(cursor) = self.cursor { + query = query.after(&Key::segments(&timeline, cursor)?)?; + } + + let entries: Vec<_> = self.shared.store.list(&query).try_collect().await?; + let mut segments: Vec = entries + .into_iter() + .filter_map(|entry| match entry.key { + Key::Segments { segment, .. } => Some(segment), + _ => None, + }) + .filter(|segment| self.cursor.is_none_or(|cursor| *segment > cursor)) + .collect(); + segments.sort_unstable(); + segments.dedup(); + + for segment in segments { + match self.shared.store.get_segments(&timeline, segment).await { + Ok(object) => self.replay(&object)?, + Err(Error::Store(err)) => return Err(Error::Store(err)), + Err(err) => { + tracing::warn!(segment, %err, "skipping unreadable timeline segment"); + continue; + } + } + self.cursor = Some(segment); + } + + Ok(()) + } + + /// Finish the replayed timeline track once the caller knows the recording ended. + /// + /// Following stops; groups already committed stay servable through [`serve`](Self::serve). + pub fn finish(self) -> Result<()> { + Ok(self.timeline.finish()?) + } + + /// Answer track and group requests on the broadcast until it closes. + /// + /// The future is independent of this reader, so it keeps serving after + /// [`finish`](Self::finish); run it on any executor. + pub fn serve(&self) -> impl Future + Send + 'static + where + T: 'static, + { + serve(self.shared.clone(), self.dynamic.clone()) + } + + /// Index one timeline object's groups, then republish each on the timeline track. + fn replay(&mut self, object: &Object) -> Result<()> { + for stored in &object.groups { + let mut group = self.decoder.group(); + let decoded = stored.frames.iter().try_for_each(|frame| group.decode(&frame.payload)); + + // Apply whatever decoded, even from a group that failed partway: the next group's + // checkpoint restates the window regardless. + let mut index = self.shared.index.lock().unwrap(); + while let Some(event) = group.next_event() { + match event { + window::Event::Push { index: at, value } => index.push(at, &value), + window::Event::Pop(range) => { + for (track, span) in index.pop(range) { + if let Ok(key) = Key::groups(track, span.bounds) { + self.shared.cache.remove(&key); + } + } + } + _ => {} + } + } + drop(index); + + if let Err(err) = decoded { + tracing::warn!(group = stored.sequence, %err, "skipping malformed timeline group"); + continue; + } + + let mut producer = match self.timeline.create_group(stored.sequence.into()) { + Ok(producer) => producer, + Err(moq_net::Error::Duplicate) => continue, + Err(err) => return Err(err.into()), + }; + let timescale = producer.timescale(); + for frame in &stored.frames { + producer.write_frame(timestamp(frame.timestamp, timescale)?, frame.payload.clone())?; + } + producer.finish()?; + } + Ok(()) + } +} + +/// Serve requested tracks until the broadcast closes. +async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic) { + let mut tracks = kio::Tasks::new(); + kio::wait(|waiter| { + loop { + match dynamic.poll_requested_track(waiter) { + Poll::Ready(Ok(request)) => tracks.push(task(Box::pin(serve_track(shared.clone(), request)))), + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } + } + let _ = tracks.poll(waiter); + Poll::Pending + }) + .await +} + +/// Accept a track the timeline names, then serve its group requests until nobody uses it. +async fn serve_track(shared: Arc>, request: track::Request) { + let name = request.name().to_string(); + if !shared.index.lock().unwrap().has_track(&name) { + request.reject(moq_net::Error::NotFound); + return; + } + + let info = match shared.store.get_info(&name).await.and_then(|info| track_info(&info)) { + Ok(info) => info, + Err(err) => { + tracing::warn!(track = name, %err, "archived track has no usable .info"); + request.reject(moq_net::Error::NotFound); + return; + } + }; + + let timescale = info.timescale; + let dynamic = request.dynamic(); + let _producer = request.accept(info); + let mut groups = kio::Tasks::new(); + + kio::wait(|waiter| { + loop { + match dynamic.poll_requested_group(waiter) { + Poll::Ready(Ok(request)) => { + let serve = serve_group(shared.clone(), name.clone(), timescale, request); + groups.push(task(Box::pin(serve))); + } + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } + } + match groups.poll(waiter).is_ready() && dynamic.poll_unused(waiter).is_ready() { + true => Poll::Ready(()), + false => Poll::Pending, + } + }) + .await +} + +/// Answer one group request from its range-named object, or reject it as never delivered. +async fn serve_group( + shared: Arc>, + track: String, + timescale: Timescale, + request: group::Request, +) { + let sequence = request.sequence(); + let span = shared.index.lock().unwrap().get(&track, sequence); + let Some(span) = span else { + request.reject(moq_net::Error::NotFound); + return; + }; + + let object = match Key::groups(track.clone(), span.bounds.clone()) { + Ok(key) => { + shared + .cache + .get_or_insert_async(&key, load(&shared.store, &track, &span)) + .await + } + Err(err) => Err(err), + }; + let object = match object { + Ok(object) => object, + Err(err) => { + tracing::warn!(track, sequence, %err, "archived group is unavailable"); + request.reject(moq_net::Error::NotFound); + return; + } + }; + + // Load validated the table against the span, so every advertised group is present. + let Ok(position) = object.groups.binary_search_by_key(&sequence, |group| group.sequence) else { + request.reject(moq_net::Error::NotFound); + return; + }; + let frames = &object.groups[position].frames; + let Some(frames) = usize::try_from(request.frame_start()) + .ok() + .and_then(|start| frames.get(start..)) + else { + request.reject(moq_net::Error::NotFound); + return; + }; + + let start = request.frame_start(); + let Ok(mut producer) = request.accept(None) else { + return; + }; + let written = producer.start_at(start).and_then(|()| { + frames.iter().try_for_each(|frame| { + let timestamp = timestamp(frame.timestamp, timescale).map_err(|_| moq_net::Error::TimestampMismatch)?; + producer.write_frame(timestamp, frame.payload.clone()) + }) + }); + match written { + Ok(()) => { + let _ = producer.finish(); + } + Err(err) => { + let _ = producer.abort(err); + } + } +} + +/// GET a span's object and require its table to hold exactly the advertised groups. +async fn load(store: &Store, track: &str, span: &Span) -> Result> { + let object = store.get_groups(track, span.bounds.clone()).await?; + check_runs(&object, &span.runs)?; + Ok(Arc::new(object)) +} + +/// Refuse a table whose sequences differ from the advertised runs. +fn check_runs(object: &Object, runs: &[RangeInclusive]) -> Result<()> { + let mut groups = object.groups.iter(); + for run in runs { + for sequence in run.clone() { + if groups.next().map(|group| group.sequence) != Some(sequence) { + return Err(Error::Table); + } + } + } + match groups.next() { + Some(_) => Err(Error::Table), + None => Ok(()), + } +} + +fn track_info(info: &crate::Info) -> Result { + let timescale = Timescale::new(info.timescale).map_err(|_| Error::Timescale(info.timescale))?; + Ok(track::Info::default() + .with_timescale(timescale) + .with_priority(info.priority)) +} + +fn timestamp(value: u64, timescale: Timescale) -> Result { + Timestamp::new(value, timescale).map_err(|_| Error::Id(value)) +} + +/// Adapt a boxed future into a [`kio::Tasks`] task. +fn task(mut future: BoxFuture<'static, ()>) -> impl FnMut(&kio::Waiter) -> Poll<()> + Send { + move |waiter| waiter.poll_future(future.as_mut()) +} + +/// Weighs a cached object by its payload bytes plus a small per-frame overhead. +#[derive(Clone)] +struct Weight; + +impl quick_cache::Weighter> for Weight { + fn weight(&self, _key: &Key, object: &Arc) -> u64 { + let frames = object.groups.iter().flat_map(|group| &group.frames); + frames.map(|frame| frame.payload.len() as u64 + 32).sum::() + 64 + } +} + +#[cfg(test)] +mod tests; diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs new file mode 100644 index 0000000000..ca817bf72a --- /dev/null +++ b/rs/moq-archive/src/reader/tests.rs @@ -0,0 +1,467 @@ +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use futures::stream::BoxStream; +use hang::timeline::{Range, Record}; +use moq_json::window; +use moq_net::{broadcast, group}; +use object_store::memory::InMemory; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, ObjectStoreExt, + PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; + +use super::*; +use crate::segment::{Frame, Group}; +use crate::{ID_MAX, Info}; + +const TIMELINE: &str = "timeline.z"; + +/// In-memory store that records every GET path. +#[derive(Debug, Clone, Default)] +struct Counting { + inner: Arc, + gets: Arc>>, +} + +impl Counting { + fn gets(&self) -> Vec { + self.gets.lock().unwrap().clone() + } +} + +impl std::fmt::Display for Counting { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Counting") + } +} + +#[async_trait::async_trait] +impl ObjectStore for Counting { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.gets.lock().unwrap().push(location.to_string()); + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// Writes archive objects the way a recording writer lays them out. +struct Archive { + store: Store, + encoder: window::Encoder, + /// Next timeline group sequence. + sequence: u64, +} + +impl Archive { + async fn new() -> Self { + Self::with_op_ratio(0).await + } + + /// `op_ratio` 0 makes every window edit its own checkpoint group. + async fn with_op_ratio(op_ratio: u32) -> Self { + let store = Store::new(Counting::default(), "rec"); + store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(op_ratio); + Self { + store, + encoder: window::Encoder::new(config), + sequence: 0, + } + } + + /// Store one track object holding `groups` of `(sequence, frame count)`. + async fn media(&self, track: &str, groups: &[(u64, usize)]) { + self.store.put_info(track, &Info::new(1, 1000).unwrap()).await.unwrap(); + let groups = groups + .iter() + .map(|&(sequence, count)| Group { + sequence, + frames: (0..count).map(|i| frame(track, sequence, i)).collect(), + }) + .collect(); + self.store.put_groups(track, &Object { groups }).await.unwrap(); + } + + /// Push `record`, pop `pop` records, and return the resulting timeline groups. + fn edit(&mut self, record: &Record, pop: u64) -> Vec { + let mut frames = Vec::new(); + let pending = self.encoder.push(record).unwrap(); + frames.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = self.encoder.pop(pop).unwrap() { + frames.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + + let mut groups: Vec = Vec::new(); + for (keyframe, payload) in frames { + if keyframe { + self.sequence += 1; + } + if groups.last().is_none_or(|group| group.sequence != self.sequence - 1) { + groups.push(Group { + sequence: self.sequence - 1, + frames: Vec::new(), + }); + } + let timestamp = record.pts; + groups.last_mut().unwrap().frames.push(Frame { timestamp, payload }); + } + groups + } + + async fn timeline(&self, segment: u64, groups: Vec) { + self.store + .put_segments(TIMELINE, segment, &Object { groups }) + .await + .unwrap(); + } + + /// Commit `record` as timeline segment `record.segment`, popping `pop` older records. + async fn commit(&mut self, record: &Record, pop: u64) { + let groups = self.edit(record, pop); + self.timeline(record.segment, groups).await; + } + + async fn raw(&self, key: &Key, bytes: &'static [u8]) { + let path = self.store.path(key).unwrap(); + self.store + .inner() + .put(&path, Bytes::from_static(bytes).into()) + .await + .unwrap(); + } +} + +fn frame(track: &str, sequence: u64, index: usize) -> Frame { + Frame { + timestamp: sequence * 100 + index as u64, + payload: Bytes::from(format!("{track}/{sequence}/{index}")), + } +} + +fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { + let mut record = Record::new(segment, segment * 2000, 2000); + for (name, ranges) in tracks { + let ranges = ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(); + record.tracks.insert(name.to_string(), ranges); + } + record +} + +async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(archive.store.clone(), &broadcast, Config::new(TIMELINE)) + .await + .unwrap(); + tokio::spawn(reader.serve()); + (broadcast, reader) +} + +/// FETCH one group starting at `frame_start`, returning each frame's timestamp and payload. +async fn fetch( + broadcast: &broadcast::Producer, + track: &str, + sequence: u64, + frame_start: u64, +) -> std::result::Result, moq_net::Error> { + let track = broadcast.consume().track(track)?; + let options = group::Fetch::default().with_frame_start(frame_start); + let mut group = track.fetch_group(sequence, options).await?; + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await? { + frames.push((frame.timestamp.value(), frame.payload)); + } + Ok(frames) +} + +fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<(u64, Bytes)> { + frames + .map(|i| { + let frame = frame(track, sequence, i); + (frame.timestamp, frame.payload) + }) + .collect() +} + +#[tokio::test] +async fn fetch_replays_original_groups() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 3), (1, 2)]).await; + archive.media("audio", &[(0, 1), (1, 1), (5, 2)]).await; + archive.media("video", &[(2, 2)]).await; + archive + .commit(&record(0, &[("video", &[(0, 1)]), ("audio", &[(0, 1), (5, 5)])]), 0) + .await; + archive.commit(&record(1, &[("video", &[(2, 2)])]), 0).await; + + let (broadcast, _reader) = open(&archive).await; + + assert_eq!( + fetch(&broadcast, "video", 0, 0).await.unwrap(), + expected("video", 0, 0..3) + ); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..2) + ); + assert_eq!( + fetch(&broadcast, "audio", 5, 0).await.unwrap(), + expected("audio", 5, 0..2) + ); + + // A requested frame_start keeps the original frame indices and timestamps. + let tail = fetch(&broadcast, "video", 1, 1).await.unwrap(); + assert_eq!(tail, expected("video", 1, 1..2)); + + let track = broadcast.consume().track("video").unwrap(); + let info = track.query().await.unwrap(); + assert_eq!(info.priority, 1); + assert_eq!(info.timescale.as_u64(), 1000); +} + +#[tokio::test] +async fn requests_download_only_their_object() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.media("audio", &[(0, 1), (1, 1), (2, 1)]).await; + archive + .commit(&record(0, &[("video", &[(0, 0)]), ("audio", &[(0, 2)])]), 0) + .await; + + let (broadcast, _reader) = open(&archive).await; + let before = archive.store.inner().gets().len(); + + for sequence in 0..3 { + assert_eq!( + fetch(&broadcast, "audio", sequence, 0).await.unwrap(), + expected("audio", sequence, 0..1) + ); + } + + let gets = archive.store.inner().gets()[before..].to_vec(); + assert_eq!( + gets, + vec![ + "rec/audio/.info".to_string(), + "rec/audio/groups/0000000000000000002.0000000000000000000".to_string(), + ], + "adjacent groups reuse one GET and never touch video" + ); +} + +#[tokio::test] +async fn unadvertised_or_unreadable_groups_are_not_found() { + let mut archive = Archive::new().await; + archive.media("audio", &[(0, 1), (1, 1), (4, 1)]).await; + // The object holds group 3, which its record does not advertise. + archive.media("video", &[(0, 1), (3, 1)]).await; + archive.raw(&Key::groups("bad", 0..=0).unwrap(), b"\x02garbage").await; + archive + .store + .put_info("bad", &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + archive + .store + .put_info("missing", &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + archive + .commit( + &record( + 0, + &[ + ("audio", &[(0, 1), (4, 4)]), + ("video", &[(0, 0), (2, 3)]), + ("bad", &[(0, 0)]), + ("missing", &[(0, 0)]), + ], + ), + 0, + ) + .await; + + let (broadcast, _reader) = open(&archive).await; + + let not_found = |result: std::result::Result, moq_net::Error>| { + assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); + }; + not_found(fetch(&broadcast, "audio", 2, 0).await); // internal gap + not_found(fetch(&broadcast, "audio", 5, 0).await); // past the last range + not_found(fetch(&broadcast, "video", 0, 0).await); // table disagrees with the record + not_found(fetch(&broadcast, "bad", 0, 0).await); // unknown envelope version + not_found(fetch(&broadcast, "missing", 0, 0).await); // no object + not_found(fetch(&broadcast, "chat", 0, 0).await); // track the timeline never named + + // Siblings remain usable. + assert_eq!( + fetch(&broadcast, "audio", 4, 0).await.unwrap(), + expected("audio", 4, 0..1) + ); +} + +#[tokio::test] +async fn refresh_follows_new_segments_and_pops() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1), (1, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 1)])]), 0).await; + + let (broadcast, mut reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 0, 0).await.is_ok()); + + archive.media("video", &[(2, 1)]).await; + // Refreshing before the commit finds nothing new; Not Found is not finality. + reader.refresh().await.unwrap(); + assert!(matches!( + fetch(&broadcast, "video", 2, 0).await, + Err(moq_net::Error::NotFound) + )); + + archive.commit(&record(1, &[("video", &[(2, 2)])]), 1).await; + reader.refresh().await.unwrap(); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..1) + ); + + // The popped record's groups are gone, although its object is still stored and was cached. + let gets = archive.store.inner().gets().len(); + assert!(matches!( + fetch(&broadcast, "video", 1, 0).await, + Err(moq_net::Error::NotFound) + )); + assert_eq!(archive.store.inner().gets().len(), gets); +} + +#[tokio::test] +async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.media("video", &[(1, 1)]).await; + archive.media("video", &[(2, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + // Segment 1 is committed to the window but its timeline object is unreadable. + archive.edit(&record(1, &[("video", &[(1, 1)])]), 0); + archive.raw(&Key::segments(TIMELINE, 1).unwrap(), b"\x01").await; + archive.commit(&record(2, &[("video", &[(2, 2)])]), 0).await; + + let (broadcast, _reader) = open(&archive).await; + + // Segment 2's checkpoint restates record 1, so its group is still served. + for sequence in 0..3 { + assert_eq!( + fetch(&broadcast, "video", sequence, 0).await.unwrap(), + expected("video", sequence, 0..1) + ); + } +} + +#[tokio::test] +async fn multi_frame_timeline_groups_decode() { + let mut archive = Archive::with_op_ratio(64).await; + archive.media("video", &[(0, 1)]).await; + archive.media("video", &[(1, 1)]).await; + let mut groups = archive.edit(&record(0, &[("video", &[(0, 0)])]), 0); + let ops = archive.edit(&record(1, &[("video", &[(1, 1)])]), 0); + assert!( + ops[0].sequence == groups[0].sequence, + "the push stays in the open group" + ); + groups[0].frames.extend(ops.into_iter().flat_map(|group| group.frames)); + archive.timeline(0, groups).await; + + let (broadcast, _reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); +} + +#[tokio::test] +async fn timeline_track_is_republished_and_finished_on_request() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + archive.commit(&record(1, &[]), 0).await; + + let (broadcast, reader) = open(&archive).await; + let track = broadcast.consume().track(TIMELINE).unwrap(); + let subscriber = track.subscribe(None).await.unwrap(); + let mut timeline = + window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); + + reader.finish().unwrap(); + + let mut segments = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let window::Event::Push { value, .. } = event { + segments.push(value.segment); + } + } + assert_eq!(segments, vec![0, 1]); +} + +#[tokio::test] +async fn open_requires_the_timeline_info() { + let store = Store::new(Counting::default(), "rec"); + let broadcast = broadcast::Info::new().produce(); + let result = Reader::open(store, &broadcast, Config::new(TIMELINE)).await; + assert!(matches!(result, Err(Error::NotFound(_)))); +} + +#[test] +fn check_runs_requires_the_exact_sequences() { + let object = Object { + groups: [0, 1, 4, ID_MAX] + .into_iter() + .map(|sequence| Group { + sequence, + frames: Vec::new(), + }) + .collect(), + }; + check_runs(&object, &[0..=1, 4..=4, ID_MAX..=ID_MAX]).unwrap(); + assert!(check_runs(&object, &[0..=1, 4..=4]).is_err(), "extra group"); + assert!( + check_runs(&object, &[0..=4, ID_MAX..=ID_MAX]).is_err(), + "missing groups" + ); + assert!(check_runs(&object, &[0..=1, 3..=4, ID_MAX..=ID_MAX]).is_err()); +} From a03664b57a235869c71e7c7e1b05d0131f6b5322 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 00:15:01 -0700 Subject: [PATCH 04/49] fix(archive): fail the writer on malformed source input (#4088) Co-authored-by: Claude Opus 5.5 --- rs/moq-archive/src/writer.rs | 104 +++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 18 deletions(-) diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 13ac32294a..41efeb8cff 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -201,8 +201,9 @@ impl Writer { /// Record until the source broadcast closes and every enrolled track ends. /// /// Then flush the final segment and finish the timeline. Fails when the timeline cannot be - /// committed or stored: the recording stops at its last durable timeline object. Returns the - /// source's error, after finishing, when the broadcast aborted. + /// committed or stored, or an enrolled track delivers a group the recording cannot represent: + /// the recording stops at its last durable timeline object. Returns the source's error, after + /// finishing, when the broadcast aborted. pub async fn run(self) -> Result<()> { let Self { control, @@ -262,6 +263,7 @@ impl Writer { recorder, timescale, largest: None, + reported: None, accepted: BTreeMap::new(), subscribed: true, _cancel: cancel, @@ -274,7 +276,7 @@ impl Writer { Some(Command::Poke) => {} }, Some(read) = reads.next(), if !reads.is_empty() => { - handle(read, &mut tracks, &mut buffered, &mut reads); + handle(read, &mut tracks, &mut buffered, &mut reads)?; } (done, result) = async { commit.as_mut().unwrap().await }, if commit.is_some() => { commit = None; @@ -393,6 +395,8 @@ struct TrackState { timescale: Timescale, /// The newest group accepted; later arrivals must exceed it. largest: Option, + /// The first-frame timestamp of the newest reported group; later groups must not precede it. + reported: Option, /// Accepted groups in sequence order: `None` while reading, `Some` once complete. Reported in /// order, so a complete group waits for every earlier accepted one. accepted: BTreeMap>, @@ -405,7 +409,9 @@ struct TrackState { impl TrackState { /// Report every complete group at the front of the accepted queue. - fn report(&mut self, name: &str, buffered: &mut HashMap>) { + /// + /// Fails on a group that starts before the previous one, which the timeline cannot place. + fn report(&mut self, name: &str, buffered: &mut HashMap>) -> Result<()> { while let Some(entry) = self.accepted.first_entry() { if entry.get().is_none() { break; @@ -414,6 +420,16 @@ impl TrackState { let (Some(first), Some(last)) = (group.frames.first(), group.frames.last()) else { continue; }; + if let Some(reported) = self.reported + && first.timestamp < reported + { + return Err(malformed( + name, + group.sequence, + format!("timestamp {} precedes {reported}", first.timestamp), + )); + } + self.reported = Some(first.timestamp); // Frame timestamps were validated while reading, so these conversions succeed. let (Ok(first), Ok(last)) = ( Timestamp::new(first.timestamp, self.timescale), @@ -428,6 +444,7 @@ impl TrackState { .or_default() .insert(group.sequence, group); } + Ok(()) } } @@ -477,21 +494,22 @@ async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future, buffered: &mut HashMap>, reads: &mut FuturesUnordered>, -) { +) -> Result<()> { let name = match read { - Read::Cancelled => return, + Read::Cancelled => return Ok(()), Read::Group { name, subscriber, result, } => { let Some(track) = tracks.get_mut(&name) else { - return; + return Ok(()); }; match result { Ok(Some(group)) => { @@ -515,24 +533,24 @@ fn handle( } Read::Frames { name, sequence, result } => { let Some(track) = tracks.get_mut(&name) else { - return; + return Ok(()); }; - let group = result - .map_err(|err| err.to_string()) - .and_then(|frames| convert(sequence, frames, track.timescale).map_err(|err| err.to_string())); - match group { - Ok(group) if !group.frames.is_empty() => { - track.accepted.insert(sequence, Some(group)); - } - Ok(_) => { - track.accepted.remove(&sequence); + match result { + Ok(frames) => { + let group = + convert(sequence, frames, track.timescale).map_err(|err| malformed(&name, sequence, err))?; + if group.frames.is_empty() { + track.accepted.remove(&sequence); + } else { + track.accepted.insert(sequence, Some(group)); + } } Err(err) => { tracing::warn!(track = %name, sequence, %err, "dropping an incomplete group"); track.accepted.remove(&sequence); } } - track.report(&name, buffered); + track.report(&name, buffered)?; name } }; @@ -544,6 +562,7 @@ fn handle( { tracks.remove(&name); } + Ok(()) } /// Convert a complete group's frames into the track's timescale. @@ -737,6 +756,10 @@ fn source_error(err: moq_net::Error) -> Error { Error::Source(err.to_string()) } +fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { + Error::Source(format!("track {track} group {sequence}: {err}")) +} + #[cfg(test)] mod tests { @@ -1074,6 +1097,51 @@ mod tests { check_objects(&store, &records).await; } + #[tokio::test] + async fn an_unrepresentable_group_fails_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + group(&video, 0, &[0]); + // One past the recording's largest group ID. + group(&video, 1 << 53, &[1000]); + video.finish().unwrap(); + source.finish(); + + assert_eq!( + writer.run().await, + Err(Error::Source(format!( + "track video group {}: {}", + 1u64 << 53, + Error::Id(1 << 53) + ))) + ); + } + + #[tokio::test] + async fn a_decreasing_timestamp_fails_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + group(&video, 0, &[1000]); + group(&video, 1, &[500]); + video.finish().unwrap(); + source.finish(); + + assert_eq!( + writer.run().await, + Err(Error::Source("track video group 1: timestamp 500 precedes 1000".into())) + ); + } + #[tokio::test] async fn removing_a_stalled_pacing_track_releases_the_recording() { let source = broadcast::Info::new().produce(); From eb55c52cd5a55b88fd493634e528b35bf4402b26 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 06:54:01 -0700 Subject: [PATCH 05/49] feat(cli): record and replay archives with `export archive` / `import archive` (#4113) Co-authored-by: Claude Opus 5.5 Co-authored-by: Grok 4.7 --- Cargo.lock | 40 +++ Cargo.toml | 1 + doc/bin/cli.md | 35 ++- quest/m1/archive/README.md | 1 - quest/m1/archive/cli.md | 43 ---- rs/moq-archive/src/writer.rs | 15 +- rs/moq-cli/Cargo.toml | 17 +- rs/moq-cli/README.md | 2 +- rs/moq-cli/src/archive.rs | 474 +++++++++++++++++++++++++++++++++++ rs/moq-cli/src/args.rs | 6 +- rs/moq-cli/src/main.rs | 18 ++ rs/moq-cli/src/subscribe.rs | 8 + 12 files changed, 607 insertions(+), 53 deletions(-) delete mode 100644 quest/m1/archive/cli.md create mode 100644 rs/moq-cli/src/archive.rs diff --git a/Cargo.lock b/Cargo.lock index 67e4ea27a0..29f7c2fd84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,6 +1471,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -4050,6 +4060,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "mdns-sd" version = "0.21.4" @@ -4298,9 +4318,11 @@ dependencies = [ "bytes", "hang", "humantime", + "moq-archive", "moq-audio", "moq-auth", "moq-hls", + "moq-json", "moq-msf", "moq-mux", "moq-relay", @@ -4310,6 +4332,7 @@ dependencies = [ "moq-tokio", "moq-transcode", "moq-video", + "object_store", "pollster", "reqwest", "rustls", @@ -6175,17 +6198,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", + "aws-lc-rs", + "base64 0.23.1", "bytes", "chrono", + "crc-fast", + "form_urlencoded", "futures-channel", "futures-core", "futures-util", "http", + "http-body-util", + "httparse", "humantime", + "hyper", "itertools 0.15.0", + "md-5", "nix", "parking_lot", "percent-encoding", + "quick-xml 0.41.0", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", "thiserror 2.0.21", "tokio", "tracing", @@ -6876,6 +6914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", + "serde", ] [[package]] @@ -7219,6 +7258,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index 3e25cae5ee..0a72c53233 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,6 +139,7 @@ loom = { version = "0.7.2", features = ["futures"] } # DNS-SD advertisement and browsing for LAN peer discovery (moq-tokio's `mdns` feature). # `async` awaits the event channel instead of blocking a thread on it. mdns-sd = { version = "0.21", features = ["async"] } +moq-archive = { version = "0.0.3", path = "rs/moq-archive" } moq-audio = { version = "0.1.2", path = "rs/moq-audio", default-features = false } moq-auth = { version = "0.1.1", path = "rs/moq-auth" } moq-binary = { version = "0.1.2", path = "rs/moq-binary" } diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 1376686f9d..7c8ecde6ca 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -18,9 +18,11 @@ or Docker; see [Install](/setup/install). | `import` | `capture` | Capture a camera, display, window, or app plus a microphone, and encode natively. | | `import` | `hls ` | Pull a remote HLS playlist. | | `import` | `rtmp`, `srt`, `rtc` | Accept pushes (`--listen`) or pull from a remote (`--connect`). | +| `import` | `archive ` | Replay a recording from an object store. | | `export` | `fmp4`, `mkv`, `ts`, `flv`, `h264`, `h265` | Write a container to stdout. | | `export` | `hls --listen` | Serve the broadcast as HLS over HTTP. | | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | +| `export` | `archive ` | Record the broadcast into an object store. | | `play` | | Decode and play in a native window with sound. | | `transcode` | | Publish a just-in-time rendition ladder next to a broadcast. | | `fetch` | `` | Write one group of a track to stdout. | @@ -178,6 +180,36 @@ refuses a listener or cluster flag. It gives up after 30 seconds, as `/fetch` does, and exits non-zero when the broadcast or group is not found, the relay refuses, or the deadline passes. +## Archive + +```bash +# Record a broadcast until it ends +moq --connect https://relay.example.com/anon --broadcast event.hang export archive s3://recordings/event + +# Replay it under another name +moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event +``` + +`export archive` records one broadcast with +[moq-archive](https://docs.rs/moq-archive), reading its catalog as it changes: +video and audio renditions pace the segments, and the catalog plus every text, +JSON, and binary track are recorded alongside. It refuses a rendition served +from another broadcast, and one that returns after the catalog dropped it. The +stage ends once the broadcast does, and it refuses a store URL that already +holds a recording. `--retention 1h` keeps only the last hour (a DVR), +deleting expired objects `--retention-grace` (default 30s) after the timeline +stops advertising them. + +`import archive` republishes a recording: the timeline replays as a live track +and every other track's groups are served on request, one object GET per group +range. By default it replays what is stored and ends the timeline there; +`--follow 2s` keeps checking for new segments of a recording still being made. + +Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, +`gs://bucket/prefix`, or `az://container/prefix`. Cloud credentials come from +the usual `AWS_*`, `GOOGLE_*`, and `AZURE_*` environment variables. The `s3`, +`gcs`, and `azure` cargo features are on by default. + ## Multiple stages Separate stages with `--` to bridge several broadcasts, or both directions, @@ -186,7 +218,8 @@ over one connection: ```bash moq --connect https://relay.example.com/anon \ import --broadcast event.hang srt --listen 0.0.0.0:9000 \ - -- export --broadcast event.hang hls --listen 0.0.0.0:8080 + -- export --broadcast event.hang hls --listen 0.0.0.0:8080 \ + -- export --broadcast event.hang archive file:///recordings/event ``` ## Redundant publishers diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 80e59ac56c..e4cdd70249 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -111,7 +111,6 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests -- [Archive endpoint](/quest/m1/archive/cli.md) - `moq ... export archive` records and `import archive` replays - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [Offline archive HLS](/quest/m1/archive/hls.md) - render playlists from the archive timeline and fetch segment media lazily - [Resume a recording](/quest/m1/archive/recovery.md) - recover the retained timeline on restart and clean up DVR orphans diff --git a/quest/m1/archive/cli.md b/quest/m1/archive/cli.md deleted file mode 100644 index cfa76925bf..0000000000 --- a/quest/m1/archive/cli.md +++ /dev/null @@ -1,43 +0,0 @@ -# [M] Archive endpoint in moq-cli - -## Goal - -`moq-cli` records and replays archives with no custom code, through an -`archive ` endpoint on the existing direction verbs: -`export archive` records a broadcast with `moq_archive::Writer`, and -`import archive` republishes a recording, serving its groups on demand -through `moq_archive::Reader`. Recording an ingest in one process is a -multi-stage line. - -```bash -moq --connect $RELAY --broadcast ev.hang export archive s3://rec/ev -moq --connect $RELAY \ - import --broadcast ev.hang srt --listen 0.0.0.0:9000 \ - -- export --broadcast ev.hang archive file:///rec/ev -moq --connect $RELAY --broadcast ev-replay.hang import archive s3://rec/ev -``` - -## Plan - -Parse the store URL with `object_store::parse_url`. Compile in every -`object_store` cloud backend (S3, GCS, Azure) behind features, with local -files always available. - -An archive stage carries exactly one broadcast; refuse a multi-broadcast -stage (an RTMP or SRT listener accepting many) rather than invent a prefix -scheme. - -Export reads its own catalog: video and audio renditions enroll as pacing -tracks, the catalog and any other track as non-pacing, and renditions added -later enroll as they appear. The writer stays catalog-agnostic. Expose the -writer's retention window and deletion grace as flags. - -Import hands the reader the stage's `broadcast::Producer`. The recording is -not live; decide with the reader whether import finishes the replayed -timeline or keeps following a growing archive. - -Update `doc/bin/cli.md` and every example invocation. - -## Closes - -- [#2281](https://github.com/moq-dev/moq/issues/2281) - close this issue when the quest finishes diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 41efeb8cff..766794a75e 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -31,7 +31,7 @@ use futures::future::BoxFuture; use futures::stream::FuturesUnordered; use futures::{FutureExt, StreamExt}; use hang::timeline::Record; -use moq_mux::timeline::{self, Deferred, DeferredDrain, Pending, Recorder}; +use moq_mux::timeline::{self, Deferred, DeferredDrain, Pending, Recorder, Reserved}; use moq_net::{Timescale, Timestamp, broadcast, group, track}; use object_store::ObjectStore; use tokio::sync::{mpsc, watch}; @@ -116,6 +116,11 @@ impl Clone for Control { } } +/// Withholds segment commits until dropped. +pub struct Reservation { + _inner: Reserved, +} + struct Shared { store: Store, source: broadcast::Consumer, @@ -342,6 +347,14 @@ impl Control { self.send(Command::Poke) } + /// Hold segment commits back until this guard drops, so a batch of tracks can enroll first. + #[must_use = "dropping the reservation releases segment commits"] + pub fn reserve(&self) -> Reservation { + Reservation { + _inner: self.deferred.reserve(), + } + } + /// Stop recording `name`, dropping its incomplete groups. The name cannot be enrolled again. pub fn remove(&self, name: &str) -> Result<()> { self.send(Command::Remove(name.to_string())) diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index de9b25aa66..7ee2a123d2 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,7 +26,7 @@ path = "src/main.rs" doc = false [features] -default = ["iroh", "cluster-lan", "noq", "websocket", "nvidia"] +default = ["iroh", "cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). cluster-lan = ["moq-relay/cluster-lan"] @@ -37,6 +37,11 @@ noq = ["moq-tokio/noq"] # congestion-control testing. qlog = ["moq-tokio/qlog"] websocket = ["moq-tokio/websocket"] +# The cloud object stores `export archive` and `import archive` reach (`s3://`, +# `gs://`, `az://`). Local `file://` URLs need none of them. +s3 = ["object_store/aws"] +gcs = ["object_store/gcp"] +azure = ["object_store/azure"] # Device capture (camera + microphone) + encode/publish. Off by default because # it pulls in moq-video + moq-audio capture: on Linux the microphone needs # ALSA/libasound at build time through cpal (the camera needs nothing, moq-v4l @@ -93,10 +98,13 @@ base64 = { workspace = true } bytes = { workspace = true } hang = { workspace = true } humantime = { workspace = true } +moq-archive = { workspace = true } moq-audio = { workspace = true, optional = true, features = ["aac"] } moq-auth = { workspace = true, features = ["serve"] } # `server` enables the HTTP egress server for `moq export hls`; the importer is always available. moq-hls = { workspace = true, features = ["server"] } +# Names the MSF catalog track `export archive` records. +moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the # CLI's own feature set (iroh, noq, cluster-lan) is what selects them. @@ -108,6 +116,7 @@ moq-srt = { workspace = true } moq-tokio = { workspace = true, default-features = false, features = ["aws-lc-rs", "tcp", "uds"] } moq-transcode = { workspace = true, optional = true, features = ["openh264"] } moq-video = { workspace = true, optional = true, features = ["openh264"] } +object_store = { workspace = true, features = ["fs"] } pollster = { workspace = true, optional = true } reqwest = { workspace = true, features = ["rustls", "json"] } rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } @@ -124,10 +133,8 @@ winit = { version = "0.30.13", optional = true } sd-notify = { workspace = true } [dev-dependencies] -# Authors an MSF-only catalog, which is the one shape that tells a completer reading -# the wrong catalog track apart from one reading the right one: `moq-mux`'s producer -# publishes hang and MSF from the same source, so both answer for an ordinary broadcast. -moq-msf = { path = "../moq-msf" } +# Decodes the timeline the archive round-trip test checks. +moq-json = { workspace = true } # `test_relay` stands up a real relay for the `fetch` tests to read through. moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } tempfile = { workspace = true } diff --git a/rs/moq-cli/README.md b/rs/moq-cli/README.md index de0241a778..95fd4ef6a7 100644 --- a/rs/moq-cli/README.md +++ b/rs/moq-cli/README.md @@ -19,7 +19,7 @@ Multi-arch images (`linux/amd64` and `linux/arm64`) are published to [Docker Hub ## Usage -`moq-cli` routes endpoints onto a shared MoQ Origin: `moq `. The MoQ side (before the verb) dials with `--connect `, self-hosts QUIC/WebTransport with `--listen `, or self-hosts raw qmux with `--listen-tcp-bind ` / `--listen-unix-bind ` (Unix only). `import` moves media into MoQ, `export` moves it out. The endpoint is a container format (`fmp4`, `ts`, `flv`, ... read from stdin / written to stdout), or a gateway (`hls`, `rtmp`, `srt`, `rtc`). A build with the `play` feature can also render a broadcast locally with `moq play`. +`moq-cli` routes endpoints onto a shared MoQ Origin: `moq `. The MoQ side (before the verb) dials with `--connect `, self-hosts QUIC/WebTransport with `--listen `, or self-hosts raw qmux with `--listen-tcp-bind ` / `--listen-unix-bind ` (Unix only). `import` moves media into MoQ, `export` moves it out. The endpoint is a container format (`fmp4`, `ts`, `flv`, ... read from stdin / written to stdout), or a gateway (`hls`, `rtmp`, `srt`, `rtc`, `archive`). A build with the `play` feature can also render a broadcast locally with `moq play`. Separate additional stages with `--` to bridge several broadcasts (or both directions) over one connection, each naming its own `--broadcast`: diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs new file mode 100644 index 0000000000..2689096655 --- /dev/null +++ b/rs/moq-cli/src/archive.rs @@ -0,0 +1,474 @@ +//! Archive endpoints: record one broadcast into an object store (export), or +//! republish a recording from one (import), through `moq-archive`. + +use std::collections::HashSet; +use std::time::Duration; + +use anyhow::Context; +use hang::moq_net; +use moq_archive::writer::{Control, Retention}; +use moq_mux::catalog::{CatalogFormat, Stream as _}; +use object_store::ObjectStore; +use url::Url; + +use crate::moq::notify_ready; + +/// How long expired objects outlive the timeline that dropped them, unless +/// `--retention-grace` says otherwise. +const GRACE: Duration = Duration::from_secs(30); + +/// `export archive` args. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct ExportArgs { + /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. + pub store: Url, + + /// Keep only this much recent content (a DVR), deleting older segments. Unset keeps everything. + #[usage(long)] + pub retention: Option, + + /// How long expired objects outlive the timeline that dropped them, so readers can finish + /// their downloads. Defaults to 30s; needs `--retention`. + #[usage(long)] + pub retention_grace: Option, +} + +/// `import archive` args. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct ImportArgs { + /// The recording to replay, as `export archive` wrote it. + pub store: Url, + + /// Keep following a recording that is still growing, checking for new segments at this + /// interval. Unset replays what is stored now and ends the timeline there. + #[usage(long)] + pub follow: Option, +} + +/// Record the broadcast `name` into `args.store` until the broadcast ends. +/// +/// Reads the broadcast's own catalog: video and audio renditions pace the segments, while the +/// catalog and every other track are recorded alongside without pacing. +pub async fn export( + origin: moq_net::origin::Consumer, + name: String, + format: CatalogFormat, + args: ExportArgs, +) -> anyhow::Result<()> { + let retention = match (args.retention, args.retention_grace) { + (Some(window), grace) => Some(Retention::new( + window.into_std(), + grace.map_or(GRACE, crate::duration::Duration::into_std), + )), + (None, Some(_)) => anyhow::bail!("`--retention-grace` needs `--retention`"), + (None, None) => None, + }; + let catalog_track = catalog_track(format)?; + let store = open(&args.store)?; + + let broadcast = origin + .routed_broadcast(&name) + .await + .with_context(|| format!("broadcast `{name}` is unavailable"))?; + let config = moq_archive::writer::Config::default().with_retention(retention); + let writer = moq_archive::Writer::new(store, broadcast.clone(), config) + .await + .with_context(|| format!("failed to start recording into {}", args.store))?; + let control = writer.control(); + control.track(catalog_track).await?; + let catalog = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; + + tracing::info!(%name, store = %args.store, "recording"); + notify_ready(); + + // The writer decides when the recording is over; the catalog only feeds it tracks. + tokio::select! { + result = writer.run() => result.context("recording failed"), + Err(err) = enroll(control, catalog) => Err(err), + } +} + +/// Republish the recording at `args.store` as the broadcast `name`. +/// +/// The timeline replays as a live track and every other track's groups are served on request. +pub async fn import(origin: moq_net::origin::Producer, name: String, args: ImportArgs) -> anyhow::Result<()> { + let store = open(&args.store)?; + let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; + let config = moq_archive::reader::Config::new(hang::timeline::DEFAULT_NAME); + let mut reader = moq_archive::Reader::open(store, &broadcast, config) + .await + .with_context(|| format!("no readable recording at {}", args.store))?; + let serve = reader.serve(); + broadcast + .announce(Default::default()) + .context("failed to announce broadcast")?; + + tracing::info!(%name, store = %args.store, "replaying"); + notify_ready(); + + // The store holds no end marker, so only the caller can say the recording is complete. + let Some(interval) = args.follow.map(crate::duration::Duration::into_std) else { + reader.finish()?; + serve.await; + return Ok(()); + }; + let refresh = async { + loop { + tokio::time::sleep(interval).await; + reader.refresh().await?; + } + }; + tokio::select! { + () = serve => Ok(()), + result = refresh => result, + } +} + +/// Open the object store `url` names, taking cloud credentials from the `AWS_*`, `GOOGLE_*`, +/// and `AZURE_*` environment variables. +fn open(url: &Url) -> anyhow::Result>> { + let credentials = std::env::vars_os() + .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) + .filter(|(key, _)| ["AWS_", "GOOGLE_", "AZURE_"].iter().any(|cloud| key.starts_with(cloud))); + let (store, prefix) = + object_store::parse_url_opts(url, credentials).with_context(|| format!("unsupported archive store {url}"))?; + Ok(moq_archive::Store::new(store, prefix)) +} + +/// The track carrying the catalog `format` reads. +fn catalog_track(format: CatalogFormat) -> anyhow::Result<&'static str> { + Ok(match format { + CatalogFormat::Hang => hang::Catalog::DEFAULT_NAME, + CatalogFormat::HangZ => hang::Catalog::COMPRESSED_NAME, + CatalogFormat::Msf => moq_msf::DEFAULT_NAME, + other => anyhow::bail!("`export archive` cannot record a {other:?} catalog"), + }) +} + +/// Enroll each track as the catalog lists it, until the catalog ends. +async fn enroll( + control: Control, + mut catalog: moq_mux::catalog::Consumer<()>, +) -> anyhow::Result<()> { + let mut tracks = Tracks::default(); + while let Some(snapshot) = catalog.next().await? { + // One rendition's subscription can outrun the rest of this snapshot. Without the + // hold, the writer closes a segment from that rendition alone and the others never + // enter the record. + let _hold = control.reserve(); + for change in tracks.update(&snapshot)? { + match change { + Change::Pacing(name) => control.pacing_track(&name).await?, + Change::Track(name) => control.track(&name).await?, + Change::Remove(name) => control.remove(&name)?, + } + } + } + Ok(()) +} + +/// A change to the recorded track set. +#[derive(Debug, PartialEq, Eq)] +enum Change { + /// Record a rendition that paces the segments. + Pacing(String), + /// Record a track without letting it pace the segments. + Track(String), + /// Stop recording a track the catalog dropped. + Remove(String), +} + +/// The catalog tracks being recorded, diffed against each new snapshot. +#[derive(Default)] +struct Tracks { + /// Every name ever enrolled. The writer never takes a name back. + enrolled: HashSet, + /// The names the latest snapshot listed. + listed: HashSet, +} + +impl Tracks { + /// The changes that bring the recording in line with `catalog`. + /// + /// Refuses a track served from another broadcast, since an archive holds exactly one, and a + /// track that returns after the catalog dropped it, since the writer cannot resume one. + fn update(&mut self, catalog: &hang::Catalog) -> anyhow::Result> { + let video = catalog + .video + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, true)); + let audio = catalog + .audio + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, true)); + let text = catalog + .text + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, false)); + let json = catalog.json.tracks.iter().map(|(name, c)| (name, &c.broadcast, false)); + let binary = catalog + .binary + .tracks + .iter() + .map(|(name, c)| (name, &c.broadcast, false)); + + let mut listed = HashSet::new(); + let mut changes = Vec::new(); + for (name, broadcast, pacing) in video.chain(audio).chain(text).chain(json).chain(binary) { + if let Some(broadcast) = broadcast { + anyhow::bail!( + "track `{name}` is served from broadcast `{broadcast}`; an archive records one broadcast" + ); + } + if !listed.insert(name.clone()) || self.listed.contains(name) { + continue; + } + anyhow::ensure!( + self.enrolled.insert(name.clone()), + "track `{name}` returned to the catalog after it was dropped; the recording cannot resume it" + ); + changes.push(match pacing { + true => Change::Pacing(name.clone()), + false => Change::Track(name.clone()), + }); + } + + let mut dropped: Vec<_> = self.listed.difference(&listed).cloned().collect(); + dropped.sort(); + changes.extend(dropped.into_iter().map(Change::Remove)); + self.listed = listed; + Ok(changes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use hang::catalog::{AudioCodec, AudioConfig, JsonConfig, Mode}; + use moq_net::{Timescale, Timestamp, group, track}; + + fn audio() -> AudioConfig { + AudioConfig::new(AudioCodec::Opus, 48_000, 2) + } + + /// Renditions pace, data tracks don't, and a later snapshot only adds what is new. + #[test] + fn renditions_pace_and_the_rest_follow() { + let mut tracks = Tracks::default(); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), audio()); + catalog + .json + .tracks + .insert("chat".into(), JsonConfig::new(Mode::Snapshot)); + + assert_eq!( + tracks.update(&catalog).unwrap(), + [Change::Pacing("audio".into()), Change::Track("chat".into())] + ); + + catalog.audio.renditions.insert("audio2".into(), audio()); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Pacing("audio2".into())]); + assert_eq!(tracks.update(&catalog).unwrap(), []); + } + + /// A dropped rendition stops pacing, and coming back is refused rather than silently lost. + #[test] + fn a_dropped_track_is_removed_for_good() { + let mut tracks = Tracks::default(); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), audio()); + tracks.update(&catalog).unwrap(); + + let removed = catalog.audio.renditions.remove("audio").unwrap(); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Remove("audio".into())]); + + catalog.audio.renditions.insert("audio".into(), removed); + let err = tracks.update(&catalog).unwrap_err().to_string(); + assert!(err.contains("cannot resume"), "{err}"); + } + + /// A rendition living in a sibling broadcast would be missing from the recording. + #[test] + fn another_broadcast_is_refused() { + let mut rendition = audio(); + rendition.broadcast = Some(moq_net::path::RelativeOwned::new("./source")); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), rendition); + + let err = Tracks::default().update(&catalog).unwrap_err().to_string(); + assert!(err.contains("broadcast `source`"), "{err}"); + } + + /// What `export archive` records, `import archive` serves back group for group, and the + /// export ends cleanly with its broadcast. + #[tokio::test] + async fn a_recording_replays_its_groups() { + // `open` reads the credential variables. + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let mut broadcast = origin.create_broadcast("live.hang").unwrap(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let track = broadcast.create_track("audio", info).unwrap(); + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), audio()); + }) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let args = ExportArgs { + store: url.clone(), + retention: None, + retention_grace: None, + }; + let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + + for sequence in 0..3 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for offset in [0, 500] { + let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); + group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); + } + group.finish().unwrap(); + } + + // Finish only once the rendition is enrolled, which writes its `.info`. + tokio::time::timeout(Duration::from_secs(10), async { + while !dir.path().join("audio/.info").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the rendition is enrolled"); + track.finish().unwrap(); + catalog.finish().unwrap(); + broadcast.finish(); + + tokio::time::timeout(Duration::from_secs(10), recording) + .await + .expect("the export ends with its broadcast") + .unwrap() + .expect("the recording succeeds"); + + let replay = moq_tokio::origin::spawn(); + let args = ImportArgs { + store: url, + follow: None, + }; + let serving = tokio::spawn(import(replay.clone(), "replay.hang".into(), args)); + + let consumer = replay.consume().routed_broadcast("replay.hang").await.unwrap(); + let audio = consumer.track("audio").unwrap(); + for sequence in 0..3 { + let mut group = audio.fetch_group(sequence, None).await.unwrap(); + for offset in [0, 500] { + let frame = group.read_frame().await.unwrap().expect("a frame"); + assert_eq!(frame.timestamp.as_millis(), u128::from(sequence * 1000 + offset)); + assert_eq!(frame.payload, format!("{sequence}+{offset}")); + } + assert!(group.read_frame().await.unwrap().is_none()); + } + + serving.abort(); + } + + /// Renditions already live in the opening catalog all land in its first segment. + /// + /// Groups are published before export starts, so the writer can drain the first rendition + /// while the next subscription is still in flight. + #[tokio::test] + async fn an_opening_snapshot_records_every_rendition() { + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let mut broadcast = origin.create_broadcast("live.hang").unwrap(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let first = broadcast.create_track("audio", info.clone()).unwrap(); + let second = broadcast.create_track("audio2", info).unwrap(); + for track in [&first, &second] { + for sequence in 0..3 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for offset in [0, 500] { + let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); + group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); + } + group.finish().unwrap(); + } + } + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), audio()); + catalog.audio.renditions.insert("audio2".into(), audio()); + }) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let args = ExportArgs { + store: url.clone(), + retention: None, + retention_grace: None, + }; + let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + + tokio::time::timeout(Duration::from_secs(10), async { + while !dir.path().join("audio/.info").exists() || !dir.path().join("audio2/.info").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both renditions are enrolled"); + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + broadcast.finish(); + + tokio::time::timeout(Duration::from_secs(10), recording) + .await + .expect("the export ends with its broadcast") + .unwrap() + .expect("the recording succeeds"); + + let store = super::open(&url).unwrap(); + let object = store + .get_segments(hang::timeline::DEFAULT_NAME, 0) + .await + .expect("segment 0"); + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } + } + let mut opening = None; + while let Some(event) = decoder.next_event() { + if let moq_json::window::Event::Push { value, .. } = event + && value.segment == 0 + { + opening = Some(value); + } + } + let opening = opening.expect("segment 0 has a record"); + assert!(opening.tracks.contains_key("audio"), "{opening:?}"); + assert!(opening.tracks.contains_key("audio2"), "{opening:?}"); + } +} diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 2e5cff514d..27d87b0f6d 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -18,7 +18,7 @@ //! conditional on the subcommand. //! - The endpoint is one subcommand: a container format (`ts`, `fmp4`, ... read //! from stdin on import, written to stdout on export) or a gateway (`hls`, -//! `rtmp`, `srt`, `rtc`). Exactly one per stage, so "which endpoint" is +//! `rtmp`, `srt`, `rtc`, `archive`). Exactly one per stage, so "which endpoint" is //! unambiguous and there's no silently-ignored flag. //! - `--` starts another stage on the same Origin and the same MoQ attachment, so //! one process can bridge several broadcasts (or both directions at once). Usage @@ -761,6 +761,8 @@ pub enum ImportSource { Srt(crate::srt::Args), /// WebRTC: WHEP client pulling a remote (`--connect`) or WHIP server accepting publishes (`--listen`). Rtc(crate::rtc::Args), + /// Replay a recording from an object store, serving its groups on demand. + Archive(crate::archive::ImportArgs), /// Capture a local source (camera, display, window, app, microphone) and /// encode natively. Run `moq devices` to list them. #[cfg(feature = "capture")] @@ -854,6 +856,8 @@ pub enum ExportSink { Srt(crate::srt::Args), /// WebRTC: WHIP client pushing to a remote (`--connect`) or WHEP server serving plays (`--listen`). Rtc(crate::rtc::Args), + /// Record the broadcast into an object store until it ends. + Archive(crate::archive::ExportArgs), } impl ExportSink { diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 27ee260451..018c7bbb3a 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -4,6 +4,7 @@ //! grammar; this module orchestrates the shared Origin and spawns the MoQ side //! plus every stage's endpoint. +mod archive; mod args; mod auth; mod complete; @@ -638,6 +639,11 @@ fn spawn_import( tasks.spawn(rtc::connect_import(target(name), url)); } } + ImportSource::Archive(args) => { + // A replay serves the retention the recording was made with. + anyhow::ensure!(max_age.is_none(), "`--max-age` does not apply to `import archive`"); + tasks.spawn(archive::import(origin.clone(), name, args)); + } #[cfg(feature = "capture")] ImportSource::Capture(capture) => { warn_if_missing_format(&name); @@ -717,6 +723,18 @@ fn spawn_export( tasks.spawn(rtc::connect_export(origin.consume(), url, name)); } } + ExportSink::Archive(args) => { + anyhow::ensure!( + export.select.is_empty(), + "`export archive` records every rendition; drop the selection flags" + ); + let format = export + .catalog_format + .map(Into::into) + .or_else(|| moq_mux::catalog::CatalogFormat::detect(&name)) + .unwrap_or_default(); + tasks.spawn(archive::export(origin.consume(), name, format, args)); + } _ => unreachable!("container formats are handled by stdout_format above"), } } diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 0680d83f8b..86cb0ae3a4 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -105,6 +105,14 @@ pub struct SelectArgs { } impl SelectArgs { + /// Whether no selection flag was given. + pub(crate) fn is_empty(&self) -> bool { + self.video_name.is_none() + && self.video_codec.is_none() + && self.audio_name.is_none() + && self.audio_codec.is_none() + } + /// Build the rendition selection shared by stdout exports and native playback. /// /// `force` takes the place of `--video-codec`, for a sink whose format implies From fc61e33c024d2662f7cae2c5e217e04aa5fc632f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 07:04:31 -0700 Subject: [PATCH 06/49] feat(archive): resume a recording on restart (#4114) Co-authored-by: Claude Opus 5.5 --- quest/m1/archive/README.md | 6 +- quest/m1/archive/dvr.md | 11 +- quest/m1/archive/proof.md | 1 - quest/m1/archive/recovery.md | 26 ---- rs/moq-archive/src/error.rs | 6 +- rs/moq-archive/src/lib.rs | 1 + rs/moq-archive/src/recover.rs | 185 ++++++++++++++++++++++ rs/moq-archive/src/writer.rs | 239 +++++++++++++++++++++++++---- rs/moq-json/src/window/encoder.rs | 33 ++++ rs/moq-json/src/window/mod.rs | 49 +++++- rs/moq-json/src/window/producer.rs | 16 +- rs/moq-mux/src/error.rs | 5 + rs/moq-mux/src/timeline.rs | 92 ++++++++++- 13 files changed, 588 insertions(+), 82 deletions(-) delete mode 100644 quest/m1/archive/recovery.md create mode 100644 rs/moq-archive/src/recover.rs diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index e4cdd70249..f16037033a 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -40,7 +40,10 @@ percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list `moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) records enrolled tracks through `Deferred`, omits failed tracks with `Pending::omit`, stores each segment's timeline groups after `Producer::flush`, and expires DVR segments -with a deletion grace. It refuses a prefix that already holds a timeline. +with a deletion grace. On a prefix that already holds a recording, it replays the +retained timeline from a checkpoint through `timeline::Producer::resume`, refuses +groups at or below each track's largest stored group, and a DVR deletes +unreferenced group objects one grace period after recovery. `moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays the timeline onto a supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` with a byte-bounded object LRU. `Reader::refresh` follows by listing timeline keys after its cursor, so gaps and @@ -113,7 +116,6 @@ owned by that prerequisite, not duplicated in archive storage. - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [Offline archive HLS](/quest/m1/archive/hls.md) - render playlists from the archive timeline and fetch segment media lazily -- [Resume a recording](/quest/m1/archive/recovery.md) - recover the retained timeline on restart and clean up DVR orphans - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback - [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 685185fa65..096633a809 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -7,10 +7,9 @@ same timeline and group-range objects as an unbounded archive. ## Plan -The recording writer owns retention and deletion grace, and -[Resume a recording](/quest/m1/archive/recovery.md) owns checkpoint recovery -and restart cleanup. This quest consumes that contract and owns viewer -seek and return-to-live behavior, not a second writer implementation. +The recording writer owns retention, deletion grace, checkpoint recovery, and +restart cleanup. This quest consumes that contract and owns viewer seek and +return-to-live behavior, not a second writer implementation. The player reads the archive timeline, FETCHes old groups through the normal miss chain, and splices back to SUBSCRIBE at the live edge without opening a @@ -27,10 +26,6 @@ The reader evicts popped spans from its object cache, but a group it already served stays in `moq_net`'s track cache until the pool reclaims it. Decide whether expiry during a seek needs a group eviction API in `moq-net`. -## Required - -- [Resume a recording](/quest/m1/archive/recovery.md) - ## Closes - [#2275](https://github.com/moq-dev/moq/issues/2275) - close this issue when the quest finishes diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md index c28c05284b..b8cb7e0f47 100644 --- a/quest/m1/archive/proof.md +++ b/quest/m1/archive/proof.md @@ -63,5 +63,4 @@ object directly without any listing or separate index object. ## Required -- [Resume a recording](/quest/m1/archive/recovery.md) - [Offline archive HLS](/quest/m1/archive/hls.md) diff --git a/quest/m1/archive/recovery.md b/quest/m1/archive/recovery.md deleted file mode 100644 index 70b302abd5..0000000000 --- a/quest/m1/archive/recovery.md +++ /dev/null @@ -1,26 +0,0 @@ -# [M] Resume a recording - -## Goal - -A writer restarting on a prefix it exclusively owns recovers the retained -timeline and continues the same recording instead of refusing the prefix. - -## Plan - -`moq_archive::Writer::new` refuses any prefix that already holds a timeline. -Replace that refusal with recovery: list and replay the timeline's -`segments/` objects from a retained checkpoint, then resume the timeline -producer at the next segment ID with the recovered window. That needs a way -to seed `moq_mux::timeline::Producer` with a window and next segment, which it -lacks today. - -For a DVR, reconcile a complete `groups/` listing of every recorded track -against the recovered records before accepting input. Wait the deletion grace -period from successful recovery, then delete unreferenced group objects left -by interrupted expiration or uploads. Failed or incomplete recovery or listing -deletes nothing. Preserve `.info` and timeline checkpoint objects. - -An unbounded archive resumes the same way without deleting anything. Source -group sequences that restart below the recovered ranges must not produce -overlapping objects; decide whether that refuses the track or starts a new -recording. diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index 85010486b9..a8ef749d83 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -78,10 +78,6 @@ pub enum Error { #[error("moq: {0}")] Moq(String), - /// The prefix already holds a recording timeline; resuming is unsupported. - #[error("prefix already holds a recording: {0}")] - Occupied(String), - /// The track was already enrolled, or is the recording's own timeline. #[error("track already enrolled: {0}")] Enrolled(String), @@ -90,7 +86,7 @@ pub enum Error { #[error("source: {0}")] Source(String), - /// The recording's timeline could not be segmented or published. + /// The recording's timeline could not be recovered, segmented, or published. #[error("timeline: {0}")] Timeline(String), diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index b582fffbbd..3d5d9bb1d8 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -19,6 +19,7 @@ mod error; pub mod info; mod path; pub mod reader; +mod recover; pub mod segment; pub mod store; pub mod writer; diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs new file mode 100644 index 0000000000..f7ccfd5ace --- /dev/null +++ b/rs/moq-archive/src/recover.rs @@ -0,0 +1,185 @@ +//! Recover a recording so a restarted [`Writer`](crate::Writer) continues it. + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::ops::RangeInclusive; + +use futures::TryStreamExt; +use hang::timeline::Record; +use moq_json::window::{self, Checkpoint}; +use object_store::ObjectStore; + +use crate::store::list::Query; +use crate::{Error, Key, Object, Result, Store}; + +/// What a restarted writer continues from. +pub(crate) struct Recovery { + /// The retained timeline window, or `None` when no timeline object exists. + pub checkpoint: Option>, + /// The next stored timeline group sequence, so the stored numbering keeps increasing. + pub sequence: u64, + /// Per track, the largest stored group; new groups must exceed it so no object overlaps. + pub floors: HashMap, + /// Stored group objects no retained record references. Only collected for a DVR. + pub orphans: Vec, +} + +/// List the whole recording, then replay its timeline from a retained checkpoint. +/// +/// A DVR (`complete`) replays far enough back to recover every retained record and reports the +/// unreferenced group objects; an archive only needs the newest checkpoint. Any listing, GET, or +/// decode failure fails recovery, so a caller never acts on a partial view. +pub(crate) async fn recover(store: &Store, timeline: &str, complete: bool) -> Result { + let entries: Vec<_> = store.list(&Query::new()).try_collect().await?; + let mut segments = Vec::new(); + let mut groups = Vec::new(); + let mut floors = HashMap::new(); + for entry in entries { + match entry.key { + Key::Segments { track, segment } if track == timeline => segments.push(segment), + Key::Groups { track, range } if track != timeline => { + raise(&mut floors, &track, *range.end()); + groups.push(Key::Groups { track, range }); + } + _ => {} + } + } + segments.sort_unstable(); + + let (checkpoint, sequence) = match (segments.first(), segments.last()) { + (Some(&first), Some(&last)) => { + if segments.len() as u64 != last - first + 1 { + return Err(Error::Timeline(format!( + "timeline segments {first}..={last} are not contiguous" + ))); + } + let (checkpoint, sequence) = replay(store, timeline, first..=last, complete).await?; + // The newest object is segment `last`, so the window must end on the next one. + // A shorter window would resume onto that segment and collide with it. + let next = last.checked_add(1).ok_or(Error::Overflow)?; + if checkpoint.range.end != next { + return Err(Error::Timeline(format!( + "recovered window ends at {}, not segment {next}", + checkpoint.range.end + ))); + } + (Some(checkpoint), sequence) + } + _ => (None, 0), + }; + + let mut referenced = HashSet::new(); + for record in checkpoint.iter().flat_map(|checkpoint| &checkpoint.records) { + for (track, ranges) in &record.tracks { + if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { + raise(&mut floors, track, last.end); + referenced.insert(Key::groups(track.clone(), first.start..=last.end)?); + } + } + } + let orphans = match complete { + true => groups.into_iter().filter(|key| !referenced.contains(key)).collect(), + false => Vec::new(), + }; + + Ok(Recovery { + checkpoint, + sequence, + floors, + orphans, + }) +} + +fn raise(floors: &mut HashMap, track: &str, group: u64) { + let floor = floors.entry(track.to_string()).or_insert(group); + *floor = (*floor).max(group); +} + +/// Replay `segments` from the newest checkpoint that restates every record `complete` needs. +/// +/// Returns the retained window and the next timeline group sequence. +async fn replay( + store: &Store, + timeline: &str, + segments: RangeInclusive, + complete: bool, +) -> Result<(Checkpoint, u64)> { + // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, + // so walk back until a checkpoint restates from there. + let mut objects = VecDeque::new(); + let mut needed = None; + for segment in segments.rev() { + let object = store.get_segments(timeline, segment).await?; + let (offset, start) = checkpoint(&object)?; + let needed = *needed.get_or_insert(offset); + objects.push_front(object); + if !complete || start <= needed { + break; + } + } + + let mut decoder = decoder(); + let mut records = BTreeMap::new(); + for object in &objects { + for stored in &object.groups { + let mut group = decoder.group(); + for frame in &stored.frames { + group.decode(&frame.payload).map_err(json_error)?; + } + } + while let Some(event) = decoder.next_event() { + match event { + window::Event::Push { index, value } => { + records.insert(index, value); + } + window::Event::Pop(range) | window::Event::Skip(range) => { + let tail = records.split_off(&range.end); + records.retain(|index, _| *index < range.start); + records.extend(tail); + } + _ => {} + } + } + } + + // The records must be a contiguous suffix of the window, and all of it for a DVR. + let range = decoder.range(); + let start = records.keys().next().copied().unwrap_or(range.end); + if records.len() as u64 != range.end - start || (complete && start != range.start) { + return Err(Error::Timeline(format!( + "cannot recover timeline window {range:?} from the retained checkpoints" + ))); + } + + let last = objects.back().and_then(|object| object.groups.last()); + let sequence = last.map_or(Ok(0), |group| group.sequence.checked_add(1).ok_or(Error::Overflow))?; + let checkpoint = Checkpoint { + range, + records: records.into_values().collect(), + }; + Ok((checkpoint, sequence)) +} + +/// The retained offset and first restated index of the checkpoint opening `object`. +fn checkpoint(object: &Object) -> Result<(u64, u64)> { + let frame = object + .groups + .first() + .and_then(|group| group.frames.first()) + .ok_or_else(|| Error::Timeline("timeline object has no checkpoint".into()))?; + let mut decoder = decoder(); + decoder.group().decode(&frame.payload).map_err(json_error)?; + let offset = decoder.range().start; + let start = match decoder.next_event() { + Some(window::Event::Skip(skipped)) => skipped.end, + _ => offset, + }; + Ok((offset, start)) +} + +fn decoder() -> window::Decoder { + window::Decoder::new(window::ConsumerConfig::default().with_compression(true)) +} + +fn json_error(err: moq_json::Error) -> Error { + Error::Timeline(err.to_string()) +} diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 766794a75e..5f05cdc083 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -6,6 +6,8 @@ //! failed, commits the record through its own timeline encoder, and stores that segment's timeline //! groups before starting the next one. The timeline therefore only advertises durable objects. //! +//! A writer started on a prefix that already holds a recording resumes it: see [`Writer::new`]. +//! //! ```no_run //! # async fn example(source: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { //! use moq_archive::object_store::memory::InMemory; @@ -37,8 +39,8 @@ use object_store::ObjectStore; use tokio::sync::{mpsc, watch}; use tokio::time::Instant; +use crate::recover::recover; use crate::segment::{Frame, Group, Object}; -use crate::store::list::Query; use crate::{Error, Info, Key, Result, Store}; /// Subscribers ask for every cached group; the publisher clamps this to its own max age. @@ -95,6 +97,8 @@ pub struct Writer { commands: mpsc::UnboundedReceiver, committer: Committer, grace: Option, + /// Deadlines for deleting expired or orphaned objects, oldest first. + deletions: VecDeque<(Instant, Vec)>, // Owns the recording's timeline track. _timeline: broadcast::Producer, } @@ -128,6 +132,8 @@ struct Shared { timeline: String, /// Every name ever enrolled. A name is never reused, so its object ranges stay increasing. enrolled: Mutex>, + /// Per track, the largest group a resumed recording already stored. + floors: HashMap, } enum Command { @@ -143,20 +149,27 @@ enum Command { } impl Writer { - /// Start a recording under `store`'s prefix, reading tracks from `source`. + /// Start a recording under `store`'s prefix, reading tracks from `source`, or resume the one + /// already there. /// - /// Refuses a prefix that already holds a timeline: resuming a recording is unsupported. + /// Resuming replays the retained timeline and continues at the next segment. A track refuses + /// any group at or below the largest one stored for it, so a source whose group sequences + /// restarted needs a new prefix. A DVR also deletes, one grace period after recovery, every + /// group object its retained records do not reference, such as interrupted expirations and + /// uploads. The writer must own the prefix exclusively. Fails, deleting nothing, when the + /// recording cannot be listed or its timeline cannot be replayed. pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { + let section = timeline::Segmenter::new(config.timeline.clone()).section(); + let recovery = recover(&store, §ion.track, config.retention.is_some()).await?; + let broadcast = broadcast::Info::new().produce(); - let timeline = timeline::Producer::new(&broadcast, config.timeline); + let timeline = match &recovery.checkpoint { + Some(checkpoint) => { + timeline::Producer::resume(&broadcast, config.timeline, checkpoint).map_err(timeline_error)? + } + None => timeline::Producer::new(&broadcast, config.timeline), + }; let deferred = timeline.deferred().map_err(timeline_error)?; - let section = timeline.section(); - - let mut existing = store.list(&Query::segments(§ion.track)?); - if let Some(entry) = existing.next().await { - return Err(Error::Occupied(store.path(&entry?.key)?.to_string())); - } - drop(existing); let replay = track::Subscription::default().with_max_age(REPLAY); let groups = broadcast @@ -175,9 +188,16 @@ impl Writer { source, timeline: section.track, enrolled: Mutex::new(HashSet::new()), + floors: recovery.floors, }); let (commands, receiver) = mpsc::unbounded_channel(); let retention = config.retention; + let mut deletions = VecDeque::new(); + if let Some(retention) = &retention + && !recovery.orphans.is_empty() + { + deletions.push_back((Instant::now() + retention.grace, recovery.orphans)); + } Ok(Self { control: Control { shared: shared.clone(), @@ -191,9 +211,11 @@ impl Writer { groups, timescale: section.timescale.into(), retention: retention.as_ref().map(|r| r.window), - window: VecDeque::new(), + window: recovery.checkpoint.map(|c| c.records.into()).unwrap_or_default(), + sequence: recovery.sequence, }, grace: retention.map(|r| r.grace), + deletions, _timeline: broadcast, }) } @@ -215,6 +237,7 @@ impl Writer { mut commands, committer, grace, + mut deletions, _timeline, } = self; let shared = control.shared.clone(); @@ -228,7 +251,6 @@ impl Writer { let mut reads = FuturesUnordered::new(); let mut committer = Some(committer); let mut commit: Option> = None; - let mut deletions: VecDeque<(Instant, Vec)> = VecDeque::new(); let mut closed = false; // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. let mut accepting = true; @@ -263,11 +285,12 @@ impl Writer { None => accepting = false, Some(Command::Enroll { name, subscriber, recorder, timescale }) => { let (cancel, cancelled) = watch::channel(()); + let largest = shared.floors.get(&name).copied(); reads.push(guard(cancelled.clone(), recv(name.clone(), subscriber)).boxed()); tracks.insert(name, TrackState { recorder, timescale, - largest: None, + largest, reported: None, accepted: BTreeMap::new(), subscribed: true, @@ -406,7 +429,7 @@ impl Control { struct TrackState { recorder: Recorder, timescale: Timescale, - /// The newest group accepted; later arrivals must exceed it. + /// The newest group accepted or already stored; later arrivals must exceed it. largest: Option, /// The first-frame timestamp of the newest reported group; later groups must not precede it. reported: Option, @@ -652,6 +675,8 @@ struct Committer { retention: Option, /// Committed records still in the timeline window, oldest first. window: VecDeque, + /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. + sequence: u64, } impl Committer { @@ -741,7 +766,8 @@ impl Committer { Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), } } - groups.push(convert(group.sequence, frames, timescale)?); + let sequence = group.sequence.checked_add(self.sequence).ok_or(Error::Overflow)?; + groups.push(convert(sequence, frames, timescale)?); } Ok(Object { groups }) } @@ -786,12 +812,14 @@ mod tests { }; use super::*; + use crate::store::list::Query; - /// An in-memory store whose group PUTs fail for one track. + /// An in-memory store whose group PUTs fail for one track, and whose listings fail at the end. #[derive(Debug, Clone)] struct Failing { inner: Arc, track: &'static str, + list: bool, } impl std::fmt::Display for Failing { @@ -837,7 +865,18 @@ mod tests { } fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { - self.inner.list(prefix) + let listed = self.inner.list(prefix); + match self.list { + true => listed + .chain(futures::stream::once(async { + Err(object_store::Error::NotImplemented { + operation: "list".into(), + implementer: "Failing".into(), + }) + })) + .boxed(), + false => listed, + } } async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { @@ -995,6 +1034,7 @@ mod tests { let failing = Failing { inner: Arc::new(InMemory::new()), track: "audio", + list: false, }; let store = Store::new(failing, "rec"); let writer = Writer::new(store.clone(), source.consume(), Config::default()) @@ -1229,25 +1269,162 @@ mod tests { ); } - #[tokio::test] - async fn an_existing_recording_is_refused() { + /// Record `video` groups `sequences`, one per second, until the source ends. + async fn record(store: &Store, config: Config, sequences: std::ops::Range) { let source = broadcast::Info::new().produce(); let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); writer.control().pacing_track("video").await.unwrap(); - group(&video, 0, &[0]); + for sequence in sequences { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } video.finish().unwrap(); source.finish(); writer.run().await.unwrap(); + } + + /// Every stored group object, across all tracks. + async fn stored_groups(store: &Store) -> HashSet { + store + .list(&Query::new()) + .try_filter_map(|entry| async move { Ok(matches!(entry.key, Key::Groups { .. }).then_some(entry.key)) }) + .try_collect() + .await + .unwrap() + } + + /// The group objects the retained records advertise. + fn referenced(records: &[Record]) -> HashSet { + records + .iter() + .flat_map(|record| &record.tracks) + .map(|(name, ranges)| Key::groups(name.clone(), ranges[0].start..=ranges.last().unwrap().end).unwrap()) + .collect() + } + + fn orphan(sequence: u64) -> Object { + Object { + groups: vec![Group { + sequence, + frames: vec![Frame { + timestamp: sequence * 1000, + payload: "orphan".into(), + }], + }], + } + } + + #[tokio::test] + async fn a_restarted_writer_resumes_the_recording() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..3).await; + // The source's cache replays groups the recording already holds. + record(&store, Config::default(), 0..6).await; + + let records = window(&store).await; + assert_eq!( + records.iter().map(|record| record.segment).collect::>(), + (0..6).collect::>() + ); + assert_eq!(ranges(&records, "video"), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, &records).await; - let again = broadcast::Info::new().produce(); - assert!(matches!( - Writer::new(store, again.consume(), Config::default()).await, - Err(Error::Occupied(_)) - )); + // The resumed timeline groups continue the stored numbering. + let mut sequences = Vec::new(); + for segment in 0..6 { + let object = store.get_segments(TIMELINE, segment).await.unwrap(); + sequences.extend(object.groups.iter().map(|group| group.sequence)); + } + assert!(sequences.windows(2).all(|pair| pair[0] < pair[1]), "{sequences:?}"); + } + + #[tokio::test] + async fn a_restarted_dvr_deletes_unreferenced_groups() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + record(&store, config.clone(), 0..6).await; + + // An interrupted expiration, an uncommitted upload, and a track no retained record names. + store.put_groups("video", &orphan(1)).await.unwrap(); + store.put_groups("video", &orphan(7)).await.unwrap(); + store.put_groups("audio", &orphan(0)).await.unwrap(); + + // Groups at or below the uncommitted upload are refused, so nothing overlaps it. + record(&store, config, 6..10).await; + + let records = window(&store).await; + assert_eq!( + records.iter().map(|record| record.segment).collect::>(), + vec![5, 6, 7] + ); + assert_eq!(ranges(&records, "video"), vec![(5, 5), (8, 8), (9, 9)]); + check_objects(&store, &records).await; + assert_eq!(stored_groups(&store).await, referenced(&records)); + store.get_info("video").await.unwrap(); + store.get_info(TIMELINE).await.unwrap(); + store.get_segments(TIMELINE, 0).await.unwrap(); + } + + #[tokio::test] + async fn a_failed_recovery_deletes_nothing() { + let inner = Arc::new(InMemory::new()); + let store = Store::new(inner.clone(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + record(&store, config.clone(), 0..6).await; + store.put_groups("video", &orphan(1)).await.unwrap(); + let before = stored_groups(&store).await; + + let failing = Failing { + inner: inner.clone(), + track: "", + list: true, + }; + let source = broadcast::Info::new().produce(); + let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; + assert!(matches!(result, Err(Error::Store(_)))); + + // A missing timeline object leaves the retained window unrecoverable. + store.delete(&Key::segments(TIMELINE, 3).unwrap()).await.unwrap(); + let result = Writer::new(store.clone(), source.consume(), config).await; + assert!(matches!(result, Err(Error::Timeline(_)))); + + assert_eq!(stored_groups(&store).await, before); + } + + #[tokio::test] + async fn a_dvr_window_longer_than_one_checkpoint_is_recovered() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(280), Duration::ZERO)); + record(&store, config, 0..300).await; + + let recovery = recover(&store, TIMELINE, true).await.unwrap(); + let checkpoint = recovery.checkpoint.unwrap(); + let records = window(&store).await; + assert!(records.len() > 256, "the window outgrows one checkpoint"); + assert_eq!(checkpoint.records, records); + assert_eq!(checkpoint.range.end, 300); + assert!(recovery.orphans.is_empty()); + assert_eq!(recovery.floors["video"], 299); + } + + #[tokio::test] + async fn a_timeline_that_does_not_end_at_the_next_segment_fails_recovery() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..6).await; + let before = stored_groups(&store).await; + + // `segments/5` still decodes, but it restates an earlier window, so resuming + // would write the next segment on top of it. + let older = store.get_segments(TIMELINE, 0).await.unwrap(); + store.delete(&Key::segments(TIMELINE, 5).unwrap()).await.unwrap(); + store.put_segments(TIMELINE, 5, &older).await.unwrap(); + + let source = broadcast::Info::new().produce(); + match Writer::new(store.clone(), source.consume(), Config::default()).await { + Err(Error::Timeline(message)) => assert!(message.contains("not segment 6"), "{message}"), + Err(err) => panic!("expected a timeline error, got {err}"), + Ok(_) => panic!("expected recovery to fail"), + } + assert_eq!(stored_groups(&store).await, before); } } diff --git a/rs/moq-json/src/window/encoder.rs b/rs/moq-json/src/window/encoder.rs index 6af1f43a5d..f06d52b988 100644 --- a/rs/moq-json/src/window/encoder.rs +++ b/rs/moq-json/src/window/encoder.rs @@ -86,6 +86,15 @@ impl ProducerConfig { } } +/// A retained window to continue, such as one replayed from stored groups. +#[derive(Debug, Clone, PartialEq)] +pub struct Checkpoint { + /// Absolute index of the oldest retained record, and of the next to be pushed. + pub range: std::ops::Range, + /// The newest retained records, oldest first, ending just before `range.end`. + pub records: Vec, +} + /// One encoded frame, and the group boundary it implies. #[derive(Clone, Debug)] #[non_exhaustive] @@ -407,6 +416,30 @@ impl Encoder { } impl Encoder { + /// Create an encoder continuing `checkpoint`, so the first edit opens a group restating it. + /// + /// Fails when the records outnumber the range, or the range exceeds the safe integer range. + pub fn resume(config: ProducerConfig, checkpoint: &Checkpoint) -> Result { + let Checkpoint { range, records } = checkpoint; + if range.start > range.end || range.end > MAX_INDEX || records.len() as u64 > range.end - range.start { + return Err(Error::Json("invalid window checkpoint".into())); + } + + let mut encoder = Self::new(config); + let skip = encoder + .config + .checkpoint_records + .map(|limit| records.len().saturating_sub(limit)) + .unwrap_or_default(); + encoder.window = records[skip..] + .iter() + .map(serde_json::to_value) + .collect::>()?; + encoder.offset = range.start; + encoder.start = range.end - encoder.window.len() as u64; + Ok(encoder) + } + /// Append one record to the back of the window. /// /// Emits a push into the open group, or a header restating the window (the new record included) diff --git a/rs/moq-json/src/window/mod.rs b/rs/moq-json/src/window/mod.rs index b318556194..51d22b18c2 100644 --- a/rs/moq-json/src/window/mod.rs +++ b/rs/moq-json/src/window/mod.rs @@ -55,7 +55,7 @@ mod producer; pub use consumer::Consumer; pub use decoder::{ConsumerConfig, Decoder, Event, Group}; -pub use encoder::{Encoded, Encoder, Pending, ProducerConfig}; +pub use encoder::{Checkpoint, Encoded, Encoder, Pending, ProducerConfig}; pub use producer::Producer; #[cfg(test)] @@ -291,6 +291,53 @@ mod test { ); } + #[test] + fn a_resumed_encoder_continues_the_window() { + let config = ProducerConfig::default().with_op_ratio(0).with_checkpoint_records(2); + let mut original = Encoder::::new(config.clone()); + let mut decoder = Decoder::::new(ConsumerConfig::default()); + for n in 0..4 { + let frame = original.push(&rec(n)).unwrap(); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + } + let frame = original.pop(1).unwrap().unwrap(); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + std::iter::from_fn(|| decoder.next_event()).for_each(drop); + + let checkpoint = Checkpoint { + range: 1..4, + records: vec![rec(1), rec(2), rec(3)], + }; + let mut resumed = Encoder::resume(config, &checkpoint).unwrap(); + assert_eq!(resumed.range(), 1..4); + assert_eq!( + resumed.window(), + vec![rec(2), rec(3)], + "trimmed to the checkpoint bound" + ); + + let frame = resumed.push(&rec(4)).unwrap(); + assert!(frame.keyframe, "the first edit restates the window"); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + assert_eq!( + std::iter::from_fn(|| decoder.next_event()).collect::>(), + vec![Event::Push { + index: 4, + value: rec(4) + }], + "a reader that kept up sees only the new record" + ); + + let invalid = Checkpoint { + range: 3..4, + records: vec![rec(2), rec(3)], + }; + assert!(Encoder::resume(ProducerConfig::default(), &invalid).is_err()); + } + #[test] fn pops_cross_the_omitted_checkpoint_prefix() { let config = ProducerConfig::default().with_checkpoint_records(2); diff --git a/rs/moq-json/src/window/producer.rs b/rs/moq-json/src/window/producer.rs index 5967f80bb5..50d604bf9c 100644 --- a/rs/moq-json/src/window/producer.rs +++ b/rs/moq-json/src/window/producer.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; use serde_json::Value; -use super::{Encoded, Encoder, ProducerConfig}; +use super::{Checkpoint, Encoded, Encoder, ProducerConfig}; use crate::Result; /// Publishes a sliding window of JSON records over a track. @@ -32,13 +32,17 @@ impl Clone for Producer { impl Producer { /// Create a producer that publishes to the given track. pub fn new(track: moq_net::track::Producer, config: ProducerConfig) -> Self { + Self::with_encoder(track, Encoder::new(config)) + } + + fn with_encoder(track: moq_net::track::Producer, encoder: Encoder) -> Self { Self { inner: Arc::new(Mutex::new(Inner { track: Track { inner: track, group: None, }, - encoder: Encoder::new(config), + encoder, finished: false, })), _marker: PhantomData, @@ -89,6 +93,14 @@ impl Producer { } impl Producer { + /// Create a producer continuing `checkpoint` on the given track. + /// + /// The first edit opens a group restating the checkpoint, so a reader that already holds those + /// records sees only what follows. Fails like [`Encoder::resume`]. + pub fn resume(track: moq_net::track::Producer, config: ProducerConfig, checkpoint: &Checkpoint) -> Result { + Ok(Self::with_encoder(track, Encoder::resume(config, checkpoint)?)) + } + /// Append one record to the back of the window. pub fn push(&mut self, value: &T) -> Result<()> { self.inner.lock().unwrap().push(value) diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 5a1637f7a0..f107b735c6 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -200,6 +200,11 @@ pub enum Error { #[error("timeline segment {0} was not yielded for deferred publication")] TimelineDeferredRecord(u64), + /// [`timeline::Producer::resume`](crate::timeline::Producer::resume) received a checkpoint + /// whose record at this window index is a different segment. + #[error("timeline checkpoint record at index {0} is a different segment")] + TimelineCheckpoint(u64), + /// Error from a muxer/demuxer that reports via `anyhow` (currently MPEG-TS). /// Boxed in an `Arc` so the enum stays `Clone` (`anyhow::Error` is not). #[error("{0}")] diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs index ecc240c092..669df15f49 100644 --- a/rs/moq-mux/src/timeline.rs +++ b/rs/moq-mux/src/timeline.rs @@ -57,6 +57,7 @@ use std::time::Duration; use hang::catalog::Archive; use hang::timeline::{DEFAULT_NAME, Range, Record, RecordExt}; +use moq_json::window::Checkpoint; use moq_net::{Timescale, Timestamp}; @@ -739,16 +740,30 @@ struct Output { impl Output { fn prepare(&mut self) -> crate::Result<()> { if self.sink.is_none() && !self.closed { - let info = moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog); - let net = self.broadcast.create_track(DEFAULT_NAME, info)?; - let config = moq_json::window::ProducerConfig::default() - .with_compression(true) - .with_checkpoint_records(CHECKPOINT_RECORDS); - self.sink = Some(moq_json::window::Producer::new(net, config)); + let net = self.create_track()?; + self.sink = Some(moq_json::window::Producer::new(net, Self::config())); } Ok(()) } + /// Create the timeline track now, continuing `checkpoint`. + fn resume(&mut self, checkpoint: &Checkpoint) -> crate::Result<()> { + let net = self.create_track()?; + self.sink = Some(moq_json::window::Producer::resume(net, Self::config(), checkpoint)?); + Ok(()) + } + + fn create_track(&self) -> crate::Result { + let info = moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog); + Ok(self.broadcast.create_track(DEFAULT_NAME, info)?) + } + + fn config() -> moq_json::window::ProducerConfig { + moq_json::window::ProducerConfig::default() + .with_compression(true) + .with_checkpoint_records(CHECKPOINT_RECORDS) + } + fn push(&mut self, record: &Record) -> crate::Result<()> { self.prepare()?; let Some(sink) = self.sink.as_mut() else { @@ -824,6 +839,31 @@ impl Producer { } } + /// A timeline for `broadcast` continuing `checkpoint`, such as a window recovered from storage. + /// + /// A timeline's window index is its segment number, so the next record is segment + /// `checkpoint.range.end`. Creates the timeline track immediately; its first group restates the + /// checkpoint. Fails when a checkpoint record is not the segment at its index, the checkpoint + /// is malformed, or the track cannot be created. + pub fn resume( + broadcast: &moq_net::broadcast::Producer, + config: Config, + checkpoint: &Checkpoint, + ) -> crate::Result { + let producer = Self::new(broadcast, config); + producer.output.lock().unwrap().resume(checkpoint)?; + + // The encoder accepted the checkpoint, so the records fit before `range.end`. + let start = checkpoint.range.end - checkpoint.records.len() as u64; + for (index, record) in (start..).zip(&checkpoint.records) { + if record.segment != index { + return Err(crate::Error::TimelineCheckpoint(index)); + } + } + producer.segmenter.state.lock().unwrap().next_segment = checkpoint.range.end; + Ok(producer) + } + /// Enroll `name` without letting it influence segmentation, returning its [`Recorder`]. /// /// Its groups are recorded into whichever segment is open when they arrive, but the track @@ -1322,6 +1362,46 @@ mod test { (broadcast, timeline) } + #[tokio::test] + async fn a_resumed_timeline_continues_the_checkpoint() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let checkpoint = Checkpoint { + range: 1..3, + records: vec![Record::new(1, 1_000, 1_000), Record::new(2, 2_000, 1_000)], + }; + let mut timeline = Producer::resume(&broadcast, Config::default(), &checkpoint).unwrap(); + + let mut video = timeline.pacing_track("video0").unwrap(); + video.record(10, ms(3_000), true); + video.record(11, ms(4_000), true); + video.end(ms(5_000)); + drop(video); + timeline.finish().unwrap(); + + assert_eq!( + drain(&broadcast, &timeline).await, + vec![ + entry(1, 1_000, 1_000, &[]), + entry(2, 2_000, 1_000, &[]), + entry(3, 3_000, 1_000, &[("video0", &[(10, 10)])]), + entry(4, 4_000, 1_000, &[("video0", &[(11, 11)])]), + ] + ); + } + + #[test] + fn a_checkpoint_record_must_be_the_segment_at_its_index() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let checkpoint = Checkpoint { + range: 0..1, + records: vec![Record::new(5, 0, 1_000)], + }; + assert!(matches!( + Producer::resume(&broadcast, Config::default(), &checkpoint), + Err(crate::Error::TimelineCheckpoint(0)) + )); + } + #[tokio::test] async fn deferred_records_are_invisible_until_committed() { let (broadcast, timeline) = setup(); From c6644eafce8bb19c4cc21992b144dd30f6f80422 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 07:14:27 -0700 Subject: [PATCH 07/49] test(hls): serve archive recordings as HLS from the timeline alone (#4115) Co-authored-by: Claude Opus 5.5 Co-authored-by: Grok 4.7 --- Cargo.lock | 4 + doc/bin/hls.md | 11 + quest/m1/archive/README.md | 8 +- quest/m1/archive/hls-window.md | 20 + quest/m1/archive/hls.md | 38 -- quest/m1/archive/proof.md | 8 +- rs/moq-hls/Cargo.toml | 4 + rs/moq-hls/src/export/archive_tests.rs | 493 +++++++++++++++++++++++++ rs/moq-hls/src/export/mod.rs | 7 +- 9 files changed, 547 insertions(+), 46 deletions(-) create mode 100644 quest/m1/archive/hls-window.md delete mode 100644 quest/m1/archive/hls.md create mode 100644 rs/moq-hls/src/export/archive_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 29f7c2fd84..f7a2745b9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4424,12 +4424,16 @@ dependencies = [ name = "moq-hls" version = "0.5.3" dependencies = [ + "async-trait", "axum", "bytes", + "futures", "hang", "humantime", "kio 0.6.0", "m3u8-rs", + "moq-archive", + "moq-json", "moq-mux", "moq-net", "moq-tokio", diff --git a/doc/bin/hls.md b/doc/bin/hls.md index d8b169ffdf..ddb9028029 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -38,6 +38,17 @@ broadcast by path: /{broadcast}/{video|audio}/{rendition}/seg/t{pts}.m4s ``` +A [`moq-archive`](https://docs.rs/moq-archive) recording replayed through its +`Reader` is served the same way, with no second stored copy. Playlists come +from the replayed timeline alone, and a segment GETs exactly one stored object +of its rendition, so switching renditions never downloads both. An +inline-parameter-set codec with no catalog `description` is the exception: +the first playlist render GETs one keyframe group to build the init segment, +then caches it. Out-of-band configs need no media GET. Set `--window` +to cover the recording. The playlist ends with `EXT-X-ENDLIST` only once the +reader's caller declares the recording finished; the store holds no completion +marker. + The init URL carries a hash of its bytes, so a reconfigured rendition gets a new one. An embedder of the library can also label the publisher's run with `Broadcaster::set_generation`. Every segment URL then carries it diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index f16037033a..fabafaf0e1 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -34,6 +34,12 @@ The segment engine is in `rs/moq-mux/src/timeline.rs`: - `moq-hls` renders live playlists from the timeline alone and FETCHes media per HTTP request (`rs/moq-hls/src/export/mod.rs:3-8`). A clean timeline finish ends every window with `EXT-X-ENDLIST` (:325-327). +- The same exporter serves a recording replayed through `moq_archive::Reader` + with no archive-specific code (`rs/moq-hls/src/export/archive_tests.rs`): + playlists read only the timeline (an inline parameter set also GETs one + keyframe group to build its init), and a segment GETs one object of its + rendition. The caller supplies the catalog, and `--window` must cover the + recording. `rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`: percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list/delete. @@ -114,8 +120,8 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests +- [Archive HLS window](/quest/m1/archive/hls-window.md) - serve a replayed recording's whole retained timeline without a server-wide `--window` - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts -- [Offline archive HLS](/quest/m1/archive/hls.md) - render playlists from the archive timeline and fetch segment media lazily - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback - [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation diff --git a/quest/m1/archive/hls-window.md b/quest/m1/archive/hls-window.md new file mode 100644 index 0000000000..33cafc2466 --- /dev/null +++ b/quest/m1/archive/hls-window.md @@ -0,0 +1,20 @@ +# [S] Archive HLS window + +## Goal + +`moq-hls` lists a replayed recording's whole retained timeline in HLS and DASH +without a server-wide `--window`, while live broadcasts keep a window within +the relay's cache. + +## Plan + +The export window (`moq_hls::export::Config::window`) trims every broadcast a +server exports. A recording served through `moq_archive::Reader` already +bounds itself: DVR expiry pops records from the replayed timeline, and an +unbounded archive never pops. Today an operator must raise `--window` past the +recording's length, which also inflates segment `Cache-Control: max-age` and +DASH `timeShiftBufferDepth` for every live broadcast on the same server. + +Decide how an export learns that a broadcast's timeline is authoritative (the +catalog's `archive` entry carrying a store or replay path, a per-broadcast +option, or a separate server) and cap the derived HTTP and DASH values. diff --git a/quest/m1/archive/hls.md b/quest/m1/archive/hls.md deleted file mode 100644 index f4a2966c22..0000000000 --- a/quest/m1/archive/hls.md +++ /dev/null @@ -1,38 +0,0 @@ -# [L] Offline archive HLS - -## Goal - -`moq-hls` serves ordinary HLS from a growing or static archive without a second -stored media copy, and renders playlists without downloading media objects. - -## Plan - -Use the archive catalog for rendition and initialization metadata, then reuse -the live HLS renderer against the timeline. Segment number, PTS, duration, -track ranges, gaps, and keyframe state provide everything needed for a media -playlist. Playlist generation and reloads read the timeline only; a regression -test must fail if they GET a media segment. - -When a player requests media, use the recording reader to GET only the selected -range-named object and transmux its groups on demand. Switching between -360p and 1080p must not download both rendition objects. Emit media URIs with -the track and inclusive group bounds from the timeline record, so the handler -can derive `groups/.` directly without listing or a -segment-ID lookup. HLS sequence numbers do not appear in storage keys. Keep -one storage object per track per timeline segment; a MoQ group need not be an -HLS segment, especially for one-group-per-frame audio. No LL-HLS parts. - -Emit `EXT-X-ENDLIST` exactly when the timeline track the exporter reads -finishes cleanly, as the live export already does -(`rs/moq-hls/src/export/mod.rs:325-327`, `rendition.rs:243-246`). The store -holds no completion marker: the reader finishes the replayed timeline track -when its caller supplies finality out of band, so a standalone or BYOB archive -without such a caller stays a reloadable playlist. - -Use the catalog supplied to the exporter. This quest does not establish which -catalog update applies to a historic group; timestamps do not provide an -explicit binding. Track immutability and update correlation belong to -[Catalog track identity](/quest/m2/catalog-tracks.md), independently of DVR. - -Prove aligned audio/video switching, missing track segments, discontinuities, -caller-supplied finality, and bounded LRU reads. diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md index b8cb7e0f47..225449edf4 100644 --- a/quest/m1/archive/proof.md +++ b/quest/m1/archive/proof.md @@ -58,9 +58,5 @@ objects survive. Failed or incomplete recovery/listing must delete nothing; restart must finish this cleanup before accepting new groups. Finally render and reload HLS playlists while rejecting every media-object GET -until a segment URI is requested. The range-bearing URI must resolve one -object directly without any listing or separate index object. - -## Required - -- [Offline archive HLS](/quest/m1/archive/hls.md) +until a segment URI is requested. A segment request must resolve one object +from the replayed timeline without any listing or separate index object. diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index e759559a79..283a3fe31e 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -36,6 +36,10 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] +async-trait = "0.1" +futures = { workspace = true } +moq-archive = { path = "../moq-archive" } +moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs new file mode 100644 index 0000000000..3d5450c584 --- /dev/null +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -0,0 +1,493 @@ +//! Serving a `moq-archive` recording as HLS. +//! +//! A [`moq_archive::Reader`] replays the recording's timeline onto a broadcast and answers FETCH +//! from range-named objects, so the exporter serves it exactly like a live broadcast. These tests +//! pin the storage traffic that composition produces: playlists read only the timeline, and a +//! segment GETs exactly one object of the requested rendition. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use futures::stream::BoxStream; +use hang::timeline::{Range, Record}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::object_store::path::Path; +use moq_archive::object_store::{ + self, CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use moq_archive::{Frame, Group, Info, Object, Store, reader}; +use moq_json::window; + +use super::*; + +const TIMELINE: &str = hang::timeline::DEFAULT_NAME; + +/// In-memory store that records every GET path. +#[derive(Debug, Clone, Default)] +struct Counting { + inner: Arc, + gets: Arc>>, +} + +impl Counting { + /// Every GET since the last call. + fn take(&self) -> Vec { + std::mem::take(&mut *self.gets.lock().unwrap()) + } +} + +impl std::fmt::Display for Counting { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Counting") + } +} + +#[async_trait::async_trait] +impl ObjectStore for Counting { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.gets.lock().unwrap().push(location.to_string()); + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// Writes recording objects the way `moq_archive::Writer` lays them out, with full control over +/// each record's timing and ranges. +struct Recording { + store: Store, + encoder: window::Encoder, + /// Next timeline group sequence. + sequence: u64, +} + +impl Recording { + async fn new(tracks: &[&str]) -> Self { + let store = Store::new(Counting::default(), "rec"); + store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); + for track in tracks { + // Legacy media is stamped in microseconds. + store.put_info(track, &Info::new(1, 1_000_000).unwrap()).await.unwrap(); + } + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(0); + Self { + store, + encoder: window::Encoder::new(config), + sequence: 0, + } + } + + /// Store one object for `track` holding `groups`, each a list of frame timestamps in micros. + async fn media(&self, track: &str, groups: &[(u64, &[u64])]) { + let groups = groups + .iter() + .map(|&(sequence, frames)| Group { + sequence, + frames: frames + .iter() + .enumerate() + .map(|(index, µs)| legacy(track, micros, index == 0)) + .collect(), + }) + .collect(); + self.store.put_groups(track, &Object { groups }).await.unwrap(); + } + + /// Commit `record` as timeline segment `record.segment`, popping `pop` older records. + async fn commit(&mut self, record: &Record, pop: u64) { + let mut payloads = Vec::new(); + let pending = self.encoder.push(record).unwrap(); + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = self.encoder.pop(pop).unwrap() { + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + + let mut groups: Vec = Vec::new(); + for (keyframe, payload) in payloads { + if keyframe || groups.is_empty() { + groups.push(Group { + sequence: self.sequence, + frames: Vec::new(), + }); + self.sequence += 1; + } + let frame = Frame { + timestamp: record.pts, + payload, + }; + groups.last_mut().unwrap().frames.push(frame); + } + self.store + .put_segments(TIMELINE, record.segment, &Object { groups }) + .await + .unwrap(); + } + + /// Every GET since the last call. + fn gets(&self) -> Vec { + self.store.inner().take() + } +} + +/// One Legacy frame: a VP8 keyframe for video (geometry the muxer can parse), filler otherwise. +fn legacy(track: &str, micros: u64, keyframe: bool) -> Frame { + let payload: &'static [u8] = match (track.starts_with("audio"), keyframe) { + (true, _) => &[0xFC, 0xFF, 0xFE], + (false, true) => &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00], + (false, false) => &[0x31, 0x00, 0x00], + }; + let frame = hang::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + payload: Bytes::from_static(payload), + }; + let mut encoded = BytesMut::new(); + frame.encode(&mut encoded).unwrap(); + Frame { + timestamp: micros, + payload: encoded.freeze(), + } +} + +fn record(segment: u64, pts: u64, duration: u64, tracks: &[(&str, u64, u64)]) -> Record { + let mut record = Record::new(segment, pts, duration); + for &(track, start, end) in tracks { + record.tracks.insert(track.to_string(), vec![Range::new(start, end)]); + } + record +} + +/// The catalog an exporter is handed: every rendition's config, plus the archive entry naming +/// the recording's timeline. Out-of-band configs, so no init needs media. +fn catalog() -> hang::Catalog { + let mut catalog = hang::Catalog::default(); + catalog.archive = Some(hang::catalog::Archive::new(TIMELINE)); + for (name, width, height) in [("360p", 640, 360), ("1080p", 1920, 1080)] { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.coded_width = Some(width); + config.coded_height = Some(height); + catalog.video.renditions.insert(name.to_string(), config); + } + let audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + catalog.audio.renditions.insert("audio".to_string(), audio); + catalog +} + +/// A recording served back through an origin: the reader's broadcast carries the supplied +/// catalog, and a broadcaster exports it. +struct Replay { + /// Taken to supply finality. + reader: Option>, + broadcaster: Arc, + _catalog: moq_json::snapshot::Producer, + _broadcast: moq_net::broadcast::Producer, + _origin: moq_net::origin::Producer, +} + +impl Replay { + async fn open(recording: &Recording, cache: u64) -> Self { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + tokio::spawn(moq_net::time::run(driver)); + let broadcast = origin.create_broadcast("rec").unwrap(); + + let track = broadcast + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut json = moq_json::snapshot::Config::default(); + json.delta_ratio = 0; + let mut catalog = moq_json::snapshot::Producer::new(track, json); + catalog.update(&self::catalog()).unwrap(); + + let config = reader::Config::new(TIMELINE).with_cache(cache); + let reader = moq_archive::Reader::open(recording.store.clone(), &broadcast, config) + .await + .unwrap(); + tokio::spawn(reader.serve()); + broadcast.announce(Default::default()).unwrap(); + for _ in 0..10 { + tokio::task::yield_now().await; + } + + let source = moq_mux::Source::new(origin.consume(), "rec"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()) + .await + .expect("a rendition becomes playable"); + + Self { + reader: Some(reader), + broadcaster, + _catalog: catalog, + _broadcast: broadcast, + _origin: origin, + } + } + + fn rendition(&self, kind: Kind, name: &str) -> Arc { + self.broadcaster + .rendition(kind, name) + .expect("rendition in the catalog") + } + + async fn playlist(&self, kind: Kind, name: &str) -> String { + let rendition = self.rendition(kind, name); + tokio::time::timeout(Duration::from_secs(5), rendition.playlist(None)) + .await + .expect("playlist renders") + .unwrap() + .expect("playlist is servable") + } + + /// Wait until `name`'s playlist satisfies `ready`, then return it. + async fn playlist_until(&self, kind: Kind, name: &str, ready: impl Fn(&str) -> bool) -> String { + for _ in 0..500 { + let playlist = self.playlist(kind, name).await; + if ready(&playlist) { + return playlist; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} playlist never became ready"); + } +} + +fn is_media(path: &str) -> bool { + path.contains("/groups/") +} + +/// Three aligned 2s segments: one keyframe group per video rendition, four audio groups each. +async fn three_segments() -> Recording { + let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; + for segment in 0..3u64 { + let pts = segment * 2_000_000; + for video in ["360p", "1080p"] { + recording.media(video, &[(segment, &[pts, pts + 1_000_000])]).await; + } + let audio: Vec<(u64, [u64; 1])> = (0..4).map(|i| (segment * 4 + i, [pts + i * 500_000])).collect(); + let audio: Vec<(u64, &[u64])> = audio + .iter() + .map(|(sequence, frames)| (*sequence, &frames[..])) + .collect(); + recording.media("audio", &audio).await; + + let tracks = [ + ("360p", segment, segment), + ("1080p", segment, segment), + ("audio", segment * 4, segment * 4 + 3), + ]; + recording + .commit(&record(segment, segment * 2000, 2000, &tracks), 0) + .await; + } + recording +} + +#[tokio::test] +async fn playlists_read_only_the_timeline_and_segments_one_object() { + let recording = three_segments().await; + let replay = Replay::open(&recording, 64 * 1024 * 1024).await; + + let master = replay.broadcaster.master_playlist(None); + assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); + + // Render and reload every playlist: aligned numbering, and not one media GET. + for _ in 0..2 { + for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { + let playlist = replay.playlist(kind, name).await; + for segment in 0..3 { + assert!(playlist.contains(&format!("seg/{segment}.m4s\n")), "{name}: {playlist}"); + } + assert!(!playlist.contains("#EXT-X-ENDLIST"), "no finality was supplied"); + } + } + let gets = recording.gets(); + assert!(gets.iter().any(|path| path.contains("/segments/")), "{gets:?}"); + assert!( + !gets.iter().any(|path| is_media(path)), + "playlists must not GET media: {gets:?}" + ); + + // Switching renditions downloads only the selected rendition's object. + let low = replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); + assert_eq!(&low[4..8], b"moof"); + assert_eq!( + recording.gets(), + [ + "rec/360p/.info", + "rec/360p/groups/0000000000000000001.0000000000000000001" + ] + ); + let high = replay + .rendition(Kind::Video, "1080p") + .segment(2) + .await + .unwrap() + .unwrap(); + assert_eq!(&high[4..8], b"moof"); + assert_eq!( + recording.gets(), + [ + "rec/1080p/.info", + "rec/1080p/groups/0000000000000000002.0000000000000000002" + ] + ); + + // An audio segment spans four groups but still costs one object GET. + let audio = replay + .rendition(Kind::Audio, "audio") + .segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + assert_eq!( + recording.gets(), + [ + "rec/audio/.info", + "rec/audio/groups/0000000000000000007.0000000000000000004" + ] + ); + + // A repeated request hits the reader's cache. + replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); + assert_eq!(recording.gets(), Vec::::new()); +} + +#[tokio::test] +async fn a_bounded_cache_rereads_evicted_objects() { + let recording = three_segments().await; + // Too small for any object, so nothing stays cached. + let replay = Replay::open(&recording, 1).await; + replay.playlist(Kind::Audio, "audio").await; + recording.gets(); + + // Each of the segment's four groups misses the cache and GETs the same object again. + let audio = replay + .rendition(Kind::Audio, "audio") + .segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + let gets: Vec = recording.gets().into_iter().filter(|path| is_media(path)).collect(); + assert_eq!( + gets, + vec!["rec/audio/groups/0000000000000000007.0000000000000000004"; 4] + ); +} + +#[tokio::test] +async fn missing_track_segments_are_gaps_and_time_jumps_are_discontinuities() { + let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; + recording.media("360p", &[(0, &[0])]).await; + recording.media("1080p", &[(0, &[0])]).await; + recording + .commit(&record(0, 0, 2000, &[("360p", 0, 0), ("1080p", 0, 0)]), 0) + .await; + // 1080p stored nothing for segment 1. + recording.media("360p", &[(1, &[2_000_000])]).await; + recording.commit(&record(1, 2000, 2000, &[("360p", 1, 1)]), 0).await; + // Content time jumps from 4s to 10s. + recording.media("360p", &[(2, &[10_000_000])]).await; + recording.media("1080p", &[(1, &[10_000_000])]).await; + recording + .commit(&record(2, 10_000, 2000, &[("360p", 2, 2), ("1080p", 1, 1)]), 0) + .await; + + let replay = Replay::open(&recording, 64 * 1024 * 1024).await; + let high = replay.playlist(Kind::Video, "1080p").await; + let expected = concat!( + "#EXTINF:2.00000,\nseg/0.m4s\n", + "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/1.m4s\n", + "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n", + ); + assert!(high.contains(expected), "{high}"); + let low = replay.playlist(Kind::Video, "360p").await; + assert!(!low.contains("#EXT-X-GAP"), "{low}"); + assert!( + low.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n"), + "{low}" + ); + + // A gap is never fetched. + recording.gets(); + let rendition = replay.rendition(Kind::Video, "1080p"); + assert!(rendition.segment(1).await.unwrap().is_none()); + assert!(!recording.gets().iter().any(|path| is_media(path))); + let after = rendition.segment(2).await.unwrap().unwrap(); + assert_eq!(&after[4..8], b"moof"); +} + +#[tokio::test] +async fn a_growing_recording_ends_only_on_caller_finality() { + let mut recording = three_segments().await; + let mut replay = Replay::open(&recording, 64 * 1024 * 1024).await; + replay.playlist(Kind::Video, "360p").await; + + // A DVR commit: segment 3 arrives and segment 0 expires. + recording.media("360p", &[(3, &[6_000_000])]).await; + recording.commit(&record(3, 6000, 2000, &[("360p", 3, 3)]), 1).await; + recording.gets(); + let mut reader = replay.reader.take().unwrap(); + reader.refresh().await.unwrap(); + let playlist = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/3.m4s\n")) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); + assert!(!playlist.contains("seg/0.m4s"), "{playlist}"); + assert!( + !playlist.contains("#EXT-X-ENDLIST"), + "a recording without finality stays live" + ); + let gets = recording.gets(); + assert!( + !gets.iter().any(|path| is_media(path)), + "following reads only the timeline: {gets:?}" + ); + + // The store holds no completion marker; the caller supplies finality. + reader.finish().unwrap(); + let playlist = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; + assert!(playlist.contains("seg/3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 4b5770a10e..8b3b424189 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -6,7 +6,10 @@ //! per-track group ranges), and media bytes move only when an HTTP client requests a segment, //! which FETCHes exactly the groups that segment covers from the relay cache and transmuxes //! them to CMAF. A broadcast whose catalog advertises no timeline can't be served this way and -//! is skipped. +//! is skipped. A `moq-archive` recording replayed onto a broadcast is served the same way: its +//! playlists read only the stored timeline, and each segment GETs one stored object. +//! An inline-parameter-set codec with no catalog `description` GETs one keyframe +//! group on the first playlist render to build its init, then caches it. //! //! The same machinery serves two kinds of consumer: //! @@ -18,6 +21,8 @@ //! yield every rendition and every finalized segment in order, for mirroring a broadcast to //! storage. +#[cfg(test)] +mod archive_tests; pub mod master; mod mpd; mod playlist; From 25a852d65dbbc947aa1151dccc231e77a7f7c402 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 10:18:33 -0700 Subject: [PATCH 08/49] feat(moq-hls): list a durable archive timeline without the live window (#4155) Co-authored-by: Claude Opus 5.5 --- doc/bin/hls.md | 13 ++++-- quest/m1/archive/README.md | 11 +++-- quest/m1/archive/hls-window.md | 20 -------- rs/moq-cli/src/hls.rs | 1 + rs/moq-hls/src/export/archive_tests.rs | 65 +++++++++++++++++++++----- rs/moq-hls/src/export/mod.rs | 15 ++++++ rs/moq-hls/src/export/mpd.rs | 42 +++++++++++------ rs/moq-hls/src/export/rendition.rs | 5 +- rs/moq-hls/src/export/renditions.rs | 31 ++++++++---- rs/moq-hls/src/export/segments.rs | 27 ++++++----- 10 files changed, 154 insertions(+), 76 deletions(-) delete mode 100644 quest/m1/archive/hls-window.md diff --git a/doc/bin/hls.md b/doc/bin/hls.md index ddb9028029..cc85030f74 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -44,10 +44,12 @@ from the replayed timeline alone, and a segment GETs exactly one stored object of its rendition, so switching renditions never downloads both. An inline-parameter-set codec with no catalog `description` is the exception: the first playlist render GETs one keyframe group to build the init segment, -then caches it. Out-of-band configs need no media GET. Set `--window` -to cover the recording. The playlist ends with `EXT-X-ENDLIST` only once the -reader's caller declares the recording finished; the store holds no completion -marker. +then caches it. Out-of-band configs need no media GET. When the catalog's +`archive` entry names a `store` and no `replay` path, its ranges are durable on +this broadcast, so the playlists list the whole retained timeline and only the +recording's own retention trims them; DASH `timeShiftBufferDepth` is the listed +span. The playlist ends with `EXT-X-ENDLIST` only once the reader's caller +declares the recording finished; the store holds no completion marker. The init URL carries a hash of its bytes, so a reconfigured rendition gets a new one. An embedder of the library can also label the publisher's run with @@ -55,7 +57,8 @@ new one. An embedder of the library can also label the publisher's run with (`seg/{generation}.{segment}.m4s`), since a restarted publisher reuses segment numbers for different media. -`--window` sets the playlist duration (default 16 s), +`--window` sets the live playlist duration (default 16 s) and caps segment +`Cache-Control: max-age` for every broadcast, `--listen-tls-cert`/`--listen-tls-key` or `--listen-tls-generate` serve HTTPS, and `--cors-origin` opens it to browsers. H.264/H.265 and AAC/Opus renditions are served. Import handles classic HLS; diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index fabafaf0e1..b9117707b2 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -38,8 +38,14 @@ The segment engine is in `rs/moq-mux/src/timeline.rs`: with no archive-specific code (`rs/moq-hls/src/export/archive_tests.rs`): playlists read only the timeline (an inline parameter set also GETs one keyframe group to build its init), and a segment GETs one object of its - rendition. The caller supplies the catalog, and `--window` must cover the - recording. + rendition. The caller supplies the catalog. +- A catalog `archive` entry with a `store` and no `replay` path declares its + ranges durable on that broadcast, so the exporter lists the whole retained + timeline and only its pops trim it (`durable` in + `rs/moq-hls/src/export/mod.rs`). DASH `timeShiftBufferDepth` becomes the + listed span, and `--window` still bounds live playlists and caps segment + `max-age`. The catalog already states durability, so no per-broadcast + option or separate server is needed. `rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`: percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list/delete. @@ -120,7 +126,6 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests -- [Archive HLS window](/quest/m1/archive/hls-window.md) - serve a replayed recording's whole retained timeline without a server-wide `--window` - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback - [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation diff --git a/quest/m1/archive/hls-window.md b/quest/m1/archive/hls-window.md deleted file mode 100644 index 33cafc2466..0000000000 --- a/quest/m1/archive/hls-window.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Archive HLS window - -## Goal - -`moq-hls` lists a replayed recording's whole retained timeline in HLS and DASH -without a server-wide `--window`, while live broadcasts keep a window within -the relay's cache. - -## Plan - -The export window (`moq_hls::export::Config::window`) trims every broadcast a -server exports. A recording served through `moq_archive::Reader` already -bounds itself: DVR expiry pops records from the replayed timeline, and an -unbounded archive never pops. Today an operator must raise `--window` past the -recording's length, which also inflates segment `Cache-Control: max-age` and -DASH `timeShiftBufferDepth` for every live broadcast on the same server. - -Decide how an export learns that a broadcast's timeline is authoritative (the -catalog's `archive` entry carrying a store or replay path, a per-broadcast -option, or a separate server) and cap the derived HTTP and DASH values. diff --git a/rs/moq-cli/src/hls.rs b/rs/moq-cli/src/hls.rs index f09a193cde..6a50170c84 100644 --- a/rs/moq-cli/src/hls.rs +++ b/rs/moq-cli/src/hls.rs @@ -34,6 +34,7 @@ pub struct ExportArgs { /// Minimum media listed in each rendition's playlist window. Keep it within the /// relay's group-cache retention, since segments are fetched from there on request. + /// A timeline durable in a catalog-named store lists everything it retains instead. #[usage(long, default = "16s")] pub window: crate::duration::Duration, diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 3d5450c584..139434de14 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -196,11 +196,19 @@ fn record(segment: u64, pts: u64, duration: u64, tracks: &[(&str, u64, u64)]) -> record } -/// The catalog an exporter is handed: every rendition's config, plus the archive entry naming -/// the recording's timeline. Out-of-band configs, so no init needs media. -fn catalog() -> hang::Catalog { +/// The archive entry a replay advertises: the recording's timeline, durable in its store. +fn durable() -> hang::catalog::Archive { + let mut archive = hang::catalog::Archive::new(TIMELINE); + archive.store = Some("memory:///rec/".parse().unwrap()); + archive.version = Some(hang::catalog::Archive::VERSION); + archive +} + +/// The catalog an exporter is handed: every rendition's config, plus `archive`. Out-of-band +/// configs, so no init needs media. +fn catalog(archive: hang::catalog::Archive) -> hang::Catalog { let mut catalog = hang::Catalog::default(); - catalog.archive = Some(hang::catalog::Archive::new(TIMELINE)); + catalog.archive = Some(archive); for (name, width, height) in [("360p", 640, 360), ("1080p", 1920, 1080)] { let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); config.coded_width = Some(width); @@ -224,7 +232,7 @@ struct Replay { } impl Replay { - async fn open(recording: &Recording, cache: u64) -> Self { + async fn open(recording: &Recording, cache: u64, archive: hang::catalog::Archive) -> Self { let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); tokio::spawn(moq_net::time::run(driver)); let broadcast = origin.create_broadcast("rec").unwrap(); @@ -235,7 +243,7 @@ impl Replay { let mut json = moq_json::snapshot::Config::default(); json.delta_ratio = 0; let mut catalog = moq_json::snapshot::Producer::new(track, json); - catalog.update(&self::catalog()).unwrap(); + catalog.update(&self::catalog(archive)).unwrap(); let config = reader::Config::new(TIMELINE).with_cache(cache); let reader = moq_archive::Reader::open(recording.store.clone(), &broadcast, config) @@ -296,8 +304,13 @@ fn is_media(path: &str) -> bool { /// Three aligned 2s segments: one keyframe group per video rendition, four audio groups each. async fn three_segments() -> Recording { + segments(3).await +} + +/// `count` aligned 2s segments, laid out like [`three_segments`]. +async fn segments(count: u64) -> Recording { let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; - for segment in 0..3u64 { + for segment in 0..count { let pts = segment * 2_000_000; for video in ["360p", "1080p"] { recording.media(video, &[(segment, &[pts, pts + 1_000_000])]).await; @@ -324,7 +337,7 @@ async fn three_segments() -> Recording { #[tokio::test] async fn playlists_read_only_the_timeline_and_segments_one_object() { let recording = three_segments().await; - let replay = Replay::open(&recording, 64 * 1024 * 1024).await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; let master = replay.broadcaster.master_playlist(None); assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); @@ -396,7 +409,7 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { async fn a_bounded_cache_rereads_evicted_objects() { let recording = three_segments().await; // Too small for any object, so nothing stays cached. - let replay = Replay::open(&recording, 1).await; + let replay = Replay::open(&recording, 1, durable()).await; replay.playlist(Kind::Audio, "audio").await; recording.gets(); @@ -433,7 +446,7 @@ async fn missing_track_segments_are_gaps_and_time_jumps_are_discontinuities() { .commit(&record(2, 10_000, 2000, &[("360p", 2, 2), ("1080p", 1, 1)]), 0) .await; - let replay = Replay::open(&recording, 64 * 1024 * 1024).await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; let high = replay.playlist(Kind::Video, "1080p").await; let expected = concat!( "#EXTINF:2.00000,\nseg/0.m4s\n", @@ -460,7 +473,7 @@ async fn missing_track_segments_are_gaps_and_time_jumps_are_discontinuities() { #[tokio::test] async fn a_growing_recording_ends_only_on_caller_finality() { let mut recording = three_segments().await; - let mut replay = Replay::open(&recording, 64 * 1024 * 1024).await; + let mut replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; replay.playlist(Kind::Video, "360p").await; // A DVR commit: segment 3 arrives and segment 0 expires. @@ -491,3 +504,33 @@ async fn a_growing_recording_ends_only_on_caller_finality() { .await; assert!(playlist.contains("seg/3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); } + +/// A durable timeline lists the whole recording past the default 16s window, and DASH offers +/// the whole listed span. A live-style entry, or a `replay` path that moves the durable ranges +/// to another broadcast, keeps the window. +#[tokio::test] +async fn a_durable_timeline_lists_past_the_window() { + let recording = segments(12).await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + let playlist = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/11.m4s\n")) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0\n"), "{playlist}"); + assert!(playlist.contains("seg/0.m4s\n"), "{playlist}"); + + for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { + replay.rendition(kind, name).init().await.unwrap(); + } + let manifest = replay.broadcaster.manifest(None).expect("manifest renders"); + assert!(manifest.contains("timeShiftBufferDepth=\"PT24.000S\""), "{manifest}"); + + let mut elsewhere = durable(); + elsewhere.replay = Some(moq_net::path::RelativeOwned::new("./recording")); + for archive in [hang::catalog::Archive::new(TIMELINE), elsewhere] { + let live = Replay::open(&recording, 64 * 1024 * 1024, archive).await; + let playlist = live + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/11.m4s\n")) + .await; + assert!(!playlist.contains("seg/0.m4s\n"), "{playlist}"); + } +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 8b3b424189..e32845e5e7 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -66,6 +66,10 @@ pub struct Config { /// Minimum duration of media listed in each rendition's playlist window. Older timeline /// records are evicted once the remaining segments still cover this span; keep it within /// the relay's group-cache retention, since segments are fetched from there on request. + /// + /// A durable timeline (its catalog `archive` entry names a `store` and no `replay` path) + /// lists everything it retains instead, since its own retention already bounds it. The + /// window still caps segment `Cache-Control: max-age` for every broadcast. pub window: Duration, } @@ -347,6 +351,9 @@ async fn watch_catalog( // records out to every rendition. if !timeline_started && let Some(archive) = catalog.archive.clone() { timeline_started = true; + if durable(&archive) { + renditions.fanout().unbound(); + } let watcher = tokio::spawn(watch_timeline(broadcast.clone(), archive, renditions.fanout())); *timeline_watcher.lock().unwrap() = Some(watcher); } @@ -363,6 +370,14 @@ async fn watch_catalog( renditions.close(); } +/// Whether every range `archive` advertises stays FETCHable from this broadcast until the +/// timeline pops it: a store makes the ranges durable, and no `replay` path means this +/// broadcast serves them. The catalog states this, so the playlists follow the timeline's own +/// retention rather than a window sized for relay caches. +fn durable(archive: &hang::catalog::Archive) -> bool { + archive.store.is_some() && archive.replay.is_none() +} + /// The broadcast's timeline watcher: read the single timeline track and fan each record out /// to every rendition's window. async fn watch_timeline( diff --git a/rs/moq-hls/src/export/mpd.rs b/rs/moq-hls/src/export/mpd.rs index bea274ff80..11a4c333ee 100644 --- a/rs/moq-hls/src/export/mpd.rs +++ b/rs/moq-hls/src/export/mpd.rs @@ -61,8 +61,9 @@ pub(crate) struct Manifest { pub availability_start: Option, /// When this render happened (`MPD@publishTime`, dynamic only). pub publish: SystemTime, - /// The playlist window (`MPD@timeShiftBufferDepth`, dynamic only). - pub window: Duration, + /// The playlist window (`MPD@timeShiftBufferDepth`, dynamic only), or `None` when the + /// timeline bounds itself and the depth is the span it lists. + pub window: Option, /// The broadcast ended: render a `static` presentation instead of a `dynamic` one. pub finished: bool, /// Video representations, in catalog order. @@ -105,6 +106,23 @@ fn frame_rate(rate: f64) -> Option { } } +/// `units` of `timescale` as a [`Duration`]. +fn duration(units: u64, timescale: u32) -> Duration { + Duration::from_nanos((u128::from(units) * 1_000_000_000 / u128::from(timescale.max(1))) as u64) +} + +/// The longest span any representation lists, oldest segment start to newest segment end. +fn listed_span<'a>(representations: impl Iterator) -> Duration { + representations + .filter_map(|rep| { + let (first, _) = rep.segments.first()?; + let (last, d) = rep.segments.last()?; + Some(duration(last + d - first, rep.timescale)) + }) + .max() + .unwrap_or_default() +} + /// The largest listed segment duration in whole seconds, for `MPD@maxSegmentDuration` (and the /// update cadence). Like HLS's target duration, derived from the segments when the publisher /// declared no bound. @@ -225,10 +243,7 @@ pub(crate) fn render_manifest(manifest: &Manifest, query: Option<&str>) -> Strin let duration = representations() .filter_map(|rep| { let (t, d) = rep.segments.last()?; - let timescale = rep.timescale.max(1) as u64; - Some(Duration::from_nanos( - ((t + d) as u128 * 1_000_000_000 / timescale as u128) as u64, - )) + Some(duration(t + d, rep.timescale)) }) .max() .unwrap_or_default(); @@ -242,14 +257,15 @@ pub(crate) fn render_manifest(manifest: &Manifest, query: Option<&str>) -> Strin // Reload cadence and live delay follow HLS conventions: players refresh about once // per segment and sit a few segments behind the live edge (bounded by the window). let update = Duration::from_secs(target); - let delay = Duration::from_secs(3 * target).min(manifest.window.max(update)); + let window = manifest.window.unwrap_or_else(|| listed_span(representations())); + let delay = Duration::from_secs(3 * target).min(window.max(update)); let _ = write!( out, " type=\"dynamic\" availabilityStartTime=\"{}\" publishTime=\"{}\" minimumUpdatePeriod=\"{}\" timeShiftBufferDepth=\"{}\" suggestedPresentationDelay=\"{}\"", humantime::format_rfc3339_millis(availability), humantime::format_rfc3339_millis(manifest.publish), xs_duration(update), - xs_duration(manifest.window), + xs_duration(window), xs_duration(delay), ); } @@ -315,7 +331,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_751_846_400_123)), publish: SystemTime::UNIX_EPOCH + Duration::from_millis(1_751_846_410_000), - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000), (2_000, 2_000)], false)], audio: vec![audio(vec![(0, 2_000), (2_000, 2_000)], false)], @@ -355,7 +371,7 @@ mod tests { let manifest = Manifest { availability_start: None, publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: true, // The window starts mid-broadcast: presentation time stays anchored at pts 0 (no // presentationTimeOffset), so the duration spans the lead-in and a live session @@ -379,7 +395,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000)], false)], audio: Vec::new(), @@ -397,7 +413,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![rep], audio: Vec::new(), @@ -420,7 +436,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000)], false)], audio: vec![audio(Vec::new(), false)], diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index f1f1a03691..abb534c590 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -343,8 +343,9 @@ impl Rendition { } /// Feed one timeline record into this rendition's window: its own ranges (empty when the - /// record carries none for it, a gap), timed by the record. - pub(crate) fn push(&self, index: u64, entry: &Entry, discontinuity: u64, window: Duration) { + /// record carries none for it, a gap), timed by the record. With no `window`, only source + /// timeline pops trim it. + pub(crate) fn push(&self, index: u64, entry: &Entry, discontinuity: u64, window: Option) { if !self.media.admits(&self.live) { return; } diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index 63815fed7c..ab66923500 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -65,6 +65,10 @@ struct Feed { anchor: Option, /// The publisher run every segment URL carries; late-created renditions inherit it. generation: Option>, + /// The playlist window duration applied on every push (see + /// [`Config::window`](super::Config::window)), or `None` when the source timeline is + /// authoritative and only its pops trim the playlists. + window: Option, } /// The producing side of a broadcast's rendition set. @@ -87,8 +91,6 @@ pub(crate) struct Producer { #[derive(Clone)] pub(crate) struct Fanout { feed: Arc>, - /// The playlist window duration (see [`Config::window`](super::Config::window)), applied on every push. - window: Duration, } impl Producer { @@ -104,8 +106,8 @@ impl Producer { closed: false, anchor: None, generation: None, + window: Some(window), })), - window, }, } } @@ -115,9 +117,10 @@ impl Producer { self.fanout.clone() } - /// The playlist window duration every rendition's window is trimmed to. - pub fn window(&self) -> Duration { - self.fanout.window + /// The playlist window duration every rendition's window is trimmed to, or `None` when + /// only the source timeline trims them. + pub fn window(&self) -> Option { + self.fanout.feed.lock().unwrap().window } /// The estimated wall-clock time of timeline `pts` 0 (see [`Feed::anchor`]); `None` until @@ -184,6 +187,12 @@ impl Producer { } impl Fanout { + /// List every record the source timeline retains, trimming only on its pops, instead of + /// a window. Call before the first record. + pub fn unbound(&self) { + self.feed.lock().unwrap().window = None; + } + /// Fan one timeline record out to every living rendition, and into the replay history. pub fn push(&self, index: u64, entry: Entry) { let mut feed = self.feed.lock().unwrap(); @@ -205,17 +214,19 @@ impl Fanout { let pts = Duration::from(entry.pts); let discontinuity = feed.discontinuities.stamp(pts, pts + entry.duration); feed.history.push_back((index, entry.clone(), discontinuity)); - while feed.history.len() >= 2 { + let window = feed.window; + while let Some(window) = window + && feed.history.len() >= 2 + { let newest = &feed.history.back().unwrap().1; let span = (Duration::from(newest.pts) + newest.duration).saturating_sub(Duration::from(feed.history[1].1.pts)); - if span < self.window { + if span < window { break; } feed.history.pop_front(); } - let window = self.window; feed.targets.retain(|target| { let Some(rendition) = target.upgrade() else { return false; @@ -335,7 +346,7 @@ impl Producer { let mut feed = self.fanout.feed.lock().unwrap(); rendition.label(feed.generation.clone()); for (index, entry, discontinuity) in &feed.history { - rendition.push(*index, entry, *discontinuity, self.fanout.window); + rendition.push(*index, entry, *discontinuity, feed.window); } if feed.ended { rendition.end(); diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 015f1f2509..6b8a92a0b6 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -166,8 +166,9 @@ impl Producer { } } - /// Append a row, evicting the front of the window past `window`. - pub fn push(&self, row: Row, window: Duration) { + /// Append a row, evicting the front of the window past `window`. With no `window`, only + /// source timeline pops remove rows. + pub fn push(&self, row: Row, window: Option) { let Ok(mut state) = self.state.write() else { return; }; @@ -187,7 +188,9 @@ impl Producer { state.rows.push_back(row); // Evict from the front while the remaining rows still cover the window. - while state.rows.len() >= 2 { + while let Some(window) = window + && state.rows.len() >= 2 + { let span = state.rows.back().unwrap().end.saturating_sub(state.rows[1].pts.into()); if span < window { break; @@ -439,8 +442,8 @@ mod tests { #[test] fn every_row_is_listed() { let live = Producer::new(); - live.push(row(0, 0, 0, 2_000), Duration::from_secs(30)); - live.push(row(1, 1, 2_000, 2_000), Duration::from_secs(30)); + live.push(row(0, 0, 0, 2_000), Some(Duration::from_secs(30))); + live.push(row(1, 1, 2_000, 2_000), Some(Duration::from_secs(30))); let window = live.window(); assert_eq!(window.sequence, 0); @@ -456,7 +459,7 @@ mod tests { #[test] fn window_evicts_and_advances_sequence() { let live = Producer::new(); - let window = Duration::from_secs(4); + let window = Some(Duration::from_secs(4)); for i in 0..6u64 { live.push(row(i, i, i * 2_000, 2_000), window); } @@ -473,7 +476,7 @@ mod tests { #[test] fn source_window_pop_removes_playlist_rows() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); for i in 0..4u64 { let mut row = row(i, i, i * 2_000, 2_000); row.index = i + 10; @@ -497,9 +500,9 @@ mod tests { #[test] fn a_skipped_source_range_clears_rows_before_the_next_segment() { let live = Producer::new(); - live.push(row(4, 4, 8_000, 2_000), Duration::from_secs(10)); + live.push(row(4, 4, 8_000, 2_000), Some(Duration::from_secs(10))); live.clear(); - live.push(row(10, 10, 20_000, 2_000), Duration::from_secs(10)); + live.push(row(10, 10, 20_000, 2_000), Some(Duration::from_secs(10))); let snapshot = live.window(); assert_eq!(snapshot.sequence, 10); @@ -567,7 +570,7 @@ mod tests { #[test] fn segment_ranges_and_gaps() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 1_000), window); // Segment 1 is a gap for this rendition: no ranges. live.push( @@ -595,7 +598,7 @@ mod tests { #[test] fn backwards_jump_resets_the_window() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 10_000, 2_000), window); live.push(row(1, 1, 12_000, 2_000), window); live.push(row(2, 2, 1_000, 2_000), window); // restart: pts rewound @@ -612,7 +615,7 @@ mod tests { #[test] fn next_after_walks_segments() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 2_000), window); live.push(row(1, 1, 2_000, 2_000), window); From fb1f7f6eea28cc1ea1c1264db5651d3c9cbc726c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 12:55:18 -0700 Subject: [PATCH 09/49] quest(archive): per-track timelines and replay catalog (#4168) Co-authored-by: Claude Opus 5.5 --- quest/m1/archive/README.md | 5 +++ quest/m1/archive/browser.md | 21 +++++----- quest/m1/archive/dvr.md | 9 +++++ quest/m1/archive/proof.md | 9 ++++- quest/m1/archive/replay-catalog.md | 48 +++++++++++++++++++++++ quest/m1/archive/track-timeline/README.md | 46 ++++++++++++++++++++++ quest/m1/archive/track-timeline/core.md | 46 ++++++++++++++++++++++ quest/m1/archive/track-timeline/js.md | 17 ++++++++ 8 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 quest/m1/archive/replay-catalog.md create mode 100644 quest/m1/archive/track-timeline/README.md create mode 100644 quest/m1/archive/track-timeline/core.md create mode 100644 quest/m1/archive/track-timeline/js.md diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index b9117707b2..89cd0aedad 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -63,6 +63,9 @@ DVR expiry recover from the next checkpoint; `Reader::finish` applies out-of-ban ### Format +[Per-track timelines](/quest/m1/archive/track-timeline/README.md) replaces the +aligned segments below with one timeline per track. + The format is the draft's [Recording section](/drafts/draft-lcurley-moq-hang.md#recording). The application chooses the object prefix, selected tracks, retention, and credentials; `moq-archive` owns the @@ -126,6 +129,8 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests +- [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge +- [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback - [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation diff --git a/quest/m1/archive/browser.md b/quest/m1/archive/browser.md index 78a0dd061c..7d08f060cf 100644 --- a/quest/m1/archive/browser.md +++ b/quest/m1/archive/browser.md @@ -9,16 +9,16 @@ layout, and FETCH behavior as native `moq-archive` users. Port the archive contract to the JS packages with memory and OPFS storage. The application explicitly enrolls video, audio, catalog, or arbitrary data tracks; -the archive does not infer them from Hang. The JS timeline already publishes -through the Window (`js/hang/src/container/timeline.ts:134`, `:165`, backed by -`js/json/src/window/`) with pacing tracks and application-driven cuts; add the -deferred commit Rust has (`Producer::deferred`, -`rs/moq-mux/src/timeline.rs:917`). - -Persist one range-named object per track per segment after its groups complete, -then publish the archive timeline record. Match the 19-digit group-bound keys, -ascending delta-encoded IDs, and sequential timeline discovery used by Rust. A typical audio segment contains many -one-group-per-frame audio groups. Match the Rust binary envelope bytes and `.info` property values, per the [Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without inferring catalog-to-group applicability. +the archive does not infer them from Hang. Record against the per-track +timelines from [JS per-track timelines](/quest/m1/archive/track-timeline/js.md). + +Commit each track independently: persist each stored span, including a frame +range of a still-open group, then publish that track's timeline record. Match +the object keys, envelope bytes, `.info` property values, and timeline +discovery the Rust writer uses after +[Rust per-track timelines](/quest/m1/archive/track-timeline/core.md), per the +[Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without +inferring catalog-to-group applicability. [Catalog track identity](/quest/m2/catalog-tracks.md) addresses that separately. Use [JavaScript FETCH](/quest/m1/js-fetch.md)'s on-demand group requests to @@ -34,3 +34,4 @@ remains downstream (moq.pro) work. ## Required - [JavaScript FETCH](/quest/m1/js-fetch.md) - generic on-demand group serving and IETF FETCH support +- [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - the timeline this archive records against diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 096633a809..48959cd37c 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -26,6 +26,15 @@ The reader evicts popped spans from its object cache, but a group it already served stays in `moq_net`'s track cache until the pool reclaims it. Decide whether expiry during a seek needs a group eviction API in `moq-net`. +`moq-hls` reads a timeline from the catalog's own broadcast, and an +`archive.replay` path only marks it non-durable. Decide whether a live +broadcast's exporter follows `replay` to the recording for rewind, or whether +viewers address the replay broadcast directly. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - seeks through per-track timelines + ## Closes - [#2275](https://github.com/moq-dev/moq/issues/2275) - close this issue when the quest finishes diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md index 225449edf4..b24575dbb1 100644 --- a/quest/m1/archive/proof.md +++ b/quest/m1/archive/proof.md @@ -7,8 +7,9 @@ reads, and timeline-only HLS generation from one multi-rendition broadcast. ## Plan -Record explicitly enrolled video, audio, catalog, and non-media tracks. Cut at -aligned keyframe boundaries, including multiple audio groups per segment, then +Record explicitly enrolled video, audio, catalog, and non-media tracks. Cut each +track on its own timeline, including many audio groups per object and a group +split across objects by frame, then replay their original sequences, timestamps, and payloads through `track::Dynamic`. @@ -60,3 +61,7 @@ restart must finish this cleanup before accepting new groups. Finally render and reload HLS playlists while rejecting every media-object GET until a segment URI is requested. A segment request must resolve one object from the replayed timeline without any listing or separate index object. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - proves the per-track format, not the aligned one diff --git a/quest/m1/archive/replay-catalog.md b/quest/m1/archive/replay-catalog.md new file mode 100644 index 0000000000..d6f5bb8ec5 --- /dev/null +++ b/quest/m1/archive/replay-catalog.md @@ -0,0 +1,48 @@ +# [S] Replay catalog + +## Goal + +A replayed recording plays as VOD HLS from the stock server: `moq export +archive`, then `moq import archive`, then `moq export hls` lists the whole +recording with no embedder supplying a catalog. The replay broadcast publishes +its recorded catalog live, stamped with the recording's `store` and `version`, +so the exporter treats its timeline as durable and lists past `--window`. A +`--follow` replay grows like an event playlist. + +Choosing which catalog applies to which media group stays with +[Catalog track identity](/quest/m2/catalog-tracks.md). + +## Plan + +Today the reader writes live groups only for timelines; the recorded catalog +is FETCH-only, so a SUBSCRIBE to it on the replay broadcast never sees a group +and `moq-hls` never finds the `archive` entry. The HLS archive tests work around +this by hand-building a catalog. + +- Republish each recorded catalog group live in its timeline's order, so the newest + one is at the live edge and a `--follow` replay picks up catalogs recorded + after it opened. +- Stamp `store` with the URL passed to `import archive`, and `version` with the + recording format. Refuse a URL carrying userinfo so credentials never land in + a catalog. `replay` stays unset: the timelines live on this broadcast. +- Keep the logic in `moq-cli`; `moq-archive` stays catalog-agnostic. Select the + catalog track with the CLI's catalog format as `export archive` does: hang + and hang.z are stamped, MSF is refused. +- The recorded catalog's `archive` entry describes the source's live + timelines, not the recording's, so replace it with the timelines the reader + replays rather than trusting the recorded ones. + +Add a CI test that records a broadcast longer than the default window, replays +it, and asserts the stock exporter lists segment 0 with a durable +`timeShiftBufferDepth`. Consider moving the HLS archive tests onto the +republished catalog instead of their hand-built one. + +Update `doc/bin/cli.md` and `doc/bin/hls.md` inline with an export, import, and +serve example. + +Cover a DVR recording whose catalog outlived its first video segment, and fail +loudly on a recording with no catalog. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - a DVR keeps its catalog's newest group diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md new file mode 100644 index 0000000000..2b4190e23b --- /dev/null +++ b/quest/m1/archive/track-timeline/README.md @@ -0,0 +1,46 @@ +# Per-track timelines + +## Goal + +Every track carries its own timeline, so tracks segment, commit, and expire +independently. A DVR keeps each track's newest group, such as a catalog that +never changes, and an append-only group stays addressable as frames arrive. An +edge like `moq-hls` derives HLS and DASH from group timestamps, so a publisher +never needs to know about HLS. + +Nothing on `main` has users yet: break the timeline, catalog `archive` entry, +and recording format in place, with no compatibility path, even though hang +0.21 released the current shape. The children merge into this line's branch, +so Rust and JS reach `main` together. + +## Plan + +Today one timeline numbers aligned segments for every track. A DVR pops whole +segments, so a static catalog expires with the first video segment, and the +writer only records a non-pacing group once it completes, so a group that never +closes is never stored. `moq_json::window` pops from the front, so one shared +timeline cannot keep an old catalog record while dropping newer video. + +Decisions: + +- One timeline track per track, live and recorded. `moq-mux` publishes them for + every broadcast; an unsubscribed track costs nothing. +- The catalog's root `archive` entry maps each track to its timeline, including + the catalog track itself. `replay`, `store`, and `version` stay beside it. +- Each track cuts on its own: automatically at a group boundary between a + minimum and maximum duration (roughly 1s and 10s), splitting a long-lived + group by frame at the maximum. Manual cuts stay as an optimization, such as a + video keyframe cutting audio so derived segments need fewer objects. +- A stored object may hold a frame range of a group, not only whole groups. +- HLS and DASH segments are derived at the edge from group timestamps, not + from storage objects. Fetching extra objects is fine when they land in the + reader's cache for the next request. + +## Quests + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the draft, `hang`, `moq-mux`, `moq-archive`, and `moq-hls` move to per-track timelines in one change +- [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust + +## Related + +- [Catalog track identity](/quest/m2/catalog-tracks.md) - the catalog's own timeline gives it timestamps, but which catalog applies to a group stays there diff --git a/quest/m1/archive/track-timeline/core.md b/quest/m1/archive/track-timeline/core.md new file mode 100644 index 0000000000..4651a338f9 --- /dev/null +++ b/quest/m1/archive/track-timeline/core.md @@ -0,0 +1,46 @@ +# [XL] Rust per-track timelines + +## Goal + +The moq-hang draft, `hang`, `moq-mux`, `moq-archive`, and `moq-hls` use one +timeline per track, per the [line's decisions](/quest/m1/archive/track-timeline/README.md). +A DVR recording keeps every track's newest group, an append-only group is +stored as its frames arrive, and `moq-hls` serves aligned HLS and DASH from +live or recorded timelines without the publisher cutting for it. + +## Plan + +These crates compile together, so they change in one PR. Update the draft's +timeline and Recording sections in the same change, in a new draft revision; +most of their rules assume one aligned segment counter (one timeline per +broadcast, cross-track boundaries, whole-segment retention, reading segment N +of track T through record N). + +- **Timeline:** a record describes one stored span of its own track: sequence, + pts, duration, and the group and frame range. Drop cross-track pacing and the + completeness wait; keep manual cuts. Retention pops each track's window + independently and always keeps its newest record. +- **Catalog:** replace the root `archive.track` with a map from track to + timeline, covering the catalog track too. +- **Writer and reader:** commit each track independently, and store frame + ranges so a never-closing group is recorded. Bump the recording `version` and + refuse the old one. Recovery, grace deletion, and FETCH replay follow the + per-track index. +- **HLS and DASH:** take segment boundaries from a reference video rendition + (an audio one when the broadcast has no video), near a target duration, and + number them so every edge and every reload agrees, including after DVR pops. Every other video rendition snaps each + boundary to its nearest group start within a tolerance (around 1s); a + segment with no start in range becomes a gap (`EXT-X-GAP` in HLS), so a + player switching renditions lands on the next real segment. Gaps are a + best-effort fallback: the draft and HLS docs say a publisher wanting HLS + export SHOULD align video GOPs across renditions. Audio and other + renditions take the groups and frames whose timestamps fall in each span, + possibly from more than one object. The reader's cache absorbs the overlap. + +Carry over the existing tests (DVR trim, durable listing, archive replay) and +add a static catalog outliving its first video segment, an append-only group +spanning several objects, audio cut independently of video, renditions that +start at different times, video renditions whose group starts never +coincide, and an audio-only broadcast. + +Update `doc/concept/hang.md` and the HLS and CLI docs this makes stale. diff --git a/quest/m1/archive/track-timeline/js.md b/quest/m1/archive/track-timeline/js.md new file mode 100644 index 0000000000..461ff1cf75 --- /dev/null +++ b/quest/m1/archive/track-timeline/js.md @@ -0,0 +1,17 @@ +# [M] JS per-track timelines + +## Goal + +`@moq/hang` publishes one timeline per track with the same records, cuts, and +catalog `archive` map as Rust, and parses them identically. + +## Plan + +Port the landed Rust shape to `js/hang/src/timeline.ts` and its catalog schema, +replacing the aligned timeline and cross-track pacing rather than keeping both. +Cover the same cut rules and a static catalog outliving other tracks' records, +and check the records against Rust output in the interop suite. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the format this mirrors From 2ae46078d22da1ee23773b03b39276f5b19c7ca3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:30:09 -0700 Subject: [PATCH 10/49] fix(moq-cli): declare moq-json dev-dependency by path (#4166) Co-authored-by: Claude Opus 5.5 From d6059370ff90af3de57d602b1025c9254fc389f3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:49:05 -0700 Subject: [PATCH 11/49] test(archive): prove recording, replay, and retention end to end (#4169) Co-authored-by: Claude Opus 5.5 --- quest/m1/archive/README.md | 5 +- quest/m1/archive/proof.md | 67 ----- quest/m1/archive/pruning.md | 17 ++ quest/m2/README.md | 1 + quest/m2/archive-s3.md | 22 ++ rs/moq-archive/src/lib.rs | 4 + rs/moq-archive/src/mock.rs | 272 +++++++++++++++++ rs/moq-archive/src/path.rs | 50 ++++ rs/moq-archive/src/proof.rs | 395 +++++++++++++++++++++++++ rs/moq-archive/src/reader/mod.rs | 25 +- rs/moq-archive/src/reader/tests.rs | 245 ++++++++++----- rs/moq-archive/src/segment.rs | 61 ++++ rs/moq-archive/src/store.rs | 212 ++++++------- rs/moq-archive/src/writer.rs | 237 +++++++++------ rs/moq-hls/src/export/archive_tests.rs | 35 ++- 15 files changed, 1290 insertions(+), 358 deletions(-) delete mode 100644 quest/m1/archive/proof.md create mode 100644 quest/m1/archive/pruning.md create mode 100644 quest/m2/archive-s3.md create mode 100644 rs/moq-archive/src/mock.rs create mode 100644 rs/moq-archive/src/proof.rs diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 89cd0aedad..1f3a9ff4d7 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -60,6 +60,9 @@ unreferenced group objects one grace period after recovery. supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` with a byte-bounded object LRU. `Reader::refresh` follows by listing timeline keys after its cursor, so gaps and DVR expiry recover from the next checkpoint; `Reader::finish` applies out-of-band finality. +`rs/moq-archive/src/proof.rs` records one multi-rendition broadcast end to end: its exact keys and +bytes match on memory, local disk, and an unordered listing, FETCH replays every group exactly, +and a rendition's playback GETs only that rendition's objects. ### Format @@ -133,7 +136,7 @@ owned by that prerequisite, not duplicated in archive storage. - [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback -- [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation +- [DVR timeline pruning](/quest/m1/archive/pruning.md) - a DVR deletes timeline objects no retained checkpoint needs ## Related diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md deleted file mode 100644 index b24575dbb1..0000000000 --- a/quest/m1/archive/proof.md +++ /dev/null @@ -1,67 +0,0 @@ -# [M] Archive proof - -## Goal - -Prove deterministic segment storage, exact FETCH replay, selective rendition -reads, and timeline-only HLS generation from one multi-rendition broadcast. - -## Plan - -Record explicitly enrolled video, audio, catalog, and non-media tracks. Cut each -track on its own timeline, including many audio groups per object and a group -split across objects by frame, then -replay their original sequences, timestamps, and payloads through -`track::Dynamic`. - -Verify the exact object keys and bytes on memory, local, and S3-compatible -`object_store` implementations, including the percent-encoded track names. A -360p or audio-only FETCH must not GET the 1080p object, while adjacent group -requests should hit the segment LRU. - -Cover the persistence boundary: a crash after segment PUT but before timeline -commit leaves invisible orphan data that a listing bootstrap ignores; a failed -or mismatched `.info` exposes no ranges; a failed independent track PUT omits -only that track while the record's other tracks stay advertised; later -segments remain usable. Catalog-to-group applicability is outside this proof. Also -cover a stalled pacing track forced to a gap, sparse group ranges, malformed -table offsets, an unknown envelope or `.info` version treated as a missing -segment, segment create collisions under the single-writer rule, a missing -tail, and a clean end without a completion marker. Accept equivalent `.info` -JSON with reordered members or different whitespace without rewriting it; -reject differing properties and malformed or unsupported metadata. - -Exercise group and segment IDs 0 and 2^53 - 1; reject 2^53 and the largest -QUIC varint in keys and reconstructed group IDs. Cover consecutive zero -deltas, sparse deltas, overflow, stopping at ID exhaustion without wrapping -or inferring clean finality, empty objects, mismatched filename bounds, and overlapping -ranges across segments. Reject decreasing or duplicate group arrivals while -allowing accepted groups to complete out of order. Verify JSON-safe timescales -and timestamps, accepting timestamp 2^53 - 1 and rejecting 2^53. -Test ordered S3 lookup at both endpoints and between ranges, unordered paginated -results, incremental cursor recovery, DVR expiration while a reader is offline, -and stale media listings preceding a new timeline commit. Following N+1 must -not refresh all media listings. Wire these cases into CI for the store, writer, -and reader implementations; do not add an unconnected standalone proof script. - -The store's own tests also cover recording-prefix isolation (`rec` beside -`rec-other`), empty prefixes, track-prefix listings, continuation pages, and -every supported pagination option. A page must not lose directory entries -silently or fail because the backend matched a neighbouring recording. Every -publicly constructible key either serializes to a path its parser accepts or -fails before storage; direct `Key::Groups` construction must not bypass range -validation. These cases belong beside the store and codec code and need no -new public API or format change. - -Crash a DVR writer after its pop becomes durable but before media deletion. -On exclusive restart, prove that expired and uncommitted group objects are -removed after the grace period while retained media, `.info`, and checkpoint -objects survive. Failed or incomplete recovery/listing must delete nothing; -restart must finish this cleanup before accepting new groups. - -Finally render and reload HLS playlists while rejecting every media-object GET -until a segment URI is requested. A segment request must resolve one object -from the replayed timeline without any listing or separate index object. - -## Required - -- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - proves the per-track format, not the aligned one diff --git a/quest/m1/archive/pruning.md b/quest/m1/archive/pruning.md new file mode 100644 index 0000000000..3e2dfcfbc6 --- /dev/null +++ b/quest/m1/archive/pruning.md @@ -0,0 +1,17 @@ +# [S] DVR timeline pruning + +## Goal + +A DVR writer deletes timeline objects that no retained checkpoint needs, so a +long-running DVR stores a bounded number of objects. + +## Plan + +`moq_archive::Writer` deletes expired group objects but never a +`segments/` timeline object, so a 24/7 DVR with 2s segments adds +about 43,000 objects a day, and every restart lists all of them. The draft +already allows it: the writer keeps the latest timeline object and enough +earlier groups to recover the retained window from a checkpoint. Delete the +oldest timeline objects no longer needed, one grace period after they stop +being needed, keeping the remaining keys contiguous, as recovery requires. +Extend the restart cleanup and its tests in `rs/moq-archive/src/writer.rs`. diff --git a/quest/m2/README.md b/quest/m2/README.md index f7cc1a225c..4cd3ed3c47 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -18,6 +18,7 @@ upstream release waits in [m4](/quest/m4/README.md). - [AV1 metadata separation](/quest/m2/av1-metadata.md) - retain metadata OBUs inline while evaluating separate delivery - [SEI separation](/quest/m2/sei/README.md) - retain inline SEI until measured savings or a metadata-only consumer justify a split - [Catalog track identity](/quest/m2/catalog-tracks.md) - compare immutable track definitions with explicit catalog-to-group binding +- [Archive S3 wire proof](/quest/m2/archive-s3.md) - the archive proof also runs through the S3 client against an in-process S3-compatible server - [Mobile ownership](/quest/m2/mobile-ownership.md) - decide whether Rust or platform code owns mobile capture, codecs, and rendering - [iOS capture](/quest/m2/mobile-capture-ios.md) - camera and screen capture if the mobile ownership decision selects Rust - [Android capture](/quest/m2/mobile-capture-android.md) - NDK/JNI capture using the existing codecs if mobile ownership selects Rust diff --git a/quest/m2/archive-s3.md b/quest/m2/archive-s3.md new file mode 100644 index 0000000000..d190490d8c --- /dev/null +++ b/quest/m2/archive-s3.md @@ -0,0 +1,22 @@ +# [S] Archive S3 wire proof + +## Goal + +The `moq-archive` proof (`rs/moq-archive/src/proof.rs`) also runs through +`object_store`'s S3 client against an in-process S3-compatible server, in CI +with no external network. + +## Plan + +The proof covers memory, local disk, and an S3-style listing fake +(`rs/moq-archive/src/mock.rs`), but not the S3 client: percent-encoded keys +such as `catalog%2Ejson` inside request URLs, `PutMode::Create` through +`If-None-Match`, `start-after` listing, and continuation tokens. Evaluate a +maintained in-process server (for example `s3s` with `s3s-fs`) on loopback; +refuse one that ignores conditional creates, since collisions would pass +silently. Gate it behind a dev-dependency feature if the build cost is large, +and wire it into at least the nightly workflow. + +## Related + +- [Timeline-indexed MoQ archives](/quest/m1/archive/README.md) diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 3d5d9bb1d8..3edecbe87e 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -17,7 +17,11 @@ pub use object_store; mod error; pub mod info; +#[cfg(test)] +mod mock; mod path; +#[cfg(test)] +mod proof; pub mod reader; mod recover; pub mod segment; diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs new file mode 100644 index 0000000000..2d7275badf --- /dev/null +++ b/rs/moq-archive/src/mock.rs @@ -0,0 +1,272 @@ +//! A test object store: any inner store plus an operation log, injected failures, and +//! S3-style paginated listing. + +use std::sync::{Arc, Mutex}; + +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; +use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; +use object_store::memory::InMemory; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, + PutOptions, PutPayload, PutResult, +}; + +/// S3's ListObjectsV2 page limit. +const MAX_KEYS: usize = 1000; + +/// One call made against a [`Mock`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Op { + Get(String), + Put(String), + Delete(String), + /// A streaming or paginated listing, with its prefix and exclusive offset. + List { + prefix: String, + offset: Option, + }, +} + +#[derive(Debug, Default)] +struct State { + ops: Vec, + /// PUTs whose path contains any of these fail. + fail_puts: Vec, + /// GETs whose path contains any of these return Not Found. + hide_gets: Vec, + /// Streaming listings end with an error after every entry. + fail_lists: bool, + /// Streaming listings yield entries in descending order, like a backend that promises none. + unordered: bool, +} + +#[derive(Debug, Clone)] +pub(crate) struct Mock { + inner: Arc, + state: Arc>, +} + +impl Mock { + pub fn new(inner: impl ObjectStore) -> Self { + Self { + inner: Arc::new(inner), + state: Default::default(), + } + } + + pub fn memory() -> Self { + Self::new(InMemory::new()) + } + + /// Share this store's objects, without its log or failures. + pub fn fork(&self) -> Self { + Self { + inner: self.inner.clone(), + state: Default::default(), + } + } + + /// Every call since the last take. + pub fn take(&self) -> Vec { + std::mem::take(&mut self.state().ops) + } + + /// Paths of every GET since the last take. + pub fn gets(&self) -> Vec { + self.take() + .into_iter() + .filter_map(|op| match op { + Op::Get(path) => Some(path), + _ => None, + }) + .collect() + } + + pub fn fail_puts(&self, pattern: &str) -> &Self { + self.state().fail_puts.push(pattern.to_string()); + self + } + + pub fn hide_gets(&self, pattern: &str) -> &Self { + self.state().hide_gets.push(pattern.to_string()); + self + } + + pub fn fail_lists(&self) -> &Self { + self.state().fail_lists = true; + self + } + + pub fn unordered(&self) -> &Self { + self.state().unordered = true; + self + } + + /// Clear every injected failure. + pub fn heal(&self) { + let mut state = self.state(); + state.fail_puts.clear(); + state.hide_gets.clear(); + state.fail_lists = false; + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap() + } + + fn log(&self, op: Op) { + self.state().ops.push(op); + } + + fn listed( + &self, + prefix: Option<&Path>, + offset: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.log(Op::List { + prefix: prefix.map(ToString::to_string).unwrap_or_default(), + offset: offset.map(ToString::to_string), + }); + let listed = match offset { + Some(offset) => self.inner.list_with_offset(prefix, offset), + None => self.inner.list(prefix), + }; + let state = self.state(); + let listed = match state.unordered { + true => futures::stream::once(async move { + let mut metas: Vec = listed.try_collect().await?; + metas.sort_by(|a, b| b.location.cmp(&a.location)); + Ok::<_, object_store::Error>(futures::stream::iter(metas.into_iter().map(Ok))) + }) + .try_flatten() + .boxed(), + false => listed, + }; + match state.fail_lists { + true => listed + .chain(futures::stream::once(async { Err(unsupported("list")) })) + .boxed(), + false => listed, + } + } +} + +fn unsupported(operation: &str) -> object_store::Error { + object_store::Error::NotImplemented { + operation: operation.into(), + implementer: "Mock".into(), + } +} + +impl std::fmt::Display for Mock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Mock({})", self.inner) + } +} + +#[async_trait::async_trait] +impl ObjectStore for Mock { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.log(Op::Put(location.to_string())); + if self.state().fail_puts.iter().any(|p| location.as_ref().contains(p)) { + return Err(unsupported("put")); + } + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.log(Op::Get(location.to_string())); + if self.state().hide_gets.iter().any(|p| location.as_ref().contains(p)) { + return Err(object_store::Error::NotFound { + path: location.to_string(), + source: "hidden".into(), + }); + } + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + let state = self.state.clone(); + let locations = locations + .inspect_ok(move |path| state.lock().unwrap().ops.push(Op::Delete(path.to_string()))) + .boxed(); + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.listed(prefix, None) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, object_store::Result> { + self.listed(prefix, Some(offset)) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// ListObjectsV2 semantics: a raw string prefix, lexical order, an exclusive `start-after` +/// offset, at most 1000 keys, and an opaque continuation token that resumes after the last key. +#[async_trait::async_trait] +impl PaginatedListStore for Mock { + async fn list_paginated( + &self, + prefix: Option<&str>, + opts: PaginatedListOptions, + ) -> object_store::Result { + if opts.delimiter.is_some() { + return Err(unsupported("delimiter")); + } + self.log(Op::List { + prefix: prefix.unwrap_or_default().to_string(), + offset: opts.offset.clone(), + }); + + let mut metas: Vec = self.inner.list(None).try_collect().await?; + metas.sort_by(|a, b| a.location.as_ref().cmp(b.location.as_ref())); + let after = opts.page_token.or(opts.offset); + metas.retain(|meta| { + let key = meta.location.as_ref(); + prefix.is_none_or(|prefix| key.starts_with(prefix)) && after.as_deref().is_none_or(|after| key > after) + }); + + let take = opts.max_keys.unwrap_or(MAX_KEYS).clamp(1, MAX_KEYS); + let page_token = (metas.len() > take).then(|| metas[take - 1].location.to_string()); + metas.truncate(take); + Ok(PaginatedListResult { + result: ListResult { + objects: metas, + common_prefixes: Vec::new(), + extensions: Default::default(), + }, + page_token, + }) + } +} diff --git a/rs/moq-archive/src/path.rs b/rs/moq-archive/src/path.rs index 86a8294e7b..4df4784b40 100644 --- a/rs/moq-archive/src/path.rs +++ b/rs/moq-archive/src/path.rs @@ -280,6 +280,10 @@ mod tests { parse_id("0009007199254740992"), Err(Error::Id(n)) if n == ID_MAX + 1 )); + // The largest QUIC varint still fits the 19-digit field. + let varint = (1u64 << 62) - 1; + assert!(matches!(format_id(varint), Err(Error::Id(n)) if n == varint)); + assert!(matches!(parse_id("4611686018427387903"), Err(Error::Id(n)) if n == varint)); assert!(matches!(parse_id("5"), Err(Error::Path(_)))); assert!(matches!(parse_id("000000000000000000X"), Err(Error::Path(_)))); } @@ -334,6 +338,52 @@ mod tests { assert!(matches!(Key::groups("v", ID_MAX + 1..=ID_MAX + 1), Err(Error::Id(_)))); } + #[test] + fn direct_construction_is_validated_when_serialized() { + let prefix = Path::from("rec"); + let empty = Key::Info { track: String::new() }; + assert!(matches!(empty.path(&prefix), Err(Error::Track))); + let segment = Key::Segments { + track: "t".to_string(), + segment: ID_MAX + 1, + }; + assert!(matches!(segment.path(&prefix), Err(Error::Id(_)))); + let varint = Key::Groups { + track: "v".to_string(), + range: 0..=(1 << 62) - 1, + }; + assert!(matches!(varint.path(&prefix), Err(Error::Id(_)))); + + // Everything that does serialize parses back to the same key. + for key in [ + Key::Info { + track: "a/b".to_string(), + }, + Key::Groups { + track: ".x".to_string(), + range: ID_MAX..=ID_MAX, + }, + Key::Segments { + track: "é".to_string(), + segment: 0, + }, + ] { + let path = key.path(&prefix).unwrap(); + assert_eq!(Key::parse(&prefix, &path).unwrap(), key); + } + } + + #[test] + fn keys_parse_under_an_empty_prefix() { + let key = Key::groups("catalog.json", 1..=2).unwrap(); + let path = key.path(&Path::ROOT).unwrap(); + assert_eq!( + path.as_ref(), + "catalog%2Ejson/groups/0000000000000000002.0000000000000000001" + ); + assert_eq!(Key::parse(&Path::ROOT, &path).unwrap(), key); + } + #[test] fn direct_inverted_range_is_rejected_when_serialized() { let (smallest, largest) = (2, 1); diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs new file mode 100644 index 0000000000..4acd6238ab --- /dev/null +++ b/rs/moq-archive/src/proof.rs @@ -0,0 +1,395 @@ +//! End to end: one multi-rendition broadcast recorded by a [`Writer`], stored byte-identically on +//! every backend, and replayed exactly through a [`Reader`]. + +use std::collections::BTreeMap; +use std::time::Duration; + +use bytes::Bytes; +use futures::TryStreamExt; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::local::LocalFileSystem; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::mock::{Mock, Op}; +use crate::reader::Config as ReaderConfig; +use crate::writer::{Config, Retention}; +use crate::{Reader, Store, Writer}; + +const TIMELINE: &str = hang::timeline::DEFAULT_NAME; +const RENDITIONS: [&str; 2] = ["video/1080p", "video/360p"]; +const PACING: [&str; 3] = ["video/1080p", "video/360p", "audio"]; +const TRACKS: [&str; 5] = ["video/1080p", "video/360p", "audio", "catalog.json", "chat"]; + +/// Frames of one source group: (millisecond timestamp, payload). +type Frames = Vec<(u64, Bytes)>; + +/// Recorded segments: three 2s segments, then a one-frame tail. +const SEGMENTS: u64 = 4; + +/// The source groups of one segment: (track, sequence, frames). +/// +/// Video renditions share a 2s GOP and audio has four groups per segment. The catalog and a +/// non-media chat track publish sparsely, including a sequence skip inside one segment and an +/// empty payload. The tail keeps any single track from cutting the final segment once the others +/// end. +fn plan(segment: u64) -> Vec<(&'static str, u64, Frames)> { + let pts = segment * 2000; + let (frames, audio) = if segment + 1 < SEGMENTS { (4, 4) } else { (1, 1) }; + let mut groups = Vec::new(); + for track in RENDITIONS { + let frames = (0..frames) + .map(|i| (pts + i * 500, Bytes::from(format!("{track} {segment}.{i}")))) + .collect(); + groups.push((track, segment, frames)); + } + for sequence in segment * 4..segment * 4 + audio { + let frames = (0..2) + .map(|i| (sequence * 500 + i * 250, Bytes::from(format!("audio {sequence}.{i}")))) + .collect(); + groups.push(("audio", sequence, frames)); + } + let catalog = |sequence: u64| Bytes::from(format!(r#"{{"version":{sequence}}}"#)); + let chat = |text: &'static str| Bytes::from_static(text.as_bytes()); + match segment { + 0 => { + groups.push(("catalog.json", 0, vec![(0, catalog(0))])); + groups.push(("chat", 0, vec![(1200, chat("hello")), (1300, Bytes::new())])); + } + 2 => { + groups.push(("catalog.json", 1, vec![(4000, catalog(1))])); + groups.push(("chat", 3, vec![(4100, chat("skip"))])); + groups.push(("chat", 5, vec![(5100, chat("bye"))])); + } + _ => {} + } + groups +} + +fn ms(value: u64) -> Timestamp { + Timestamp::new(value, Timescale::MILLI).unwrap() +} + +fn write(track: &track::Producer, sequence: u64, frames: &Frames) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for (timestamp, payload) in frames { + group.write_frame(ms(*timestamp), payload.clone()).unwrap(); + } + group.finish().unwrap(); +} + +/// Let the writer read and report everything already published. Reads never touch the store. +async fn settle() { + for _ in 0..256 { + tokio::task::yield_now().await; + } +} + +/// Record every segment of [`plan`] into `store`. +async fn record(store: &Store) { + let source = broadcast::Info::new().produce(); + let tracks: BTreeMap<&str, track::Producer> = TRACKS + .iter() + .map(|&name| { + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + (name, source.create_track(name, info).unwrap()) + }) + .collect(); + + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + for name in TRACKS { + match PACING.contains(&name) { + true => control.pacing_track(name).await.unwrap(), + false => control.track(name).await.unwrap(), + } + } + let run = tokio::spawn(writer.run()); + + // A non-pacing track joins whichever segment is open when its group arrives, so publish each + // segment's pacing groups (closing the previous segment) before its sparse groups. + for segment in 0..SEGMENTS { + let (pacing, sparse): (Vec<_>, Vec<_>) = plan(segment) + .into_iter() + .partition(|(name, _, _)| PACING.contains(name)); + for (name, sequence, frames) in pacing { + write(&tracks[name], sequence, &frames); + } + settle().await; + for (name, sequence, frames) in sparse { + write(&tracks[name], sequence, &frames); + } + settle().await; + } + for track in tracks.values() { + track.finish().unwrap(); + } + source.finish(); + run.await.unwrap().unwrap(); +} + +/// Every object under the store's prefix, by path relative to the store root. +async fn objects(store: &Store) -> BTreeMap { + let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); + let mut objects = BTreeMap::new(); + for meta in metas { + let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); + objects.insert(meta.location.to_string(), bytes); + } + objects +} + +fn id(value: u64) -> String { + format!("{value:019}") +} + +/// The exact layout [`plan`] produces: no manifest, index, or completion marker. +fn layout() -> Vec { + let mut keys = Vec::new(); + let groups = |track: &str, ranges: &[(u64, u64)]| { + ranges + .iter() + .map(|&(smallest, largest)| format!("rec/{track}/groups/{}.{}", id(largest), id(smallest))) + .collect::>() + }; + keys.push("rec/audio/.info".to_string()); + keys.extend(groups("audio", &[(0, 3), (4, 7), (8, 11), (12, 12)])); + keys.push("rec/catalog%2Ejson/.info".to_string()); + keys.extend(groups("catalog%2Ejson", &[(0, 0), (1, 1)])); + keys.push("rec/chat/.info".to_string()); + keys.extend(groups("chat", &[(0, 0), (3, 5)])); + keys.push("rec/timeline%2Ez/.info".to_string()); + keys.extend((0..SEGMENTS).map(|segment| format!("rec/timeline%2Ez/segments/{}", id(segment)))); + for track in ["video%2F1080p", "video%2F360p"] { + keys.push(format!("rec/{track}/.info")); + keys.extend(groups(track, &[(0, 0), (1, 1), (2, 2), (3, 3)])); + } + keys.sort(); + keys +} + +/// FETCH one whole group, returning its frames. +async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> moq_net::Result { + let track = broadcast.consume().track(track)?; + let mut group = track.fetch_group(sequence, group::Fetch::default()).await?; + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await? { + frames.push(( + frame.timestamp.convert(Timescale::MILLI).unwrap().value(), + frame.payload, + )); + } + Ok(frames) +} + +#[tokio::test] +async fn recordings_are_byte_identical_on_every_backend() { + let memory = Store::new(Mock::memory(), "rec"); + record(&memory).await; + let expected = objects(&memory).await; + assert_eq!(expected.keys().cloned().collect::>(), layout()); + assert_eq!( + &expected["rec/video%2F360p/.info"][..], + br#"{"version":1,"priority":0,"timescale":1000}"# + ); + + let dir = tempfile::tempdir().unwrap(); + let local = Store::new(Mock::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap()), "rec"); + record(&local).await; + assert_eq!(objects(&local).await, expected, "local disk"); + + // A backend with no listing order, and a sibling recording sharing the prefix's stem. + let unordered = Mock::memory(); + unordered.unordered(); + let sibling = Store::new(unordered.clone(), "rec-other"); + record(&sibling).await; + let unordered = Store::new(unordered, "rec"); + record(&unordered).await; + assert_eq!(objects(&unordered).await, expected, "unordered listing"); + assert_eq!(objects(&sibling).await.len(), expected.len()); + + // Every group object holds exactly the source groups, in order, byte for byte. + let mut stored: BTreeMap<(String, u64), Frames> = BTreeMap::new(); + for (path, bytes) in &expected { + let key = crate::Key::parse(&Path::from("rec"), &Path::parse(path).unwrap()).unwrap(); + if let crate::Key::Groups { track, range } = key { + let object = crate::Object::decode_groups(bytes.clone(), range).unwrap(); + for group in object.groups { + let frames = group.frames.into_iter().map(|f| (f.timestamp, f.payload)).collect(); + stored.insert((track.clone(), group.sequence), frames); + } + } + } + let source: BTreeMap<(String, u64), Frames> = (0..SEGMENTS) + .flat_map(plan) + .map(|(track, sequence, frames)| ((track.to_string(), sequence), frames)) + .collect(); + assert_eq!(stored, source); +} + +#[tokio::test] +async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + record(&store).await; + + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(store, &broadcast, ReaderConfig::new(TIMELINE)) + .await + .unwrap(); + tokio::spawn(reader.serve()); + mock.take(); + + // Low-rendition playback never downloads the 1080p object. + for sequence in 0..SEGMENTS { + let expected = plan(sequence) + .into_iter() + .find(|(name, ..)| *name == "video/360p") + .unwrap() + .2; + assert_eq!(fetch(&broadcast, "video/360p", sequence).await.unwrap(), expected); + } + let object = + |track: &str, smallest: u64, largest: u64| format!("rec/{track}/groups/{}.{}", id(largest), id(smallest)); + // Each track request also GETs that track's `.info`, but nothing of another track. + let media = |gets: Vec, track: &str| { + let prefix = format!("rec/{track}/"); + assert!(gets.iter().all(|path| path.starts_with(&prefix)), "{gets:?}"); + gets.into_iter() + .filter(|path| path.contains("/groups/")) + .collect::>() + }; + assert_eq!( + media(mock.gets(), "video%2F360p"), + [ + object("video%2F360p", 0, 0), + object("video%2F360p", 1, 1), + object("video%2F360p", 2, 2), + object("video%2F360p", 3, 3), + ] + ); + + // Audio-only playback: four adjacent groups per GET, the rest from the cache. + for segment in 0..SEGMENTS { + for (_, sequence, frames) in plan(segment).into_iter().filter(|(name, ..)| *name == "audio") { + assert_eq!(fetch(&broadcast, "audio", sequence).await.unwrap(), frames); + } + } + assert_eq!( + media(mock.gets(), "audio"), + [ + object("audio", 0, 3), + object("audio", 4, 7), + object("audio", 8, 11), + object("audio", 12, 12), + ] + ); + + // Every other enrolled group replays its original sequence, timestamps, and payloads. + for segment in 0..SEGMENTS { + for (name, sequence, frames) in plan(segment) { + assert_eq!( + fetch(&broadcast, name, sequence).await.unwrap(), + frames, + "{name} {sequence}" + ); + } + } + for (track, sequence) in [("chat", 1), ("chat", 4), ("chat", 6), ("audio", 13), ("video/1080p", 4)] { + assert!( + matches!(fetch(&broadcast, track, sequence).await, Err(moq_net::Error::NotFound)), + "{track} {sequence} was never recorded" + ); + } + assert!( + !mock.take().iter().any(|op| matches!(op, Op::List { .. })), + "a group request resolves its object without listing" + ); +} + +#[tokio::test] +async fn an_offline_reader_follows_dvr_expiry() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let source = broadcast::Info::new().produce(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let video = source.create_track("video", info).unwrap(); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + + let frames = |sequence: u64| vec![(sequence * 1000, Bytes::from(format!("video {sequence}")))]; + let committed = |segment: u64| { + let store = store.clone(); + async move { + while store.get_segments(TIMELINE, segment).await.is_err() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + }; + + for sequence in 0..3 { + write(&video, sequence, &frames(sequence)); + } + committed(1).await; + + let broadcast = broadcast::Info::new().produce(); + let mut reader = Reader::open(store.clone(), &broadcast, ReaderConfig::new(TIMELINE)) + .await + .unwrap(); + tokio::spawn(reader.serve()); + assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), frames(0)); + + // The reader is offline while the DVR commits and expires several segments. + for sequence in 3..10 { + write(&video, sequence, &frames(sequence)); + } + committed(8).await; + // Segments 7 and 8 hold the 2s window. + while store.get_groups("video", 6..=6).await.is_ok() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + + mock.take(); + reader.refresh().await.unwrap(); + let ops = mock.take(); + let (list, gets) = ops.split_first().unwrap(); + assert_eq!( + list, + &Op::List { + prefix: "rec/timeline%2Ez/segments".to_string(), + offset: Some(format!("rec/timeline%2Ez/segments/{}", id(1))), + } + ); + let segments: Vec<_> = (2..=8) + .map(|segment| Op::Get(format!("rec/timeline%2Ez/segments/{}", id(segment)))) + .collect(); + assert_eq!(gets, segments, "only the new timeline keys are read"); + + // Expired groups are gone from the index, so they cost no media GET. + for sequence in 0..7 { + assert!( + matches!( + fetch(&broadcast, "video", sequence).await, + Err(moq_net::Error::NotFound) + ), + "group {sequence} expired" + ); + } + let gets = mock.gets(); + assert!(!gets.iter().any(|path| path.contains("/groups/")), "{gets:?}"); + for sequence in 7..9 { + assert_eq!(fetch(&broadcast, "video", sequence).await.unwrap(), frames(sequence)); + } + + video.finish().unwrap(); + source.finish(); + run.await.unwrap().unwrap(); +} diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs index 64b22c9fab..5be88a433c 100644 --- a/rs/moq-archive/src/reader/mod.rs +++ b/rs/moq-archive/src/reader/mod.rs @@ -23,6 +23,7 @@ mod index; +use std::collections::HashMap; use std::future::Future; use std::ops::RangeInclusive; use std::sync::{Arc, Mutex}; @@ -79,6 +80,8 @@ struct Shared { store: Store, index: Mutex, cache: quick_cache::sync::Cache, Weight>, + /// Each track's `.info`, which is immutable, so a re-requested track costs no GET. + infos: Mutex>, } impl Reader { @@ -97,6 +100,7 @@ impl Reader { store, index: Mutex::default(), cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), + infos: Mutex::default(), }); let mut reader = Self { @@ -237,13 +241,20 @@ async fn serve_track(shared: Arc>, request: track::Req return; } - let info = match shared.store.get_info(&name).await.and_then(|info| track_info(&info)) { - Ok(info) => info, - Err(err) => { - tracing::warn!(track = name, %err, "archived track has no usable .info"); - request.reject(moq_net::Error::NotFound); - return; - } + let cached = shared.infos.lock().unwrap().get(&name).cloned(); + let info = match cached { + Some(info) => info, + None => match shared.store.get_info(&name).await.and_then(|info| track_info(&info)) { + Ok(info) => { + shared.infos.lock().unwrap().insert(name.clone(), info.clone()); + info + } + Err(err) => { + tracing::warn!(track = name, %err, "archived track has no usable .info"); + request.reject(moq_net::Error::NotFound); + return; + } + }, }; let timescale = info.timescale; diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index ca817bf72a..c560788199 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -1,89 +1,19 @@ -use std::sync::{Arc, Mutex}; - use bytes::Bytes; -use futures::stream::BoxStream; use hang::timeline::{Range, Record}; use moq_json::window; use moq_net::{broadcast, group}; -use object_store::memory::InMemory; -use object_store::path::Path; -use object_store::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, ObjectStoreExt, - PutMultipartOptions, PutOptions, PutPayload, PutResult, -}; +use object_store::ObjectStoreExt; use super::*; +use crate::mock::{Mock, Op}; use crate::segment::{Frame, Group}; use crate::{ID_MAX, Info}; const TIMELINE: &str = "timeline.z"; -/// In-memory store that records every GET path. -#[derive(Debug, Clone, Default)] -struct Counting { - inner: Arc, - gets: Arc>>, -} - -impl Counting { - fn gets(&self) -> Vec { - self.gets.lock().unwrap().clone() - } -} - -impl std::fmt::Display for Counting { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Counting") - } -} - -#[async_trait::async_trait] -impl ObjectStore for Counting { - async fn put_opts( - &self, - location: &Path, - payload: PutPayload, - opts: PutOptions, - ) -> object_store::Result { - self.inner.put_opts(location, payload, opts).await - } - - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOptions, - ) -> object_store::Result> { - self.inner.put_multipart_opts(location, opts).await - } - - async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { - self.gets.lock().unwrap().push(location.to_string()); - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: BoxStream<'static, object_store::Result>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { - self.inner.list(prefix) - } - - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } -} - /// Writes archive objects the way a recording writer lays them out. struct Archive { - store: Store, + store: Store, encoder: window::Encoder, /// Next timeline group sequence. sequence: u64, @@ -96,7 +26,7 @@ impl Archive { /// `op_ratio` 0 makes every window edit its own checkpoint group. async fn with_op_ratio(op_ratio: u32) -> Self { - let store = Store::new(Counting::default(), "rec"); + let store = Store::new(Mock::memory(), "rec"); store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); let config = window::ProducerConfig::default() .with_compression(true) @@ -188,7 +118,7 @@ fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { record } -async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { +async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { let broadcast = broadcast::Info::new().produce(); let reader = Reader::open(archive.store.clone(), &broadcast, Config::new(TIMELINE)) .await @@ -269,7 +199,7 @@ async fn requests_download_only_their_object() { .await; let (broadcast, _reader) = open(&archive).await; - let before = archive.store.inner().gets().len(); + archive.store.inner().take(); for sequence in 0..3 { assert_eq!( @@ -278,7 +208,7 @@ async fn requests_download_only_their_object() { ); } - let gets = archive.store.inner().gets()[before..].to_vec(); + let gets = archive.store.inner().gets(); assert_eq!( gets, vec![ @@ -365,12 +295,12 @@ async fn refresh_follows_new_segments_and_pops() { ); // The popped record's groups are gone, although its object is still stored and was cached. - let gets = archive.store.inner().gets().len(); + archive.store.inner().take(); assert!(matches!( fetch(&broadcast, "video", 1, 0).await, Err(moq_net::Error::NotFound) )); - assert_eq!(archive.store.inner().gets().len(), gets); + assert_eq!(archive.store.inner().gets(), Vec::::new()); } #[tokio::test] @@ -396,6 +326,161 @@ async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { } } +#[tokio::test] +async fn a_missing_tail_is_retried_on_the_next_refresh() { + let mut archive = Archive::new().await; + for segment in 0..3 { + archive.media("video", &[(segment, 1)]).await; + archive + .commit(&record(segment, &[("video", &[(segment, segment)])]), 0) + .await; + } + // Listed, but not yet readable. + archive.store.inner().hide_gets("segments/0000000000000000002"); + + let (broadcast, mut reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); + assert!(matches!( + fetch(&broadcast, "video", 2, 0).await, + Err(moq_net::Error::NotFound) + )); + + archive.store.inner().heal(); + archive.store.inner().take(); + reader.refresh().await.unwrap(); + assert_eq!( + archive.store.inner().take(), + [ + Op::List { + prefix: "rec/timeline%2Ez/segments".to_string(), + offset: Some("rec/timeline%2Ez/segments/0000000000000000001".to_string()), + }, + Op::Get("rec/timeline%2Ez/segments/0000000000000000002".to_string()), + ], + "the cursor stays before the missing tail" + ); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..1) + ); +} + +#[tokio::test] +async fn following_lists_only_new_timeline_keys() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + let (broadcast, mut reader) = open(&archive).await; + + for segment in 1..4 { + // A media object stored ahead of its commit is invisible until the timeline names it. + archive.media("video", &[(segment, 1)]).await; + reader.refresh().await.unwrap(); + assert!(matches!( + fetch(&broadcast, "video", segment, 0).await, + Err(moq_net::Error::NotFound) + )); + + archive + .commit(&record(segment, &[("video", &[(segment, segment)])]), 0) + .await; + archive.store.inner().take(); + reader.refresh().await.unwrap(); + let previous = format!("rec/timeline%2Ez/segments/{:019}", segment - 1); + assert_eq!( + archive.store.inner().take(), + [ + Op::List { + prefix: "rec/timeline%2Ez/segments".to_string(), + offset: Some(previous), + }, + Op::Get(format!("rec/timeline%2Ez/segments/{segment:019}")), + ], + "following segment {segment} touches no media listing" + ); + assert_eq!( + fetch(&broadcast, "video", segment, 0).await.unwrap(), + expected("video", segment, 0..1) + ); + } +} + +#[tokio::test] +async fn an_unordered_listing_replays_in_segment_order() { + let mut archive = Archive::new().await; + archive.store.inner().unordered(); + for segment in 0..4 { + archive.media("video", &[(segment, 1)]).await; + let pop = u64::from(segment >= 2); + archive + .commit(&record(segment, &[("video", &[(segment, segment)])]), pop) + .await; + } + + let (broadcast, mut reader) = open(&archive).await; + + // The cursor is the newest segment, not the last one listed. + archive.media("video", &[(4, 1)]).await; + archive.commit(&record(4, &[("video", &[(4, 4)])]), 1).await; + archive.store.inner().take(); + reader.refresh().await.unwrap(); + assert_eq!( + archive.store.inner().gets(), + ["rec/timeline%2Ez/segments/0000000000000000004"] + ); + + for segment in 0..3 { + assert!(matches!( + fetch(&broadcast, "video", segment, 0).await, + Err(moq_net::Error::NotFound) + )); + } + for segment in 3..5 { + assert_eq!( + fetch(&broadcast, "video", segment, 0).await.unwrap(), + expected("video", segment, 0..1) + ); + } +} + +#[tokio::test] +async fn a_track_without_usable_info_is_not_found() { + let mut archive = Archive::new().await; + archive.media("audio", &[(0, 1)]).await; + let object = |track| Object { + groups: vec![Group { + sequence: 0, + frames: vec![frame(track, 0, 0)], + }], + }; + archive.store.put_groups("bare", &object("bare")).await.unwrap(); + archive.store.put_groups("future", &object("future")).await.unwrap(); + archive + .raw( + &Key::info("future").unwrap(), + br#"{"version":2,"priority":0,"timescale":1000}"#, + ) + .await; + archive + .commit( + &record(0, &[("audio", &[(0, 0)]), ("bare", &[(0, 0)]), ("future", &[(0, 0)])]), + 0, + ) + .await; + + let (broadcast, _reader) = open(&archive).await; + for track in ["bare", "future"] { + assert!( + matches!(fetch(&broadcast, track, 0, 0).await, Err(moq_net::Error::NotFound)), + "{track}" + ); + } + assert_eq!( + fetch(&broadcast, "audio", 0, 0).await.unwrap(), + expected("audio", 0, 0..1) + ); +} + #[tokio::test] async fn multi_frame_timeline_groups_decode() { let mut archive = Archive::with_op_ratio(64).await; @@ -440,7 +525,7 @@ async fn timeline_track_is_republished_and_finished_on_request() { #[tokio::test] async fn open_requires_the_timeline_info() { - let store = Store::new(Counting::default(), "rec"); + let store = Store::new(Mock::memory(), "rec"); let broadcast = broadcast::Info::new().produce(); let result = Reader::open(store, &broadcast, Config::new(TIMELINE)).await; assert!(matches!(result, Err(Error::NotFound(_)))); diff --git a/rs/moq-archive/src/segment.rs b/rs/moq-archive/src/segment.rs index c2fd68c5aa..6b844dcdee 100644 --- a/rs/moq-archive/src/segment.rs +++ b/rs/moq-archive/src/segment.rs @@ -346,6 +346,39 @@ mod tests { assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Id(_)))); } + /// A table with one frameless group per delta. + fn deltas(deltas: &[u64]) -> Bytes { + let mut bytes = BytesMut::new(); + write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, deltas.len() as u64).unwrap(); + for &delta in deltas { + write_varint(&mut bytes, delta).unwrap(); + write_varint(&mut bytes, 0).unwrap(); + } + bytes.freeze() + } + + fn sequences(bytes: Bytes) -> Result> { + Ok(Object::decode(bytes)? + .groups + .iter() + .map(|group| group.sequence) + .collect()) + } + + #[test] + fn delta_extremes() { + let varint = (1u64 << 62) - 1; + assert_eq!(sequences(deltas(&[0, 0, 0])).unwrap(), vec![0, 1, 2]); + assert_eq!(sequences(deltas(&[0, 9, 0])).unwrap(), vec![0, 10, 11]); + assert_eq!(sequences(deltas(&[0, ID_MAX - 1])).unwrap(), vec![0, ID_MAX]); + assert_eq!(sequences(deltas(&[ID_MAX])).unwrap(), vec![ID_MAX]); + assert!(matches!(sequences(deltas(&[0, ID_MAX])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[ID_MAX + 1])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[varint])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[ID_MAX, varint])), Err(Error::Id(_)))); + } + #[test] fn timestamp_bounds() { assert!( @@ -413,6 +446,34 @@ mod tests { assert!(matches!(Object::decode(&b"\x01"[..]), Err(Error::Table))); } + #[test] + fn frame_offsets_must_tile_the_payload() { + // One group of two frames at (offset, length), followed by `payload` bytes. + let table = |frames: [(u64, u64); 2], payload: usize| { + let mut bytes = BytesMut::new(); + for value in [1, 1, 0, 2] { + write_varint(&mut bytes, value).unwrap(); + } + for (offset, length) in frames { + for value in [0, offset, length] { + write_varint(&mut bytes, value).unwrap(); + } + } + bytes.extend_from_slice(&vec![b'x'; payload]); + Object::decode(bytes.freeze()) + }; + assert!(table([(0, 1), (1, 2)], 3).is_ok()); + assert!(matches!(table([(0, 1), (0, 2)], 3), Err(Error::Table)), "overlap"); + assert!(matches!(table([(0, 1), (2, 1)], 3), Err(Error::Table)), "gap"); + assert!(matches!(table([(1, 1), (2, 1)], 3), Err(Error::Table)), "late start"); + assert!(matches!(table([(0, 1), (1, 3)], 3), Err(Error::Table)), "past the end"); + let varint = (1u64 << 62) - 1; + assert!( + matches!(table([(0, 1), (1, varint)], 3), Err(Error::Table)), + "huge length" + ); + } + #[test] fn filename_bounds_must_match_the_table() { let object = object(vec![ diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 3a9092703e..c79db85180 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -343,113 +343,16 @@ mod tests { use std::num::NonZeroUsize; use futures::TryStreamExt; - use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; + use object_store::ObjectStoreExt; use object_store::memory::InMemory; use object_store::path::Path; - use object_store::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, - PutOptions, PutPayload, PutResult, - }; use super::*; use crate::ID_MAX; + use crate::mock::Mock; use crate::path::encode_track; use crate::segment::{Frame, Group}; - /// In-memory store with a trivial offset-based paginated listing implementation. - /// Page tokens are decimal indexes into the filtered, sorted key list. - #[derive(Debug, Clone)] - struct PaginatedMemory { - inner: InMemory, - } - - impl std::fmt::Display for PaginatedMemory { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "PaginatedMemory") - } - } - - #[async_trait::async_trait] - impl ObjectStore for PaginatedMemory { - async fn put_opts( - &self, - location: &Path, - payload: PutPayload, - opts: PutOptions, - ) -> object_store::Result { - self.inner.put_opts(location, payload, opts).await - } - - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOptions, - ) -> object_store::Result> { - self.inner.put_multipart_opts(location, opts).await - } - - async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: BoxStream<'static, object_store::Result>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { - self.inner.list(prefix) - } - - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } - } - - #[async_trait::async_trait] - impl PaginatedListStore for PaginatedMemory { - async fn list_paginated( - &self, - prefix: Option<&str>, - opts: PaginatedListOptions, - ) -> object_store::Result { - let mut metas: Vec = self.inner.list(None).try_collect().await?; - metas.sort_by(|a, b| a.location.as_ref().cmp(b.location.as_ref())); - if let Some(prefix) = prefix { - metas.retain(|meta| meta.location.as_ref().starts_with(prefix)); - } - if let Some(offset) = opts.offset.as_deref() { - metas.retain(|meta| meta.location.as_ref() > offset); - } - let start: usize = match opts.page_token { - Some(token) => token.parse().map_err(|err| object_store::Error::Generic { - store: "PaginatedMemory", - source: Box::new(err), - })?, - None => 0, - }; - let remaining = &metas[start.min(metas.len())..]; - let take = opts.max_keys.unwrap_or(remaining.len()).min(remaining.len()); - let objects = remaining[..take].to_vec(); - let next = start.saturating_add(take); - let page_token = (next < metas.len()).then(|| next.to_string()); - Ok(PaginatedListResult { - result: ListResult { - objects, - common_prefixes: Vec::new(), - extensions: Default::default(), - }, - page_token, - }) - } - } - fn memory() -> Store { Store::new(InMemory::new(), "rec") } @@ -520,6 +423,30 @@ mod tests { assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 1 }"#); } + #[tokio::test] + async fn malformed_or_unsupported_existing_info_is_refused_and_kept() { + let store = memory(); + let info = Info::new(0, 1_000).unwrap(); + for (track, existing, check) in [ + ( + "v2", + &br#"{"version":2,"priority":0,"timescale":1000}"#[..], + (|err| matches!(err, Error::Version(2))) as fn(&Error) -> bool, + ), + ("junk", b"not json", |err| matches!(err, Error::Json(_))), + ("zero", br#"{"version":1,"priority":0,"timescale":0}"#, |err| { + matches!(err, Error::Timescale(0)) + }), + ] { + let path = store.path(&Key::info(track).unwrap()).unwrap(); + store.inner().put(&path, existing.to_vec().into()).await.unwrap(); + let err = store.put_info(track, &info).await.unwrap_err(); + assert!(check(&err), "{track}: {err}"); + let kept = store.inner().get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(&kept[..], existing, "{track} is not rewritten"); + } + } + #[tokio::test] async fn info_property_mismatch_is_a_hard_error() { let store = memory(); @@ -818,7 +745,7 @@ mod tests { #[tokio::test] async fn paginated_listing_walks_pages_and_excludes_siblings() { - let store = Store::new(PaginatedMemory { inner: InMemory::new() }, "rec"); + let store = Store::new(Mock::memory(), "rec"); for segment in 0..3 { store .put_segments("timeline.z", segment, &one_group(segment, b"t")) @@ -827,7 +754,6 @@ mod tests { } store .inner() - .inner .put( &Path::from("rec-other/video/.info"), Bytes::from_static(b"sibling").into(), @@ -896,6 +822,90 @@ mod tests { assert_eq!(paged, streamed); } + /// The first object listed after `groups_from(group)`, one S3 page of one key. + async fn lookup(store: &Store, group: u64) -> Option> { + let query = Query::groups_from("video", group) + .unwrap() + .page_size(NonZeroUsize::new(1).unwrap()); + let page = store.list_paginated(&query).await.unwrap(); + match page.entries.first().map(|entry| &entry.key) { + Some(Key::Groups { range, .. }) => Some(range.clone()), + Some(key) => panic!("unexpected {key:?}"), + None => None, + } + } + + #[tokio::test] + async fn ordered_lookup_finds_the_covering_object() { + let store = Store::new(Mock::memory(), "rec"); + for range in [0..=2, 5..=7, 10..=10, ID_MAX..=ID_MAX] { + let groups = range.clone().filter(|sequence| *sequence != 6); + let object = Object { + groups: groups + .map(|sequence| Group { + sequence, + frames: vec![frame(0, b"g")], + }) + .collect(), + }; + store.put_groups("video", &object).await.unwrap(); + } + // A sibling track sorts after `video/groups/` and must never be returned. + store.put_groups("video-alt", &one_group(3, b"a")).await.unwrap(); + + // Largest-first filenames make the first key at or past the group its only candidate. + for (group, found) in [ + (0, Some(0..=2)), + (1, Some(0..=2)), + (2, Some(0..=2)), + (3, Some(5..=7)), + (5, Some(5..=7)), + (6, Some(5..=7)), + (7, Some(5..=7)), + (8, Some(10..=10)), + (10, Some(10..=10)), + (11, Some(ID_MAX..=ID_MAX)), + (ID_MAX, Some(ID_MAX..=ID_MAX)), + ] { + assert_eq!(lookup(&store, group).await, found, "group {group}"); + } + store + .delete(&Key::groups("video", ID_MAX..=ID_MAX).unwrap()) + .await + .unwrap(); + assert_eq!(lookup(&store, 11).await, None); + assert!(Query::groups_from("video", ID_MAX + 1).is_err()); + } + + #[tokio::test] + async fn empty_prefix_lists_the_whole_store() { + let store = Store::new(Mock::memory(), ""); + store.put_info("catalog.json", &Info::new(0, 1).unwrap()).await.unwrap(); + store.put_groups("video", &one_group(4, b"a")).await.unwrap(); + assert_eq!(store.paginated_prefix(None), None); + + let expected = + std::collections::HashSet::from([Key::info("catalog.json").unwrap(), Key::groups("video", 4..=4).unwrap()]); + let streamed: std::collections::HashSet = store + .list(&Query::new()) + .map_ok(|entry| entry.key) + .try_collect() + .await + .unwrap(); + assert_eq!(streamed, expected); + let page = store.list_paginated(&Query::new()).await.unwrap(); + assert_eq!( + page.entries + .into_iter() + .map(|entry| entry.key) + .collect::>(), + expected + ); + assert!(page.next.is_none()); + let page = store.list_paginated(&Query::groups("video").unwrap()).await.unwrap(); + assert_eq!(page.entries.len(), 1); + } + #[test] fn directory_results_are_not_silently_dropped() { let store = memory(); diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 5f05cdc083..9ed25d86b7 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -705,6 +705,11 @@ impl Committer { let segment = pending.segment; let record = (*pending).clone(); + let pts = Timestamp::new( + record.pts, + Timescale::new(self.timescale).map_err(|_| Error::Timescale(self.timescale))?, + ) + .map_err(|_| Error::Id(record.pts))?; self.timeline.push(pending).map_err(timeline_error)?; self.window.push_back(record); @@ -714,7 +719,7 @@ impl Committer { } self.timeline.flush().map_err(timeline_error)?; - let object = self.read_timeline()?; + let object = self.read_timeline(pts)?; store.put_segments(&shared.timeline, segment, &object).await?; let mut keys = Vec::new(); @@ -747,8 +752,11 @@ impl Committer { expired } - /// Collect the timeline groups completed since the last segment. - fn read_timeline(&mut self) -> Result { + /// Collect the timeline groups completed since the last segment, stamped at `pts`. + /// + /// The live timeline track stamps frames with the wall clock; storing the segment's content + /// time instead keeps a recording's bytes a function of its content alone. + fn read_timeline(&mut self, pts: Timestamp) -> Result { let waiter = kio::Waiter::noop(); let timescale = self.groups.info().timescale; let mut groups = Vec::new(); @@ -759,7 +767,10 @@ impl Committer { let mut frames = Vec::new(); loop { match group.poll_read_frame(&waiter) { - Poll::Ready(Ok(Some(frame))) => frames.push(frame), + Poll::Ready(Ok(Some(mut frame))) => { + frame.timestamp = pts; + frames.push(frame); + } Poll::Ready(Ok(None)) => break, Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), // Flushing closed every group, so an open one is a bug. @@ -803,91 +814,12 @@ fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { mod tests { use futures::TryStreamExt; - use futures::stream::BoxStream; use object_store::memory::InMemory; - use object_store::path::Path; - use object_store::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, PutMultipartOptions, PutOptions, - PutPayload, PutResult, - }; use super::*; + use crate::mock::Mock; use crate::store::list::Query; - /// An in-memory store whose group PUTs fail for one track, and whose listings fail at the end. - #[derive(Debug, Clone)] - struct Failing { - inner: Arc, - track: &'static str, - list: bool, - } - - impl std::fmt::Display for Failing { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Failing") - } - } - - #[async_trait::async_trait] - impl ObjectStore for Failing { - async fn put_opts( - &self, - location: &Path, - payload: PutPayload, - opts: PutOptions, - ) -> object_store::Result { - if location.as_ref().contains(&format!("/{}/groups/", self.track)) { - return Err(object_store::Error::NotImplemented { - operation: "put".into(), - implementer: "Failing".into(), - }); - } - self.inner.put_opts(location, payload, opts).await - } - - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOptions, - ) -> object_store::Result> { - self.inner.put_multipart_opts(location, opts).await - } - - async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: BoxStream<'static, object_store::Result>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { - let listed = self.inner.list(prefix); - match self.list { - true => listed - .chain(futures::stream::once(async { - Err(object_store::Error::NotImplemented { - operation: "list".into(), - implementer: "Failing".into(), - }) - })) - .boxed(), - false => listed, - } - } - - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } - } - const TIMELINE: &str = hang::timeline::DEFAULT_NAME; fn ms(v: u64) -> Timestamp { @@ -1031,12 +963,9 @@ mod tests { let video = track(&source, "video"); let audio = track(&source, "audio"); - let failing = Failing { - inner: Arc::new(InMemory::new()), - track: "audio", - list: false, - }; - let store = Store::new(failing, "rec"); + let mock = Mock::memory(); + mock.fail_puts("/audio/groups/"); + let store = Store::new(mock, "rec"); let writer = Writer::new(store.clone(), source.consume(), Config::default()) .await .unwrap(); @@ -1125,6 +1054,39 @@ mod tests { run.abort(); } + #[tokio::test] + async fn accepted_groups_may_complete_out_of_order() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + + let mut first = video.create_group(group::Info { sequence: 0 }).unwrap(); + first.write_frame(ms(0), "0@0").unwrap(); + group(&video, 1, &[1000]); + group(&video, 2, &[2000]); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(window(&store).await.is_empty(), "group 1 waits for group 0"); + + first.write_frame(ms(500), "0@500").unwrap(); + first.finish().unwrap(); + video.finish().unwrap(); + source.finish(); + run.await.unwrap().unwrap(); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2)]); + check_objects(&store, &records).await; + let object = store.get_groups("video", 0..=0).await.unwrap(); + let frames: Vec<_> = object.groups[0].frames.iter().map(|f| f.timestamp).collect(); + assert_eq!(frames, vec![0, 500]); + } + #[tokio::test] async fn decreasing_arrivals_are_refused() { let source = broadcast::Info::new().produce(); @@ -1365,20 +1327,101 @@ mod tests { store.get_segments(TIMELINE, 0).await.unwrap(); } + #[tokio::test] + async fn a_dvr_crash_between_pop_and_delete_is_cleaned_on_restart() { + let store = Store::new(InMemory::new(), "rec"); + + // A grace longer than the test keeps every expired object past the crash. + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let config = + Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + // Segment 5 stays open, so the newest durable timeline object is segment 4. + while window(&store).await.last().map(|record| record.segment) != Some(4) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + run.abort(); + let _ = run.await; + + let retained = window(&store).await; + assert_eq!( + retained.iter().map(|record| record.segment).collect::>(), + vec![3, 4] + ); + let expired: HashSet<_> = (0..3).map(|s| Key::groups("video", s..=s).unwrap()).collect(); + assert_eq!(stored_groups(&store).await, &referenced(&retained) | &expired); + // An upload the crash left uncommitted. + store.put_groups("video", &orphan(5)).await.unwrap(); + + let grace = Duration::from_millis(200); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let started = Instant::now(); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + assert!( + stored_groups(&store).await.is_superset(&expired), + "expired objects outlive the grace, for readers holding the old timeline" + ); + + writer.control().pacing_track("video").await.unwrap(); + // The source replays the uncommitted group; it is refused rather than overwritten. + for sequence in 5..9 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.finish(); + writer.run().await.unwrap(); + assert!(started.elapsed() >= grace); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(6, 6), (7, 7), (8, 8)]); + check_objects(&store, &records).await; + assert_eq!(stored_groups(&store).await, referenced(&records)); + store.get_info("video").await.unwrap(); + store.get_info(TIMELINE).await.unwrap(); + for segment in 0..=7 { + store.get_segments(TIMELINE, segment).await.unwrap(); + } + } + + #[tokio::test] + async fn an_archive_restart_leaves_uncommitted_groups_unadvertised() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..3).await; + // Media stored after the last timeline commit: a crash before its record. + store.put_groups("video", &orphan(3)).await.unwrap(); + + record(&store, Config::default(), 3..6).await; + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2), (4, 4), (5, 5)]); + check_objects(&store, &records).await; + // An archive deletes nothing; the orphan stays invisible. + assert!( + stored_groups(&store) + .await + .contains(&Key::groups("video", 3..=3).unwrap()) + ); + assert!(!referenced(&records).contains(&Key::groups("video", 3..=3).unwrap())); + } + #[tokio::test] async fn a_failed_recovery_deletes_nothing() { - let inner = Arc::new(InMemory::new()); - let store = Store::new(inner.clone(), "rec"); + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); record(&store, config.clone(), 0..6).await; store.put_groups("video", &orphan(1)).await.unwrap(); let before = stored_groups(&store).await; - let failing = Failing { - inner: inner.clone(), - track: "", - list: true, - }; + let failing = mock.fork(); + failing.fail_lists(); let source = broadcast::Info::new().produce(); let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; assert!(matches!(result, Err(Error::Store(_)))); diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 35e05e2592..40f327c8f6 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -5,6 +5,7 @@ //! pin the storage traffic that composition produces: playlists read only the timeline, and a //! segment GETs exactly one object of the requested rendition. +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -24,11 +25,14 @@ use super::*; const TIMELINE: &str = hang::timeline::DEFAULT_NAME; -/// In-memory store that records every GET path. +/// In-memory store that records every GET path and counts listings. #[derive(Debug, Clone, Default)] struct Counting { inner: Arc, gets: Arc>>, + lists: Arc, + /// Fail every media GET, so nothing can quietly depend on one. + reject_media: Arc, } impl Counting { @@ -36,6 +40,15 @@ impl Counting { fn take(&self) -> Vec { std::mem::take(&mut *self.gets.lock().unwrap()) } + + /// Listings since the store was created. + fn lists(&self) -> usize { + self.lists.load(Ordering::SeqCst) + } + + fn reject_media(&self, reject: bool) { + self.reject_media.store(reject, Ordering::SeqCst); + } } impl std::fmt::Display for Counting { @@ -65,6 +78,12 @@ impl ObjectStore for Counting { async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { self.gets.lock().unwrap().push(location.to_string()); + if self.reject_media.load(Ordering::SeqCst) && is_media(location.as_ref()) { + return Err(object_store::Error::NotImplemented { + operation: "media GET".into(), + implementer: "Counting".into(), + }); + } self.inner.get_opts(location, options).await } @@ -76,6 +95,7 @@ impl ObjectStore for Counting { } fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.lists.fetch_add(1, Ordering::SeqCst); self.inner.list(prefix) } @@ -342,7 +362,9 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { let master = replay.broadcaster.master_playlist(None); assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); - // Render and reload every playlist: aligned numbering, and not one media GET. + // Render and reload every playlist with media GETs refused: aligned numbering, and not one + // media GET. + recording.store.inner().reject_media(true); for _ in 0..2 { for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { let playlist = replay.playlist(kind, name).await; @@ -358,6 +380,9 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { !gets.iter().any(|path| is_media(path)), "playlists must not GET media: {gets:?}" ); + recording.store.inner().reject_media(false); + // A range-bearing segment URI resolves its object directly: no listing, no index object. + let lists = recording.store.inner().lists(); // Switching renditions downloads only the selected rendition's object. let low = replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); @@ -400,10 +425,10 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { ] ); - // A repeated request hits the reader's object cache. The parked track re-subscribes - // upstream to confirm its warm cache, which re-reads only the small `.info`. + // A repeated request hits the reader's cache, including the immutable `.info`. replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); - assert_eq!(recording.gets(), ["rec/360p/.info"]); + assert_eq!(recording.gets(), Vec::::new()); + assert_eq!(recording.store.inner().lists(), lists, "segments never list"); } #[tokio::test] From e2722289efbecdf6ebddfe1c1293bba76d94b001 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 26 Sep 2026 10:08:19 -0700 Subject: [PATCH 12/49] quest(archive/track-timeline): claim line Co-Authored-By: Claude Opus 5.5 From e356f682bc54b75182fab86a4fcaa2e868358be8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 26 Sep 2026 16:14:56 -0700 Subject: [PATCH 13/49] quest(archive): plan idle flush on max(wall, pts) (#4301) Co-authored-by: Claude Opus 5.5 --- quest/m1/archive/README.md | 1 + quest/m1/archive/flush.md | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 quest/m1/archive/flush.md diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index c3208cf9ad..b932c53e7a 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -134,6 +134,7 @@ owned by that prerequisite, not duplicated in archive storage. - [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge - [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay +- [Idle flush](/quest/m1/archive/flush.md) - idle tracks are recorded within a bounded wall-clock delay, and `flush()` forces a track's pending record out - [Paced replay](/quest/m1/archive/paced-replay.md) - a replay pushes its groups to live subscribers on one shared clock, so any live player plays it - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback diff --git a/quest/m1/archive/flush.md b/quest/m1/archive/flush.md new file mode 100644 index 0000000000..0b9c3fe980 --- /dev/null +++ b/quest/m1/archive/flush.md @@ -0,0 +1,39 @@ +# [M] Idle tracks still get recorded + +## Goal + +A recording closes the record holding every frame within a bounded wall-clock +delay of its arrival, and hands it to the committer, even when its track goes +idle, and a caller can force one track's pending +record out with `flush()`. An append-only group that stops growing, or a track +whose publisher stops sending, no longer waits for the next frame or the end of +the recording. + +## Plan + +Decided: + +- Time decisions use elapsed = max(wall clock, pts) everywhere, so the existing + cutting rule does the work with no new setting: an open group idle past + `duration_max` (10 s by default) splits between frames, and a finished sparse + group (minimum 0) stores at once. Peer timestamps alone can never stall a + cut, since the local wall clock bounds them. A forward pts leap can close a + record early; a new record then starts at the leaping frame, so each leap + costs at most one extra record. Say in the PR how repeated leaps are bounded. +- The segmenter stays pure: it takes `now` as an input and exposes the next + wall-clock deadline at which a record must close. Callers arm the timer. The + moq-archive writer uses tokio time; moq-mux's `Recorder` must stay wasm-safe + (moq-mux builds for wasm32), so it uses moq-net's runtime `Timer` or + `web_async` time, never `tokio::time`. +- Manual flush is per track: `Control::flush(name)` on the writer, beside the + broadcast-wide `cut`, and `Recorder::flush()` on a timeline. +- The bound covers closing a record, not upload latency: a slow object-store + PUT still delays the durable commit, which the writer already serializes. +- Tests inject `now` into the segmenter and run the writer on paused tokio + time, per the repository's mock-time rule. Cover an idle open group, an idle + track after a finished group, a manual flush mid-group, and a publisher whose + timestamps run ahead of or behind the wall clock. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the per-track segmenter and writer this extends From 70bafeaf612b09fc21196e52f79f4d0fd5571d3a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 26 Sep 2026 19:04:08 -0700 Subject: [PATCH 14/49] quest(archive): the JS timeline recorder flushes idle tracks too (#4306) Co-authored-by: Claude Opus 5.5 --- quest/m1/archive/flush.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quest/m1/archive/flush.md b/quest/m1/archive/flush.md index 0b9c3fe980..61ad9751c3 100644 --- a/quest/m1/archive/flush.md +++ b/quest/m1/archive/flush.md @@ -27,6 +27,9 @@ Decided: `web_async` time, never `tokio::time`. - Manual flush is per track: `Control::flush(name)` on the writer, beside the broadcast-wide `cut`, and `Recorder::flush()` on a timeline. +- JS mirrors it: `@moq/hang`'s `Timeline.Recorder` gets the same idle + deadline and a `flush()`, so Rust and JS timelines stay one API. Without it, + an idle browser track keeps today's unbounded behavior (Codex on #4301). - The bound covers closing a record, not upload latency: a slow object-store PUT still delays the durable commit, which the writer already serializes. - Tests inject `now` into the segmenter and run the writer on paused tokio @@ -37,3 +40,4 @@ Decided: ## Required - [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the per-track segmenter and writer this extends +- [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - the JS recorder this extends, so the flush lands on the per-track shape rather than the aligned one it replaces From d83b3808e5b22524ebff0b1b5dfc3e66f5e0632e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 27 Sep 2026 16:47:56 -0700 Subject: [PATCH 15/49] feat(archive)!: per-track timelines (#4280) Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + doc/bin/cli.md | 23 +- doc/bin/hls.md | 29 +- doc/concept/hang.md | 4 +- doc/lib/rs/hang.md | 2 +- doc/lib/rs/index.md | 2 +- drafts/draft-lcurley-moq-hang.md | 296 +- quest/m1/archive/README.md | 135 +- quest/m1/archive/browser.md | 3 +- quest/m1/archive/dvr.md | 4 - quest/m1/archive/idle-flush.md | 24 + quest/m1/archive/paced-replay.md | 4 - quest/m1/archive/replay-catalog.md | 4 - quest/m1/archive/track-timeline/README.md | 20 +- quest/m1/archive/track-timeline/core.md | 46 - quest/m1/archive/track-timeline/js.md | 4 - rs/hang/fixtures/catalog-clock.json | 6 +- rs/hang/src/catalog/archive.rs | 62 +- rs/hang/src/catalog/root.rs | 18 +- rs/hang/src/lib.rs | 2 +- rs/hang/src/timeline.rs | 222 +- rs/moq-archive/Cargo.toml | 5 + rs/moq-archive/benches/writer.rs | 78 + rs/moq-archive/src/error.rs | 8 +- rs/moq-archive/src/info.rs | 20 +- rs/moq-archive/src/lib.rs | 8 +- rs/moq-archive/src/path.rs | 155 +- rs/moq-archive/src/proof.rs | 148 +- rs/moq-archive/src/reader/index.rs | 249 +- rs/moq-archive/src/reader/mod.rs | 325 ++- rs/moq-archive/src/reader/tests.rs | 509 ++-- rs/moq-archive/src/recover.rs | 144 +- rs/moq-archive/src/segment.rs | 204 +- rs/moq-archive/src/store.rs | 225 +- rs/moq-archive/src/writer.rs | 1435 +++++---- rs/moq-audio/src/encode/capture.rs | 13 +- rs/moq-cli/src/archive.rs | 101 +- rs/moq-hls/src/export/archive_tests.rs | 237 +- rs/moq-hls/src/export/mod.rs | 283 +- rs/moq-hls/src/export/rendition.rs | 241 +- rs/moq-hls/src/export/renditions.rs | 45 +- rs/moq-hls/src/export/segments.rs | 114 +- rs/moq-hls/src/export/spans.rs | 359 +++ rs/moq-hls/src/import.rs | 7 - rs/moq-mux/src/catalog/producer.rs | 104 +- rs/moq-mux/src/catalog/tracks.rs | 19 +- rs/moq-mux/src/container/flv/import_test.rs | 8 +- rs/moq-mux/src/container/fmp4/import.rs | 107 +- rs/moq-mux/src/container/fmp4/import_test.rs | 200 +- rs/moq-mux/src/container/mkv/import_test.rs | 8 +- rs/moq-mux/src/container/producer.rs | 115 +- rs/moq-mux/src/error.rs | 48 +- rs/moq-mux/src/timeline.rs | 2733 ++++++------------ rs/moq-rtc/src/codec/vp8.rs | 4 +- rs/moq-transcode/src/catalog.rs | 5 +- rs/moq-video/src/encode/producer.rs | 18 +- 56 files changed, 4500 insertions(+), 4693 deletions(-) create mode 100644 quest/m1/archive/idle-flush.md delete mode 100644 quest/m1/archive/track-timeline/core.md create mode 100644 rs/moq-archive/benches/writer.rs create mode 100644 rs/moq-hls/src/export/spans.rs diff --git a/Cargo.lock b/Cargo.lock index 8c4203a9a5..e329e57aed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4179,6 +4179,7 @@ version = "0.0.6" dependencies = [ "async-trait", "bytes", + "criterion", "futures", "hang", "kio 0.6.0", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index af778b0422..7d881cb6c2 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -211,19 +211,22 @@ moq --connect https://relay.example.com/anon --broadcast event-replay.hang impor ``` `export archive` records one broadcast with -[moq-archive](https://docs.rs/moq-archive), reading its catalog as it changes: -video and audio renditions pace the segments, and the catalog plus every text, -JSON, and binary track are recorded alongside. It refuses a rendition served +[moq-archive](https://docs.rs/moq-archive), reading its catalog as it changes. +Every track gets its own timeline, stored in spans cut at group boundaries +between 2s and 10s. The catalog and every text, JSON, and binary track are +sparse data, so each of their groups is stored as soon as it finishes, and a +group that never closes is stored in pieces as it grows. It refuses a rendition served from another broadcast, and one that returns after the catalog dropped it. The -stage ends once the broadcast does, and it refuses a store URL that already -holds a recording. `--retention 1h` keeps only the last hour (a DVR), +stage ends once the broadcast does. A store URL that already holds a +recording is continued: each track resumes after its newest stored span. `--retention 1h` keeps only the last hour (a DVR), deleting expired objects `--retention-grace` (default 30s) after the timeline -stops advertising them. +stops advertising them. Every track keeps at least its newest span, so a catalog +that never changes outlives the video it was published with. -`import archive` republishes a recording: the timeline replays as a live track -and every other track's groups are served on request, one object GET per group -range. By default it replays what is stored and ends the timeline there; -`--follow 2s` keeps checking for new segments of a recording still being made. +`import archive` republishes a recording: each track's timeline replays as a +live track and every track's groups are served on request, one object GET per +stored span. By default it replays what is stored and ends the timelines there; +`--follow 2s` keeps checking for new spans of a recording still being made. Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. Cloud credentials come from diff --git a/doc/bin/hls.md b/doc/bin/hls.md index cc85030f74..d73c9a2009 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -17,16 +17,21 @@ moq --connect https://relay.example.com/anon --broadcast my-stream.hang export h moq --connect https://relay.example.com/anon --broadcast my-stream.hang import hls https://example.com/live/master.m3u8 ``` -Export never subscribes to media. It reads the broadcast's -[timeline track](/concept/hang#catalog), a log of complete segments aligned -across every rendition, to build playlists, then fetches exactly the groups a -requested segment covers from the relay's cache and transmuxes them to CMAF on -demand. So a segment is servable for as long as the relay's +Export never subscribes to media. It reads each rendition's +[timeline track](/concept/hang#catalog), a small index of that rendition's +groups, to build playlists, then fetches exactly the frames a requested segment +covers from the relay's cache and transmuxes them to CMAF on demand. So a +segment is servable for as long as the relay's [cache](/bin/relay/config#cache) retains it, and idle renditions cost nothing. -Because segments are aligned, the same number names the same span of content in -every media playlist; a record with nothing for a rendition renders as -`EXT-X-GAP` and a jump in content time as `EXT-X-DISCONTINUITY`. A broadcast -whose catalog advertises no timeline is skipped. One server exposes every +Segment boundaries come from one reference rendition (the first video +rendition by name, or the first audio one without video) and are numbered by +its records, so every edge and every reload agree. Every other video rendition +snaps each boundary to its nearest keyframe within about a second, and a +segment with none in range renders as `EXT-X-GAP`; audio takes every frame +inside the segment's span. A jump in content time renders as +`EXT-X-DISCONTINUITY`. Gaps are a fallback: a publisher wanting clean HLS +export should align video GOPs across renditions. A broadcast +whose catalog advertises no timelines is skipped. One server exposes every broadcast by path: ```text @@ -40,12 +45,12 @@ broadcast by path: A [`moq-archive`](https://docs.rs/moq-archive) recording replayed through its `Reader` is served the same way, with no second stored copy. Playlists come -from the replayed timeline alone, and a segment GETs exactly one stored object -of its rendition, so switching renditions never downloads both. An +from the replayed timelines alone, and a segment GETs only its rendition's +stored objects, so switching renditions never downloads both. An inline-parameter-set codec with no catalog `description` is the exception: the first playlist render GETs one keyframe group to build the init segment, then caches it. Out-of-band configs need no media GET. When the catalog's -`archive` entry names a `store` and no `replay` path, its ranges are durable on +`archive` entry names a `store` and no `replay` path, its spans are durable on this broadcast, so the playlists list the whole retained timeline and only the recording's own retention trims them; DASH `timeShiftBufferDepth` is the listed span. The playlist ends with `EXT-X-ENDLIST` only once the reader's caller diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 20f3bfc0ed..96b9090348 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -57,8 +57,8 @@ A few things the catalog can express beyond decoder config: - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Stalled renditions.** A publisher can flag a rendition as temporarily bad so players prefer another one without the track disappearing. First-party video publishers set this flag after more than three frame intervals of source silence or encoding lag while subscribed, and clear it after three on-time completed frames or when idle. Browser and native capture poll while waiting; FLV and MPEG-TS importers observe video silence as container data arrives. The shared detector is `hang::catalog::stalled::Detector` in Rust and `Catalog.Stalled.Detector` in JavaScript. It is a playback diagnostic, not an authorization or routing signal. -- **Archive.** A broadcast may advertise an `archive` entry naming its timeline track (a small index of each complete aligned segment) and, if recorded, the replay MoQ path, object-store URL, and format version. The timeline is what lets the [HLS gateway](/bin/hls) build playlists without subscribing to media. -- **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a segment index. +- **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and, if recorded, the replay MoQ path, object-store URL, and format version. Every track is cut by one rule, at group boundaries between 2s and 10s (a zero minimum for sparse data such as the catalog), and tracks commit and expire independently. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. +- **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a timeline. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. ## Text diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index b25a36a24a..dfde9ad195 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -12,7 +12,7 @@ The [hang media format](/concept/hang) on top of `moq-net`: a live catalog describing renditions with WebCodecs-style decoder configs, and containers that carry a timestamp with every frame. -- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), stalled hints, cross-broadcast rendition references, and the `archive` entry (timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. +- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), stalled hints, cross-broadcast rendition references, and the `archive` entry (each track's timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. - **Containers.** `legacy`, `cmaf`, and `loc`, decoded for you; unknown kinds pass through untouched. - **`Ordered`** reads a media track as timestamped frames, reorders groups, and skips ones that fall past your max age. - **Codecs described**: H.264, H.265, VP8, VP9, AV1, AAC, Opus, PCM. diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index 2a02438809..0d75772357 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -19,7 +19,7 @@ The reference implementation. Every crate is on | [moq-uring](https://docs.rs/moq-uring) | Experimental Linux io\_uring worker: one pinned thread per ring serving moq-lite over its own QUIC stack. | | [hang](/lib/rs/hang) | The media layer: catalog, containers, ordered frame delivery. | | [moq-mux](/lib/rs/moq-mux) | Import and export fMP4/CMAF, MPEG-TS, Matroska, FLV, and Annex-B. | -| [moq-archive](https://docs.rs/moq-archive) | Versioned hang recordings on any `object_store` backend: track layout, `.info` JSON, and segment objects. | +| [moq-archive](https://docs.rs/moq-archive) | Versioned hang recordings on any `object_store` backend: track layout, `.info` JSON, per-track timelines, and span objects. | | [moq-video](/lib/rs/moq-video) | Native capture, hardware encode/decode (Apple, Windows, NVIDIA, VAAPI, V4L2, Android), and GPU rendering. | | [moq-v4l](https://docs.rs/moq-v4l) | Safe Video4Linux 2 bindings with the kernel headers checked in, so a build needs no libclang. | | [moq-audio](/lib/rs/moq-audio) | Microphone and speaker, Opus/PCM/AAC codecs, echo cancellation. | diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 04fffe9264..831b889405 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -141,7 +141,7 @@ For example, a chat entry should name a chat track, not carry individual chat me This way catalog updates are rare and a client MAY choose to not subscribe. The `json` and `binary` sections ({{data}}) are how a track like that is listed. -This specification defines audio, video, and text media tracks, plus an optional `archive` entry ({{archive-catalog}}) naming the timeline track ({{timeline}}) that indexes their segments, an optional root `clock` entry ({{clock}}) mapping content time to wall time, and the application data tracks in {{data}}. +This specification defines audio, video, and text media tracks, plus an optional `archive` entry ({{archive-catalog}}) naming the timeline track ({{timeline}}) that indexes each track, an optional root `clock` entry ({{clock}}) mapping content time to wall time, and the application data tracks in {{data}}. ## Clock {#clock} The catalog's root `clock` field is the broadcast's one continuous clock: @@ -155,12 +155,12 @@ type ClockSchema = { The `wall` field is the wall-clock time of PTS zero, in `timescale` units since the moq epoch, 2020-01-01T00:00:00Z. A consumer derives the wall-clock time of any media timestamp as `wall + pts` after converting that timestamp into this timescale, and Unix time by adding the epoch back (for HLS `EXT-X-PROGRAM-DATE-TIME` or DASH `availabilityStartTime`). -Every media track and the archive index refer to this one mapping after timescale conversion; there are no competing wall epochs. +Every media track and every timeline refer to this one mapping after timescale conversion; there are no competing wall epochs. The `timescale` field is the units per second for `wall`, defaulting to 1000000 (microseconds) when absent. A zero `timescale` is invalid, as is a `wall` outside the JSON-safe integer range; both are refused rather than truncated. -The mapping is fixed for the broadcast and independent of `archive`: a live-only publisher exposes its clock without creating a segment index. +The mapping is fixed for the broadcast and independent of `archive`: a live-only publisher exposes its clock without creating a timeline. A discontinuity marker is a delivery event, not a new epoch, and a system-clock adjustment never retimes the mapping. ## Video @@ -616,7 +616,7 @@ Some metadata tracks are compressed. Compression is signalled per track, never inferred from the track's name. A data track ({{data}}) declares it with the `compression` field ({{field-compression}}). -The two tracks this specification defines as always compressed, `catalog.json.z` ({{catalog}}) and the timeline track ({{timeline}}), are identified by their role instead. +The tracks this specification defines as always compressed, `catalog.json.z` ({{catalog}}) and the timeline tracks ({{timeline}}), are identified by their role instead. The `.z` suffix on those names is a naming convention, and a consumer MUST NOT treat it as a signal. Such a track is compressed per {{moqflate}}: each group is one raw DEFLATE stream, sync flushed at each frame boundary. @@ -624,22 +624,21 @@ The suffix is the declaration. # Timeline {#timeline} -The timeline track is the broadcast's segment index. +A timeline track indexes one track's content. MoQ groups carry only an opaque sequence number; the timestamps live inside the media frames. -The timeline republishes the broadcast's segmentation as metadata: one record per segment, mapping a span of content time to the group ranges that carry it on each media track. -A consumer can answer "which groups cover time T on track X" and "where is the live edge" from a few bytes per segment, without downloading media. -This is sufficient to render an HLS or DASH playlist, seek a VOD recording, or index an archive. +A timeline republishes a track's spans as metadata: one record per span, mapping a span of content time to the frame positions that carry it. +A consumer can answer "which groups cover time T on track X" and "where is the live edge" from a few bytes per span, without downloading media. +This is sufficient to derive an HLS or DASH playlist, seek a VOD recording, or index an archive. -The timeline is optional. -There is one timeline per broadcast, because its purpose is that segments are aligned across the broadcast's tracks: segment N covers the same span of content time on every track, which is what HLS requires of switchable renditions. -A broadcast that does not need aligned segments simply omits it. +Timelines are optional, and a track has at most one. +Each track's timeline is cut, published, and expired independently; nothing in one timeline waits for another track. ## Catalog Section {#archive-catalog} -The catalog's root `archive` field is the one name for the segment index, and for any durable recording of those ranges: +The catalog's root `archive` field names every timeline, and any durable recording of their spans: ~~~ type ArchiveSchema = { - "track": string, + "timelines": Map, "timescale": number | undefined, "durationMax": number | undefined, "replay": string | undefined, @@ -648,41 +647,38 @@ type ArchiveSchema = { } ~~~ -The `track` field names the MoQ track carrying the segment records. -The name `timeline.z` is RECOMMENDED; a consumer MUST use the advertised name rather than assuming it. -A live publisher without a store advertises `archive` with the timeline fields (`track`, `timescale`, `durationMax`) alone. -Every range the timeline advertises is FETCHable; with a store they are also durable. -There is no sibling `timeline` entry and no generation: a client that must tell recordings apart compares `replay` and `store`. +The `timelines` field maps each indexed track to the MoQ track carrying its records. +Any track may be indexed, including the catalog track itself and data tracks ({{data}}). +Naming a timeline after its track with a `.timeline.z` suffix is RECOMMENDED; a consumer MUST use the advertised names rather than assuming them. +A live publisher without a store advertises `archive` with the timeline fields (`timelines`, `timescale`, `durationMax`) alone. +Every span a timeline advertises is FETCHable; with a store it is also durable. +There is no generation: a client that must tell recordings apart compares `replay` and `store`. The `timescale` field is the units per second for the records' `pts` and `duration` values, and for `durationMax`. If absent, it defaults to 1000 (milliseconds). A zero `timescale` is invalid and MUST be refused. -The `durationMax` field, if present, is the declared upper bound on a segment's `duration`, in `timescale` units. -A publisher that controls its encoder knows its keyframe cadence up front, so a consumer can size buffers or write an HLS `EXT-X-TARGETDURATION` from the catalog alone, before observing a single segment. -The value MUST NOT change for the life of the broadcast, and a publisher MUST NOT emit a record whose `duration` exceeds it. -A publisher that cannot honor that MUST omit the field rather than emit a record contradicting it. - -The field is absent when the media decides the segmentation instead, which is the common case: a real-time encoder places keyframes on demand and a single GOP may be minutes long, and a publisher importing a source it does not control cannot promise anything about that source. -A consumer needing a bound then derives one from the records it has seen, raising it as longer segments arrive. +The `durationMax` field, if present, is the declared upper bound on a record's `duration`, in `timescale` units. +A publisher MUST NOT emit a record whose `duration` exceeds it; splitting a group between frames ({{timeline-cutting}}) makes the bound achievable for any content, and a record whose frames are farther apart than the bound ends at the bound, leaving the rest of the gap uncovered. +The value MUST NOT change for the life of the broadcast. -Wall-clock mapping is the catalog root `clock` ({{clock}}), not this section: a consumer derives the wall-clock time of any segment as `clock.wall + pts` after converting `pts` into the clock's timescale. +Wall-clock mapping is the catalog root `clock` ({{clock}}), not this section: a consumer derives the wall-clock time of any record as `clock.wall + pts` after converting `pts` into the clock's timescale. The `replay` field, if present, is a relative MoQ broadcast path ({{field-broadcast}}) the archive is served back from. -Absent, the timeline lives on the catalog's own broadcast. +Absent, the timelines live on the catalog's own broadcast. A wildcard replay path names no generation. The `store` field, if present, is the object-store URL the recording objects ({{recording}}) live under. Authorization for `replay` and `store` is external. The `version` field is the recording object format version ({{recording-track}}). -It is 1 for this specification. +It is 2 for this specification. A publisher that exposes a store MUST set it; a publisher that does not MUST omit it. A catalog that composes another broadcast's renditions MUST preserve that child's `archive` entry instead of synthesizing one. ## Track Framing {#timeline-framing} -The timeline track is a sliding window of records. +A timeline track is a sliding window of records. An unbounded publisher only appends records, while a DVR publisher also removes records from the front as they expire. The first frame of each group is a UTF-8 JSON object containing a checkpoint of the retained window: @@ -707,74 +703,67 @@ A consumer that missed records reports their absolute index range as skipped bef The frames are DEFLATE-compressed ({{!RFC1951}}) within each group. The publisher ends each frame's compressed data with an empty sync-flush block (the `0x00 0x00 0xff 0xff` trailer is removed, as in {{?RFC7692}}), so a consumer decompresses frames incrementally from the group's first frame. -The `.z` suffix on the RECOMMENDED track name marks this compression, mirroring the catalog's `catalog.json.z` sibling. +The `.z` suffix on the RECOMMENDED track names marks this compression, mirroring the catalog's `catalog.json.z` sibling. ## Records {#timeline-records} -Each record describes one complete segment: +Each record describes one span of its track: ~~~ type TimelineRecord = { - "segment": number, + "sequence": number, "pts": number, "duration": number, - "tracks": Map | undefined, + "start": TimelinePosition, + "end": TimelinePosition, + "keyframe": boolean | undefined, } -type TimelineRange = { - "start": number, - "end": number, - "keyframe": boolean | undefined, +type TimelinePosition = { + "group": number, + "frame": number | undefined, } ~~~ -The `segment` field is the segment's number. -Numbers are consecutive within a broadcast, anchoring HLS `EXT-X-MEDIA-SEQUENCE`; they are explicit rather than implied by record order so a reader joining mid-stream, or reading a windowed recording, keeps stable numbering. +The `sequence` field is the record's number, consecutive within its timeline and equal to the record's window index. -The `pts` field is the segment's start and `duration` its length, both in the timeline's timescale. +The `pts` field is the timestamp of the span's first frame and `duration` its length, both in the timescale of the catalog section. The next record's `pts` equals `pts + duration` unless content time itself jumped; a consumer SHOULD treat such a jump as a discontinuity. -The `tracks` field maps each participating track name to the group ranges it contributes. -Each range covers groups `start` through `end` inclusive, as used by moq-lite FETCH and SUBSCRIBE. -More than one range means the group sequence is discontinuous inside the segment: the skipped groups never existed. -A track absent from the map has no content for the span (a gap; HLS `EXT-X-GAP`). +A position names frame `frame` of group `group`, as used by moq-lite FETCH and SUBSCRIBE; `frame` defaults to 0. +Positions order by group, then frame. +The span holds every frame from `start` (inclusive) to `end` (exclusive). +An `end` at frame 0 of group `g` holds every frame of group `g - 1` and none of `g`, whether or not `g` exists; an `end` at a later frame of `g` holds the frames of `g` before it. +Every group from `start.group` to the last group holding a frame exists and holds at least one frame of the span. +A record starts where the previous one ended, unless the track skipped group sequences between them. -Participating tracks need not be audio or video. -A catalog, or an application's own metadata track such as a chat log, is listed exactly like a media track, which is what lets a recording ({{recording}}) address all of them the same way. -A consumer that only wants renditions therefore MUST select tracks by consulting the catalog rather than by assuming every name in the map is media. -A record MUST tolerate and SHOULD preserve unknown fields, like the catalog. +The `keyframe` field states whether the span's first frame is a keyframe, i.e. whether a player can join or switch renditions there. +If absent, it defaults to true. -The `keyframe` field states whether the range's first group starts with a keyframe, i.e. whether a player can join or switch renditions there. -If absent, it defaults to true; a publisher sets `false` when a source resumes without one, so an exporter knows not to advertise the segment as independently decodable. +A record MUST tolerate and SHOULD preserve unknown fields, like the catalog. -## Segmentation {#timeline-segmentation} -A segment is a span of content time shared by every media track. -A track contributes every group whose start falls inside the span, so a segment boundary SHOULD land on a group start: every group already begins with a keyframe ({{container}}), so a boundary at a group start lets each track contribute whole groups and remain independently decodable. -A segment MAY span multiple groups of a track (short groups packed into a longer segment). +## Cutting {#timeline-cutting} +Where a span ends is publisher policy. +A publisher MUST NOT publish a record until its span is final: every frame between `start` and `end` has been published. +The newest record is therefore the live edge of its track. -How boundaries are chosen is publisher policy: following a source's existing segmentation (an imported HLS playlist, CMAF segments on disk), or pacing by a minimum duration. -A publisher pacing itself SHOULD end a segment at the earliest point that is a group start on every enrolled track and at least the minimum past the segment's start, which makes the track with the coarsest groups pace the broadcast and leaves no track's group split across a boundary. -A minimum is always satisfiable, whereas a maximum is not: a single group longer than it cannot be divided. -Where no such point exists because two tracks have different coarse cadences, a publisher MUST choose one of them rather than a point interior to any track's group. +The RECOMMENDED policy is one rule for every track: -Whatever the policy, a publisher MUST NOT emit a record until the segment is complete: every *pacing* track's groups for the span are known. -Records are therefore self-contained and immediately servable, and the newest record is the live edge. -A pacing track that has produced nothing for the span holds the record back; a publisher that knows a track has stopped for good closes it, and the record then simply omits it (a gap). +- A record ends at the first group boundary at least a minimum duration past its own start, so short groups (audio) pack into one record and long ones (video) get one each. +- A group still open a maximum duration past the record's start is split between frames, so a group that never closes (an append-log such as a `moq-json` stream) is indexed as it grows; `durationMax` advertises this maximum. +- A skipped group sequence ends the record at its newest known content, not where the next group starts, so a pause is not counted as content. -A pacing track is one whose groups arrive continuously, which is what makes them usable as boundaries. -A track that publishes on its own schedule cannot pace: a catalog emits a group only when the renditions change, so a timeline waiting for it would stall the moment it went quiet. -Such a track is *non-pacing*: its groups are listed in whichever segment is open when they arrive, but it never determines a boundary and never holds a record back. -A publisher SHOULD record its catalog this way, so a recording can resolve the renditions in effect at any segment. +A minimum of 2 seconds and a maximum of 10 seconds are RECOMMENDED. +Sparse data that publishes on its own schedule, such as a catalog, SHOULD use a minimum of zero: each group is then its own record, ended as soon as the group finishes, so its newest group is indexed without waiting for another that may never come. -Placement of a non-pacing track's groups is therefore by arrival rather than by content time: nothing waits for them, so a group that arrives after its segment has already been published is listed in the next one. -When a segment closes, every non-pacing group that arrived while it was open belongs to that segment regardless of the timestamp basis carried by the non-pacing track. -The frames still carry their own timestamps, so no timing information is lost. -A non-pacing track's timestamps do not extend the final segment's duration. -A non-pacing track whose group never closes (an append-log such as a `moq-json` stream) is listed once, in the segment its group opened in. -A publisher that needs such content addressable per segment SHOULD roll the group at segment boundaries, which costs the shared compression window but makes each segment self-contained. +A publisher MAY add boundaries of its own, such as following a source's segmentation or cutting audio where video starts a group, so a derived segment needs fewer records. +The final record of an ended track has no successor; its `duration` runs to the newest known content, which a publisher SHOULD carry past the start of the last frame when it knows where that frame ends. -A group that starts before the first boundary belongs to the first segment. -The final segment of an ended broadcast has no closing boundary; its `duration` runs to the newest known content. -A publisher SHOULD carry the end of the last group's content into that value, since a publisher that knows only where each group *started* would report a duration one group short, and zero for a final segment that is a single group. +## Derived Formats {#timeline-derived} +Segmented formats such as HLS and DASH need one segment numbering across renditions, which per-track timelines do not provide on their own. +An edge deriving one takes the segment boundaries from a reference rendition's records, the first video rendition by name with a timeline, or the first such audio rendition when there is none, and numbers the segments by that rendition's record `sequence`, so every edge and every reload agree. +Another video rendition snaps each boundary to its nearest record that starts a group on a keyframe; a segment with no such record nearby has no content on that rendition (a gap; HLS `EXT-X-GAP`). +Other renditions take the frames whose timestamps fall inside each segment's span. +A publisher wanting such an export SHOULD start video groups at the same timestamps across renditions. # MPEG-TS Service Information {#mpegts-si} @@ -834,21 +823,18 @@ Carrying the source's time rather than synthesizing one keeps the clock consiste A live broadcast is bounded history: moq-lite {{moql}} serves the present with SUBSCRIBE and the recent past with FETCH, both ending at the publisher's cache. A *recording* is the persistent tier, writing a broadcast to a filesystem or object store so it can be served back long after the live session ended. -A recording is addressed by segment. -The timeline ({{timeline}}) names every segment and the groups that carry it. -Each track's object is addressed by its smallest and largest stored group IDs; the timeline supplies those bounds without reading media. -A reader that wants segment N of a track issues one whole-object GET, with no range header and no second request, which is what both an HLS or DASH origin and a player seeking a VOD recording need. +A recording is addressed by record. +Each recorded track has its own timeline ({{timeline}}), and each of its records is stored as one object named by the record's `sequence`. +A reader that wants a record of a track issues one whole-object GET, with no range header and no second request, which is what both an HLS or DASH origin and a player seeking a VOD recording need. -A recording covers the tracks the timeline describes, plus the catalog and the timeline itself. -A broadcast with no timeline has no segments and cannot be recorded this way. +A recording covers the tracks its writer selects, including the catalog, plus each one's timeline. ## Layout {#recording-layout} A recording is a set of objects under a common prefix: ~~~ //.info -//groups/. -//segments/ +//segments/ ~~~ The common prefix is application-defined and is not interpreted by this format. @@ -856,22 +842,17 @@ The common prefix is application-defined and is not interpreted by this format. `` is the track's UTF-8 name with every byte outside `A-Z a-z 0-9 _ -` percent-encoded as `%` followed by two uppercase hexadecimal digits. For example, `catalog.json` becomes `catalog%2Ejson`. An encoded name never contains `/` or begins with `.`, so a track cannot address anything outside the prefix. -`.info`, `groups`, and `segments` are reserved within a track directory. +`.info` and `segments` are reserved within a track directory. + +A recording names each track's timeline by appending `.timeline.z` to the track's name, so a recording is recoverable from its objects alone; a recorded track's own name MUST NOT end in `.timeline.z`. +For a recorded track, `segments/` holds the frames of record `sequence` of its timeline. +For a timeline track, `segments/` holds the complete Window groups ({{timeline-framing}}) closed while committing record `sequence` of the track it indexes, at least one per object. -`` and `` are the inclusive first and last group sequence numbers in the object. -Recorded group and segment IDs MUST be integers in the range 0 through 9007199254740991 (2^53 - 1), so timeline JSON preserves them exactly. -Each filename field is written as exactly 19 decimal digits, padded with leading zeros, so lexical and numeric order agree. +`` is written as exactly 19 decimal digits, padded with leading zeros, so lexical and numeric order agree. +Recorded sequences, group IDs, and frame indices MUST be integers in the range 0 through 9007199254740991 (2^53 - 1), so timeline JSON preserves them exactly. Writers and readers MUST reject IDs outside this range, including reconstructed group IDs in the binary table. -A range-named object MUST contain at least one group, and its table's first and last sequences MUST match the filename. -For each track, object ranges MUST be nonoverlapping and strictly increasing in timeline segment order; gaps are allowed both within and between objects. -A track with no stored groups for a segment has no object or range in that record. -There are no empty index objects or duplicate copies addressed by segment number. - -This layout applies to every recorded track, including the catalog, except the recording-owned timeline track identified by the catalog's `archive` field ({{archive-catalog}}). -The timeline uses `segments/` with the same binary envelope and at least one complete Window group per object. -`` is the committed timeline segment ID, encoded as 19 zero-padded decimal digits within the recording ID range. -After its first segment, the recording MUST commit consecutive segment IDs, including all-gap segments. -A writer MUST stop before allocating an ID beyond the limit; IDs never wrap. +After its first record, a timeline MUST commit consecutive sequences. +A writer MUST stop before allocating a sequence beyond the limit; sequences never wrap. Each track has its own objects so a reader can fetch one rendition without downloading the others. ## Track Objects {#recording-track} @@ -879,30 +860,31 @@ Each track has its own objects so a reader can fetch one rendition without downl ~~~ { - "version": 1, + "version": 2, "priority": 0, "timescale": 1000000 } ~~~ All three fields are required integers. -`version` identifies this recording format and MUST be 1. +`version` identifies this recording format and MUST be 2. `priority` and `timescale` have the meanings of moq-lite `TRACK_INFO` {{moql}}: `priority` is in the range 0 through 255, and `timescale` is in the range 1 through 9007199254740991, so JSON consumers can preserve it exactly. A reader MUST preserve integer values exactly. `Publisher Max Age` is not stored; a reader supplies its own serving policy. -The track object MUST be durable before its first segment object is stored, including for the timeline track. +The track object MUST be durable before its first segment object is stored, including for a timeline track. It is immutable for the lifetime of the recording. A reader MUST refuse an unknown version or invalid track properties. On an existing `.info`, a writer MUST validate and compare the parsed `version`, `priority`, and `timescale` values; JSON whitespace and member order do not affect equality. Different property values MUST fail enrollment, and the existing object MUST NOT be rewritten. ## Segment Objects {#recording-segments} -A segment object holds one track's complete groups for one segment: +A segment object holds one record's frames: ~~~ Segment Object { - Version (i) = 1 + Version (i) = 2 + Frame Start (i) Group Count (i) Group Table { Sequence Delta (i) @@ -918,11 +900,11 @@ Segment Object { ~~~ Fields annotated `(i)` are variable-length integers using the QUIC encoding ({{!RFC9000}}, Section 16). +`Frame Start` is the index within its group of the first group's first stored frame; every later group starts at frame 0. `Group Count` gives the number of group entries; each `Frame Count` gives the number of frame entries in that group. The first `Sequence Delta` is the absolute group sequence number. Each subsequent value is the group sequence number minus the previous sequence number minus one; zero therefore means the next consecutive group. Groups MUST have strictly ascending sequence numbers; reconstruction MUST reject values exceeding the recording ID limit ({{recording-layout}}). -For range-named objects, the sequences MUST match the ranges in the timeline. Frame entries appear in their original order within each group. `Timestamp` is the frame's absolute timestamp in the track's `timescale` units, not a delta. It MUST be in the range 0 through 9007199254740991 (2^53 - 1), so browser replay preserves it exactly; writers and readers MUST reject larger values. @@ -934,103 +916,92 @@ A reader can locate a group or a frame index directly from the table without par A decoder MUST validate the complete table before accessing any payload. Counts and entries MUST fit in the retrieved object, and every offset and length MUST describe a range within its payload bytes, with overflow checked before arithmetic. -An unknown version or malformed table is a missing segment, not an invalid recording. +An unknown version or malformed table is a missing record, not an invalid recording. -A segment object MUST contain whole groups. -Segmentation forbids a boundary inside a track's group ({{timeline-segmentation}}), so a reader never has to stitch a group across objects. +A recorded track's object MUST hold exactly the frames of its record's span ({{timeline-records}}): its first group and `Frame Start` match `start`, its groups are consecutive and none is empty, and its last group ends at `end`. +A group may span several objects when a record was split between frames; a reader stitches it from consecutive records. Segment objects are immutable once written and are parseable without the timeline. -The timeline uses the same envelope and stores only complete Window groups ({{timeline-framing}}) closed while committing that segment. -Its own groups are discovered by listing, not included in the record's `tracks`, avoiding a record that must index itself. +The timeline objects use the same envelope with a `Frame Start` of zero. +Their own groups are discovered by listing, not indexed by a record, avoiding a record that must index itself. A reader replays their checkpoints and operations in group order, preserving group boundaries and their independent DEFLATE windows. Neither timeline nor media objects are appended to or rewritten. One writer owns a recording prefix. An existing segment object key MAY be reused only for identical object bytes; a conflicting create MUST fail. ## Writer Behavior {#recording-writer} -A writer subscribes to the broadcast and buffers the in-progress segment independently of the publisher or relay cache. -It writes a track's segment object once that track's groups for the span are known. -Per-track objects are written independently: a track whose content is complete does not wait for a slower one. +A writer subscribes to the broadcast and buffers each track's open span independently of the publisher or relay cache. +It cuts each track's timeline itself ({{timeline-cutting}}) and commits each track independently: a track whose span is complete does not wait for any other track. The recording contract MUST NOT depend on a relay retaining those groups while storage catches up. -A writer MUST make each track's `.info` and segment object durable before publishing the timeline record that references them. -The recording owns its timeline encoder so it can publish the ranges actually stored. - -If a track's object cannot be made durable, the writer MUST omit only that track from the record's `tracks`; successfully stored tracks retain their ranges. -The segment number and timing are unchanged, including when every track is omitted. -The writer pushes the resulting record and every subsequent record through its own Window encoder. -It MUST NOT copy source frames whose Window position or group-local DEFLATE dictionary depends on a record it changed. - A writer MUST accept new groups for each track in strictly increasing sequence order and MUST refuse a duplicate or decreasing sequence; it does not reorder arrivals. -Groups already accepted MAY complete in any order; the writer buffers them and writes their table in sequence order. -An incomplete group omitted by an application-forced segment cut MUST NOT be inserted into a later object if that would overlap or precede an earlier object's range. -The application MAY force a cut or remove a stalled track; storage does not impose a timeout. -A writer MUST NOT insert objects for earlier segments after committing a later segment. +Groups already accepted MAY complete in any order; a group's frames are indexed once every earlier accepted group has finished, and a group still open is indexed frame by frame as it grows. +A writer MUST NOT insert frames before the end of a record it already committed. -For segment N, the writer publishes the stored ranges, applies any retention pops, closes the timeline's current Window group, and stores the complete groups under the timeline track's `segments/N` key, with N encoded as specified in {{recording-layout}}. -It MUST make that object durable before committing the next segment or deleting expired objects. +To commit record N of a track, the writer makes the track's `segments/N` object durable, publishes the record through the track's timeline encoder, applies any retention pops, closes the timeline's current Window group, and stores the complete groups under the timeline track's `segments/N` key. +It MUST make that timeline object durable before committing the track's next record or deleting expired objects. +If a track's object cannot be made durable, the writer drops the record: its frames are lost, and the next record takes sequence N. If the timeline object cannot be made durable, the recording stops at the preceding durable timeline object. +The recording owns its timeline encoders so they publish only what was stored. -On a clean end, the writer MUST flush the final partial segment, finish the timeline track, and make its final complete groups durable. +On a clean end, the writer MUST close each track's final record, finish its timeline track, and make its final complete groups durable. There is no completion marker; object listing alone does not distinguish a clean end from an interruption. ## Retention {#recording-retention} A recording has one of two retention modes: -- An *archive* is unbounded and retains every complete segment until explicitly deleted. -- A *DVR* retains a configured duration of complete segments and expires the oldest whole segments as newer ones become durable. +- An *archive* is unbounded and retains every record until explicitly deleted. +- A *DVR* retains a configured duration of each track's records and expires each track's oldest records as newer ones become durable. An application offering DVR without an explicit retention value SHOULD default to at least 30 seconds. -The writer removes the oldest segment only when the remaining complete segments still cover the configured duration. +The writer removes a track's oldest record only while the track's remaining records still span the configured duration, measured from the second-oldest record's `pts` to the newest record's end. +A track's newest record is never removed, so a catalog that never changes outlives the media recorded with it. Retention is measured from timeline records, not from wall-clock arrival or relay cache state. -Trimming occurs only as part of committing a new segment, before that segment's timeline object is finalized ({{recording-writer}}). -A recording MUST NOT trim after its final segment is committed. -Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired segments' objects. -The writer MUST retain the latest timeline object, even for an all-gap segment, and enough earlier timeline groups to recover the retained window from a checkpoint. +Trimming occurs only as part of committing a new record of the same track, before that record's timeline object is finalized ({{recording-writer}}). +A recording MUST NOT trim after a track's final record is committed. +Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired records' objects. +The writer MUST retain each timeline's latest object and enough earlier timeline groups to recover the retained window from a checkpoint. The index therefore never advertises media the retention process has already deleted, although media objects MAY temporarily outlive the index. Relay cache eviction does not change the recording timeline. -Before accepting new groups, a restarting DVR writer MUST recover the complete retained timeline and list every recorded track's `groups/` prefix under exclusive ownership of the recording prefix. -After waiting the configured deletion grace period from successful recovery, it MUST delete group objects whose keys are absent from the recovered retained records, including expired objects and uncommitted uploads left by a crash. -It MUST NOT perform this cleanup if timeline recovery or listing fails or is incomplete, or while another writer can create or commit objects. -This completes previously committed expiration; it does not pop additional records after a final segment. -Timeline objects needed for checkpoint recovery and `.info` objects are not candidates for this cleanup. - ## Bootstrap and Recovery {#recording-recovery} -A reader or restarting writer lists the timeline track's `segments/` prefix and replays its objects in numeric segment order, including retention operations, from a retained checkpoint. -The catalog supplies the timeline track's name ({{archive-catalog}}). -The recovered records determine the committed track object keys; a missing or malformed referenced object MUST NOT be served. -Objects not referenced by the recovered timeline do not advertise content on their own. +A reader or restarting writer lists each timeline track's `segments/` prefix and replays its objects in numeric sequence order, including retention operations, from a retained checkpoint. +The catalog supplies the timeline names ({{archive-catalog}}); a restarting writer finds them by their suffix ({{recording-layout}}). +The recovered records determine the committed object keys; a missing or malformed referenced object MUST NOT be served. +Objects not referenced by a recovered timeline do not advertise content on their own. Listing is not an atomic snapshot across tracks; absence from an earlier listing MUST NOT override a successful GET of an object referenced by a later durable timeline record. -After replaying segment N below the recording ID limit, a reader follows an active recording by GET of `segments/N+1`, without refreshing media listings. +Before accepting new groups, a restarting writer MUST recover each retained timeline under exclusive ownership of the recording prefix. +A track's objects at or past its next sequence were never referenced by a timeline, so the writer deletes them before reusing their keys. +It resumes each track after the end of its newest committed record, including partway through a group that record split. +A restarting DVR writer also lists every recorded track's `segments/` prefix and, after waiting the configured deletion grace period from successful recovery, deletes objects whose keys are absent from the recovered retained records, including expired objects left by interrupted expiration. +It MUST NOT perform this cleanup if timeline recovery or listing fails or is incomplete, or while another writer can create or commit objects. +This completes previously committed expiration; it does not pop additional records after a final record. +Timeline objects needed for checkpoint recovery and `.info` objects are not candidates for this cleanup. + +After replaying a timeline's record N below the recording ID limit, a reader follows an active recording by GET of that timeline's `segments/N+1`, without refreshing media listings. At the limit there is no next key; this does not establish a clean recording end. -Not Found does not distinguish a pending commit, an expired DVR segment, or an interrupted recording. +Not Found does not distinguish a pending commit, an expired DVR record, or an interrupted recording. A reader that has fallen behind retention MUST bootstrap again from a retained checkpoint. Alternatively, a reader MAY list timeline keys after its last replayed key, consume all pages, sort the results, and replay them in order. A continuation token is used only within that enumeration; the next refresh starts from the last replayed key. -A reader MUST NOT advance its replay cursor past a missing segment without recovering from a retained checkpoint. -Retention operations evict expired ranges from the reader's cached index; incremental listing alone does not report deletions. +A reader MUST NOT advance its replay cursor past a missing object without recovering from a retained checkpoint. +Retention operations evict expired records from the reader's cached index; incremental listing alone does not report deletions. ## Reader Behavior {#recording-reader} -Reading segment N of track T is: take the minimum and maximum group IDs from T's ranges in record N, GET `groups/.`, and parse the groups. -The table MUST contain exactly the group sequences advertised for that track in the record, including its gaps. +Reading record N of track T is a GET of T's `segments/N`; the table MUST hold exactly the frames the record names. Nothing on that path reads media the consumer did not ask for, and nothing requires a second request. A reader MAY serve moq-lite FETCH from a recording. -Given a group, a reader locates the candidate object from a cached filename range index or the timeline's track ranges, then uses the table for the group and any requested `frame_start` {{moql}}. -A filename listing requires no media GETs. -On a backend guaranteeing lexical listing order and exclusive offsets, listing after `groups/` (19 padded digits, without the dot) selects the first candidate with an upper bound at least the requested group. -A reader MUST check the lower bound and committed timeline membership before serving it; an internal gap is resolved by the candidate's table. -An unordered listing MUST be collected and sorted before selecting a candidate; taking its first result is insufficient. +Given a group, a reader finds the retained records whose spans hold it, in order, and serves their frames from the requested `frame_start` {{moql}}. +A group whose earliest retained record starts after the requested frame is not served. +A group whose records do not reach its end yet, such as an append-log still being recorded, grows as later records commit, and ends where the recording does once the reader learns it ended. The reader reconstructs moq-lite FRAME headers from the stored timestamps and lengths, preserving the original payload bytes. A group absent from the recording is a normal FETCH failure. Recording catalog groups does not establish which catalog update applies to a media group; this format does not define that correlation. -A reader deriving a presentation-ordered format renders it from the timeline and transmuxes segment objects on demand. -Nothing derived needs to be stored: the playlist or manifest is a function of the timeline, and a media segment is a function of one recorded object. -An HLS segment URI can carry the track and both group bounds, allowing the handler to resolve the exact object without listing or a segment-ID index. -HLS sequence numbers remain timeline metadata and need not occur in object names. +A reader deriving a presentation-ordered format renders it from the timelines ({{timeline-derived}}) and transmuxes record objects on demand. +Nothing derived needs to be stored: the playlist or manifest is a function of the timelines, and a media segment is a function of the records its span covers. # Rooms @@ -1062,7 +1033,7 @@ Clamping a reference that escapes above that root would silently redirect the su TODO Security A consumer parsing a recording ({{recording}}) is parsing data at rest that it did not necessarily write. -It MUST validate the group/frame table against the bytes actually retrieved before allocating from its counts or accessing payloads ({{recording-segments}}), and treat a malformed object as a missing segment rather than letting it invalidate the recording. +It MUST validate the group/frame table against the bytes actually retrieved before allocating from its counts or accessing payloads ({{recording-segments}}), and treat a malformed object as a missing record rather than letting it invalidate the recording. It MUST reject a track object with a zero `timescale`. Varint fields are subject to the same limits as moq-lite {{moql}}. A recording inherits the confidentiality and integrity properties of the storage holding it; encryption at rest is transparent to the format and out of scope. @@ -1078,6 +1049,15 @@ This document has no IANA actions. # Appendix A: Changelog {:numbered="false"} +## moq-hang-04 +{:numbered="false"} + +- Replaced the broadcast's one aligned timeline with one timeline per track: the catalog `archive` entry's `track` became a `timelines` map from each indexed track, the catalog included, to its timeline track. +- Replaced the segment record with a per-track record: `sequence`, `pts`, `duration`, and a `start`/`end` range of group and frame positions, dropping cross-track pacing and completeness. +- One cutting rule for every track: a record ends at the first group boundary past a minimum (2 seconds RECOMMENDED, zero for sparse data such as a catalog) and splits a group between frames at a maximum (10 seconds RECOMMENDED), so a group that never closes is indexed as it grows and `durationMax` bounds every record. +- Recording format version 2: each track stores record N at `segments/N`, beside its timeline's `segments/N`, with a `Frame Start` field in the segment object. Tracks commit and expire independently, and a DVR keeps each track's newest record. +- Described deriving HLS and DASH at the edge from a reference rendition's records. + ## moq-hang-03 {:numbered="false"} diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index c3208cf9ad..a0944420f2 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -19,105 +19,77 @@ landed. The rest of the line is additive on top of them. ### Landed -The segment engine is in `rs/moq-mux/src/timeline.rs`: - -- `Segmenter` (:477) builds aligned `{ segment, pts, duration, tracks }` - records (`rs/hang/src/timeline.rs:105-121`) from pacing and non-pacing - tracks, with explicit `cut(pts)` boundaries. -- `Producer::deferred` (:917) returns a `Deferred` (:619) whose `next` yields - a cancellation-safe `Pending` record (:702). Dropping it requeues the record, - `Pending::gap()` (:712) turns it into a gap, and `Producer::push` (:948) - commits it into the visible timeline in segment order. -- The timeline track is a `moq_json::window::Producer` (:742). - `Producer::pop` (:983) trims the oldest records and `finish` (:1015) closes - the track. -- `moq-hls` renders live playlists from the timeline alone and FETCHes media - per HTTP request (`rs/moq-hls/src/export/mod.rs:3-8`). A clean timeline - finish ends every window with `EXT-X-ENDLIST` (:325-327). +Every track has its own timeline (`rs/moq-mux/src/timeline.rs`): + +- A `Segmenter` builds one track's `{ sequence, pts, duration, start, end }` + records (`rs/hang/src/timeline.rs`) from its frame reports: a record ends at + the first group start past a minimum, a sparse track (zero minimum) records + each group as it finishes, a group open past `durationMax` splits between + frames, and `cut(pts)` adds application boundaries. +- A `Producer` publishes records onto a timeline track, a + `moq_json::window::Producer` whose `pop` trims the oldest and `flush` closes + the open group. `Timelines` pairs them for live publishing and names each + track's timeline in the catalog's root `archive` entry. +- `moq-hls` renders playlists from the timelines alone and FETCHes media per + HTTP request. Segment boundaries come from a reference rendition's records; + other renditions resolve each segment against their own timelines + (`rs/moq-hls/src/export/spans.rs`). A clean reference timeline finish ends + every window with `EXT-X-ENDLIST`. - The same exporter serves a recording replayed through `moq_archive::Reader` with no archive-specific code (`rs/moq-hls/src/export/archive_tests.rs`): - playlists read only the timeline (an inline parameter set also GETs one - keyframe group to build its init), and a segment GETs one object of its - rendition. The caller supplies the catalog. + playlists read only the timelines, and a segment GETs only its rendition's + objects. The caller supplies the catalog. - A catalog `archive` entry with a `store` and no `replay` path declares its - ranges durable on that broadcast, so the exporter lists the whole retained + spans durable on that broadcast, so the exporter lists the whole retained timeline and only its pops trim it (`durable` in - `rs/moq-hls/src/export/mod.rs`). DASH `timeShiftBufferDepth` becomes the - listed span, and `--window` still bounds live playlists and caps segment - `max-age`. The catalog already states durability, so no per-broadcast - option or separate server is needed. - -`rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`: -percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list/delete. -`moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) records enrolled tracks -through `Deferred`, omits failed tracks with `Pending::omit`, stores each -segment's timeline groups after `Producer::flush`, and expires DVR segments -with a deletion grace. On a prefix that already holds a recording, it replays the -retained timeline from a checkpoint through `timeline::Producer::resume`, refuses -groups at or below each track's largest stored group, and a DVR deletes -unreferenced group objects one grace period after recovery. -`moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays the timeline onto a -supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` with a byte-bounded -object LRU. `Reader::refresh` follows by listing timeline keys after its cursor, so gaps and -DVR expiry recover from the next checkpoint; `Reader::finish` applies out-of-band finality. -`rs/moq-archive/src/proof.rs` records one multi-rendition broadcast end to end: its exact keys and -bytes match on memory, local disk, and an unordered listing, FETCH replays every group exactly, -and a rendition's playback GETs only that rendition's objects. + `rs/moq-hls/src/export/mod.rs`). + +`rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`. +`moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) cuts each enrolled track +with its own `Segmenter`, streams frames as they arrive, and commits each track +independently: the record's object, the record, retention pops, then the +timeline's `segments/`. A failed object PUT drops that record. On a prefix +that already holds a recording, it replays each retained timeline through +`timeline::Producer::resume`, deletes objects past each track's committed tail, +resumes after the newest record's end (partway through a split group), and a +DVR deletes unreferenced objects one grace period after recovery. +`moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays every timeline +onto a supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` +with a byte-bounded object LRU, stitching a group split across records and +growing a group whose records do not reach its end yet. `Reader::refresh` +follows by listing each timeline's keys after its cursor; `Reader::finish` +applies out-of-band finality. `rs/moq-archive/src/proof.rs` records one +multi-rendition broadcast end to end. ### Format -[Per-track timelines](/quest/m1/archive/track-timeline/README.md) replaces the -aligned segments below with one timeline per track. - The format is the draft's -[Recording section](/drafts/draft-lcurley-moq-hang.md#recording). +[Recording section](/drafts/draft-lcurley-moq-hang.md#recording), version 2. The application chooses the object prefix, selected tracks, retention, and credentials; `moq-archive` owns the portable layout and codecs: ```text //.info -//groups/. -//segments/ +//segments/ ``` -`.info` is versioned JSON with the immutable priority and timescale. A segment -object is a versioned binary envelope: a group/frame table with timestamps and -payload offsets, then the original payloads. One GET populates an LRU with the -adjacent groups, while audio-only or low-rendition playback never downloads -unrelated tracks. - -There is no `.head`, manifest, `.complete`, or `.timeline` object. The archive -timeline uses the same envelope under `segments/`, with consecutive -IDs for incremental GETs. Other tracks use `groups/.` with -19-digit zero-padded inclusive bounds, strictly increasing nonoverlapping -ranges, and no segment number or secondary index file. The catalog's `archive` entry names the -timeline track, plus any replay path, store URL, and format version. Listing -bootstraps recovery; following the next timeline key updates the cached ranges. - -### Writer and reader - -The application passes a `broadcast::Consumer` and opts arbitrary track names -in as pacing or non-pacing, including catalog and JSON tracks; `moq-archive` -never parses a media catalog. For each closed segment the writer PUTs one -object per participating track, drops the ranges of any track whose PUT -failed, then pushes the record. A clean source end flushes the final partial -segment and finishes the timeline; there is no completion marker. - -The reader takes a `broadcast::Producer` and uses `track::Dynamic` -(`rs/moq-net/src/model/track.rs:1652`) to answer FETCH for the tracks and -groups the timeline advertises: map the request to the track's group-range key, GET -once, validate, cache, and replay the original timestamps. Any ingest that -produces a broadcast (RTMP, SRT, WHIP) is archivable without its own -implementation. +A recorded track's `segments/` holds record `n` of its timeline; its +timeline, named with a `.timeline.z` suffix, stores the window groups committed +with record `n` under its own `segments/`. `.info` is versioned JSON with +the immutable priority and timescale. A segment object is a versioned binary +envelope: a frame start, a group/frame table with timestamps and payload +offsets, then the original payloads. One GET populates an LRU with the adjacent +groups, while audio-only or low-rendition playback never downloads unrelated +tracks. There is no `.head`, manifest, `.complete`, or index object. ### Retention -An unbounded archive only pushes records. During each new segment commit, a -DVR pops expired records before closing and storing that segment's timeline -groups, then waits the configured grace period before deleting expired objects. Timeline -objects use `segments/` for the segment being committed. Retention stops after -the final segment; no existing object is rewritten. HLS is a derived view of -the archive, never a second stored copy. +An unbounded archive only pushes records. A DVR pops each track's expired +records while committing that track's next record, before storing its timeline +groups, then waits the configured grace period before deleting expired objects. +Each track keeps its newest record, so a static catalog outlives the video +recorded with it. HLS is a derived view of the archive, never a second stored +copy. ### Managed boundary @@ -133,6 +105,7 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests - [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge +- [Flush idle archive groups](/quest/m1/archive/idle-flush.md) - the writer stores a quiet append-only group on a wall-clock bound, and a track can be flushed on demand - [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay - [Paced replay](/quest/m1/archive/paced-replay.md) - a replay pushes its groups to live subscribers on one shared clock, so any live player plays it - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts diff --git a/quest/m1/archive/browser.md b/quest/m1/archive/browser.md index 7d08f060cf..a6d501c3ca 100644 --- a/quest/m1/archive/browser.md +++ b/quest/m1/archive/browser.md @@ -15,8 +15,7 @@ timelines from [JS per-track timelines](/quest/m1/archive/track-timeline/js.md). Commit each track independently: persist each stored span, including a frame range of a still-open group, then publish that track's timeline record. Match the object keys, envelope bytes, `.info` property values, and timeline -discovery the Rust writer uses after -[Rust per-track timelines](/quest/m1/archive/track-timeline/core.md), per the +discovery the Rust writer uses, per the [Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without inferring catalog-to-group applicability. [Catalog track identity](/quest/m2/catalog-tracks.md) addresses that separately. diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 48959cd37c..9161ca2c3a 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -31,10 +31,6 @@ whether expiry during a seek needs a group eviction API in `moq-net`. broadcast's exporter follows `replay` to the recording for rewind, or whether viewers address the replay broadcast directly. -## Required - -- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - seeks through per-track timelines - ## Closes - [#2275](https://github.com/moq-dev/moq/issues/2275) - close this issue when the quest finishes diff --git a/quest/m1/archive/idle-flush.md b/quest/m1/archive/idle-flush.md new file mode 100644 index 0000000000..935858a45f --- /dev/null +++ b/quest/m1/archive/idle-flush.md @@ -0,0 +1,24 @@ +# [M] Flush idle archive groups + +## Goal + +`moq_archive::Writer` stores a group that stops receiving frames within a +bounded time, instead of waiting for the next frame or the track's end. A +sparse append-only group, such as a log that goes quiet, is durable and +listed on its timeline while it is still open. An application can also flush +or cut one track on demand. + +## Plan + +- Drive each track's `Segmenter` from a timer as well as its frame reports: a + record left open past a bound closes after its newest reported frame, the + way `duration_max` splits a long group, and the group's later frames start + the next record. +- Advance time on max(wall clock, pts), so a peer can neither stall the flush + by withholding frames nor skip it by lying about timestamps. +- Add a per-track flush and cut to `moq_archive::Control` beside the + broadcast-wide `cut`. + +## Required + +- [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - the writer cuts and commits each track independently diff --git a/quest/m1/archive/paced-replay.md b/quest/m1/archive/paced-replay.md index e125e77c59..c17cd658e6 100644 --- a/quest/m1/archive/paced-replay.md +++ b/quest/m1/archive/paced-replay.md @@ -25,7 +25,3 @@ who joins late joins mid-replay. - Test with paused time: record, replay paced, and assert a plain subscriber gets every group in order at media pace, that a late subscriber starts at the current group, and that two tracks stay aligned. - -## Required - -- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the reader's per-track index this paces diff --git a/quest/m1/archive/replay-catalog.md b/quest/m1/archive/replay-catalog.md index 435ff5a053..3435387cbf 100644 --- a/quest/m1/archive/replay-catalog.md +++ b/quest/m1/archive/replay-catalog.md @@ -43,7 +43,3 @@ serve example. Cover a DVR recording whose catalog outlived its first video segment, and fail loudly on a recording with no catalog. - -## Required - -- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - a DVR keeps its catalog's newest group diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index 2b4190e23b..fc9fc5e645 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -15,11 +15,13 @@ so Rust and JS reach `main` together. ## Plan -Today one timeline numbers aligned segments for every track. A DVR pops whole -segments, so a static catalog expires with the first video segment, and the -writer only records a non-pacing group once it completes, so a group that never -closes is never stored. `moq_json::window` pops from the front, so one shared -timeline cannot keep an old catalog record while dropping newer video. +The Rust side has landed: a `hang::timeline::Record` is one span of its own +track (`sequence`, `pts`, `duration`, `start`/`end` group and frame positions), +`moq_mux::timeline::Timelines` publishes one timeline per enrolled track, +`moq-archive` writes recording version 2 (`/segments/` beside its +timeline's `segments/`), and `moq-hls` derives segments from a reference +rendition's records (`rs/moq-hls/src/export/spans.rs`). The draft is updated +in moq-hang-04. Decisions: @@ -27,9 +29,10 @@ Decisions: every broadcast; an unsubscribed track costs nothing. - The catalog's root `archive` entry maps each track to its timeline, including the catalog track itself. `replay`, `store`, and `version` stay beside it. -- Each track cuts on its own: automatically at a group boundary between a - minimum and maximum duration (roughly 1s and 10s), splitting a long-lived - group by frame at the maximum. Manual cuts stay as an optimization, such as a +- Each track cuts on its own by one rule: at a group boundary between a + minimum and maximum duration (2s and 10s by default, a zero minimum only for + sparse data such as the catalog), splitting a long-lived group by frame at + the maximum. Manual cuts stay as an optimization, such as a video keyframe cutting audio so derived segments need fewer objects. - A stored object may hold a frame range of a group, not only whole groups. - HLS and DASH segments are derived at the edge from group timestamps, not @@ -38,7 +41,6 @@ Decisions: ## Quests -- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the draft, `hang`, `moq-mux`, `moq-archive`, and `moq-hls` move to per-track timelines in one change - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust ## Related diff --git a/quest/m1/archive/track-timeline/core.md b/quest/m1/archive/track-timeline/core.md deleted file mode 100644 index 4651a338f9..0000000000 --- a/quest/m1/archive/track-timeline/core.md +++ /dev/null @@ -1,46 +0,0 @@ -# [XL] Rust per-track timelines - -## Goal - -The moq-hang draft, `hang`, `moq-mux`, `moq-archive`, and `moq-hls` use one -timeline per track, per the [line's decisions](/quest/m1/archive/track-timeline/README.md). -A DVR recording keeps every track's newest group, an append-only group is -stored as its frames arrive, and `moq-hls` serves aligned HLS and DASH from -live or recorded timelines without the publisher cutting for it. - -## Plan - -These crates compile together, so they change in one PR. Update the draft's -timeline and Recording sections in the same change, in a new draft revision; -most of their rules assume one aligned segment counter (one timeline per -broadcast, cross-track boundaries, whole-segment retention, reading segment N -of track T through record N). - -- **Timeline:** a record describes one stored span of its own track: sequence, - pts, duration, and the group and frame range. Drop cross-track pacing and the - completeness wait; keep manual cuts. Retention pops each track's window - independently and always keeps its newest record. -- **Catalog:** replace the root `archive.track` with a map from track to - timeline, covering the catalog track too. -- **Writer and reader:** commit each track independently, and store frame - ranges so a never-closing group is recorded. Bump the recording `version` and - refuse the old one. Recovery, grace deletion, and FETCH replay follow the - per-track index. -- **HLS and DASH:** take segment boundaries from a reference video rendition - (an audio one when the broadcast has no video), near a target duration, and - number them so every edge and every reload agrees, including after DVR pops. Every other video rendition snaps each - boundary to its nearest group start within a tolerance (around 1s); a - segment with no start in range becomes a gap (`EXT-X-GAP` in HLS), so a - player switching renditions lands on the next real segment. Gaps are a - best-effort fallback: the draft and HLS docs say a publisher wanting HLS - export SHOULD align video GOPs across renditions. Audio and other - renditions take the groups and frames whose timestamps fall in each span, - possibly from more than one object. The reader's cache absorbs the overlap. - -Carry over the existing tests (DVR trim, durable listing, archive replay) and -add a static catalog outliving its first video segment, an append-only group -spanning several objects, audio cut independently of video, renditions that -start at different times, video renditions whose group starts never -coincide, and an audio-only broadcast. - -Update `doc/concept/hang.md` and the HLS and CLI docs this makes stale. diff --git a/quest/m1/archive/track-timeline/js.md b/quest/m1/archive/track-timeline/js.md index 461ff1cf75..30bc460798 100644 --- a/quest/m1/archive/track-timeline/js.md +++ b/quest/m1/archive/track-timeline/js.md @@ -11,7 +11,3 @@ Port the landed Rust shape to `js/hang/src/timeline.ts` and its catalog schema, replacing the aligned timeline and cross-track pacing rather than keeping both. Cover the same cut rules and a static catalog outliving other tracks' records, and check the records against Rust output in the interop suite. - -## Required - -- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the format this mirrors diff --git a/rs/hang/fixtures/catalog-clock.json b/rs/hang/fixtures/catalog-clock.json index 1db86f67f8..2a63ca1dbd 100644 --- a/rs/hang/fixtures/catalog-clock.json +++ b/rs/hang/fixtures/catalog-clock.json @@ -20,5 +20,9 @@ } }, "clock": { "wall": 1751846400000000, "timescale": 1000000 }, - "archive": { "track": "timeline.z", "timescale": 1000, "durationMax": 2000 } + "archive": { + "timelines": { "video": "video.timeline.z", "audio": "audio.timeline.z" }, + "timescale": 1000, + "durationMax": 2000 + } } diff --git a/rs/hang/src/catalog/archive.rs b/rs/hang/src/catalog/archive.rs index 2cf3a5e8e2..9c7199761a 100644 --- a/rs/hang/src/catalog/archive.rs +++ b/rs/hang/src/catalog/archive.rs @@ -1,29 +1,31 @@ +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; /// The moq epoch (2020-01-01T00:00:00Z) in Unix-epoch milliseconds. pub const MOQ_EPOCH_UNIX_MILLIS: u64 = 1_577_836_800_000; -/// Discovers the broadcast's segment index and any durable archive. +/// Discovers each track's timeline and any durable archive. /// -/// A live publisher sets the timeline fields (`track`, `timescale`, `duration_max`) alone, and a -/// recording also names the replay broadcast and object store those ranges live under. Every -/// advertised range is FETCHable; with a store they are durable. +/// A live publisher sets the timeline fields (`timelines`, `timescale`, `duration_max`) alone, +/// and a recording also names the replay broadcast and object store those spans live under. +/// Every advertised span is FETCHable; with a store they are durable. #[serde_with::skip_serializing_none] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] #[non_exhaustive] pub struct Archive { - /// The MoQ track carrying the broadcast's segment records. - pub track: String, + /// Each indexed track's timeline track, keyed by the indexed track's name. + pub timelines: BTreeMap, - /// Units per second for the timeline's timestamps. Defaults to milliseconds. + /// Units per second for every timeline's timestamps. Defaults to milliseconds. #[serde( default = "Archive::default_timescale", deserialize_with = "super::deserialize_timescale_or_default" )] pub timescale: u32, - /// The declared upper bound on a segment's duration, in [`timescale`](Self::timescale) units. + /// The declared upper bound on a record's duration, in [`timescale`](Self::timescale) units. pub duration_max: Option, /// The MoQ broadcast the archive is served back from, relative to this catalog, if any. @@ -41,17 +43,17 @@ pub struct Archive { impl Archive { /// The recording object format advertised in [`version`](Self::version). - pub const VERSION: u32 = 1; + pub const VERSION: u32 = 2; /// The default timeline timescale, milliseconds. pub const fn default_timescale() -> u32 { 1000 } - /// An archive naming `track` as its timeline, with no duration bound or durable storage. - pub fn new(track: impl Into) -> Self { + /// An archive with no timelines, duration bound, or durable storage. + pub fn new() -> Self { Self { - track: track.into(), + timelines: BTreeMap::new(), timescale: Self::default_timescale(), duration_max: None, replay: None, @@ -61,23 +63,34 @@ impl Archive { } } +impl Default for Archive { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod test { use super::*; + fn live() -> Archive { + let mut archive = Archive::new(); + archive + .timelines + .insert("video".to_string(), "video.timeline.z".to_string()); + archive + } + #[test] fn live_publisher_is_timeline_fields_alone() { - let json = serde_json::to_string(&Archive::new("timeline.z")).unwrap(); - assert_eq!(json, r#"{"track":"timeline.z","timescale":1000}"#); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - Archive::new("timeline.z") - ); + let json = serde_json::to_string(&live()).unwrap(); + assert_eq!(json, r#"{"timelines":{"video":"video.timeline.z"},"timescale":1000}"#); + assert_eq!(serde_json::from_str::(&json).unwrap(), live()); } #[test] fn recording_roundtrips_replay_store_and_version() { - let mut archive = Archive::new("timeline.z"); + let mut archive = live(); archive.replay = Some(moq_net::path::RelativeOwned::new("./recordings/clip")); archive.store = Some("https://objects.example/rec/".parse().unwrap()); archive.version = Some(Archive::VERSION); @@ -85,20 +98,25 @@ mod test { let json = serde_json::to_string(&archive).unwrap(); assert_eq!( json, - r#"{"track":"timeline.z","timescale":1000,"replay":"recordings/clip","store":"https://objects.example/rec/","version":1}"# + r#"{"timelines":{"video":"video.timeline.z"},"timescale":1000,"replay":"recordings/clip","store":"https://objects.example/rec/","version":2}"# ); assert_eq!(serde_json::from_str::(&json).unwrap(), archive); } #[test] fn zero_timescale_is_refused() { - serde_json::from_str::(r#"{"track":"timeline.z","timescale":0}"#) + serde_json::from_str::(r#"{"timelines":{},"timescale":0}"#) .expect_err("a zero timescale must not decode"); } #[test] fn invalid_store_url_is_refused() { - serde_json::from_str::(r#"{"track":"timeline.z","store":"not a url"}"#) + serde_json::from_str::(r#"{"timelines":{},"store":"not a url"}"#) .expect_err("an invalid store URL must not decode"); } + + #[test] + fn a_single_shared_timeline_is_refused() { + serde_json::from_str::(r#"{"track":"timeline.z"}"#).expect_err("the old shape names no timelines"); + } } diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 8941e3835f..ab99c0c1a1 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; /// A catalog track, created by a broadcaster to describe the tracks available in a broadcast. /// /// The base catalog carries the media sections (`video`, `audio`, `text`), the optional -/// `archive` (the segment index and any durable recording), the optional `clock` (the +/// `archive` (each track's timeline and any durable recording), the optional `clock` (the /// broadcast's one wall-clock mapping), and the data sections /// (`json`, `binary`) for application tracks that aren't media. /// Applications extend it with their own root sections (e.g. `scte35`) through `E`. The catalog @@ -42,14 +42,14 @@ pub struct Catalog { #[serde(default)] pub audio: Audio, - /// The broadcast's segment index and any durable archive, if the publisher offers one. + /// Each track's timeline and any durable archive, if the publisher offers them. /// See [`Archive`](crate::catalog::Archive) and the [`timeline`](crate::timeline) module. pub archive: Option, /// The broadcast's one continuous clock, if the publisher exposes one. /// /// `wall` is the wall-clock time of PTS zero in `timescale` units since the moq epoch - /// (2020-01-01); every media track and the archive index refer to this mapping after + /// (2020-01-01); every media track and every timeline refer to this mapping after /// timescale conversion. Independent of [`archive`](Self::archive): a live-only publisher /// exposes its clock without creating a segment index. See /// [`Clock`](crate::catalog::Clock). @@ -685,7 +685,7 @@ mod test { assert_eq!(clock.wall.scale().as_u64(), 1_000_000); let archive = catalog.archive.expect("the fixture carries an archive"); - assert_eq!(archive.track, "timeline.z"); + assert_eq!(archive.timelines["video"], "video.timeline.z"); assert_eq!(archive.timescale, 1000); // Archive PTS 2000 (ms) lands 2s after the wall epoch, whatever the track timescale. @@ -701,7 +701,10 @@ mod test { #[test] fn archive_roundtrips_at_the_root() { - let mut archive = crate::catalog::Archive::new("timeline.z"); + let mut archive = crate::catalog::Archive::new(); + archive + .timelines + .insert("video".to_string(), "video.timeline.z".to_string()); archive.duration_max = Some(2000); archive.replay = Some(moq_net::path::RelativeOwned::new("recordings/clip")); archive.version = Some(crate::catalog::Archive::VERSION); @@ -712,7 +715,10 @@ mod test { }; let json = catalog.to_json().expect("failed to encode"); - assert!(json.contains(r#""archive":{"track":"timeline.z""#), "{json}"); + assert!( + json.contains(r#""archive":{"timelines":{"video":"video.timeline.z"}"#), + "{json}" + ); assert!( !json.contains(r#""timeline":"#), "the old root key must not appear: {json}" diff --git a/rs/hang/src/lib.rs b/rs/hang/src/lib.rs index f4705d0aa4..cfe65b273e 100644 --- a/rs/hang/src/lib.rs +++ b/rs/hang/src/lib.rs @@ -22,7 +22,7 @@ pub mod catalog; /// The container is the contents of each media track. pub mod container; -/// The timeline indexes the broadcast's aligned segments. +/// Per-track timelines index each track's spans. pub mod timeline; /// Export the moq-net version we use. diff --git a/rs/hang/src/timeline.rs b/rs/hang/src/timeline.rs index 757cfd255d..71e1cc750b 100644 --- a/rs/hang/src/timeline.rs +++ b/rs/hang/src/timeline.rs @@ -1,47 +1,47 @@ -//! The timeline track: the broadcast's segment index, one record per aligned segment. +//! Per-track timelines: each track's index of stored spans, one record per span. //! //! MoQ groups carry only an opaque sequence number; the timestamps live inside the media -//! frames. The timeline track republishes the broadcast's segmentation as metadata: each -//! record describes one *segment*, a span of content time shared by every media track, and -//! maps it to the group ranges that carry it on each track. A consumer can answer "which -//! groups cover time T on track X" (and "where is the live edge") from a few bytes per -//! segment without downloading media. That is exactly what an HLS/DASH origin needs to render -//! playlists without touching media bytes, and the index a VOD player seeks with. +//! frames. A timeline track republishes one track's spans as metadata: each record maps a span +//! of content time to the group and frame positions that carry it. A consumer can answer +//! "which groups cover time T on track X" (and "where is the live edge") from a few bytes per +//! span without downloading media. That is what an HLS/DASH origin needs to render playlists +//! without touching media bytes, the index a VOD player seeks with, and the object index of a +//! recording. //! -//! ## Segments +//! ## Spans //! -//! A segment covers the same span of content time on every track of the broadcast, which is -//! what HLS requires of switchable renditions. Segment boundaries land on video keyframes -//! (every group already opens on one), so a video track contributes whole groups to each -//! segment; an audio track packs however many of its shorter groups fall inside the span. -//! A record is published once the segment is *complete*: every participating track's groups -//! for the span are known, so the record is self-contained and immediately servable. +//! Every track has its own timeline, so tracks cut, commit, and expire independently. A span +//! usually holds whole groups; a group that outlives the publisher's maximum span is split by +//! frame, so an append-only group that never closes is still indexed as its frames arrive. +//! Spans are contiguous in position: a record starts where the previous one ended unless the +//! track skipped group sequences in between. //! -//! There is one timeline per broadcast, advertised by the catalog's root -//! [`Archive`](crate::catalog::Archive) entry (its `track` field names this track, -//! [`DEFAULT_NAME`](crate::timeline::DEFAULT_NAME) by convention). A broadcast that doesn't need aligned segments simply -//! doesn't publish one. +//! The catalog's root [`Archive`](crate::catalog::Archive) entry maps each track to its +//! timeline track, conventionally the track name plus [`SUFFIX`](crate::timeline::SUFFIX). //! -//! On the wire the track is a DEFLATE-compressed `moq-json` window (see `moq_json::window`). +//! On the wire each timeline is a DEFLATE-compressed `moq-json` window (see `moq_json::window`). //! Each group starts with a checkpoint and continues with push/pop operations; group rolls are an //! encoding detail that consumers do not surface as duplicate records. Like the catalog, a record //! tolerates and preserves unknown fields: extend it by flattening a //! [`Record`](crate::timeline::Record) into your own struct via //! [`RecordExt`](crate::timeline::RecordExt). -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use crate::Result; -/// The conventional name for a broadcast's timeline track (the `.z` marks the -/// DEFLATE-compressed stream, like the catalog's `.json.z` sibling). +/// The conventional suffix appended to a track name to name its timeline track (the `.z` marks +/// the DEFLATE-compressed stream, like the catalog's `.json.z` sibling). /// -/// A publisher records the actual name in the catalog's root -/// [`Archive`](crate::catalog::Archive) entry (`timeline.track`); a consumer reads it from the -/// catalog rather than assuming, so this is only a default. -pub const DEFAULT_NAME: &str = "timeline.z"; +/// A publisher records the actual names in the catalog's root +/// [`Archive`](crate::catalog::Archive) entry; a consumer reads them from the catalog rather than +/// assuming, so this is only a default. +pub const SUFFIX: &str = ".timeline.z"; + +/// The conventional timeline track name for `track`: the name plus [`SUFFIX`]. +pub fn default_name(track: &str) -> String { + format!("{track}{SUFFIX}") +} /// The application extension carried alongside a record's base fields. /// @@ -51,74 +51,69 @@ pub const DEFAULT_NAME: &str = "timeline.z"; pub trait RecordExt: serde::Serialize + serde::de::DeserializeOwned + Default + Clone + Send + Unpin + 'static {} impl RecordExt for () {} -/// A contiguous run of groups a track contributes to a segment, `start` through `end` -/// inclusive. +/// A frame position within a track: frame `frame` of group `group`. /// -/// A track's segment entry is an array of these: more than one range means the group -/// sequence is discontinuous inside the segment (groups that never existed, e.g. a gappy -/// source). -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct Range { - /// The first group of the run, as used by FETCH/SUBSCRIBE on the media track. - pub start: u64, - - /// The last group of the run, inclusive. - pub end: u64, - - /// Whether the run's first group starts with a keyframe, i.e. whether a player can join or - /// switch renditions at this range. Defaults to `true` (and is omitted from the JSON when - /// true); a publisher sets `false` when a source resumes without one, so an exporter knows - /// not to advertise the segment as independently decodable. - #[serde(default = "Range::default_keyframe", skip_serializing_if = "Clone::clone")] - pub keyframe: bool, +/// Positions order by group, then frame. `frame` is omitted from the JSON when zero, so a +/// position at a group start reads as just the group. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct Position { + /// The group sequence, as used by FETCH/SUBSCRIBE on the track. + pub group: u64, + + /// The frame index within the group. + #[serde(default, skip_serializing_if = "is_zero")] + pub frame: u64, +} + +fn is_zero(v: &u64) -> bool { + *v == 0 } -impl Range { - fn default_keyframe() -> bool { - true +impl Position { + /// Frame `frame` of group `group`. + pub const fn new(group: u64, frame: u64) -> Self { + Self { group, frame } } - /// A range covering groups `start` through `end` inclusive, starting on a keyframe. - pub fn new(start: u64, end: u64) -> Self { - Self { - start, - end, - keyframe: true, - } + /// The start of group `group`. + pub const fn group(group: u64) -> Self { + Self { group, frame: 0 } } } -/// One timeline record: a complete aligned segment, mapping its span of content time to the -/// group ranges that carry it on each track. +/// One timeline record: a span of one track, mapping its content time to frame positions. /// -/// Records are self-contained: `pts` and `duration` bound the span (no peeking at the next -/// record), and `tracks` names every participating media track's group ranges. A track absent -/// from `tracks` has no content for the span (a gap; HLS `EXT-X-GAP`). `pts`/`duration` are in -/// the timescale declared by the catalog's [`Archive`](crate::catalog::Archive) entry -/// (default milliseconds). Extend with a typed [`RecordExt`]. +/// The span holds every frame from [`start`](Self::start) (inclusive) to [`end`](Self::end) +/// (exclusive). An `end` at frame zero of group `g` means the span holds every frame of group +/// `g - 1` and nothing of `g`, whether or not `g` exists. `pts`/`duration` are in the timescale +/// declared by the catalog's [`Archive`](crate::catalog::Archive) entry (default milliseconds). +/// Extend with a typed [`RecordExt`]. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde( rename_all = "camelCase", bound(serialize = "E: serde::Serialize", deserialize = "E: serde::de::DeserializeOwned") )] pub struct Record { - /// The segment's number. Consecutive within a broadcast, so it anchors HLS - /// `EXT-X-MEDIA-SEQUENCE`; explicit (rather than implied by record order) so a reader - /// joining mid-stream or across a windowed recording keeps stable numbering. - pub segment: u64, + /// The record's number, consecutive within its track's timeline and equal to its window index. + pub sequence: u64, - /// The segment's start, in the timeline's timescale. + /// The span's start, the timestamp of its first frame, in the timeline's timescale. pub pts: u64, - /// The segment's duration, in the timeline's timescale. The next record's `pts` is + /// The span's duration, in the timeline's timescale. The next record's `pts` is /// `pts + duration` unless content time itself jumped (a discontinuity). pub duration: u64, - /// The group ranges each participating media track contributes to this segment, keyed by - /// track name. A track absent from the map has no content in this span. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub tracks: BTreeMap>, + /// The first frame of the span, inclusive. + pub start: Position, + + /// The end of the span, exclusive. + pub end: Position, + + /// Whether the span's first frame is a keyframe, i.e. whether a player can join or switch + /// renditions at this record. Defaults to `true` (and is omitted from the JSON when true). + #[serde(default = "default_keyframe", skip_serializing_if = "Clone::clone")] + pub keyframe: bool, /// The application extension, flattened into the record's JSON object (nothing for the /// default `()`). See [`RecordExt`]. @@ -126,19 +121,33 @@ pub struct Record { pub ext: E, } +fn default_keyframe() -> bool { + true +} + impl Record { - /// A record with no tracks and the default (empty) extension; fill in - /// [`tracks`](Self::tracks) afterward. - pub fn new(segment: u64, pts: u64, duration: u64) -> Self { + /// A keyframe record spanning `start..end` with the default (empty) extension. + pub fn new(sequence: u64, pts: u64, duration: u64, start: Position, end: Position) -> Self { Self { - segment, + sequence, pts, duration, - tracks: BTreeMap::new(), + start, + end, + keyframe: true, ext: E::default(), } } + /// The groups holding at least one frame of the span, in order. + pub fn groups(&self) -> std::ops::RangeInclusive { + let last = match self.end.frame { + 0 => self.end.group.saturating_sub(1), + _ => self.end.group, + }; + self.start.group..=last + } + /// Parse a record from a slice of bytes. pub fn from_slice(v: &[u8]) -> Result { Ok(serde_json::from_slice(v)?) @@ -156,54 +165,38 @@ mod test { #[test] fn roundtrip() { - let mut record = Record::<()>::new(3, 84_000, 6_000); - record.tracks.insert("video".to_string(), vec![Range::new(42, 42)]); - record - .tracks - .insert("audio".to_string(), vec![Range::new(512, 540), Range::new(550, 560)]); - + let record = Record::<()>::new(3, 84_000, 6_000, Position::group(42), Position::group(44)); let json = record.to_vec().unwrap(); assert_eq!( std::str::from_utf8(&json).unwrap(), - concat!( - r#"{"segment":3,"pts":84000,"duration":6000,"tracks":{"#, - r#""audio":[{"start":512,"end":540},{"start":550,"end":560}],"#, - r#""video":[{"start":42,"end":42}]}}"# - ) + r#"{"sequence":3,"pts":84000,"duration":6000,"start":{"group":42},"end":{"group":44}}"# ); assert_eq!(Record::<()>::from_slice(&json).unwrap(), record); + assert_eq!(record.groups(), 42..=43); } #[test] - fn keyframe_false_roundtrips_and_true_is_omitted() { - let mut range = Range::new(7, 9); - range.keyframe = false; - let mut record = Record::<()>::new(0, 0, 2_000); - record.tracks.insert("video".to_string(), vec![range]); - + fn a_frame_split_roundtrips() { + let mut record = Record::<()>::new(0, 0, 10_000, Position::new(7, 300), Position::new(7, 600)); + record.keyframe = false; let json = record.to_vec().unwrap(); assert_eq!( std::str::from_utf8(&json).unwrap(), - r#"{"segment":0,"pts":0,"duration":2000,"tracks":{"video":[{"start":7,"end":9,"keyframe":false}]}}"# + r#"{"sequence":0,"pts":0,"duration":10000,"start":{"group":7,"frame":300},"end":{"group":7,"frame":600},"keyframe":false}"# ); + assert_eq!(Record::<()>::from_slice(&json).unwrap(), record); + assert_eq!(record.groups(), 7..=7); - let decoded = Record::<()>::from_slice(&json).unwrap(); - assert_eq!(decoded, record); // An omitted flag decodes as the keyframe default. let plain: Record = - Record::from_slice(br#"{"segment":0,"pts":0,"duration":1,"tracks":{"v":[{"start":1,"end":1}]}}"#).unwrap(); - assert!(plain.tracks["v"][0].keyframe); + Record::from_slice(br#"{"sequence":0,"pts":0,"duration":1,"start":{"group":1},"end":{"group":2}}"#) + .unwrap(); + assert!(plain.keyframe); } #[test] - fn empty_tracks_is_omitted() { - let record = Record::<()>::new(1, 1_000, 500); - let json = record.to_vec().unwrap(); - assert_eq!( - std::str::from_utf8(&json).unwrap(), - r#"{"segment":1,"pts":1000,"duration":500}"# - ); - assert_eq!(Record::<()>::from_slice(&json).unwrap(), record); + fn default_name_appends_the_suffix() { + assert_eq!(default_name("video0"), "video0.timeline.z"); } #[test] @@ -216,17 +209,12 @@ mod test { } impl RecordExt for Ext {} - let record = Record { - segment: 2, - pts: 14_000, - duration: 2_000, - tracks: BTreeMap::new(), - ext: Ext { discontinuity: true }, - }; + let mut record = Record::::new(2, 14_000, 2_000, Position::group(1), Position::group(2)); + record.ext.discontinuity = true; let json = record.to_vec().unwrap(); assert_eq!( std::str::from_utf8(&json).unwrap(), - r#"{"segment":2,"pts":14000,"duration":2000,"discontinuity":true}"# + r#"{"sequence":2,"pts":14000,"duration":2000,"start":{"group":1},"end":{"group":2},"discontinuity":true}"# ); assert_eq!(Record::::from_slice(&json).unwrap(), record); } diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 595b0db05e..12fc9c2bd9 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -31,6 +31,11 @@ tracing = { workspace = true } [dev-dependencies] async-trait = "0.1" +criterion = { workspace = true } object_store = { workspace = true, features = ["fs"] } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } + +[[bench]] +name = "writer" +harness = false diff --git a/rs/moq-archive/benches/writer.rs b/rs/moq-archive/benches/writer.rs new file mode 100644 index 0000000000..8df214a484 --- /dev/null +++ b/rs/moq-archive/benches/writer.rs @@ -0,0 +1,78 @@ +//! `Writer::run` recording every enrolled track into memory, swept over the number of tracks and +//! the groups each one publishes. Throughput is per frame, so a per-event cost that grows with the +//! track table shows up as a slope across the track counts. +//! +//! Run with `cargo bench -p moq-archive --bench writer`. + +use std::time::{Duration, Instant}; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::{Store, Writer, writer}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; + +const FRAMES_PER_GROUP: u64 = 2; + +/// A writer with `tracks` enrolled tracks, each already holding `groups` one-second groups. +async fn setup(tracks: usize, groups: u64) -> Writer { + let source = broadcast::Info::new().produce(); + let store = Store::new(InMemory::new(), "bench"); + let writer = Writer::new(store, source.consume(), writer::Config::default()) + .await + .unwrap(); + let control = writer.control(); + let config = moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)); + + for index in 0..tracks { + let name = format!("track{index}"); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let track = source.create_track(name.as_str(), info).unwrap(); + control.track(&name, config.clone()).await.unwrap(); + for sequence in 0..groups { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for frame in 0..FRAMES_PER_GROUP { + let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); + group.write_frame(pts, vec![0u8; 64]).unwrap(); + } + group.finish().unwrap(); + } + track.finish().unwrap(); + } + source.close(); + writer +} + +fn writer(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + let mut bench = c.benchmark_group("writer"); + bench.sample_size(10); + bench.warm_up_time(Duration::from_secs(1)); + bench.measurement_time(Duration::from_secs(2)); + for tracks in [1, 10, 100] { + for groups in [10, 100] { + bench.throughput(Throughput::Elements(tracks as u64 * groups * FRAMES_PER_GROUP)); + bench.bench_function(BenchmarkId::new(format!("{groups} groups"), tracks), |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let writer = runtime.block_on(setup(tracks, groups)); + let start = Instant::now(); + runtime.block_on(writer.run()).unwrap(); + total += start.elapsed(); + } + total + }); + }); + } + } + bench.finish(); +} + +criterion_group!(benches, writer); +criterion_main!(benches); diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index a8ef749d83..0b7b396ba9 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -10,7 +10,7 @@ pub enum Error { #[error("not found: {0}")] NotFound(String), - /// The recording format version is not 1. + /// The recording format version is not the current one. #[error("unknown version {0}")] Version(u64), @@ -38,9 +38,9 @@ pub enum Error { #[error("group sequences are not strictly ascending")] Sequence, - /// A group range is empty, reversed, or does not match an object's table. - #[error("invalid or mismatched group bounds {smallest}..={largest}")] - Bounds { smallest: u64, largest: u64 }, + /// An object's table does not hold exactly the frames its record names. + #[error("object does not match its record")] + Span, /// The binary table is truncated, overlapping, gapped, or out of range. #[error("malformed table")] diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index 75e92acc20..c17d7d7ad9 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -17,7 +17,7 @@ pub struct Info { } impl Info { - /// Format version 1 with `priority` and `timescale`. + /// The current format version with `priority` and `timescale`. pub fn new(priority: u8, timescale: u64) -> Result { let info = Self { version: VERSION, @@ -62,16 +62,16 @@ mod tests { fn roundtrip() { let info = Info::new(0, 1_000_000).unwrap(); let bytes = info.encode().unwrap(); - assert_eq!(bytes.as_ref(), br#"{"version":1,"priority":0,"timescale":1000000}"#); + assert_eq!(bytes.as_ref(), br#"{"version":2,"priority":0,"timescale":1000000}"#); assert_eq!(Info::decode(&bytes).unwrap(), info); } #[test] fn whitespace_and_member_order_do_not_affect_equality() { - let info = Info::decode(br#"{"timescale":1000,"priority":7,"version":1}"#).unwrap(); + let info = Info::decode(br#"{"timescale":1000,"priority":7,"version":2}"#).unwrap(); assert_eq!( info, - Info::decode(br#"{ "version": 1, "priority": 7, "timescale": 1000 }"#).unwrap() + Info::decode(br#"{ "version": 2, "priority": 7, "timescale": 1000 }"#).unwrap() ); assert_eq!(info.priority, 7); assert_eq!(info.timescale, 1000); @@ -79,10 +79,10 @@ mod tests { #[test] fn unknown_version_is_refused() { - assert!(matches!( - Info::decode(br#"{"version":2,"priority":0,"timescale":1000}"#), - Err(Error::Version(2)) - )); + for version in [1, 3] { + let json = format!(r#"{{"version":{version},"priority":0,"timescale":1000}}"#); + assert!(matches!(Info::decode(json.as_bytes()), Err(Error::Version(v)) if v == version)); + } } #[test] @@ -96,11 +96,11 @@ mod tests { #[test] fn unknown_members_and_missing_fields_are_refused() { assert!(matches!( - Info::decode(br#"{"version":1,"priority":0,"timescale":1,"extra":true}"#), + Info::decode(br#"{"version":2,"priority":0,"timescale":1,"extra":true}"#), Err(Error::Json(_)) )); assert!(matches!( - Info::decode(br#"{"version":1,"priority":0}"#), + Info::decode(br#"{"version":2,"priority":0}"#), Err(Error::Json(_)) )); } diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 3edecbe87e..d1bc87a73c 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -6,9 +6,9 @@ //! broadcast. Callers that need runtime dispatch //! supply `Arc`; the archive API itself stays generic. //! -//! Group bounds are finite inclusive ranges in first-to-last order: +//! Every object is keyed by its track and a record sequence: //! ``` -//! let key = moq_archive::Key::groups("video", 5..=7)?; +//! let key = moq_archive::Key::segments("video", 7)?; //! assert_eq!(key.track(), "video"); //! # Ok::<(), moq_archive::Error>(()) //! ``` @@ -37,10 +37,10 @@ pub use store::Store; pub use writer::Writer; /// Recording format version written into `.info` and the binary envelope. -pub const VERSION: u64 = 1; +pub const VERSION: u64 = 2; /// Largest group, segment, or timestamp value the format allows (`2^53 - 1`). pub const ID_MAX: u64 = (1 << 53) - 1; -/// Decimal width of group and segment ID filename fields. +/// Decimal width of segment ID filename fields. pub(crate) const ID_WIDTH: usize = 19; diff --git a/rs/moq-archive/src/path.rs b/rs/moq-archive/src/path.rs index 4df4784b40..576067c3ff 100644 --- a/rs/moq-archive/src/path.rs +++ b/rs/moq-archive/src/path.rs @@ -1,5 +1,3 @@ -use std::ops::RangeInclusive; - use object_store::path::{Path, PathPart}; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; @@ -28,7 +26,7 @@ pub(crate) fn decode_track(encoded: &str) -> Result { Ok(decoded.into_owned()) } -/// Write a group or segment ID as 19 zero-padded decimal digits. +/// Write a segment ID as 19 zero-padded decimal digits. pub(crate) fn format_id(id: u64) -> Result { check_id(id)?; Ok(format!("{id:0width$}", width = ID_WIDTH)) @@ -56,18 +54,12 @@ pub enum Key { /// The unencoded track name. track: String, }, - /// `/groups/.` - Groups { - /// The unencoded track name. - track: String, - /// Inclusive group sequences in the object, from first to last. - range: RangeInclusive, - }, - /// `/segments/` + /// `/segments/`: record `segment` of a track's timeline, or for a + /// timeline track, the window groups closed while committing that record. Segments { /// The unencoded track name. track: String, - /// The committed timeline segment ID. + /// The record's sequence. segment: u64, }, } @@ -80,15 +72,7 @@ impl Key { Ok(Self::Info { track }) } - /// A groups object named by its inclusive first-to-last sequence range. - pub fn groups(track: impl Into, range: RangeInclusive) -> Result { - let track = track.into(); - encode_track(&track)?; - check_range(&range)?; - Ok(Self::Groups { track, range }) - } - - /// A timeline object at `segments/`. + /// The object at `segments/`. pub fn segments(track: impl Into, segment: u64) -> Result { let track = track.into(); encode_track(&track)?; @@ -99,7 +83,7 @@ impl Key { /// The unencoded track name this key belongs to. pub fn track(&self) -> &str { match self { - Self::Info { track } | Self::Groups { track, .. } | Self::Segments { track, .. } => track, + Self::Info { track } | Self::Segments { track, .. } => track, } } @@ -108,10 +92,6 @@ impl Key { let path = push(prefix, &encode_track(self.track())?)?; match self { Self::Info { .. } => push(&path, ".info"), - Self::Groups { range, .. } => { - let path = push(&path, "groups")?; - push(&path, &range_name(range)?) - } Self::Segments { segment, .. } => { let path = push(&path, "segments")?; push(&path, &format_id(*segment)?) @@ -142,14 +122,6 @@ fn parse_parts<'a>(mut parts: impl Iterator>, location: &Pat } Ok(Key::Info { track }) } - "groups" => { - let name = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; - if parts.next().is_some() { - return Err(Error::Path(location.to_string())); - } - let range = parse_range(name.as_ref())?; - Ok(Key::Groups { track, range }) - } "segments" => { let name = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; if parts.next().is_some() { @@ -168,21 +140,11 @@ pub(crate) fn track_prefix(track: &str) -> Result { push(&Path::ROOT, &encode_track(track)?) } -pub(crate) fn groups_prefix(prefix: &Path, track: &str) -> Result { - let path = append(prefix, &track_prefix(track)?); - push(&path, "groups") -} - pub(crate) fn segments_prefix(prefix: &Path, track: &str) -> Result { let path = append(prefix, &track_prefix(track)?); push(&path, "segments") } -pub(crate) fn groups_offset(prefix: &Path, track: &str, group: u64) -> Result { - let path = groups_prefix(prefix, track)?; - push(&path, &format_id(group)?) -} - fn append(base: &Path, relative: &Path) -> Path { let mut path = base.clone(); path.extend(relative.parts()); @@ -194,35 +156,6 @@ pub(crate) fn push(base: &Path, segment: &str) -> Result { Ok(base.clone().join(part)) } -fn range_name(range: &RangeInclusive) -> Result { - check_range(range)?; - let (smallest, largest) = (*range.start(), *range.end()); - Ok(format!("{}.{}", format_id(largest)?, format_id(smallest)?)) -} - -fn parse_range(name: &str) -> Result> { - let (largest, smallest) = name.split_once('.').ok_or_else(|| Error::Path(name.to_string()))?; - if smallest.contains('.') { - return Err(Error::Path(name.to_string())); - } - let largest = parse_id(largest)?; - let smallest = parse_id(smallest)?; - if largest < smallest { - return Err(Error::Bounds { smallest, largest }); - } - Ok(smallest..=largest) -} - -pub(crate) fn check_range(range: &RangeInclusive) -> Result<()> { - let (smallest, largest) = (*range.start(), *range.end()); - check_id(smallest)?; - check_id(largest)?; - if range.is_empty() { - return Err(Error::Bounds { smallest, largest }); - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -293,7 +226,7 @@ mod tests { let prefix = Path::from("rec/1"); for key in [ Key::info("catalog.json").unwrap(), - Key::groups("video", 5..=10).unwrap(), + Key::segments("video", 5).unwrap(), Key::segments("timeline.z", 0).unwrap(), Key::segments("timeline.z", ID_MAX).unwrap(), ] { @@ -308,36 +241,6 @@ mod tests { assert_eq!(path.as_ref(), "rec/catalog%2Ejson/.info"); } - #[test] - fn inverted_range_is_rejected() { - let (smallest, largest) = (2, 1); - assert!(matches!( - Key::groups("v", smallest..=largest), - Err(Error::Bounds { - smallest: 2, - largest: 1 - }) - )); - } - - #[test] - fn exhausted_range_is_rejected() { - let mut range = 1..=1; - assert_eq!(range.next(), Some(1)); - assert!(matches!(Key::groups("v", range), Err(Error::Bounds { .. }))); - } - - #[test] - fn range_id_endpoints_are_valid() { - let key = Key::groups("v", 0..=ID_MAX).unwrap(); - assert_eq!( - key.path(&Path::from("rec")).unwrap().as_ref(), - "rec/v/groups/0009007199254740991.0000000000000000000" - ); - assert!(matches!(Key::groups("v", 0..=ID_MAX + 1), Err(Error::Id(_)))); - assert!(matches!(Key::groups("v", ID_MAX + 1..=ID_MAX + 1), Err(Error::Id(_)))); - } - #[test] fn direct_construction_is_validated_when_serialized() { let prefix = Path::from("rec"); @@ -348,20 +251,15 @@ mod tests { segment: ID_MAX + 1, }; assert!(matches!(segment.path(&prefix), Err(Error::Id(_)))); - let varint = Key::Groups { - track: "v".to_string(), - range: 0..=(1 << 62) - 1, - }; - assert!(matches!(varint.path(&prefix), Err(Error::Id(_)))); // Everything that does serialize parses back to the same key. for key in [ Key::Info { track: "a/b".to_string(), }, - Key::Groups { + Key::Segments { track: ".x".to_string(), - range: ID_MAX..=ID_MAX, + segment: ID_MAX, }, Key::Segments { track: "é".to_string(), @@ -375,40 +273,15 @@ mod tests { #[test] fn keys_parse_under_an_empty_prefix() { - let key = Key::groups("catalog.json", 1..=2).unwrap(); + let key = Key::segments("catalog.json", 2).unwrap(); let path = key.path(&Path::ROOT).unwrap(); - assert_eq!( - path.as_ref(), - "catalog%2Ejson/groups/0000000000000000002.0000000000000000001" - ); + assert_eq!(path.as_ref(), "catalog%2Ejson/segments/0000000000000000002"); assert_eq!(Key::parse(&Path::ROOT, &path).unwrap(), key); } #[test] - fn direct_inverted_range_is_rejected_when_serialized() { - let (smallest, largest) = (2, 1); - let key = Key::Groups { - track: "v".to_string(), - range: smallest..=largest, - }; - assert!(matches!( - key.path(&Path::from("rec")), - Err(Error::Bounds { - smallest: 2, - largest: 1 - }) - )); - - let key = Key::Groups { - track: "v".to_string(), - range: 0..=ID_MAX + 1, - }; - assert!(matches!(key.path(&Path::from("rec")), Err(Error::Id(_)))); - } - - #[test] - fn groups_offset_has_no_dot() { - let path = groups_offset(&Path::from("rec"), "video", 5).unwrap(); - assert_eq!(path.as_ref(), "rec/video/groups/0000000000000000005"); + fn a_retired_directory_is_not_a_key() { + let path = Path::from("video/groups/0000000000000000001.0000000000000000001"); + assert!(matches!(Key::parse(&Path::ROOT, &path), Err(Error::Path(_)))); } } diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs index 4acd6238ab..3f8f800d56 100644 --- a/rs/moq-archive/src/proof.rs +++ b/rs/moq-archive/src/proof.rs @@ -16,23 +16,21 @@ use crate::reader::Config as ReaderConfig; use crate::writer::{Config, Retention}; use crate::{Reader, Store, Writer}; -const TIMELINE: &str = hang::timeline::DEFAULT_NAME; const RENDITIONS: [&str; 2] = ["video/1080p", "video/360p"]; -const PACING: [&str; 3] = ["video/1080p", "video/360p", "audio"]; +const MEDIA: [&str; 3] = ["video/1080p", "video/360p", "audio"]; const TRACKS: [&str; 5] = ["video/1080p", "video/360p", "audio", "catalog.json", "chat"]; /// Frames of one source group: (millisecond timestamp, payload). type Frames = Vec<(u64, Bytes)>; -/// Recorded segments: three 2s segments, then a one-frame tail. +/// Source segments: three 2s segments, then a one-frame tail. const SEGMENTS: u64 = 4; /// The source groups of one segment: (track, sequence, frames). /// -/// Video renditions share a 2s GOP and audio has four groups per segment. The catalog and a -/// non-media chat track publish sparsely, including a sequence skip inside one segment and an -/// empty payload. The tail keeps any single track from cutting the final segment once the others -/// end. +/// Video renditions share a 2s GOP and audio has four 500ms groups per segment, which its own +/// timeline packs into 2s records by the default minimum. The catalog and a non-media chat track publish sparsely, +/// including a sequence skip and an empty payload. fn plan(segment: u64) -> Vec<(&'static str, u64, Frames)> { let pts = segment * 2000; let (frames, audio) = if segment + 1 < SEGMENTS { (4, 4) } else { (1, 1) }; @@ -85,6 +83,25 @@ async fn settle() { } } +/// Media cut at 1s, so one-second test groups get a record each. +fn media() -> moq_mux::timeline::Config { + moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)) +} + +fn timeline(track: &str) -> String { + hang::timeline::default_name(track) +} + +/// Every recorded track's timeline, as the catalog's `archive` entry names them. +fn config() -> ReaderConfig { + ReaderConfig::new( + TRACKS + .iter() + .map(|track| (track.to_string(), timeline(track))) + .collect(), + ) +} + /// Record every segment of [`plan`] into `store`. async fn record(store: &Store) { let source = broadcast::Info::new().produce(); @@ -103,24 +120,17 @@ async fn record(store: &Store) { .unwrap(); let control = writer.control(); for name in TRACKS { - match PACING.contains(&name) { - true => control.pacing_track(name).await.unwrap(), - false => control.track(name).await.unwrap(), - } + let config = match MEDIA.contains(&name) { + true => moq_mux::timeline::Config::default(), + false => moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO), + }; + control.track(name, config).await.unwrap(); } let run = tokio::spawn(writer.run()); - // A non-pacing track joins whichever segment is open when its group arrives, so publish each - // segment's pacing groups (closing the previous segment) before its sparse groups. + // Each track cuts its own timeline, so the interleave across tracks changes nothing stored. for segment in 0..SEGMENTS { - let (pacing, sparse): (Vec<_>, Vec<_>) = plan(segment) - .into_iter() - .partition(|(name, _, _)| PACING.contains(name)); - for (name, sequence, frames) in pacing { - write(&tracks[name], sequence, &frames); - } - settle().await; - for (name, sequence, frames) in sparse { + for (name, sequence, frames) in plan(segment) { write(&tracks[name], sequence, &frames); } settle().await; @@ -128,7 +138,7 @@ async fn record(store: &Store) { for track in tracks.values() { track.finish().unwrap(); } - source.finish(); + source.close(); run.await.unwrap().unwrap(); } @@ -147,26 +157,24 @@ fn id(value: u64) -> String { format!("{value:019}") } +/// Each track's record count: a GOP per video record, four audio groups per record, and a group per +/// sparse record. +const RECORDS: [(&str, u64); 5] = [ + ("video%2F1080p", 4), + ("video%2F360p", 4), + ("audio", 4), + ("catalog%2Ejson", 2), + ("chat", 3), +]; + /// The exact layout [`plan`] produces: no manifest, index, or completion marker. fn layout() -> Vec { let mut keys = Vec::new(); - let groups = |track: &str, ranges: &[(u64, u64)]| { - ranges - .iter() - .map(|&(smallest, largest)| format!("rec/{track}/groups/{}.{}", id(largest), id(smallest))) - .collect::>() - }; - keys.push("rec/audio/.info".to_string()); - keys.extend(groups("audio", &[(0, 3), (4, 7), (8, 11), (12, 12)])); - keys.push("rec/catalog%2Ejson/.info".to_string()); - keys.extend(groups("catalog%2Ejson", &[(0, 0), (1, 1)])); - keys.push("rec/chat/.info".to_string()); - keys.extend(groups("chat", &[(0, 0), (3, 5)])); - keys.push("rec/timeline%2Ez/.info".to_string()); - keys.extend((0..SEGMENTS).map(|segment| format!("rec/timeline%2Ez/segments/{}", id(segment)))); - for track in ["video%2F1080p", "video%2F360p"] { - keys.push(format!("rec/{track}/.info")); - keys.extend(groups(track, &[(0, 0), (1, 1), (2, 2), (3, 3)])); + for (track, records) in RECORDS { + for directory in [track.to_string(), format!("{track}%2Etimeline%2Ez")] { + keys.push(format!("rec/{directory}/.info")); + keys.extend((0..records).map(|record| format!("rec/{directory}/segments/{}", id(record)))); + } } keys.sort(); keys @@ -194,7 +202,7 @@ async fn recordings_are_byte_identical_on_every_backend() { assert_eq!(expected.keys().cloned().collect::>(), layout()); assert_eq!( &expected["rec/video%2F360p/.info"][..], - br#"{"version":1,"priority":0,"timescale":1000}"# + br#"{"version":2,"priority":0,"timescale":1000}"# ); let dir = tempfile::tempdir().unwrap(); @@ -212,12 +220,15 @@ async fn recordings_are_byte_identical_on_every_backend() { assert_eq!(objects(&unordered).await, expected, "unordered listing"); assert_eq!(objects(&sibling).await.len(), expected.len()); - // Every group object holds exactly the source groups, in order, byte for byte. + // The media objects hold exactly the source groups, in order, byte for byte. let mut stored: BTreeMap<(String, u64), Frames> = BTreeMap::new(); for (path, bytes) in &expected { let key = crate::Key::parse(&Path::from("rec"), &Path::parse(path).unwrap()).unwrap(); - if let crate::Key::Groups { track, range } = key { - let object = crate::Object::decode_groups(bytes.clone(), range).unwrap(); + if let crate::Key::Segments { track, .. } = key + && !track.ends_with(hang::timeline::SUFFIX) + { + let object = crate::Object::decode(bytes.clone()).unwrap(); + assert_eq!(object.frame_start, 0, "no source group outlived the maximum"); for group in object.groups { let frames = group.frames.into_iter().map(|f| (f.timestamp, f.payload)).collect(); stored.insert((track.clone(), group.sequence), frames); @@ -238,9 +249,7 @@ async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { record(&store).await; let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(store, &broadcast, ReaderConfig::new(TIMELINE)) - .await - .unwrap(); + let reader = Reader::open(store, &broadcast, config()).await.unwrap(); tokio::spawn(reader.serve()); mock.take(); @@ -253,24 +262,18 @@ async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { .2; assert_eq!(fetch(&broadcast, "video/360p", sequence).await.unwrap(), expected); } - let object = - |track: &str, smallest: u64, largest: u64| format!("rec/{track}/groups/{}.{}", id(largest), id(smallest)); + let object = |track: &str, record: u64| format!("rec/{track}/segments/{}", id(record)); // Each track request also GETs that track's `.info`, but nothing of another track. let media = |gets: Vec, track: &str| { let prefix = format!("rec/{track}/"); assert!(gets.iter().all(|path| path.starts_with(&prefix)), "{gets:?}"); gets.into_iter() - .filter(|path| path.contains("/groups/")) + .filter(|path| path.contains("/segments/")) .collect::>() }; assert_eq!( media(mock.gets(), "video%2F360p"), - [ - object("video%2F360p", 0, 0), - object("video%2F360p", 1, 1), - object("video%2F360p", 2, 2), - object("video%2F360p", 3, 3), - ] + (0..4).map(|record| object("video%2F360p", record)).collect::>() ); // Audio-only playback: four adjacent groups per GET, the rest from the cache. @@ -281,12 +284,7 @@ async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { } assert_eq!( media(mock.gets(), "audio"), - [ - object("audio", 0, 3), - object("audio", 4, 7), - object("audio", 8, 11), - object("audio", 12, 12), - ] + (0..4).map(|record| object("audio", record)).collect::>() ); // Every other enrolled group replays its original sequence, timestamps, and payloads. @@ -322,14 +320,14 @@ async fn an_offline_reader_follows_dvr_expiry() { let video = source.create_track("video", info).unwrap(); let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); let run = tokio::spawn(writer.run()); let frames = |sequence: u64| vec![(sequence * 1000, Bytes::from(format!("video {sequence}")))]; let committed = |segment: u64| { let store = store.clone(); async move { - while store.get_segments(TIMELINE, segment).await.is_err() { + while store.get_segments(&timeline("video"), segment).await.is_err() { tokio::time::sleep(Duration::from_millis(5)).await; } } @@ -341,19 +339,18 @@ async fn an_offline_reader_follows_dvr_expiry() { committed(1).await; let broadcast = broadcast::Info::new().produce(); - let mut reader = Reader::open(store.clone(), &broadcast, ReaderConfig::new(TIMELINE)) - .await - .unwrap(); + let config = ReaderConfig::new([("video".to_string(), timeline("video"))].into()); + let mut reader = Reader::open(store.clone(), &broadcast, config).await.unwrap(); tokio::spawn(reader.serve()); assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), frames(0)); - // The reader is offline while the DVR commits and expires several segments. + // The reader is offline while the DVR commits and expires several records. for sequence in 3..10 { write(&video, sequence, &frames(sequence)); } committed(8).await; - // Segments 7 and 8 hold the 2s window. - while store.get_groups("video", 6..=6).await.is_ok() { + // Records 7 and 8 hold the 2s window. + while store.get_segments("video", 6).await.is_ok() { tokio::time::sleep(Duration::from_millis(5)).await; } @@ -364,12 +361,12 @@ async fn an_offline_reader_follows_dvr_expiry() { assert_eq!( list, &Op::List { - prefix: "rec/timeline%2Ez/segments".to_string(), - offset: Some(format!("rec/timeline%2Ez/segments/{}", id(1))), + prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), + offset: Some(format!("rec/video%2Etimeline%2Ez/segments/{}", id(1))), } ); let segments: Vec<_> = (2..=8) - .map(|segment| Op::Get(format!("rec/timeline%2Ez/segments/{}", id(segment)))) + .map(|segment| Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{}", id(segment)))) .collect(); assert_eq!(gets, segments, "only the new timeline keys are read"); @@ -384,12 +381,15 @@ async fn an_offline_reader_follows_dvr_expiry() { ); } let gets = mock.gets(); - assert!(!gets.iter().any(|path| path.contains("/groups/")), "{gets:?}"); + assert!( + !gets.iter().any(|path| path.starts_with("rec/video/segments/")), + "{gets:?}" + ); for sequence in 7..9 { assert_eq!(fetch(&broadcast, "video", sequence).await.unwrap(), frames(sequence)); } video.finish().unwrap(); - source.finish(); + source.close(); run.await.unwrap().unwrap(); } diff --git a/rs/moq-archive/src/reader/index.rs b/rs/moq-archive/src/reader/index.rs index bedbd7b9e2..4cafc63d2c 100644 --- a/rs/moq-archive/src/reader/index.rs +++ b/rs/moq-archive/src/reader/index.rs @@ -1,175 +1,170 @@ use std::collections::{BTreeMap, HashMap}; -use std::ops::{Range, RangeInclusive}; +use std::ops::Range; -use hang::timeline::Record; +use hang::timeline::{Position, Record}; -use crate::path::check_range; +use crate::path::check_id; -/// One track's stored object for one record: its filename bounds and the exact runs it holds. -#[derive(Debug, Clone, PartialEq, Eq)] +/// One committed record: the object at `segments/` and the frames it holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Span { - /// Inclusive first-to-last group sequence, the object's filename bounds. - pub bounds: RangeInclusive, - /// The advertised runs, ascending and nonoverlapping; groups between them never existed. - pub runs: Vec>, + /// The record's sequence, which names its object. + pub sequence: u64, + /// The first frame, inclusive. + pub start: Position, + /// The end, exclusive. + pub end: Position, } impl Span { - /// Build from a record's ranges, refusing empty, reversed, out-of-range, or unordered runs. - fn new(ranges: &[hang::timeline::Range]) -> Option { - let mut runs: Vec> = Vec::with_capacity(ranges.len()); - for range in ranges { - let run = range.start..=range.end; - check_range(&run).ok()?; - if let Some(prev) = runs.last() - && run.start() <= prev.end() - { - return None; - } - runs.push(run); - } - let bounds = *runs.first()?.start()..=*runs.last()?.end(); - Some(Self { bounds, runs }) + /// Build from a record, refusing an empty span or IDs past the recording range. + fn new(record: &Record) -> Option { + check_id(record.sequence).ok()?; + check_id(record.start.group).ok()?; + check_id(record.end.group).ok()?; + (record.start < record.end).then_some(Self { + sequence: record.sequence, + start: record.start, + end: record.end, + }) } +} - fn contains(&self, group: u64) -> bool { - self.runs.iter().any(|run| run.contains(&group)) - } +/// One track's committed spans. +#[derive(Default)] +struct Spans { + /// Each span keyed by its start. + spans: BTreeMap, + /// Each span's start keyed by its sequence, so a pop can find it. + sequences: BTreeMap, } -/// The committed group ranges advertised by the replayed timeline window. +/// The spans each track's replayed timeline window commits. #[derive(Default)] pub(super) struct Index { - /// Per track, each span keyed by its smallest group sequence. - tracks: HashMap>, - /// Per window index, the `(track, smallest)` spans that record added, so a pop can evict them. - records: BTreeMap>, + tracks: HashMap, } impl Index { - /// Add a record's spans at window `index`. + /// Add `track`'s record. /// - /// A track whose ranges are malformed or overlap another record's span is left out, so its - /// groups behave like ones the source never delivered; the record's other tracks stay usable. - pub fn push(&mut self, index: u64, record: &Record) { - let mut added = Vec::new(); - for (track, ranges) in &record.tracks { - let Some(span) = Span::new(ranges) else { - tracing::warn!(track, segment = record.segment, "ignoring malformed archive ranges"); - continue; - }; - - let spans = self.tracks.entry(track.clone()).or_default(); - let smallest = *span.bounds.start(); - let before = spans.range(..=span.bounds.end()).next_back(); - if before.is_some_and(|(_, prev)| prev.bounds.end() >= span.bounds.start()) { - tracing::warn!(track, segment = record.segment, "ignoring overlapping archive ranges"); - continue; - } - - spans.insert(smallest, span); - added.push((track.clone(), smallest)); + /// A malformed record or one overlapping another is left out, so its frames behave like ones + /// the source never delivered; the track's other records stay usable. + pub fn push(&mut self, track: &str, record: &Record) { + let Some(span) = Span::new(record) else { + tracing::warn!(track, sequence = record.sequence, "ignoring a malformed archive record"); + return; + }; + + let spans = self.tracks.entry(track.to_string()).or_default(); + let before = spans.spans.range(..span.end).next_back(); + if before.is_some_and(|(_, prev)| prev.end > span.start) || spans.sequences.contains_key(&span.sequence) { + tracing::warn!( + track, + sequence = record.sequence, + "ignoring an overlapping archive record" + ); + return; } - self.records.insert(index, added); + spans.spans.insert(span.start, span); + spans.sequences.insert(span.sequence, span.start); } - /// Remove the records at these window indices, returning the evicted spans by track. - pub fn pop(&mut self, indices: Range) -> Vec<(String, Span)> { - let popped: Vec = self.records.range(indices).map(|(index, _)| *index).collect(); - let mut evicted = Vec::new(); - for index in popped { - for (track, smallest) in self.records.remove(&index).unwrap_or_default() { - let span = self.tracks.get_mut(&track).and_then(|spans| spans.remove(&smallest)); - if let Some(span) = span { - evicted.push((track, span)); - } - } - } - evicted + /// Remove `track`'s records with these sequences, returning the evicted spans. + pub fn pop(&mut self, track: &str, sequences: Range) -> Vec { + let Some(spans) = self.tracks.get_mut(track) else { + return Vec::new(); + }; + let popped: Vec = spans + .sequences + .range(sequences) + .map(|(sequence, _)| *sequence) + .collect(); + popped + .into_iter() + .filter_map(|sequence| { + let start = spans.sequences.remove(&sequence)?; + spans.spans.remove(&start) + }) + .collect() } - /// Whether any replayed record named this track. - pub fn has_track(&self, track: &str) -> bool { - self.tracks.contains_key(track) - } - - /// The span advertising `group` on `track`, if the retained window commits it. - pub fn get(&self, track: &str, group: u64) -> Option { - let (_, span) = self.tracks.get(track)?.range(..=group).next_back()?; - span.contains(group).then(|| span.clone()) + /// The spans holding frames of `group` on `track`, in order. + pub fn group(&self, track: &str, group: u64) -> Vec { + let Some(spans) = self.tracks.get(track) else { + return Vec::new(); + }; + let head = Position::group(group); + let mut found: Vec = spans + .spans + .range(..Position::group(group.saturating_add(1))) + .rev() + .map(|(_, span)| *span) + .take_while(|span| span.end > head) + .collect(); + found.reverse(); + found } } #[cfg(test)] mod tests { - use hang::timeline::Range; - use super::*; use crate::ID_MAX; - fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { - let mut record = Record::new(segment, segment * 1000, 1000); - for (name, ranges) in tracks { - let ranges = ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(); - record.tracks.insert(name.to_string(), ranges); - } - record + fn record(sequence: u64, start: Position, end: Position) -> Record { + Record::new(sequence, sequence * 1000, 1000, start, end) + } + + fn at(group: u64) -> Position { + Position::group(group) } #[test] - fn lookup_respects_bounds_and_internal_gaps() { + fn lookup_finds_every_span_of_a_group() { let mut index = Index::default(); - index.push(0, &record(0, &[("audio", &[(0, 2), (5, 6)]), ("video", &[(0, 0)])])); - index.push(1, &record(1, &[("audio", &[(7, 9)])])); - - assert_eq!(index.get("audio", 1).unwrap().bounds, 0..=6); - assert_eq!(index.get("audio", 6).unwrap().runs, vec![0..=2, 5..=6]); - assert!(index.get("audio", 3).is_none(), "internal gap"); - assert_eq!(index.get("audio", 8).unwrap().bounds, 7..=9); - assert!(index.get("audio", 10).is_none()); - assert!(index.get("video", 1).is_none()); - assert!(index.get("chat", 0).is_none()); - assert!(index.has_track("video")); - assert!(!index.has_track("chat")); + index.push("audio", &record(0, at(0), at(3))); + index.push("audio", &record(1, at(5), at(6))); + // Group 7 split across two records, then whole groups again. + index.push("video", &record(0, at(7), Position::new(7, 3))); + index.push("video", &record(1, Position::new(7, 3), at(9))); + + let sequences = |spans: Vec| spans.iter().map(|span| span.sequence).collect::>(); + assert_eq!(sequences(index.group("audio", 1)), vec![0]); + assert!(index.group("audio", 3).is_empty(), "the gap between records"); + assert_eq!(sequences(index.group("audio", 5)), vec![1]); + assert!(index.group("audio", 6).is_empty()); + assert_eq!(sequences(index.group("video", 7)), vec![0, 1]); + assert_eq!(sequences(index.group("video", 8)), vec![1]); + assert!(index.group("chat", 0).is_empty()); } #[test] fn pop_evicts_only_the_popped_records() { let mut index = Index::default(); - index.push(0, &record(0, &[("video", &[(0, 1)])])); - index.push(1, &record(1, &[("video", &[(2, 3)])])); + index.push("video", &record(0, at(0), at(2))); + index.push("video", &record(1, at(2), at(4))); - let evicted = index.pop(0..1); + let evicted = index.pop("video", 0..1); assert_eq!(evicted.len(), 1); - assert_eq!(evicted[0].1.bounds, 0..=1); - assert!(index.get("video", 0).is_none()); - assert!(index.get("video", 2).is_some()); - assert!(index.pop(0..1).is_empty(), "already popped"); - assert!(index.has_track("video"), "a popped track stays known"); + assert_eq!(evicted[0].start, at(0)); + assert!(index.group("video", 0).is_empty()); + assert_eq!(index.group("video", 2).len(), 1); + assert!(index.pop("video", 0..1).is_empty(), "already popped"); } #[test] - fn malformed_and_overlapping_tracks_are_ignored() { + fn malformed_and_overlapping_records_are_ignored() { let mut index = Index::default(); - index.push(0, &record(0, &[("video", &[(4, 6)]), ("audio", &[(3, 1)])])); - assert!(index.get("audio", 1).is_none(), "reversed run"); - - // Overlapping the earlier video span drops only the video entry. - index.push(1, &record(1, &[("video", &[(6, 8)]), ("audio", &[(10, 11)])])); - assert_eq!(index.get("video", 6).unwrap().bounds, 4..=6); - assert!(index.get("video", 7).is_none()); - assert!(index.get("audio", 10).is_some()); - - // Unordered runs and IDs past the recording range. - index.push( - 2, - &record(2, &[("chat", &[(5, 6), (1, 2)]), ("data", &[(ID_MAX, ID_MAX + 1)])]), - ); - assert!(index.get("chat", 5).is_none()); - assert!(index.get("data", ID_MAX).is_none()); - - // Popping a record whose track was ignored leaves the earlier span alone. - index.pop(1..2); - assert!(index.get("video", 4).is_some()); + index.push("video", &record(0, at(4), at(7))); + index.push("video", &record(1, at(6), at(9))); + assert_eq!(index.group("video", 8), Vec::new(), "overlapping the earlier span"); + index.push("video", &record(2, at(3), at(5))); + assert_eq!(index.group("video", 3), Vec::new(), "overlapping the later span"); + + index.push("audio", &record(0, at(3), at(3))); + assert!(index.group("audio", 3).is_empty(), "an empty span"); + index.push("data", &record(0, at(ID_MAX + 1), at(ID_MAX + 2))); + assert!(index.group("data", ID_MAX + 1).is_empty(), "past the recording range"); } } diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs index 5be88a433c..98b8feebb7 100644 --- a/rs/moq-archive/src/reader/mod.rs +++ b/rs/moq-archive/src/reader/mod.rs @@ -1,9 +1,10 @@ //! Serve an archive's groups through a caller-supplied broadcast. //! -//! A [`Reader`] replays the archive timeline onto its timeline track and answers FETCH for every -//! group that timeline commits. Each request maps to one range-named object, fetched with a single -//! GET, validated against the record that advertised it, and kept in a byte-bounded cache so -//! adjacent groups reuse it. A request for one track never downloads another track's object. +//! A [`Reader`] replays each track's timeline onto its timeline track and answers FETCH for every +//! frame those timelines commit. A group maps to the records holding it, each one object fetched +//! with a single GET, validated against its record, and kept in a byte-bounded cache so adjacent +//! groups reuse it. A request for one track never downloads another track's object. A group whose +//! records do not reach its end yet, such as an append-only log, grows as later records commit. //! //! The reader never infers that a recording ended: it keeps following until the caller, who may //! know finality out of band, calls [`Reader::finish`]. @@ -13,7 +14,8 @@ //! use moq_archive::reader::{Config, Reader}; //! //! let broadcast = moq_net::broadcast::Info::new().produce(); -//! let mut reader = Reader::open(store, &broadcast, Config::new("timeline.z")).await?; +//! let timelines = [("video".to_string(), "video.timeline.z".to_string())].into(); +//! let mut reader = Reader::open(store, &broadcast, Config::new(timelines)).await?; //! let serve = reader.serve(); //! // Spawn `serve` on any executor, then call `reader.refresh()` to follow a growing archive. //! # drop(serve); @@ -23,18 +25,18 @@ mod index; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::future::Future; -use std::ops::RangeInclusive; use std::sync::{Arc, Mutex}; use std::task::Poll; use futures::TryStreamExt; use futures::future::BoxFuture; -use hang::timeline::Record; +use hang::timeline::{Position, Record}; use moq_json::window; use moq_net::{Timescale, Timestamp, broadcast, group, track}; use object_store::ObjectStore; +use tokio::sync::watch; use self::index::{Index, Span}; use crate::store::list::Query; @@ -44,17 +46,17 @@ use crate::{Error, Key, Object, Result, Store}; #[derive(Debug, Clone)] #[non_exhaustive] pub struct Config { - /// The timeline track name, from the catalog's `archive` entry. - pub timeline: String, + /// Each served track's timeline track, from the catalog's `archive` entry. + pub timelines: BTreeMap, /// Upper bound on cached object bytes shared by every track. pub cache: u64, } impl Config { - /// Read the timeline named `timeline` with a 64 MiB object cache. - pub fn new(timeline: impl Into) -> Self { + /// Serve the tracks `timelines` indexes with a 64 MiB object cache. + pub fn new(timelines: BTreeMap) -> Self { Self { - timeline: timeline.into(), + timelines, cache: 64 * 1024 * 1024, } } @@ -66,11 +68,18 @@ impl Config { } } -/// Serves an archive through a [`broadcast::Producer`] and follows its timeline. +/// Serves an archive through a [`broadcast::Producer`] and follows its timelines. pub struct Reader { shared: Arc>, dynamic: broadcast::Dynamic, - timeline: track::Producer, + timelines: Vec, +} + +/// One track's timeline, replayed onto its own timeline track. +struct Timeline { + /// The track it indexes. + track: String, + producer: track::Producer, decoder: window::Decoder, /// The last timeline segment replayed. cursor: Option, @@ -82,16 +91,26 @@ struct Shared { cache: quick_cache::sync::Cache, Weight>, /// Each track's `.info`, which is immutable, so a re-requested track costs no GET. infos: Mutex>, + /// Notified whenever the index changes; holds whether the caller declared the recording ended. + changed: watch::Sender, } impl Reader { - /// Create the timeline track on `broadcast`, then replay every retained timeline object. + /// Create every timeline track on `broadcast`, then replay each track's retained timeline. /// - /// Fails if the timeline's `.info` is missing or invalid, the timeline track already exists, - /// or the store cannot list the timeline. + /// Fails if a timeline's `.info` is missing or invalid, a timeline track already exists, or the + /// store cannot list a timeline. pub async fn open(store: Store, broadcast: &broadcast::Producer, config: Config) -> Result { - let info = track_info(&store.get_info(&config.timeline).await?)?; - let timeline = broadcast.create_track(config.timeline, info)?; + let mut timelines = Vec::new(); + for (track, timeline) in config.timelines { + let info = track_info(&store.get_info(&timeline).await?)?; + timelines.push(Timeline { + track, + producer: broadcast.create_track(timeline, info)?, + decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), + cursor: None, + }); + } let items = usize::try_from(config.cache / (1024 * 1024)) .unwrap_or(usize::MAX) @@ -101,14 +120,13 @@ impl Reader { index: Mutex::default(), cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), infos: Mutex::default(), + changed: watch::Sender::new(false), }); let mut reader = Self { shared, dynamic: broadcast.dynamic(), - timeline, - decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), - cursor: None, + timelines, }; reader.refresh().await?; Ok(reader) @@ -116,18 +134,51 @@ impl Reader { /// Replay timeline objects committed since the last refresh. /// - /// Lists timeline keys after the last replayed segment, then GETs them in segment order. Each - /// stored group opens with a window checkpoint, so a segment that expired or is malformed is - /// skipped and the next one recovers the retained window, evicting whatever it popped. A + /// Lists each timeline's keys after its last replayed segment, then GETs them in segment order. + /// Each stored group opens with a window checkpoint, so a segment that expired or is malformed + /// is skipped and the next one recovers the retained window, evicting whatever it popped. A /// malformed newest object is retried on the next refresh. pub async fn refresh(&mut self) -> Result<()> { - let timeline = self.timeline.name().to_string(); - let mut query = Query::segments(&timeline)?; + for timeline in &mut self.timelines { + timeline.refresh(&self.shared).await?; + } + Ok(()) + } + + /// Finish every replayed timeline track once the caller knows the recording ended. + /// + /// Following stops, and a group still growing is served as it stands. Groups already committed + /// stay servable through [`serve`](Self::serve). + pub fn finish(self) -> Result<()> { + self.shared.changed.send_replace(true); + for timeline in self.timelines { + timeline.producer.finish()?; + } + Ok(()) + } + + /// Answer track and group requests on the broadcast until it closes. + /// + /// The future is independent of this reader, so it keeps serving after + /// [`finish`](Self::finish); run it on any executor. + pub fn serve(&self) -> impl Future + Send + 'static + where + T: 'static, + { + let tracks = self.timelines.iter().map(|timeline| timeline.track.clone()).collect(); + serve(self.shared.clone(), self.dynamic.clone(), tracks) + } +} + +impl Timeline { + async fn refresh(&mut self, shared: &Shared) -> Result<()> { + let name = self.producer.name().to_string(); + let mut query = Query::segments(&name)?; if let Some(cursor) = self.cursor { - query = query.after(&Key::segments(&timeline, cursor)?)?; + query = query.after(&Key::segments(&name, cursor)?)?; } - let entries: Vec<_> = self.shared.store.list(&query).try_collect().await?; + let entries: Vec<_> = shared.store.list(&query).try_collect().await?; let mut segments: Vec = entries .into_iter() .filter_map(|entry| match entry.key { @@ -140,11 +191,11 @@ impl Reader { segments.dedup(); for segment in segments { - match self.shared.store.get_segments(&timeline, segment).await { - Ok(object) => self.replay(&object)?, + match shared.store.get_segments(&name, segment).await { + Ok(object) => self.replay(shared, &object)?, Err(Error::Store(err)) => return Err(Error::Store(err)), Err(err) => { - tracing::warn!(segment, %err, "skipping unreadable timeline segment"); + tracing::warn!(timeline = name, segment, %err, "skipping unreadable timeline segment"); continue; } } @@ -154,40 +205,30 @@ impl Reader { Ok(()) } - /// Finish the replayed timeline track once the caller knows the recording ended. - /// - /// Following stops; groups already committed stay servable through [`serve`](Self::serve). - pub fn finish(self) -> Result<()> { - Ok(self.timeline.finish()?) - } - - /// Answer track and group requests on the broadcast until it closes. - /// - /// The future is independent of this reader, so it keeps serving after - /// [`finish`](Self::finish); run it on any executor. - pub fn serve(&self) -> impl Future + Send + 'static - where - T: 'static, - { - serve(self.shared.clone(), self.dynamic.clone()) - } - /// Index one timeline object's groups, then republish each on the timeline track. - fn replay(&mut self, object: &Object) -> Result<()> { + fn replay(&mut self, shared: &Shared, object: &Object) -> Result<()> { for stored in &object.groups { let mut group = self.decoder.group(); let decoded = stored.frames.iter().try_for_each(|frame| group.decode(&frame.payload)); // Apply whatever decoded, even from a group that failed partway: the next group's // checkpoint restates the window regardless. - let mut index = self.shared.index.lock().unwrap(); + let mut index = shared.index.lock().unwrap(); while let Some(event) = group.next_event() { match event { - window::Event::Push { index: at, value } => index.push(at, &value), - window::Event::Pop(range) => { - for (track, span) in index.pop(range) { - if let Ok(key) = Key::groups(track, span.bounds) { - self.shared.cache.remove(&key); + window::Event::Push { index: at, value } if at == value.sequence => index.push(&self.track, &value), + window::Event::Push { index: at, value } => { + tracing::warn!( + track = self.track, + at, + sequence = value.sequence, + "ignoring a misnumbered record" + ); + } + window::Event::Pop(range) | window::Event::Skip(range) => { + for span in index.pop(&self.track, range) { + if let Ok(key) = Key::segments(self.track.clone(), span.sequence) { + shared.cache.remove(&key); } } } @@ -195,13 +236,14 @@ impl Reader { } } drop(index); + shared.changed.send_modify(|_| {}); if let Err(err) = decoded { tracing::warn!(group = stored.sequence, %err, "skipping malformed timeline group"); continue; } - let mut producer = match self.timeline.create_group(stored.sequence.into()) { + let mut producer = match self.producer.create_group(stored.sequence.into()) { Ok(producer) => producer, Err(moq_net::Error::Duplicate) => continue, Err(err) => return Err(err.into()), @@ -217,26 +259,29 @@ impl Reader { } /// Serve requested tracks until the broadcast closes. -async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic) { - let mut tracks = kio::Tasks::new(); +async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic, tracks: Vec) { + let mut serving = kio::Tasks::new(); kio::wait(|waiter| { loop { match dynamic.poll_requested_track(waiter) { - Poll::Ready(Ok(request)) => tracks.push(task(Box::pin(serve_track(shared.clone(), request)))), + Poll::Ready(Ok(request)) => { + let known = tracks.iter().any(|track| track == request.name()); + serving.push(task(Box::pin(serve_track(shared.clone(), request, known)))) + } Poll::Ready(Err(_)) => return Poll::Ready(()), Poll::Pending => break, } } - let _ = tracks.poll(waiter); + let _ = serving.poll(waiter); Poll::Pending }) .await } -/// Accept a track the timeline names, then serve its group requests until nobody uses it. -async fn serve_track(shared: Arc>, request: track::Request) { +/// Accept a track a timeline indexes, then serve its group requests until nobody uses it. +async fn serve_track(shared: Arc>, request: track::Request, known: bool) { let name = request.name().to_string(); - if !shared.index.lock().unwrap().has_track(&name) { + if !known { request.reject(moq_net::Error::NotFound); return; } @@ -281,7 +326,10 @@ async fn serve_track(shared: Arc>, request: track::Req .await } -/// Answer one group request from its range-named object, or reject it as never delivered. +/// Answer one group request from the records holding it, or reject it as never delivered. +/// +/// A group whose records stop short of its end keeps its producer open and grows as later records +/// commit, until one reaches the group's end or the caller finishes the reader. async fn serve_group( shared: Arc>, track: String, @@ -289,85 +337,106 @@ async fn serve_group( request: group::Request, ) { let sequence = request.sequence(); - let span = shared.index.lock().unwrap().get(&track, sequence); - let Some(span) = span else { + let start = request.frame_start(); + let mut changed = shared.changed.subscribe(); + let mut spans = shared.index.lock().unwrap().group(&track, sequence); + + // The earliest retained record must hold the requested frame; an expired head is gone. + if spans + .first() + .is_none_or(|span| span.start > Position::new(sequence, start)) + { request.reject(moq_net::Error::NotFound); return; - }; - - let object = match Key::groups(track.clone(), span.bounds.clone()) { - Ok(key) => { - shared - .cache - .get_or_insert_async(&key, load(&shared.store, &track, &span)) - .await - } - Err(err) => Err(err), - }; - let object = match object { - Ok(object) => object, + } + // Validate the first object before accepting, so an unreadable group is a plain miss. Keep it, + // so a cache too small to hold it costs no second GET. + let mut first = match object(&shared, &track, &spans[0]).await { + Ok(object) => Some((spans[0].sequence, object)), Err(err) => { tracing::warn!(track, sequence, %err, "archived group is unavailable"); request.reject(moq_net::Error::NotFound); return; } }; - - // Load validated the table against the span, so every advertised group is present. - let Ok(position) = object.groups.binary_search_by_key(&sequence, |group| group.sequence) else { - request.reject(moq_net::Error::NotFound); + let Ok(mut producer) = request.accept(None) else { return; }; - let frames = &object.groups[position].frames; - let Some(frames) = usize::try_from(request.frame_start()) - .ok() - .and_then(|start| frames.get(start..)) - else { - request.reject(moq_net::Error::NotFound); + if producer.start_at(start).is_err() { return; - }; + } - let start = request.frame_start(); - let Ok(mut producer) = request.accept(None) else { - return; - }; - let written = producer.start_at(start).and_then(|()| { - frames.iter().try_for_each(|frame| { - let timestamp = timestamp(frame.timestamp, timescale).map_err(|_| moq_net::Error::TimestampMismatch)?; - producer.write_frame(timestamp, frame.payload.clone()) - }) - }); - match written { - Ok(()) => { + let mut next = start; + loop { + for span in &spans { + if span.end <= Position::new(sequence, next) { + continue; + } + let loaded = match first.take_if(|(loaded, _)| *loaded == span.sequence) { + Some((_, object)) => Ok(object), + None => object(&shared, &track, span).await, + }; + let object = match loaded { + Ok(object) if span.start <= Position::new(sequence, next) => object, + Ok(_) => { + let _ = producer.abort(moq_net::Error::NotFound); + return; + } + Err(err) => { + tracing::warn!(track, sequence, %err, "archived group is unavailable"); + let _ = producer.abort(moq_net::Error::NotFound); + return; + } + }; + // The span was validated against its object, so the group is present. + let Some(stored) = object.groups.iter().find(|group| group.sequence == sequence) else { + continue; + }; + let first = match object.groups[0].sequence == sequence { + true => object.frame_start, + false => 0, + }; + for (index, frame) in (first..).zip(&stored.frames) { + if index < next { + continue; + } + let Ok(timestamp) = timestamp(frame.timestamp, timescale) else { + let _ = producer.abort(moq_net::Error::TimestampMismatch); + return; + }; + if let Err(err) = producer.write_frame(timestamp, frame.payload.clone()) { + let _ = producer.abort(err); + return; + } + next = index + 1; + } + } + + let complete = spans.last().is_some_and(|span| span.end.group > sequence); + if complete || *changed.borrow_and_update() { let _ = producer.finish(); + return; } - Err(err) => { - let _ = producer.abort(err); + // The reader dropping ends the recording as surely as finishing it. + if changed.changed().await.is_err() { + let _ = producer.finish(); + return; } + spans = shared.index.lock().unwrap().group(&track, sequence); } } -/// GET a span's object and require its table to hold exactly the advertised groups. -async fn load(store: &Store, track: &str, span: &Span) -> Result> { - let object = store.get_groups(track, span.bounds.clone()).await?; - check_runs(&object, &span.runs)?; - Ok(Arc::new(object)) -} - -/// Refuse a table whose sequences differ from the advertised runs. -fn check_runs(object: &Object, runs: &[RangeInclusive]) -> Result<()> { - let mut groups = object.groups.iter(); - for run in runs { - for sequence in run.clone() { - if groups.next().map(|group| group.sequence) != Some(sequence) { - return Err(Error::Table); - } - } - } - match groups.next() { - Some(_) => Err(Error::Table), - None => Ok(()), - } +/// A span's object, from the cache or one GET validated against the span. +async fn object(shared: &Shared, track: &str, span: &Span) -> Result> { + let key = Key::segments(track.to_string(), span.sequence)?; + shared + .cache + .get_or_insert_async(&key, async { + let object = shared.store.get_segments(track, span.sequence).await?; + object.check_span(span.start, span.end)?; + Ok(Arc::new(object)) + }) + .await } fn track_info(info: &crate::Info) -> Result { diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index c560788199..df90591724 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -1,22 +1,35 @@ +use std::collections::HashMap; +use std::time::Duration; + use bytes::Bytes; -use hang::timeline::{Range, Record}; +use hang::timeline::{Position, Record}; use moq_json::window; use moq_net::{broadcast, group}; use object_store::ObjectStoreExt; use super::*; +use crate::Info; use crate::mock::{Mock, Op}; use crate::segment::{Frame, Group}; -use crate::{ID_MAX, Info}; -const TIMELINE: &str = "timeline.z"; +fn timeline(track: &str) -> String { + hang::timeline::default_name(track) +} + +/// One track's timeline encoder, as a recording writer drives it. +struct Encoder { + encoder: window::Encoder, + /// Next timeline group sequence. + group: u64, + /// Next record sequence. + record: u64, +} /// Writes archive objects the way a recording writer lays them out. struct Archive { store: Store, - encoder: window::Encoder, - /// Next timeline group sequence. - sequence: u64, + op_ratio: u32, + encoders: HashMap, } impl Archive { @@ -26,38 +39,87 @@ impl Archive { /// `op_ratio` 0 makes every window edit its own checkpoint group. async fn with_op_ratio(op_ratio: u32) -> Self { - let store = Store::new(Mock::memory(), "rec"); - store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); - let config = window::ProducerConfig::default() - .with_compression(true) - .with_op_ratio(op_ratio); Self { - store, - encoder: window::Encoder::new(config), - sequence: 0, + store: Store::new(Mock::memory(), "rec"), + op_ratio, + encoders: HashMap::new(), } } - /// Store one track object holding `groups` of `(sequence, frame count)`. - async fn media(&self, track: &str, groups: &[(u64, usize)]) { - self.store.put_info(track, &Info::new(1, 1000).unwrap()).await.unwrap(); - let groups = groups - .iter() - .map(|&(sequence, count)| Group { - sequence, - frames: (0..count).map(|i| frame(track, sequence, i)).collect(), + async fn encoder(&mut self, track: &str) -> &mut Encoder { + if !self.encoders.contains_key(track) { + self.store.put_info(track, &Info::new(1, 1000).unwrap()).await.unwrap(); + self.store + .put_info(&timeline(track), &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(self.op_ratio); + self.encoders.insert( + track.to_string(), + Encoder { + encoder: window::Encoder::new(config), + group: 0, + record: 0, + }, + ); + } + self.encoders.get_mut(track).unwrap() + } + + /// The next record of `track`, spanning `start..end` where every group holds `frames` frames. + /// Stores its object unless `store` is false. + async fn record(&mut self, track: &str, start: Position, end: Position, frames: u64) -> Record { + let encoder = self.encoder(track).await; + let sequence = encoder.record; + encoder.record += 1; + let record = Record::new(sequence, start.group * 100, 100, start, end); + + let last = match end.frame { + 0 => end.group - 1, + _ => end.group, + }; + let groups = (start.group..=last) + .map(|group| { + let first = if group == start.group { start.frame } else { 0 }; + let stop = if group == end.group { end.frame } else { frames }; + Group { + sequence: group, + frames: (first..stop).map(|i| frame(track, group, i)).collect(), + } }) .collect(); - self.store.put_groups(track, &Object { groups }).await.unwrap(); + let object = Object { + frame_start: start.frame, + groups, + }; + self.store.put_segments(track, sequence, &object).await.unwrap(); + record + } + + /// Store and commit a record of whole groups `groups`, each holding `frames` frames. + async fn add(&mut self, track: &str, groups: std::ops::Range, frames: u64) -> Record { + let record = self + .record( + track, + Position::group(groups.start), + Position::group(groups.end), + frames, + ) + .await; + self.commit(track, &record, 0).await; + record } /// Push `record`, pop `pop` records, and return the resulting timeline groups. - fn edit(&mut self, record: &Record, pop: u64) -> Vec { + async fn edit(&mut self, track: &str, record: &Record, pop: u64) -> Vec { + let encoder = self.encoder(track).await; let mut frames = Vec::new(); - let pending = self.encoder.push(record).unwrap(); + let pending = encoder.encoder.push(record).unwrap(); frames.push((pending.keyframe, pending.payload.clone())); pending.commit(); - if let Some(pending) = self.encoder.pop(pop).unwrap() { + if let Some(pending) = encoder.encoder.pop(pop).unwrap() { frames.push((pending.keyframe, pending.payload.clone())); pending.commit(); } @@ -65,11 +127,11 @@ impl Archive { let mut groups: Vec = Vec::new(); for (keyframe, payload) in frames { if keyframe { - self.sequence += 1; + encoder.group += 1; } - if groups.last().is_none_or(|group| group.sequence != self.sequence - 1) { + if groups.last().is_none_or(|group| group.sequence != encoder.group - 1) { groups.push(Group { - sequence: self.sequence - 1, + sequence: encoder.group - 1, frames: Vec::new(), }); } @@ -79,17 +141,17 @@ impl Archive { groups } - async fn timeline(&self, segment: u64, groups: Vec) { + async fn timeline(&self, track: &str, segment: u64, groups: Vec) { self.store - .put_segments(TIMELINE, segment, &Object { groups }) + .put_segments(&timeline(track), segment, &Object::new(groups)) .await .unwrap(); } - /// Commit `record` as timeline segment `record.segment`, popping `pop` older records. - async fn commit(&mut self, record: &Record, pop: u64) { - let groups = self.edit(record, pop); - self.timeline(record.segment, groups).await; + /// Commit `record` as its track's timeline segment, popping `pop` older records. + async fn commit(&mut self, track: &str, record: &Record, pop: u64) { + let groups = self.edit(track, record, pop).await; + self.timeline(track, record.sequence, groups).await; } async fn raw(&self, key: &Key, bytes: &'static [u8]) { @@ -100,29 +162,31 @@ impl Archive { .await .unwrap(); } + + fn config(&self) -> Config { + Config::new( + self.encoders + .keys() + .map(|track| (track.clone(), timeline(track))) + .collect(), + ) + } } -fn frame(track: &str, sequence: u64, index: usize) -> Frame { +fn frame(track: &str, sequence: u64, index: u64) -> Frame { Frame { - timestamp: sequence * 100 + index as u64, + timestamp: sequence * 100 + index, payload: Bytes::from(format!("{track}/{sequence}/{index}")), } } -fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { - let mut record = Record::new(segment, segment * 2000, 2000); - for (name, ranges) in tracks { - let ranges = ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(); - record.tracks.insert(name.to_string(), ranges); - } - record +async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { + open_with(archive, archive.config()).await } -async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { +async fn open_with(archive: &Archive, config: Config) -> (broadcast::Producer, Reader) { let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(archive.store.clone(), &broadcast, Config::new(TIMELINE)) - .await - .unwrap(); + let reader = Reader::open(archive.store.clone(), &broadcast, config).await.unwrap(); tokio::spawn(reader.serve()); (broadcast, reader) } @@ -144,7 +208,7 @@ async fn fetch( Ok(frames) } -fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<(u64, Bytes)> { +fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<(u64, Bytes)> { frames .map(|i| { let frame = frame(track, sequence, i); @@ -153,16 +217,17 @@ fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<( .collect() } +fn not_found(result: std::result::Result, moq_net::Error>) { + assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); +} + #[tokio::test] async fn fetch_replays_original_groups() { let mut archive = Archive::new().await; - archive.media("video", &[(0, 3), (1, 2)]).await; - archive.media("audio", &[(0, 1), (1, 1), (5, 2)]).await; - archive.media("video", &[(2, 2)]).await; - archive - .commit(&record(0, &[("video", &[(0, 1)]), ("audio", &[(0, 1), (5, 5)])]), 0) - .await; - archive.commit(&record(1, &[("video", &[(2, 2)])]), 0).await; + archive.add("video", 0..2, 3).await; + archive.add("video", 2..3, 2).await; + archive.add("audio", 0..2, 1).await; + archive.add("audio", 5..6, 2).await; let (broadcast, _reader) = open(&archive).await; @@ -181,7 +246,7 @@ async fn fetch_replays_original_groups() { // A requested frame_start keeps the original frame indices and timestamps. let tail = fetch(&broadcast, "video", 1, 1).await.unwrap(); - assert_eq!(tail, expected("video", 1, 1..2)); + assert_eq!(tail, expected("video", 1, 1..3)); let track = broadcast.consume().track("video").unwrap(); let info = track.query().await.unwrap(); @@ -190,13 +255,91 @@ async fn fetch_replays_original_groups() { } #[tokio::test] -async fn requests_download_only_their_object() { +async fn a_group_split_across_records_is_served_whole() { let mut archive = Archive::new().await; - archive.media("video", &[(0, 1)]).await; - archive.media("audio", &[(0, 1), (1, 1), (2, 1)]).await; - archive - .commit(&record(0, &[("video", &[(0, 0)]), ("audio", &[(0, 2)])]), 0) + let head = archive + .record("video", Position::group(7), Position::new(7, 3), 5) + .await; + archive.commit("video", &head, 0).await; + let tail = archive + .record("video", Position::new(7, 3), Position::group(9), 5) .await; + archive.commit("video", &tail, 0).await; + + let (broadcast, _reader) = open(&archive).await; + assert_eq!( + fetch(&broadcast, "video", 7, 0).await.unwrap(), + expected("video", 7, 0..5) + ); + assert_eq!( + fetch(&broadcast, "video", 7, 4).await.unwrap(), + expected("video", 7, 4..5) + ); + assert_eq!( + fetch(&broadcast, "video", 8, 0).await.unwrap(), + expected("video", 8, 0..5) + ); +} + +#[tokio::test] +async fn an_open_group_grows_as_records_commit() { + let mut archive = Archive::new().await; + let head = archive.record("log", Position::group(0), Position::new(0, 2), 0).await; + archive.commit("log", &head, 0).await; + + let (broadcast, mut reader) = open(&archive).await; + let track = broadcast.consume().track("log").unwrap(); + let mut group = tokio::time::timeout(Duration::from_secs(1), track.fetch_group(0, group::Fetch::default())) + .await + .expect("fetch") + .unwrap(); + for index in 0..2 { + let frame = group.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.payload, frame_payload("log", 0, index)); + } + let pending = tokio::time::timeout(Duration::from_millis(50), group.read_frame()).await; + assert!(pending.is_err(), "the group waits for its next record"); + + let more = archive.record("log", Position::new(0, 2), Position::new(0, 3), 0).await; + archive.commit("log", &more, 0).await; + reader.refresh().await.unwrap(); + let frame = group.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.payload, frame_payload("log", 0, 2)); + + // Finishing the reader ends the group where the recording stopped. + reader.finish().unwrap(); + assert!(group.read_frame().await.unwrap().is_none()); +} + +fn frame_payload(track: &str, sequence: u64, index: u64) -> Bytes { + frame(track, sequence, index).payload +} + +#[tokio::test] +async fn an_expired_head_is_not_found() { + let mut archive = Archive::new().await; + let head = archive + .record("video", Position::group(7), Position::new(7, 3), 5) + .await; + archive.commit("video", &head, 0).await; + let tail = archive + .record("video", Position::new(7, 3), Position::group(8), 5) + .await; + archive.commit("video", &tail, 1).await; + + let (broadcast, _reader) = open(&archive).await; + not_found(fetch(&broadcast, "video", 7, 0).await); + assert_eq!( + fetch(&broadcast, "video", 7, 3).await.unwrap(), + expected("video", 7, 3..5) + ); +} + +#[tokio::test] +async fn requests_download_only_their_object() { + let mut archive = Archive::new().await; + archive.add("video", 0..1, 1).await; + archive.add("audio", 0..3, 1).await; let (broadcast, _reader) = open(&archive).await; archive.store.inner().take(); @@ -213,7 +356,7 @@ async fn requests_download_only_their_object() { gets, vec![ "rec/audio/.info".to_string(), - "rec/audio/groups/0000000000000000002.0000000000000000000".to_string(), + "rec/audio/segments/0000000000000000000".to_string(), ], "adjacent groups reuse one GET and never touch video" ); @@ -222,46 +365,33 @@ async fn requests_download_only_their_object() { #[tokio::test] async fn unadvertised_or_unreadable_groups_are_not_found() { let mut archive = Archive::new().await; - archive.media("audio", &[(0, 1), (1, 1), (4, 1)]).await; - // The object holds group 3, which its record does not advertise. - archive.media("video", &[(0, 1), (3, 1)]).await; - archive.raw(&Key::groups("bad", 0..=0).unwrap(), b"\x02garbage").await; - archive - .store - .put_info("bad", &Info::new(0, 1000).unwrap()) - .await - .unwrap(); - archive - .store - .put_info("missing", &Info::new(0, 1000).unwrap()) - .await - .unwrap(); + archive.add("audio", 0..2, 1).await; + archive.add("audio", 4..5, 1).await; + // The object holds groups 0 and 1, but its record names group 2 too. + let mismatched = archive.record("video", Position::group(0), Position::group(2), 1).await; archive .commit( - &record( - 0, - &[ - ("audio", &[(0, 1), (4, 4)]), - ("video", &[(0, 0), (2, 3)]), - ("bad", &[(0, 0)]), - ("missing", &[(0, 0)]), - ], - ), + "video", + &Record::new(0, 0, 100, Position::group(0), Position::group(3)), 0, ) .await; + let bad = archive.record("bad", Position::group(0), Position::group(1), 1).await; + archive.raw(&Key::segments("bad", 0).unwrap(), b"\x02garbage").await; + archive.commit("bad", &bad, 0).await; + let missing = Record::new(0, 0, 100, Position::group(0), Position::group(1)); + archive.encoder("missing").await; + archive.commit("missing", &missing, 0).await; + assert_eq!(mismatched.end, Position::group(2)); let (broadcast, _reader) = open(&archive).await; - let not_found = |result: std::result::Result, moq_net::Error>| { - assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); - }; - not_found(fetch(&broadcast, "audio", 2, 0).await); // internal gap - not_found(fetch(&broadcast, "audio", 5, 0).await); // past the last range + not_found(fetch(&broadcast, "audio", 2, 0).await); // between records + not_found(fetch(&broadcast, "audio", 5, 0).await); // past the last record not_found(fetch(&broadcast, "video", 0, 0).await); // table disagrees with the record - not_found(fetch(&broadcast, "bad", 0, 0).await); // unknown envelope version + not_found(fetch(&broadcast, "bad", 0, 0).await); // malformed envelope not_found(fetch(&broadcast, "missing", 0, 0).await); // no object - not_found(fetch(&broadcast, "chat", 0, 0).await); // track the timeline never named + not_found(fetch(&broadcast, "chat", 0, 0).await); // a track no timeline indexes // Siblings remain usable. assert_eq!( @@ -271,23 +401,19 @@ async fn unadvertised_or_unreadable_groups_are_not_found() { } #[tokio::test] -async fn refresh_follows_new_segments_and_pops() { +async fn refresh_follows_new_records_and_pops() { let mut archive = Archive::new().await; - archive.media("video", &[(0, 1), (1, 1)]).await; - archive.commit(&record(0, &[("video", &[(0, 1)])]), 0).await; + archive.add("video", 0..2, 1).await; let (broadcast, mut reader) = open(&archive).await; assert!(fetch(&broadcast, "video", 0, 0).await.is_ok()); - archive.media("video", &[(2, 1)]).await; + let next = archive.record("video", Position::group(2), Position::group(3), 1).await; // Refreshing before the commit finds nothing new; Not Found is not finality. reader.refresh().await.unwrap(); - assert!(matches!( - fetch(&broadcast, "video", 2, 0).await, - Err(moq_net::Error::NotFound) - )); + not_found(fetch(&broadcast, "video", 2, 0).await); - archive.commit(&record(1, &[("video", &[(2, 2)])]), 1).await; + archive.commit("video", &next, 1).await; reader.refresh().await.unwrap(); assert_eq!( fetch(&broadcast, "video", 2, 0).await.unwrap(), @@ -296,28 +422,25 @@ async fn refresh_follows_new_segments_and_pops() { // The popped record's groups are gone, although its object is still stored and was cached. archive.store.inner().take(); - assert!(matches!( - fetch(&broadcast, "video", 1, 0).await, - Err(moq_net::Error::NotFound) - )); + not_found(fetch(&broadcast, "video", 1, 0).await); assert_eq!(archive.store.inner().gets(), Vec::::new()); } #[tokio::test] async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { let mut archive = Archive::new().await; - archive.media("video", &[(0, 1)]).await; - archive.media("video", &[(1, 1)]).await; - archive.media("video", &[(2, 1)]).await; - archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; - // Segment 1 is committed to the window but its timeline object is unreadable. - archive.edit(&record(1, &[("video", &[(1, 1)])]), 0); - archive.raw(&Key::segments(TIMELINE, 1).unwrap(), b"\x01").await; - archive.commit(&record(2, &[("video", &[(2, 2)])]), 0).await; + archive.add("video", 0..1, 1).await; + // Record 1 is committed to the window but its timeline object is unreadable. + let lost = archive.record("video", Position::group(1), Position::group(2), 1).await; + archive.edit("video", &lost, 0).await; + archive + .raw(&Key::segments(timeline("video"), 1).unwrap(), b"\x02") + .await; + archive.add("video", 2..3, 1).await; let (broadcast, _reader) = open(&archive).await; - // Segment 2's checkpoint restates record 1, so its group is still served. + // Record 2's checkpoint restates record 1, so its group is still served. for sequence in 0..3 { assert_eq!( fetch(&broadcast, "video", sequence, 0).await.unwrap(), @@ -329,21 +452,15 @@ async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { #[tokio::test] async fn a_missing_tail_is_retried_on_the_next_refresh() { let mut archive = Archive::new().await; - for segment in 0..3 { - archive.media("video", &[(segment, 1)]).await; - archive - .commit(&record(segment, &[("video", &[(segment, segment)])]), 0) - .await; + for group in 0..3 { + archive.add("video", group..group + 1, 1).await; } // Listed, but not yet readable. archive.store.inner().hide_gets("segments/0000000000000000002"); let (broadcast, mut reader) = open(&archive).await; assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); - assert!(matches!( - fetch(&broadcast, "video", 2, 0).await, - Err(moq_net::Error::NotFound) - )); + not_found(fetch(&broadcast, "video", 2, 0).await); archive.store.inner().heal(); archive.store.inner().take(); @@ -352,10 +469,10 @@ async fn a_missing_tail_is_retried_on_the_next_refresh() { archive.store.inner().take(), [ Op::List { - prefix: "rec/timeline%2Ez/segments".to_string(), - offset: Some("rec/timeline%2Ez/segments/0000000000000000001".to_string()), + prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), + offset: Some("rec/video%2Etimeline%2Ez/segments/0000000000000000001".to_string()), }, - Op::Get("rec/timeline%2Ez/segments/0000000000000000002".to_string()), + Op::Get("rec/video%2Etimeline%2Ez/segments/0000000000000000002".to_string()), ], "the cursor stays before the missing tail" ); @@ -368,39 +485,35 @@ async fn a_missing_tail_is_retried_on_the_next_refresh() { #[tokio::test] async fn following_lists_only_new_timeline_keys() { let mut archive = Archive::new().await; - archive.media("video", &[(0, 1)]).await; - archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + archive.add("video", 0..1, 1).await; let (broadcast, mut reader) = open(&archive).await; - for segment in 1..4 { + for group in 1..4 { // A media object stored ahead of its commit is invisible until the timeline names it. - archive.media("video", &[(segment, 1)]).await; + let record = archive + .record("video", Position::group(group), Position::group(group + 1), 1) + .await; reader.refresh().await.unwrap(); - assert!(matches!( - fetch(&broadcast, "video", segment, 0).await, - Err(moq_net::Error::NotFound) - )); + not_found(fetch(&broadcast, "video", group, 0).await); - archive - .commit(&record(segment, &[("video", &[(segment, segment)])]), 0) - .await; + archive.commit("video", &record, 0).await; archive.store.inner().take(); reader.refresh().await.unwrap(); - let previous = format!("rec/timeline%2Ez/segments/{:019}", segment - 1); + let previous = format!("rec/video%2Etimeline%2Ez/segments/{:019}", group - 1); assert_eq!( archive.store.inner().take(), [ Op::List { - prefix: "rec/timeline%2Ez/segments".to_string(), + prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), offset: Some(previous), }, - Op::Get(format!("rec/timeline%2Ez/segments/{segment:019}")), + Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{group:019}")), ], - "following segment {segment} touches no media listing" + "following record {group} touches no media listing" ); assert_eq!( - fetch(&broadcast, "video", segment, 0).await.unwrap(), - expected("video", segment, 0..1) + fetch(&broadcast, "video", group, 0).await.unwrap(), + expected("video", group, 0..1) ); } } @@ -409,36 +522,32 @@ async fn following_lists_only_new_timeline_keys() { async fn an_unordered_listing_replays_in_segment_order() { let mut archive = Archive::new().await; archive.store.inner().unordered(); - for segment in 0..4 { - archive.media("video", &[(segment, 1)]).await; - let pop = u64::from(segment >= 2); - archive - .commit(&record(segment, &[("video", &[(segment, segment)])]), pop) + for group in 0..4 { + let record = archive + .record("video", Position::group(group), Position::group(group + 1), 1) .await; + archive.commit("video", &record, u64::from(group >= 2)).await; } let (broadcast, mut reader) = open(&archive).await; // The cursor is the newest segment, not the last one listed. - archive.media("video", &[(4, 1)]).await; - archive.commit(&record(4, &[("video", &[(4, 4)])]), 1).await; + let record = archive.record("video", Position::group(4), Position::group(5), 1).await; + archive.commit("video", &record, 1).await; archive.store.inner().take(); reader.refresh().await.unwrap(); assert_eq!( archive.store.inner().gets(), - ["rec/timeline%2Ez/segments/0000000000000000004"] + ["rec/video%2Etimeline%2Ez/segments/0000000000000000004"] ); - for segment in 0..3 { - assert!(matches!( - fetch(&broadcast, "video", segment, 0).await, - Err(moq_net::Error::NotFound) - )); + for group in 0..3 { + not_found(fetch(&broadcast, "video", group, 0).await); } - for segment in 3..5 { + for group in 3..5 { assert_eq!( - fetch(&broadcast, "video", segment, 0).await.unwrap(), - expected("video", segment, 0..1) + fetch(&broadcast, "video", group, 0).await.unwrap(), + expected("video", group, 0..1) ); } } @@ -446,34 +555,23 @@ async fn an_unordered_listing_replays_in_segment_order() { #[tokio::test] async fn a_track_without_usable_info_is_not_found() { let mut archive = Archive::new().await; - archive.media("audio", &[(0, 1)]).await; - let object = |track| Object { - groups: vec![Group { - sequence: 0, - frames: vec![frame(track, 0, 0)], - }], - }; - archive.store.put_groups("bare", &object("bare")).await.unwrap(); - archive.store.put_groups("future", &object("future")).await.unwrap(); + archive.add("audio", 0..1, 1).await; + archive.add("bare", 0..1, 1).await; + archive.add("future", 0..1, 1).await; + let path = archive.store.path(&Key::info("bare").unwrap()).unwrap(); + archive.store.inner().delete(&path).await.unwrap(); + let path = archive.store.path(&Key::info("future").unwrap()).unwrap(); + archive.store.inner().delete(&path).await.unwrap(); archive .raw( &Key::info("future").unwrap(), - br#"{"version":2,"priority":0,"timescale":1000}"#, - ) - .await; - archive - .commit( - &record(0, &[("audio", &[(0, 0)]), ("bare", &[(0, 0)]), ("future", &[(0, 0)])]), - 0, + br#"{"version":3,"priority":0,"timescale":1000}"#, ) .await; let (broadcast, _reader) = open(&archive).await; for track in ["bare", "future"] { - assert!( - matches!(fetch(&broadcast, track, 0, 0).await, Err(moq_net::Error::NotFound)), - "{track}" - ); + not_found(fetch(&broadcast, track, 0, 0).await); } assert_eq!( fetch(&broadcast, "audio", 0, 0).await.unwrap(), @@ -484,69 +582,50 @@ async fn a_track_without_usable_info_is_not_found() { #[tokio::test] async fn multi_frame_timeline_groups_decode() { let mut archive = Archive::with_op_ratio(64).await; - archive.media("video", &[(0, 1)]).await; - archive.media("video", &[(1, 1)]).await; - let mut groups = archive.edit(&record(0, &[("video", &[(0, 0)])]), 0); - let ops = archive.edit(&record(1, &[("video", &[(1, 1)])]), 0); + let first = archive.record("video", Position::group(0), Position::group(1), 1).await; + let second = archive.record("video", Position::group(1), Position::group(2), 1).await; + let mut groups = archive.edit("video", &first, 0).await; + let ops = archive.edit("video", &second, 0).await; assert!( ops[0].sequence == groups[0].sequence, "the push stays in the open group" ); groups[0].frames.extend(ops.into_iter().flat_map(|group| group.frames)); - archive.timeline(0, groups).await; + archive.timeline("video", 0, groups).await; let (broadcast, _reader) = open(&archive).await; assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); } #[tokio::test] -async fn timeline_track_is_republished_and_finished_on_request() { +async fn timeline_tracks_are_republished_and_finished_on_request() { let mut archive = Archive::new().await; - archive.media("video", &[(0, 1)]).await; - archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; - archive.commit(&record(1, &[]), 0).await; + archive.add("video", 0..1, 1).await; + archive.add("video", 1..2, 1).await; + archive.add("audio", 0..1, 1).await; let (broadcast, reader) = open(&archive).await; - let track = broadcast.consume().track(TIMELINE).unwrap(); + let track = broadcast.consume().track(&timeline("video")).unwrap(); let subscriber = track.subscribe(None).await.unwrap(); let mut timeline = window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); reader.finish().unwrap(); - let mut segments = Vec::new(); + let mut records = Vec::new(); while let Some(event) = timeline.next().await.unwrap() { if let window::Event::Push { value, .. } = event { - segments.push(value.segment); + records.push(value.sequence); } } - assert_eq!(segments, vec![0, 1]); + assert_eq!(records, vec![0, 1]); } #[tokio::test] -async fn open_requires_the_timeline_info() { +async fn open_requires_every_timeline_info() { let store = Store::new(Mock::memory(), "rec"); let broadcast = broadcast::Info::new().produce(); - let result = Reader::open(store, &broadcast, Config::new(TIMELINE)).await; + let config = Config::new([("video".to_string(), timeline("video"))].into()); + let result = Reader::open(store, &broadcast, config).await; assert!(matches!(result, Err(Error::NotFound(_)))); } - -#[test] -fn check_runs_requires_the_exact_sequences() { - let object = Object { - groups: [0, 1, 4, ID_MAX] - .into_iter() - .map(|sequence| Group { - sequence, - frames: Vec::new(), - }) - .collect(), - }; - check_runs(&object, &[0..=1, 4..=4, ID_MAX..=ID_MAX]).unwrap(); - assert!(check_runs(&object, &[0..=1, 4..=4]).is_err(), "extra group"); - assert!( - check_runs(&object, &[0..=4, ID_MAX..=ID_MAX]).is_err(), - "missing groups" - ); - assert!(check_runs(&object, &[0..=1, 3..=4, ID_MAX..=ID_MAX]).is_err()); -} diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs index f7ccfd5ace..98aeeb5af6 100644 --- a/rs/moq-archive/src/recover.rs +++ b/rs/moq-archive/src/recover.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::ops::RangeInclusive; use futures::TryStreamExt; -use hang::timeline::Record; +use hang::timeline::{Position, Record}; use moq_json::window::{self, Checkpoint}; use object_store::ObjectStore; @@ -12,97 +12,103 @@ use crate::store::list::Query; use crate::{Error, Key, Object, Result, Store}; /// What a restarted writer continues from. +#[derive(Default)] pub(crate) struct Recovery { - /// The retained timeline window, or `None` when no timeline object exists. - pub checkpoint: Option>, + /// Each recorded track's retained timeline, keyed by the recorded track's name. + pub tracks: HashMap, + /// Committed objects no retained record references, such as an interrupted expiration. Only + /// collected for a DVR, which deletes them after its grace. + pub orphans: Vec, + /// Objects at or past a track's next record: uploads a crash left before their commit. No + /// timeline ever referenced them, and the resumed writer reuses their keys. + pub uncommitted: Vec, +} + +/// One track's retained timeline. +pub(crate) struct Resume { + /// The retained window. + pub checkpoint: Checkpoint, /// The next stored timeline group sequence, so the stored numbering keeps increasing. pub sequence: u64, - /// Per track, the largest stored group; new groups must exceed it so no object overlaps. - pub floors: HashMap, - /// Stored group objects no retained record references. Only collected for a DVR. - pub orphans: Vec, } -/// List the whole recording, then replay its timeline from a retained checkpoint. +impl Resume { + /// Where the newest committed record ended: the resumed track refuses anything before it. + pub fn floor(&self) -> Option { + self.checkpoint.records.last().map(|record| record.end) + } +} + +/// List the whole recording, then replay each track's timeline from a retained checkpoint. /// /// A DVR (`complete`) replays far enough back to recover every retained record and reports the -/// unreferenced group objects; an archive only needs the newest checkpoint. Any listing, GET, or -/// decode failure fails recovery, so a caller never acts on a partial view. -pub(crate) async fn recover(store: &Store, timeline: &str, complete: bool) -> Result { +/// unreferenced objects; an archive only needs the newest checkpoint. Any listing, GET, or decode +/// failure fails recovery, so a caller never acts on a partial view. +pub(crate) async fn recover(store: &Store, complete: bool) -> Result { let entries: Vec<_> = store.list(&Query::new()).try_collect().await?; - let mut segments = Vec::new(); - let mut groups = Vec::new(); - let mut floors = HashMap::new(); + let mut segments: BTreeMap> = BTreeMap::new(); for entry in entries { - match entry.key { - Key::Segments { track, segment } if track == timeline => segments.push(segment), - Key::Groups { track, range } if track != timeline => { - raise(&mut floors, &track, *range.end()); - groups.push(Key::Groups { track, range }); - } - _ => {} + if let Key::Segments { track, segment } = entry.key { + segments.entry(track).or_default().push(segment); } } - segments.sort_unstable(); - - let (checkpoint, sequence) = match (segments.first(), segments.last()) { - (Some(&first), Some(&last)) => { - if segments.len() as u64 != last - first + 1 { - return Err(Error::Timeline(format!( - "timeline segments {first}..={last} are not contiguous" - ))); - } - let (checkpoint, sequence) = replay(store, timeline, first..=last, complete).await?; - // The newest object is segment `last`, so the window must end on the next one. - // A shorter window would resume onto that segment and collide with it. - let next = last.checked_add(1).ok_or(Error::Overflow)?; - if checkpoint.range.end != next { - return Err(Error::Timeline(format!( - "recovered window ends at {}, not segment {next}", - checkpoint.range.end - ))); - } - (Some(checkpoint), sequence) + + let mut recovery = Recovery::default(); + for (timeline, ids) in &mut segments { + let Some(track) = timeline.strip_suffix(hang::timeline::SUFFIX) else { + continue; + }; + ids.sort_unstable(); + let (first, last) = (ids[0], ids[ids.len() - 1]); + if ids.len() as u64 != last - first + 1 { + return Err(Error::Timeline(format!( + "{timeline} segments {first}..={last} are not contiguous" + ))); } - _ => (None, 0), - }; + let resume = replay(store, timeline, first..=last, complete).await?; + // The newest object is segment `last`, so the window must end on the next one. A shorter + // window would resume onto that segment and collide with it. + let next = last.checked_add(1).ok_or(Error::Overflow)?; + if resume.checkpoint.range.end != next { + return Err(Error::Timeline(format!( + "{timeline} window ends at {}, not segment {next}", + resume.checkpoint.range.end + ))); + } + recovery.tracks.insert(track.to_string(), resume); + } - let mut referenced = HashSet::new(); - for record in checkpoint.iter().flat_map(|checkpoint| &checkpoint.records) { - for (track, ranges) in &record.tracks { - if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { - raise(&mut floors, track, last.end); - referenced.insert(Key::groups(track.clone(), first.start..=last.end)?); + for (track, ids) in segments { + if track.ends_with(hang::timeline::SUFFIX) { + continue; + } + let resume = recovery.tracks.get(&track); + let next = resume.map_or(0, |resume| resume.checkpoint.range.end); + let retained: HashSet = resume + .iter() + .flat_map(|resume| &resume.checkpoint.records) + .map(|record| record.sequence) + .collect(); + for id in ids { + let key = Key::segments(track.clone(), id)?; + if id >= next { + recovery.uncommitted.push(key); + } else if complete && !retained.contains(&id) { + recovery.orphans.push(key); } } } - let orphans = match complete { - true => groups.into_iter().filter(|key| !referenced.contains(key)).collect(), - false => Vec::new(), - }; - - Ok(Recovery { - checkpoint, - sequence, - floors, - orphans, - }) -} -fn raise(floors: &mut HashMap, track: &str, group: u64) { - let floor = floors.entry(track.to_string()).or_insert(group); - *floor = (*floor).max(group); + Ok(recovery) } /// Replay `segments` from the newest checkpoint that restates every record `complete` needs. -/// -/// Returns the retained window and the next timeline group sequence. async fn replay( store: &Store, timeline: &str, segments: RangeInclusive, complete: bool, -) -> Result<(Checkpoint, u64)> { +) -> Result { // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, // so walk back until a checkpoint restates from there. let mut objects = VecDeque::new(); @@ -146,7 +152,7 @@ async fn replay( let start = records.keys().next().copied().unwrap_or(range.end); if records.len() as u64 != range.end - start || (complete && start != range.start) { return Err(Error::Timeline(format!( - "cannot recover timeline window {range:?} from the retained checkpoints" + "cannot recover {timeline} window {range:?} from the retained checkpoints" ))); } @@ -156,7 +162,7 @@ async fn replay( range, records: records.into_values().collect(), }; - Ok((checkpoint, sequence)) + Ok(Resume { checkpoint, sequence }) } /// The retained offset and first restated index of the checkpoint opening `object`. diff --git a/rs/moq-archive/src/segment.rs b/rs/moq-archive/src/segment.rs index 6b844dcdee..a032bf191a 100644 --- a/rs/moq-archive/src/segment.rs +++ b/rs/moq-archive/src/segment.rs @@ -1,12 +1,11 @@ -use std::ops::RangeInclusive; - use bytes::{Buf, BufMut, Bytes, BytesMut}; +use hang::timeline::Position; use moq_net::VarInt; -use crate::path::{check_id, check_range}; +use crate::path::check_id; use crate::{Error, Result, VERSION}; -/// One complete group in a segment object, in sequence order. +/// One group's stored frames in a segment object, in sequence order. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Group { /// Group sequence number. @@ -27,15 +26,50 @@ pub struct Frame { /// A versioned group/frame table followed by concatenated payload bytes. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Object { + /// The index of the first group's first stored frame within that group; later groups start at + /// frame zero. + pub frame_start: u64, /// Groups in strictly ascending sequence order. pub groups: Vec, } impl Object { - /// Inclusive group sequences from first to last. - pub fn bounds(&self) -> Result> { + /// An object whose first group starts at frame zero. + pub fn new(groups: Vec) -> Self { + Self { frame_start: 0, groups } + } + + /// The position of the first stored frame. + pub fn start(&self) -> Result { validate(&self.groups)?; - Ok(self.groups[0].sequence..=self.groups.last().unwrap().sequence) + Ok(Position::new(self.groups[0].sequence, self.frame_start)) + } + + /// Require this object to hold exactly the frames `start..end` of a record, with no group + /// missing or empty. + pub fn check_span(&self, start: Position, end: Position) -> Result<()> { + if self.start()? != start { + return Err(Error::Span); + } + for (expected, group) in (start.group..).zip(&self.groups) { + if group.sequence != expected || group.frames.is_empty() { + return Err(Error::Span); + } + } + let last = self.groups.last().expect("validated as non-empty"); + let first = match self.groups.len() { + 1 => self.frame_start, + _ => 0, + }; + let stop = first.checked_add(last.frames.len() as u64).ok_or(Error::Overflow)?; + let exact = match end.frame { + 0 => end.group == last.sequence + 1, + frame => end.group == last.sequence && frame == stop, + }; + if !exact { + return Err(Error::Span); + } + Ok(()) } /// Encode the binary envelope. The first sequence is absolute; later ones are `current - previous - 1`. @@ -44,6 +78,7 @@ impl Object { let mut table = BytesMut::new(); write_varint(&mut table, VERSION)?; + write_varint(&mut table, self.frame_start)?; write_varint(&mut table, self.groups.len() as u64)?; let mut payload = BytesMut::new(); @@ -82,6 +117,7 @@ impl Object { return Err(Error::Version(version)); } + let frame_start = read_varint(&mut buf)?; let group_count = read_count(&mut buf, 2)?; if group_count == 0 { return Err(Error::Empty); @@ -157,28 +193,7 @@ impl Object { return Err(Error::Table); } - Ok(Self { groups }) - } - - /// Decode and require the table's sequences to match `range`. - pub fn decode_groups(buf: impl Buf, range: RangeInclusive) -> Result { - check_range(&range)?; - let object = Self::decode(buf)?; - object.check_bounds(range)?; - Ok(object) - } - - /// Require this object's sequences to match a range-named key. - pub fn check_bounds(&self, range: RangeInclusive) -> Result<()> { - check_range(&range)?; - let got = self.bounds()?; - if got != range { - return Err(Error::Bounds { - smallest: *got.start(), - largest: *got.end(), - }); - } - Ok(()) + Ok(Self { frame_start, groups }) } } @@ -233,7 +248,14 @@ mod tests { } fn object(groups: Vec) -> Object { - Object { groups } + Object::new(groups) + } + + fn group(sequence: u64, frames: usize) -> Group { + Group { + sequence, + frames: (0..frames as u64).map(|i| frame(i, b"x")).collect(), + } } #[test] @@ -254,8 +276,14 @@ mod tests { ]); let bytes = original.encode().unwrap(); assert_eq!(Object::decode(&bytes[..]).unwrap(), original); - assert_eq!(original.bounds().unwrap(), 0..=4); - original.check_bounds(0..=4).unwrap(); + assert_eq!(original.start().unwrap(), Position::group(0)); + + let split = Object { + frame_start: 300, + ..original + }; + assert_eq!(Object::decode(split.encode().unwrap()).unwrap(), split); + assert_eq!(split.start().unwrap(), Position::new(0, 300)); } #[test] @@ -278,7 +306,8 @@ mod tests { .unwrap(); let mut buf = &bytes[..]; - assert_eq!(read_varint(&mut buf).unwrap(), 1); // version + assert_eq!(read_varint(&mut buf).unwrap(), 2); // version + assert_eq!(read_varint(&mut buf).unwrap(), 0); // frame start assert_eq!(read_varint(&mut buf).unwrap(), 3); // group count assert_eq!(read_varint(&mut buf).unwrap(), 5); // absolute assert_eq!(read_varint(&mut buf).unwrap(), 1); // one frame @@ -297,7 +326,8 @@ mod tests { fn empty_object_is_rejected() { assert!(matches!(object(vec![]).encode(), Err(Error::Empty))); let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, 0).unwrap(); assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Empty))); } @@ -337,7 +367,8 @@ mod tests { #[test] fn reconstructed_ids_must_stay_in_range() { let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, 2).unwrap(); write_varint(&mut bytes, ID_MAX).unwrap(); write_varint(&mut bytes, 0).unwrap(); // no frames @@ -349,7 +380,8 @@ mod tests { /// A table with one frameless group per delta. fn deltas(deltas: &[u64]) -> Bytes { let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, deltas.len() as u64).unwrap(); for &delta in deltas { write_varint(&mut bytes, delta).unwrap(); @@ -401,10 +433,13 @@ mod tests { #[test] fn unknown_version_is_refused() { - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 2).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Version(2)))); + // Version 1 objects held whole groups under range-named keys. + for version in [1, 3] { + let mut bytes = BytesMut::new(); + write_varint(&mut bytes, version).unwrap(); + write_varint(&mut bytes, 1).unwrap(); + assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Version(v)) if v == version)); + } } #[test] @@ -425,7 +460,8 @@ mod tests { // group_count equals remaining bytes, so a remaining-bytes check would // accept it, but each group needs at least two varints. let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, 4).unwrap(); bytes.extend_from_slice(&[0, 0, 0, 0]); assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Table))); @@ -434,16 +470,12 @@ mod tests { #[test] fn offset_overflow_and_truncated_table() { let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 4).unwrap(); // length 4, no payload + for value in [VERSION, 0, 1, 0, 1, 0, 0, 4] { + write_varint(&mut bytes, value).unwrap(); // length 4, no payload + } assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Table))); - assert!(matches!(Object::decode(&b"\x01"[..]), Err(Error::Table))); + assert!(matches!(Object::decode(&b"\x02"[..]), Err(Error::Table))); } #[test] @@ -451,7 +483,7 @@ mod tests { // One group of two frames at (offset, length), followed by `payload` bytes. let table = |frames: [(u64, u64); 2], payload: usize| { let mut bytes = BytesMut::new(); - for value in [1, 1, 0, 2] { + for value in [VERSION, 0, 1, 0, 2] { write_varint(&mut bytes, value).unwrap(); } for (offset, length) in frames { @@ -475,34 +507,50 @@ mod tests { } #[test] - fn filename_bounds_must_match_the_table() { - let object = object(vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 7, - frames: vec![frame(1, b"b")], - }, - ]); - let bytes = object.encode().unwrap(); - assert!(Object::decode_groups(&bytes[..], 5..=7).is_ok()); - assert!(matches!( - Object::decode_groups(&bytes[..], 6..=7), - Err(Error::Bounds { - smallest: 5, - largest: 7 - }) - )); - let (smallest, largest) = (7, 5); - assert!(matches!( - Object::decode_groups(&bytes[..], smallest..=largest), - Err(Error::Bounds { - smallest: 7, - largest: 5 - }) - )); + fn a_span_must_match_the_record() { + let whole = object(vec![group(5, 2), group(6, 1)]); + whole.check_span(Position::group(5), Position::group(7)).unwrap(); + assert_eq!( + whole.check_span(Position::group(5), Position::group(8)), + Err(Error::Span) + ); + assert_eq!( + whole.check_span(Position::group(4), Position::group(7)), + Err(Error::Span) + ); + assert_eq!( + whole.check_span(Position::group(5), Position::new(6, 2)), + Err(Error::Span), + "a partial end names its frame count" + ); + + // A frame split: frames 3..5 of group 7. + let split = Object { + frame_start: 3, + groups: vec![group(7, 2)], + }; + split.check_span(Position::new(7, 3), Position::new(7, 5)).unwrap(); + assert_eq!( + split.check_span(Position::new(7, 3), Position::new(7, 4)), + Err(Error::Span) + ); + + // A split's tail continues into whole groups. + let tail = Object { + frame_start: 3, + groups: vec![group(7, 2), group(8, 4)], + }; + tail.check_span(Position::new(7, 3), Position::new(8, 4)).unwrap(); + tail.check_span(Position::new(7, 3), Position::group(9)).unwrap(); + + // Gaps and empty groups never appear inside a record. + let gap = object(vec![group(5, 1), group(7, 1)]); + assert_eq!(gap.check_span(Position::group(5), Position::group(8)), Err(Error::Span)); + let empty = object(vec![group(5, 1), group(6, 0)]); + assert_eq!( + empty.check_span(Position::group(5), Position::group(7)), + Err(Error::Span) + ); } #[test] diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index c79db85180..97cfe04590 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -1,9 +1,8 @@ use std::num::NonZeroUsize; -use std::ops::RangeInclusive; use bytes::Bytes; -use futures::StreamExt; use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; use object_store::path::Path; use object_store::{ListResult, ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutPayload}; @@ -25,7 +24,7 @@ use crate::{Error, Result}; /// /// use moq_archive::store::list::Query; /// -/// let query = Query::segments("timeline.z")? +/// let query = Query::segments("video.timeline.z")? /// .page_size(NonZeroUsize::new(100).unwrap()); /// # Ok::<(), moq_archive::Error>(()) /// ``` @@ -76,12 +75,7 @@ pub mod list { Ok(Self::new().prefix(path::track_prefix(track)?)) } - /// List group objects for one track. - pub fn groups(track: &str) -> Result { - Ok(Self::new().prefix(path::groups_prefix(&Path::ROOT, track)?)) - } - - /// List timeline segment objects for one track. + /// List segment objects for one track. pub fn segments(track: &str) -> Result { Ok(Self::new().prefix(path::segments_prefix(&Path::ROOT, track)?)) } @@ -91,11 +85,6 @@ pub mod list { Ok(self.offset(key.path(&Path::ROOT)?)) } - /// List group objects beginning at the exclusive lexical offset for `group`. - pub fn groups_from(track: &str, group: u64) -> Result { - Ok(Self::groups(track)?.offset(path::groups_offset(&Path::ROOT, track, group)?)) - } - pub(crate) fn next(&self, page_token: String) -> Self { let mut next = self.clone(); next.page_token = Some(page_token); @@ -189,32 +178,39 @@ impl Store { Info::decode(&self.get_bytes(&path).await?) } - /// Create a range-named groups object. A collision is accepted only when the bytes match. - pub async fn put_groups(&self, track: &str, object: &Object) -> Result { - let key = Key::groups(track, object.bounds()?)?; - self.put_segment(&key, object.encode()?).await?; - Ok(key) - } - - /// Fetch a groups object and require its table to match the filename bounds. - pub async fn get_groups(&self, track: &str, range: RangeInclusive) -> Result { - let path = self.path(&Key::groups(track, range.clone())?)?; - Object::decode_groups(self.get_bytes(&path).await?, range) - } - - /// Create a timeline object at `segments/`. A collision is accepted only when the bytes match. + /// Create the object at `segments/`. A collision is accepted only when the bytes match. pub async fn put_segments(&self, track: &str, segment: u64, object: &Object) -> Result { let key = Key::segments(track, segment)?; self.put_segment(&key, object.encode()?).await?; Ok(key) } - /// Fetch and validate a timeline object. + /// Fetch and decode the object at `segments/`. pub async fn get_segments(&self, track: &str, segment: u64) -> Result { let path = self.path(&Key::segments(track, segment)?)?; Object::decode(self.get_bytes(&path).await?) } + /// Each recorded track's timeline track, found by listing the `.info` objects a [`Writer`] + /// creates: a timeline is named by [`hang::timeline::default_name`]. + /// + /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. + /// + /// [`Writer`]: crate::Writer + pub async fn timelines(&self) -> Result> { + let entries: Vec = self.list(&Query::new()).try_collect().await?; + Ok(entries + .into_iter() + .filter_map(|entry| match entry.key { + Key::Info { track } => { + let indexed = track.strip_suffix(hang::timeline::SUFFIX)?.to_string(); + Some((indexed, track)) + } + _ => None, + }) + .collect()) + } + /// Delete the object at `key`. pub async fn delete(&self, key: &Key) -> Result<()> { let path = self.path(key)?; @@ -365,27 +361,23 @@ mod tests { } fn one_group(sequence: u64, payload: &'static [u8]) -> Object { - Object { - groups: vec![Group { - sequence, - frames: vec![frame(sequence, payload)], - }], - } + Object::new(vec![Group { + sequence, + frames: vec![frame(sequence, payload)], + }]) } fn two_groups() -> Object { - Object { - groups: vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 7, - frames: vec![frame(1, b"bb")], - }, - ], - } + Object::new(vec![ + Group { + sequence: 5, + frames: vec![frame(0, b"a")], + }, + Group { + sequence: 6, + frames: vec![frame(1, b"bb")], + }, + ]) } async fn names(store: &Store, query: &Query) -> BTreeSet { @@ -409,7 +401,7 @@ mod tests { .inner() .put( &path, - br#"{ "timescale": 1000, "priority": 1, "version": 1 }"#.as_ref().into(), + br#"{ "timescale": 1000, "priority": 1, "version": 2 }"#.as_ref().into(), ) .await .unwrap(); @@ -420,7 +412,7 @@ mod tests { .bytes() .await .unwrap(); - assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 1 }"#); + assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 2 }"#); } #[tokio::test] @@ -429,12 +421,12 @@ mod tests { let info = Info::new(0, 1_000).unwrap(); for (track, existing, check) in [ ( - "v2", - &br#"{"version":2,"priority":0,"timescale":1000}"#[..], - (|err| matches!(err, Error::Version(2))) as fn(&Error) -> bool, + "v1", + &br#"{"version":1,"priority":0,"timescale":1000}"#[..], + (|err| matches!(err, Error::Version(1))) as fn(&Error) -> bool, ), ("junk", b"not json", |err| matches!(err, Error::Json(_))), - ("zero", br#"{"version":1,"priority":0,"timescale":0}"#, |err| { + ("zero", br#"{"version":2,"priority":0,"timescale":0}"#, |err| { matches!(err, Error::Timescale(0)) }), ] { @@ -469,38 +461,24 @@ mod tests { } #[tokio::test] - async fn groups_put_get_and_identical_collision() { + async fn segments_put_get_and_identical_collision() { let store = memory(); let object = two_groups(); - let range = object.bounds().unwrap(); - let key = store.put_groups("video", &object).await.unwrap(); - assert_eq!(key, Key::groups("video", range.clone()).unwrap()); + let key = store.put_segments("video", 3, &object).await.unwrap(); + assert_eq!(key, Key::segments("video", 3).unwrap()); assert_eq!( store.path(&key).unwrap().as_ref(), - "rec/video/groups/0000000000000000007.0000000000000000005" + "rec/video/segments/0000000000000000003" ); - assert_eq!(store.get_groups("video", range).await.unwrap(), object); - store.put_groups("video", &object).await.unwrap(); - } + assert_eq!(store.get_segments("video", 3).await.unwrap(), object); + store.put_segments("video", 3, &object).await.unwrap(); - #[tokio::test] - async fn groups_collision_with_different_bytes_fails() { - let store = memory(); - store.put_groups("video", &two_groups()).await.unwrap(); - let other = Object { - groups: vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"X")], - }, - Group { - sequence: 7, - frames: vec![frame(1, b"bb")], - }, - ], - }; + let other = Object::new(vec![Group { + sequence: 5, + frames: vec![frame(0, b"X")], + }]); assert!(matches!( - store.put_groups("video", &other).await, + store.put_segments("video", 3, &other).await, Err(Error::Conflict(_)) )); } @@ -519,11 +497,11 @@ mod tests { } #[tokio::test] - async fn listing_names_build_a_range_index() { + async fn listing_names_every_object() { let store = memory(); store.put_info("video", &Info::new(0, 1).unwrap()).await.unwrap(); - store.put_groups("video", &one_group(1, b"a")).await.unwrap(); - store.put_groups("video", &one_group(3, b"b")).await.unwrap(); + store.put_segments("video", 1, &one_group(1, b"a")).await.unwrap(); + store.put_segments("video", 3, &one_group(3, b"b")).await.unwrap(); store.put_segments("timeline.z", 2, &one_group(0, b"t")).await.unwrap(); let listed: Vec<_> = store.list(&Query::new()).try_collect().await.unwrap(); @@ -534,8 +512,8 @@ mod tests { vec![ Key::segments("timeline.z", 2).unwrap(), Key::info("video").unwrap(), - Key::groups("video", 1..=1).unwrap(), - Key::groups("video", 3..=3).unwrap(), + Key::segments("video", 1).unwrap(), + Key::segments("video", 3).unwrap(), ] ); } @@ -575,7 +553,7 @@ mod tests { async fn listing_query_is_recording_relative() { let store = memory(); store.put_info("video", &Info::new(0, 1).unwrap()).await.unwrap(); - store.put_groups("video", &one_group(1, b"a")).await.unwrap(); + store.put_segments("video", 1, &one_group(1, b"a")).await.unwrap(); store.put_info("video-alt", &Info::new(0, 1).unwrap()).await.unwrap(); let query = Query::track("video").unwrap(); @@ -584,7 +562,7 @@ mod tests { paths, BTreeSet::from([ "video/.info".to_string(), - "video/groups/0000000000000000001.0000000000000000001".to_string(), + "video/segments/0000000000000000001".to_string(), ]) ); assert_eq!( @@ -629,16 +607,6 @@ mod tests { #[tokio::test] async fn paginated_query_preserves_every_supported_option() { let store = memory(); - let groups = Query::groups_from("video", 5).unwrap(); - assert_eq!( - store.paginated_prefix(groups.prefix.as_ref()).as_deref(), - Some("rec/video/groups/") - ); - assert_eq!( - store.paginated_options(&groups).offset.as_deref(), - Some("rec/video/groups/0000000000000000005") - ); - let query = Query::segments("timeline.z") .unwrap() .after(&Key::segments("timeline.z", 1).unwrap()) @@ -822,70 +790,15 @@ mod tests { assert_eq!(paged, streamed); } - /// The first object listed after `groups_from(group)`, one S3 page of one key. - async fn lookup(store: &Store, group: u64) -> Option> { - let query = Query::groups_from("video", group) - .unwrap() - .page_size(NonZeroUsize::new(1).unwrap()); - let page = store.list_paginated(&query).await.unwrap(); - match page.entries.first().map(|entry| &entry.key) { - Some(Key::Groups { range, .. }) => Some(range.clone()), - Some(key) => panic!("unexpected {key:?}"), - None => None, - } - } - - #[tokio::test] - async fn ordered_lookup_finds_the_covering_object() { - let store = Store::new(Mock::memory(), "rec"); - for range in [0..=2, 5..=7, 10..=10, ID_MAX..=ID_MAX] { - let groups = range.clone().filter(|sequence| *sequence != 6); - let object = Object { - groups: groups - .map(|sequence| Group { - sequence, - frames: vec![frame(0, b"g")], - }) - .collect(), - }; - store.put_groups("video", &object).await.unwrap(); - } - // A sibling track sorts after `video/groups/` and must never be returned. - store.put_groups("video-alt", &one_group(3, b"a")).await.unwrap(); - - // Largest-first filenames make the first key at or past the group its only candidate. - for (group, found) in [ - (0, Some(0..=2)), - (1, Some(0..=2)), - (2, Some(0..=2)), - (3, Some(5..=7)), - (5, Some(5..=7)), - (6, Some(5..=7)), - (7, Some(5..=7)), - (8, Some(10..=10)), - (10, Some(10..=10)), - (11, Some(ID_MAX..=ID_MAX)), - (ID_MAX, Some(ID_MAX..=ID_MAX)), - ] { - assert_eq!(lookup(&store, group).await, found, "group {group}"); - } - store - .delete(&Key::groups("video", ID_MAX..=ID_MAX).unwrap()) - .await - .unwrap(); - assert_eq!(lookup(&store, 11).await, None); - assert!(Query::groups_from("video", ID_MAX + 1).is_err()); - } - #[tokio::test] async fn empty_prefix_lists_the_whole_store() { let store = Store::new(Mock::memory(), ""); store.put_info("catalog.json", &Info::new(0, 1).unwrap()).await.unwrap(); - store.put_groups("video", &one_group(4, b"a")).await.unwrap(); + store.put_segments("video", 4, &one_group(4, b"a")).await.unwrap(); assert_eq!(store.paginated_prefix(None), None); let expected = - std::collections::HashSet::from([Key::info("catalog.json").unwrap(), Key::groups("video", 4..=4).unwrap()]); + std::collections::HashSet::from([Key::info("catalog.json").unwrap(), Key::segments("video", 4).unwrap()]); let streamed: std::collections::HashSet = store .list(&Query::new()) .map_ok(|entry| entry.key) @@ -902,7 +815,7 @@ mod tests { expected ); assert!(page.next.is_none()); - let page = store.list_paginated(&Query::groups("video").unwrap()).await.unwrap(); + let page = store.list_paginated(&Query::segments("video").unwrap()).await.unwrap(); assert_eq!(page.entries.len(), 1); } @@ -923,9 +836,7 @@ mod tests { async fn id_endpoints_are_valid_keys() { let store = memory(); let object = one_group(ID_MAX, b"z"); - store.put_groups("v", &object).await.unwrap(); store.put_segments("t", ID_MAX, &object).await.unwrap(); - assert_eq!(store.get_groups("v", ID_MAX..=ID_MAX).await.unwrap(), object); assert_eq!(store.get_segments("t", ID_MAX).await.unwrap(), object); } @@ -937,8 +848,8 @@ mod tests { let info = Info::new(3, 90_000).unwrap(); store.put_info("audio", &info).await.unwrap(); let object = two_groups(); - store.put_groups("audio", &object).await.unwrap(); + store.put_segments("audio", 0, &object).await.unwrap(); assert_eq!(store.get_info("audio").await.unwrap(), info); - assert_eq!(store.get_groups("audio", 5..=7).await.unwrap(), object); + assert_eq!(store.get_segments("audio", 0).await.unwrap(), object); } } diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 9ed25d86b7..b0c7885f65 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -1,10 +1,11 @@ //! Record explicitly selected tracks of a [`broadcast::Consumer`] into a [`Store`]. //! -//! The application enrolls each track as pacing or non-pacing through a [`Control`]; the writer -//! never parses a catalog. Every complete group feeds the timeline segmenter. For each closed -//! segment the writer stores one object per participating track, omits any track whose object -//! failed, commits the record through its own timeline encoder, and stores that segment's timeline -//! groups before starting the next one. The timeline therefore only advertises durable objects. +//! The application enrolls each track through a [`Control`]; the writer never parses a catalog. +//! Every track has its own timeline: frames feed the track's segmenter as they arrive, and for +//! each closed record the writer stores the track's object, commits the record through the +//! track's own timeline encoder, and stores that record's timeline groups before committing the +//! track's next one. A timeline therefore only advertises durable objects, and no track waits for +//! another. //! //! A writer started on a prefix that already holds a recording resumes it: see [`Writer::new`]. //! @@ -15,9 +16,11 @@ //! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); //! let writer = moq_archive::Writer::new(store, source, Default::default()).await?; //! let control = writer.control(); -//! control.pacing_track("video").await?; -//! control.pacing_track("audio").await?; -//! control.track("catalog.json").await?; +//! let media = moq_mux::timeline::Config::default(); +//! control.track("video", media.clone()).await?; +//! control.track("audio", media.clone()).await?; +//! // Sparse data: each catalog group is its own record. +//! control.track("catalog.json", media.with_duration_min(std::time::Duration::ZERO)).await?; //! writer.run().await?; //! # Ok(()) //! # } @@ -32,37 +35,29 @@ use std::time::Duration; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; use futures::{FutureExt, StreamExt}; -use hang::timeline::Record; -use moq_mux::timeline::{self, Deferred, DeferredDrain, Pending, Recorder, Reserved}; +use hang::timeline::{Position, Record}; +use moq_mux::timeline::{self, Segmenter}; use moq_net::{Timescale, Timestamp, broadcast, group, track}; use object_store::ObjectStore; use tokio::sync::{mpsc, watch}; use tokio::time::Instant; -use crate::recover::recover; +use crate::recover::{Resume, recover}; use crate::segment::{Frame, Group, Object}; use crate::{Error, Info, Key, Result, Store}; /// Subscribers ask for every cached group; the publisher clamps this to its own max age. const REPLAY: Duration = Duration::from_secs(u32::MAX as u64); -/// How a [`Writer`] segments and retains its recording. +/// How a [`Writer`] retains its recording. Each track's cutting is chosen when it enrolls. #[derive(Clone, Debug, Default)] #[non_exhaustive] pub struct Config { - /// Segment pacing for the recording's timeline. - pub timeline: timeline::Config, - /// Expire old segments (a DVR), or keep everything when `None` (an archive). + /// Expire old records (a DVR), or keep everything when `None` (an archive). pub retention: Option, } impl Config { - /// Set [`timeline`](Self::timeline). - pub fn with_timeline(mut self, timeline: timeline::Config) -> Self { - self.timeline = timeline; - self - } - /// Set [`retention`](Self::retention). pub fn with_retention(mut self, retention: impl Into>) -> Self { self.retention = retention.into(); @@ -74,7 +69,7 @@ impl Config { #[derive(Clone, Debug)] #[non_exhaustive] pub struct Retention { - /// Keep at least this much content, measured from timeline records. + /// Keep at least this much of each track's content, measured from its timeline records. pub window: Duration, /// How long expired objects outlive the timeline that stopped advertising them, so readers /// holding an older timeline can finish their GETs. @@ -94,129 +89,86 @@ impl Retention { /// [`run`](Self::run). pub struct Writer { control: Control, - commands: mpsc::UnboundedReceiver, - committer: Committer, - grace: Option, + commands: mpsc::UnboundedReceiver>, /// Deadlines for deleting expired or orphaned objects, oldest first. deletions: VecDeque<(Instant, Vec)>, - // Owns the recording's timeline track. - _timeline: broadcast::Producer, } /// Enrolls, cuts, and removes tracks on a [`Writer`]. pub struct Control { shared: Arc>, - deferred: Deferred, - commands: mpsc::UnboundedSender, + commands: mpsc::UnboundedSender>, } impl Clone for Control { fn clone(&self) -> Self { Self { shared: self.shared.clone(), - deferred: self.deferred.clone(), commands: self.commands.clone(), } } } -/// Withholds segment commits until dropped. -pub struct Reservation { - _inner: Reserved, -} - struct Shared { store: Store, source: broadcast::Consumer, - /// The recording-owned timeline track, which no source track may shadow. - timeline: String, - /// Every name ever enrolled. A name is never reused, so its object ranges stay increasing. + retention: Option, + /// Owns the recording's timeline tracks, read back to store their groups. + timelines: broadcast::Producer, + /// Each track's retained timeline from a resumed recording, taken when it enrolls again. + recovered: Mutex>, + /// Every name ever enrolled. A name is never reused, so its records stay consecutive. enrolled: Mutex>, - /// Per track, the largest group a resumed recording already stored. - floors: HashMap, + /// Enrollments between their first check and their command, so the recording does not end + /// under one that is about to arrive. + enrolling: watch::Sender, } -enum Command { - Enroll { - name: String, - subscriber: Box, - recorder: Recorder, - timescale: Timescale, - }, +enum Command { + Enroll(Box>), + Cut(Timestamp), Remove(String), - /// A cut may have completed a segment. - Poke, } impl Writer { /// Start a recording under `store`'s prefix, reading tracks from `source`, or resume the one /// already there. /// - /// Resuming replays the retained timeline and continues at the next segment. A track refuses - /// any group at or below the largest one stored for it, so a source whose group sequences - /// restarted needs a new prefix. A DVR also deletes, one grace period after recovery, every - /// group object its retained records do not reference, such as interrupted expirations and - /// uploads. The writer must own the prefix exclusively. Fails, deleting nothing, when the - /// recording cannot be listed or its timeline cannot be replayed. + /// Resuming replays each track's retained timeline, and a re-enrolled track continues at its + /// next record, refusing anything at or before the end of its newest committed one. A source + /// whose group sequences restarted therefore needs a new prefix. Objects a crash left before + /// their commit are deleted now, since the resumed records reuse their keys. A DVR also + /// deletes, one grace period after recovery, every object its retained records do not + /// reference, such as interrupted expirations. The writer must own the prefix exclusively. + /// Fails, deleting nothing, when the recording cannot be listed or a timeline cannot be + /// replayed. pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { - let section = timeline::Segmenter::new(config.timeline.clone()).section(); - let recovery = recover(&store, §ion.track, config.retention.is_some()).await?; - - let broadcast = broadcast::Info::new().produce(); - let timeline = match &recovery.checkpoint { - Some(checkpoint) => { - timeline::Producer::resume(&broadcast, config.timeline, checkpoint).map_err(timeline_error)? - } - None => timeline::Producer::new(&broadcast, config.timeline), - }; - let deferred = timeline.deferred().map_err(timeline_error)?; + let recovery = recover(&store, config.retention.is_some()).await?; + for key in &recovery.uncommitted { + store.delete(key).await?; + } - let replay = track::Subscription::default().with_max_age(REPLAY); - let groups = broadcast - .consume() - .track(§ion.track) - .map_err(source_error)? - .subscribe(replay) - .await - .map_err(source_error)? - .ordered(); - let info = Info::new(groups.info().priority, groups.info().timescale.as_u64())?; - store.put_info(§ion.track, &info).await?; + let mut deletions = VecDeque::new(); + if let Some(retention) = &config.retention + && !recovery.orphans.is_empty() + { + deletions.push_back((Instant::now() + retention.grace, recovery.orphans)); + } let shared = Arc::new(Shared { store, source, - timeline: section.track, + retention: config.retention, + timelines: broadcast::Info::new().produce(), + recovered: Mutex::new(recovery.tracks), enrolled: Mutex::new(HashSet::new()), - floors: recovery.floors, + enrolling: watch::Sender::new(0), }); let (commands, receiver) = mpsc::unbounded_channel(); - let retention = config.retention; - let mut deletions = VecDeque::new(); - if let Some(retention) = &retention - && !recovery.orphans.is_empty() - { - deletions.push_back((Instant::now() + retention.grace, recovery.orphans)); - } Ok(Self { - control: Control { - shared: shared.clone(), - deferred, - commands, - }, + control: Control { shared, commands }, commands: receiver, - committer: Committer { - shared, - timeline, - groups, - timescale: section.timescale.into(), - retention: retention.as_ref().map(|r| r.window), - window: recovery.checkpoint.map(|c| c.records.into()).unwrap_or_default(), - sequence: recovery.sequence, - }, - grace: retention.map(|r| r.grace), deletions, - _timeline: broadcast, }) } @@ -227,54 +179,50 @@ impl Writer { /// Record until the source broadcast closes and every enrolled track ends. /// - /// Then flush the final segment and finish the timeline. Fails when the timeline cannot be - /// committed or stored, or an enrolled track delivers a group the recording cannot represent: - /// the recording stops at its last durable timeline object. Returns the source's error, after - /// finishing, when the broadcast aborted. + /// Then flush each track's final record and finish its timeline. Fails when a timeline cannot + /// be committed or stored, or an enrolled track delivers a frame the recording cannot + /// represent: the recording stops at each track's last durable timeline object. Returns the + /// source's error, after finishing, when the broadcast aborted. pub async fn run(self) -> Result<()> { let Self { control, mut commands, - committer, - grace, mut deletions, - _timeline, } = self; let shared = control.shared.clone(); let source = shared.source.clone(); - let mut segments = Segments::Live(control.deferred.clone()); + let grace = shared.retention.as_ref().map(|retention| retention.grace); + let mut enrolling = shared.enrolling.subscribe(); drop(control); - let mut tracks: HashMap = HashMap::new(); - // Reported groups waiting for the record that names them, kept past a track's removal. - let mut buffered: HashMap> = HashMap::new(); + let mut tracks: HashMap>> = HashMap::new(); let mut reads = FuturesUnordered::new(); - let mut committer = Some(committer); - let mut commit: Option> = None; + let mut commits: FuturesUnordered> = FuturesUnordered::new(); let mut closed = false; // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. let mut accepting = true; + // The track the last event touched. Each event touches at most one, so the loop never scans + // the whole table. + let mut touched: Option = None; loop { - if commit.is_none() - && let Some(pending) = segments.next() + if let Some(name) = touched.take() + && let Some(track) = tracks.get_mut(&name) { - let objects = take_objects(&pending, &mut buffered); - let committer = committer.take().expect("idle committer"); - commit = Some(committer.commit(pending, objects).boxed()); + if let Some(commit) = track.commit()? { + commits.push(commit); + } + if track.finish() { + tracks.remove(&name); + } } - if commit.is_none() && closed && tracks.is_empty() { + let idle = closed && tracks.is_empty() && commits.is_empty(); + if idle && *enrolling.borrow_and_update() == 0 { // Refuse late commands, so an enrollment racing the end fails instead of vanishing. commands.close(); if !accepting { - match segments { - Segments::Live(deferred) => { - segments = Segments::Drain(deferred.finish()); - continue; - } - Segments::Drain(_) => break, - } + break; } } @@ -283,38 +231,39 @@ impl Writer { biased; command = commands.recv(), if accepting => match command { None => accepting = false, - Some(Command::Enroll { name, subscriber, recorder, timescale }) => { - let (cancel, cancelled) = watch::channel(()); - let largest = shared.floors.get(&name).copied(); - reads.push(guard(cancelled.clone(), recv(name.clone(), subscriber)).boxed()); - tracks.insert(name, TrackState { - recorder, - timescale, - largest, - reported: None, - accepted: BTreeMap::new(), - subscribed: true, - _cancel: cancel, - cancelled, - }); + Some(Command::Enroll(mut track)) => { + let subscriber = track.subscriber.take().expect("an enrolling track carries its subscriber"); + reads.push(guard(track.cancelled.clone(), recv(track.name.clone(), subscriber)).boxed()); + touched = Some(track.name.clone()); + tracks.insert(track.name.clone(), track); + } + Some(Command::Cut(pts)) => { + for track in tracks.values_mut() { + track.segmenter.cut(pts); + } } Some(Command::Remove(name)) => { - tracks.remove(&name); + if let Some(track) = tracks.get_mut(&name) { + track.remove(); + touched = Some(name); + } } - Some(Command::Poke) => {} }, Some(read) = reads.next(), if !reads.is_empty() => { - handle(read, &mut tracks, &mut buffered, &mut reads)?; + touched = handle(read, &mut tracks, &mut reads)?; } - (done, result) = async { commit.as_mut().unwrap().await }, if commit.is_some() => { - commit = None; - committer = Some(done); + Some((name, committer, result)) = commits.next(), if !commits.is_empty() => { + if let Some(track) = tracks.get_mut(&name) { + track.committer = Some(committer); + touched = Some(name); + } let expired = result?; if let Some(grace) = grace && !expired.is_empty() { deletions.push_back((Instant::now() + grace, expired)); } } _ = source.closed(), if !closed => closed = true, + _ = enrolling.changed(), if idle && accepting => {} _ = tokio::time::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { let (_, keys) = deletions.pop_front().unwrap(); delete(&shared.store, keys).await; @@ -322,166 +271,274 @@ impl Writer { } } - let Segments::Drain(drain) = segments else { - unreachable!("the loop only ends after draining"); - }; - drain.result().map_err(timeline_error)?; - let mut committer = committer.expect("idle committer"); - committer.timeline.finish().map_err(timeline_error)?; - - // Complete expirations already committed to the timeline; nothing new expires after the - // final segment. + // Complete expirations already committed to the timelines; nothing new expires after the + // final records. for (deadline, keys) in deletions { tokio::time::sleep_until(deadline).await; delete(&shared.store, keys).await; } - if source.is_finished() { - Ok(()) - } else { - Err(source_error(source.closed().await)) + // A broadcast ends without a cause unless it was aborted. + match source.closed().await { + moq_net::Error::Dropped => Ok(()), + err => Err(source_error(err)), } } } impl Control { - /// Enroll `name` without letting it influence segmentation, such as a catalog or metadata track. - /// - /// Subscribes to the track and creates its `.info` before accepting any group. Fails when the - /// name was already enrolled, is the recording's timeline, or `.info` conflicts. - pub async fn track(&self, name: &str) -> Result<()> { - self.enroll(name, false).await - } - - /// Enroll the continuously publishing track `name` as a pacing track. + /// Enroll the track `name`, its records cut by `config`. /// - /// It votes on segment boundaries and holds every segment back until its groups are known. A - /// pacing track that stalls without closing stalls the recording by design; apply a deadline - /// with [`cut`](Self::cut) or [`remove`](Self::remove). - pub async fn pacing_track(&self, name: &str) -> Result<()> { - self.enroll(name, true).await + /// Media keeps the default minimum; sparse data such as a catalog uses zero, so each group is + /// stored as soon as it finishes. Subscribes to the track and creates its `.info` and its + /// timeline's before accepting any group. Fails when the name was already enrolled, names a + /// timeline, or `.info` conflicts. + pub async fn track(&self, name: &str, config: timeline::Config) -> Result<()> { + if name.ends_with(hang::timeline::SUFFIX) || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { + return Err(Error::Enrolled(name.to_string())); + } + self.shared.enrolling.send_modify(|count| *count += 1); + let result = self.subscribe(name, config).await; + self.shared.enrolling.send_modify(|count| *count -= 1); + if result.is_err() { + self.shared.enrolled.lock().unwrap().remove(name); + } + result } - /// Declare a segment boundary at `pts`, overriding the configured minimum-duration pacing. - /// - /// The first cut takes over boundaries for good, as with [`timeline::Producer::cut`]. + /// Declare a boundary at `pts` on every enrolled track; see [`Segmenter::cut`]. pub fn cut(&self, pts: Timestamp) -> Result<()> { - self.deferred.cut(pts).map_err(timeline_error)?; - self.send(Command::Poke) + self.send(Command::Cut(pts)) } - /// Hold segment commits back until this guard drops, so a batch of tracks can enroll first. - #[must_use = "dropping the reservation releases segment commits"] - pub fn reserve(&self) -> Reservation { - Reservation { - _inner: self.deferred.reserve(), - } - } - - /// Stop recording `name`, dropping its incomplete groups. The name cannot be enrolled again. + /// Stop recording `name`, storing what already arrived and dropping groups that could not be + /// indexed yet. The name cannot be enrolled again. pub fn remove(&self, name: &str) -> Result<()> { self.send(Command::Remove(name.to_string())) } - async fn enroll(&self, name: &str, pacing: bool) -> Result<()> { - if name == self.shared.timeline || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { - return Err(Error::Enrolled(name.to_string())); - } - let result = self.subscribe(name, pacing).await; - if result.is_err() { - self.shared.enrolled.lock().unwrap().remove(name); - } - result - } - - async fn subscribe(&self, name: &str, pacing: bool) -> Result<()> { + async fn subscribe(&self, name: &str, config: timeline::Config) -> Result<()> { + let shared = &self.shared; let replay = track::Subscription::default().with_max_age(REPLAY); - let subscriber = self - .shared + let subscriber = shared .source .track(name) .map_err(source_error)? - .subscribe(replay) + .subscribe(replay.clone()) .await .map_err(source_error)?; let timescale = subscriber.info().timescale; let info = Info::new(subscriber.info().priority, timescale.as_u64())?; - self.shared.store.put_info(name, &info).await?; - - let recorder = match pacing { - true => self.deferred.pacing_track(name), - false => self.deferred.track(name), + shared.store.put_info(name, &info).await?; + + let timeline = hang::timeline::default_name(name); + let output = shared + .timelines + .create_track(timeline.as_str(), timeline::Producer::info()) + .map_err(timeline_error_net)?; + let groups = shared + .timelines + .consume() + .track(&timeline) + .map_err(timeline_error_net)? + .subscribe(replay) + .await + .map_err(timeline_error_net)? + .ordered(); + let info = Info::new(groups.info().priority, groups.info().timescale.as_u64())?; + shared.store.put_info(&timeline, &info).await?; + + let resume = shared.recovered.lock().unwrap().remove(name); + let (output, segmenter, window, offset, floor) = match resume { + Some(resume) => { + let floor = resume.floor(); + let output = timeline::Producer::resume(output, &resume.checkpoint).map_err(timeline_error)?; + let segmenter = Segmenter::new(config).with_sequence(resume.checkpoint.range.end); + let window = resume.checkpoint.records.into(); + (output, segmenter, window, resume.sequence, floor) + } + None => ( + timeline::Producer::new(output), + Segmenter::new(config), + VecDeque::new(), + 0, + None, + ), }; - self.send(Command::Enroll { + + let (cancel, cancelled) = watch::channel(()); + let track = Track { name: name.to_string(), - subscriber: Box::new(subscriber), - recorder, timescale, - }) + segmenter, + floor, + largest: None, + reported: None, + accepted: BTreeMap::new(), + frames: VecDeque::new(), + records: VecDeque::new(), + subscriber: Some(Box::new(subscriber)), + subscribed: true, + closed: false, + committer: Some(Committer { + shared: shared.clone(), + name: name.to_string(), + timeline, + output, + groups, + window, + offset, + }), + cancel: Some(cancel), + cancelled, + }; + self.send(Command::Enroll(Box::new(track))) } - fn send(&self, command: Command) -> Result<()> { + fn send(&self, command: Command) -> Result<()> { self.commands.send(command).map_err(|_| Error::Closed) } } -/// One enrolled track's accepted groups. -struct TrackState { - recorder: Recorder, +/// A group accepted from the source, not yet fully reported. +#[derive(Default)] +struct Incoming { + /// The index of the next frame to arrive. + next: u64, + /// Frames that arrived but were not reported, with their indices. + pending: Vec<(u64, Frame)>, + /// No more frames will arrive. + finished: bool, +} + +/// One enrolled track: its source reads, its segmenter, and its committer. +struct Track { + name: String, timescale: Timescale, - /// The newest group accepted or already stored; later arrivals must exceed it. + segmenter: Segmenter, + /// A resumed track's committed end; earlier content is already stored. + floor: Option, + /// The newest accepted group; later groups must exceed it. largest: Option, /// The first-frame timestamp of the newest reported group; later groups must not precede it. reported: Option, - /// Accepted groups in sequence order: `None` while reading, `Some` once complete. Reported in - /// order, so a complete group waits for every earlier accepted one. - accepted: BTreeMap>, + /// Accepted groups in sequence order. Reported front first, so a later group's frames wait + /// for every earlier accepted group to finish. + accepted: BTreeMap, + /// Reported frames waiting for the record that holds them. + frames: VecDeque<(Position, Frame)>, + /// Closed records waiting to be committed, oldest first. + records: VecDeque, + /// Moved into the first read once enrolled. + subscriber: Option>, /// The subscription is still delivering groups. subscribed: bool, + /// The segmenter was closed: every remaining record is queued. + closed: bool, + /// `None` while a commit is in flight. + committer: Option>, /// Dropping the sender cancels every read for this track. - _cancel: watch::Sender<()>, + cancel: Option>, cancelled: watch::Receiver<()>, } -impl TrackState { - /// Report every complete group at the front of the accepted queue. +impl Track { + /// Report every frame the segmenter can place: the front group's arrivals, then each later + /// group once the ones before it finish. /// /// Fails on a group that starts before the previous one, which the timeline cannot place. - fn report(&mut self, name: &str, buffered: &mut HashMap>) -> Result<()> { - while let Some(entry) = self.accepted.first_entry() { - if entry.get().is_none() { - break; + fn report(&mut self) -> Result<()> { + while let Some(mut entry) = self.accepted.first_entry() { + let sequence = *entry.key(); + let incoming = entry.get_mut(); + for (index, frame) in incoming.pending.drain(..) { + if index == 0 { + if let Some(reported) = self.reported + && frame.timestamp < reported + { + return Err(malformed( + &self.name, + sequence, + format!("timestamp {} precedes {reported}", frame.timestamp), + )); + } + self.reported = Some(frame.timestamp); + } + // Frame timestamps were validated while reading, so this conversion succeeds. + let pts = Timestamp::new(frame.timestamp, self.timescale).map_err(|_| Error::Id(frame.timestamp))?; + let position = Position::new(sequence, index); + self.segmenter + .frame(position, pts, index == 0) + .map_err(|err| malformed(&self.name, sequence, err))?; + self.frames.push_back((position, frame)); } - let group = entry.remove().expect("checked above"); - let (Some(first), Some(last)) = (group.frames.first(), group.frames.last()) else { - continue; - }; - if let Some(reported) = self.reported - && first.timestamp < reported - { - return Err(malformed( - name, - group.sequence, - format!("timestamp {} precedes {reported}", first.timestamp), - )); + if !incoming.finished { + break; } - self.reported = Some(first.timestamp); - // Frame timestamps were validated while reading, so these conversions succeed. - let (Ok(first), Ok(last)) = ( - Timestamp::new(first.timestamp, self.timescale), - Timestamp::new(last.timestamp, self.timescale), - ) else { - continue; - }; - self.recorder.record(group.sequence, first, true); - self.recorder.end(last); - buffered - .entry(name.to_string()) - .or_default() - .insert(group.sequence, group); + entry.remove(); + self.segmenter.finish_group(sequence); } Ok(()) } + + /// Stop reading: drop the groups that were never reported and close the segmenter. + fn remove(&mut self) { + self.cancel = None; + self.subscribed = false; + self.accepted.clear(); + } + + /// Start committing the next closed record, if the committer is idle. + fn commit(&mut self) -> Result>> { + if !self.subscribed && self.accepted.is_empty() && !self.closed { + self.segmenter.close(); + self.closed = true; + } + self.records.extend(self.segmenter.by_ref()); + if self.committer.is_none() { + return Ok(None); + } + let Some(record) = self.records.pop_front() else { + return Ok(None); + }; + + let mut groups: Vec = Vec::new(); + while let Some((position, _)) = self.frames.front() + && *position < record.end + { + let (position, frame) = self.frames.pop_front().expect("checked above"); + match groups.last_mut() { + Some(group) if group.sequence == position.group => group.frames.push(frame), + _ => groups.push(Group { + sequence: position.group, + frames: vec![frame], + }), + } + } + let object = Object { + frame_start: record.start.frame, + groups, + }; + object.check_span(record.start, record.end)?; + + let committer = self.committer.take().expect("checked above"); + Ok(Some(committer.commit(record, object).boxed())) + } + + /// Finish the timeline once the track ended and every record is durable, returning whether the + /// track is done. + fn finish(&mut self) -> bool { + if !self.closed || !self.records.is_empty() { + return false; + } + let Some(committer) = self.committer.as_mut() else { + return false; + }; + if let Err(err) = committer.output.finish() { + tracing::warn!(track = %self.name, %err, "failed to finish the timeline"); + } + true + } } // Each read is already boxed as a future, so the variant sizes cost nothing extra. @@ -492,10 +549,10 @@ enum Read { subscriber: Box, result: moq_net::Result>, }, - Frames { + Frame { name: String, - sequence: u64, - result: moq_net::Result>, + group: Box, + result: moq_net::Result>, }, Cancelled, } @@ -509,20 +566,12 @@ async fn recv(name: String, mut subscriber: Box) -> Read { } } -async fn frames(name: String, mut group: group::Consumer) -> Read { - let sequence = group.sequence; - let mut frames = Vec::new(); - let result = loop { - match group.read_frame().await { - Ok(Some(frame)) => frames.push(frame), - Ok(None) => break Ok(frames), - Err(err) => break Err(err), - } - }; - Read::Frames { name, sequence, result } +async fn frame(name: String, mut group: Box) -> Read { + let result = group.read_frame().await; + Read::Frame { name, group, result } } -/// Resolve to [`Read::Cancelled`] once the track's state drops. +/// Resolve to [`Read::Cancelled`] once the track stops reading. async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { tokio::select! { read = read => read, @@ -530,232 +579,174 @@ async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future( read: Read, - tracks: &mut HashMap, - buffered: &mut HashMap>, + tracks: &mut HashMap>>, reads: &mut FuturesUnordered>, -) -> Result<()> { - let name = match read { - Read::Cancelled => return Ok(()), +) -> Result> { + match read { + Read::Cancelled => Ok(None), Read::Group { name, subscriber, result, } => { let Some(track) = tracks.get_mut(&name) else { - return Ok(()); + return Ok(None); }; match result { Ok(Some(group)) => { let sequence = group.sequence; - if track.largest.is_some_and(|largest| sequence <= largest) { - tracing::debug!(track = %name, sequence, "refusing a duplicate or decreasing group"); + let stored = track.floor.is_some_and(|floor| Position::group(sequence + 1) <= floor); + if track.largest.is_some_and(|largest| sequence <= largest) || stored { + tracing::debug!(track = %name, sequence, "refusing a duplicate, decreasing, or stored group"); } else { + crate::path::check_id(sequence).map_err(|err| malformed(&name, sequence, err))?; track.largest = Some(sequence); - track.accepted.insert(sequence, None); - reads.push(guard(track.cancelled.clone(), frames(name.clone(), group)).boxed()); + track.accepted.insert(sequence, Incoming::default()); + reads.push(guard(track.cancelled.clone(), frame(name.clone(), Box::new(group))).boxed()); } - reads.push(guard(track.cancelled.clone(), recv(name.clone(), subscriber)).boxed()); + reads.push(guard(track.cancelled.clone(), recv(name, subscriber)).boxed()); + // An accepted group holds no frames yet. + Ok(None) + } + Ok(None) => { + track.subscribed = false; + Ok(Some(name)) } - Ok(None) => track.subscribed = false, Err(err) => { tracing::warn!(track = %name, %err, "track ended"); track.subscribed = false; + Ok(Some(name)) } } - name } - Read::Frames { name, sequence, result } => { + Read::Frame { name, group, result } => { let Some(track) = tracks.get_mut(&name) else { - return Ok(()); + return Ok(None); }; + let sequence = group.sequence; + let Some(incoming) = track.accepted.get_mut(&sequence) else { + return Ok(None); + }; + let touched = name.clone(); match result { - Ok(frames) => { - let group = - convert(sequence, frames, track.timescale).map_err(|err| malformed(&name, sequence, err))?; - if group.frames.is_empty() { - track.accepted.remove(&sequence); - } else { - track.accepted.insert(sequence, Some(group)); + Ok(Some(frame)) => { + let index = incoming.next; + incoming.next += 1; + let timestamp = frame + .timestamp + .convert(track.timescale) + .map_err(|_| malformed(&name, sequence, Error::Overflow))? + .value(); + crate::path::check_id(timestamp).map_err(|err| malformed(&name, sequence, err))?; + // A resumed track already stored the head of the group it stopped inside. + if track.floor.is_none_or(|floor| Position::new(sequence, index) >= floor) { + let frame = Frame { + timestamp, + payload: frame.payload, + }; + incoming.pending.push((index, frame)); } + reads.push(guard(track.cancelled.clone(), self::frame(name, group)).boxed()); } + Ok(None) => incoming.finished = true, Err(err) => { - tracing::warn!(track = %name, sequence, %err, "dropping an incomplete group"); - track.accepted.remove(&sequence); + tracing::warn!(track = %name, sequence, %err, "keeping the frames of an incomplete group"); + incoming.finished = true; } } - track.report(&name, buffered)?; - name + track.report()?; + Ok(Some(touched)) } - }; - - // A track whose subscription ended closes once its last accepted group settles. - if tracks - .get(&name) - .is_some_and(|track| !track.subscribed && track.accepted.is_empty()) - { - tracks.remove(&name); } - Ok(()) -} - -/// Convert a complete group's frames into the track's timescale. -fn convert(sequence: u64, frames: Vec, timescale: Timescale) -> Result { - let frames = frames - .into_iter() - .map(|frame| { - let timestamp = frame.timestamp.convert(timescale).map_err(|_| Error::Overflow)?.value(); - crate::path::check_id(timestamp)?; - Ok(Frame { - timestamp, - payload: frame.payload, - }) - }) - .collect::>()?; - crate::path::check_id(sequence)?; - Ok(Group { sequence, frames }) -} - -/// Take the buffered groups a pending record names, one object per track. -/// -/// A track missing any advertised group cannot produce an object whose table matches its ranges, -/// so it gets `None` and is omitted. -fn take_objects( - pending: &Pending, - buffered: &mut HashMap>, -) -> Vec<(String, Option)> { - pending - .tracks - .iter() - .map(|(name, ranges)| { - let track = buffered.entry(name.clone()).or_default(); - let mut groups = Vec::new(); - let mut complete = true; - for range in ranges { - for sequence in range.start..=range.end { - match track.remove(&sequence) { - Some(group) => groups.push(group), - None => complete = false, - } - } - } - (name.clone(), complete.then_some(Object { groups })) - }) - .collect() } -/// An in-flight segment commit, returning the committer and the expired objects to delete. -type Commit = BoxFuture<'static, (Committer, Result>)>; - -/// Deferred records to commit: live, then the terminal drain. -enum Segments { - Live(Deferred), - Drain(DeferredDrain), -} - -impl Segments { - fn next(&self) -> Option { - match self { - Self::Live(deferred) => deferred.next(), - Self::Drain(drain) => drain.next(), - } - } -} +/// An in-flight record commit, returning the track, its committer, and the expired objects. +type Commit = BoxFuture<'static, (String, Committer, Result>)>; -/// Commits one segment at a time: media objects, the record, retention, then the timeline object. +/// Commits one track's records in order: the object, the record, retention, then the timeline +/// object. struct Committer { shared: Arc>, - timeline: timeline::Producer, - /// The recording's own timeline track, read back to store its complete groups. + name: String, + /// The track's timeline track. + timeline: String, + output: timeline::Producer, + /// The timeline track read back, to store its complete groups. groups: track::Ordered, - /// Units per second of record `pts` and `duration`. - timescale: u64, - retention: Option, /// Committed records still in the timeline window, oldest first. window: VecDeque, /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. - sequence: u64, + offset: u64, } impl Committer { - /// Commit `pending`, returning the committer and the expired objects to delete after the grace. - async fn commit(mut self, pending: Pending, objects: Vec<(String, Option)>) -> (Self, Result>) { - let result = self.commit_inner(pending, objects).await; - (self, result) + /// Commit `record` with its `object`. + async fn commit(mut self, record: Record, object: Object) -> (String, Self, Result>) { + let result = self.commit_inner(record, object).await; + (self.name.clone(), self, result) } - async fn commit_inner(&mut self, mut pending: Pending, objects: Vec<(String, Option)>) -> Result> { + async fn commit_inner(&mut self, mut record: Record, object: Object) -> Result> { let shared = self.shared.clone(); let store = &shared.store; - let puts = objects.iter().map(|(name, object)| async move { - let result = match object { - Some(object) => store.put_groups(name, object).await.map(|_| ()), - None => Err(Error::Timeline("advertised groups are not buffered".into())), - }; - (name, result) - }); - for (name, result) in futures::future::join_all(puts).await { - if let Err(err) = result { - tracing::warn!(track = %name, segment = pending.segment, %err, "omitting a track that was not stored"); - pending = pending.omit(name); - } + + // Number by the window rather than the segmenter, so a record dropped below leaves no hole. + record.sequence = self.output.range().end; + if let Err(err) = store.put_segments(&self.name, record.sequence, &object).await { + // The timeline only advertises durable objects, so this span is lost. The key may have + // landed anyway; clear it so the next record can take it. + tracing::warn!(track = %self.name, sequence = record.sequence, %err, "dropping a record that was not stored"); + let _ = store.delete(&Key::segments(self.name.clone(), record.sequence)?).await; + return Ok(Vec::new()); } - let segment = pending.segment; - let record = (*pending).clone(); - let pts = Timestamp::new( - record.pts, - Timescale::new(self.timescale).map_err(|_| Error::Timescale(self.timescale))?, - ) - .map_err(|_| Error::Id(record.pts))?; - self.timeline.push(pending).map_err(timeline_error)?; - self.window.push_back(record); + self.output.push(&record).map_err(timeline_error)?; + let pts = Timestamp::new(record.pts, TIMESCALE).map_err(|_| Error::Id(record.pts))?; + self.window.push_back(record.clone()); let expired = self.expire(); if !expired.is_empty() { - self.timeline.pop(expired.len() as u64).map_err(timeline_error)?; + self.output.pop(expired.len() as u64).map_err(timeline_error)?; } - self.timeline.flush().map_err(timeline_error)?; + self.output.flush().map_err(timeline_error)?; let object = self.read_timeline(pts)?; - store.put_segments(&shared.timeline, segment, &object).await?; + store.put_segments(&self.timeline, record.sequence, &object).await?; - let mut keys = Vec::new(); - for record in expired { - for (name, ranges) in &record.tracks { - if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { - keys.push(Key::groups(name.clone(), first.start..=last.end)?); - } - } - } - Ok(keys) + expired + .iter() + .map(|record| Key::segments(self.name.clone(), record.sequence)) + .collect() } - /// Pop the oldest records while the rest still cover the retention window. + /// Pop the oldest records while the rest still cover the retention window. The newest record + /// always stays, such as a catalog that never changes. fn expire(&mut self) -> Vec { - let Some(retention) = self.retention else { + let Some(retention) = self.shared.retention.as_ref().map(|retention| retention.window) else { return Vec::new(); }; - let target = (retention.as_micros() * self.timescale as u128).div_ceil(1_000_000); - let mut total: u128 = self.window.iter().map(|record| record.duration as u128).sum(); + let target = (retention.as_micros() * TIMESCALE.as_u64() as u128).div_ceil(1_000_000); let mut expired = Vec::new(); while self.window.len() > 1 { - let front = self.window.front().expect("non-empty").duration as u128; - if total - front < target { + let newest = self.window.back().expect("non-empty"); + let end = newest.pts as u128 + newest.duration as u128; + if end.saturating_sub(self.window[1].pts as u128) < target { break; } - total -= front; expired.push(self.window.pop_front().expect("non-empty")); } expired } - /// Collect the timeline groups completed since the last segment, stamped at `pts`. + /// Collect the timeline groups completed since the last record, stamped at `pts`. /// - /// The live timeline track stamps frames with the wall clock; storing the segment's content - /// time instead keeps a recording's bytes a function of its content alone. + /// The live timeline track stamps frames with the wall clock; storing the record's content time + /// instead keeps a recording's bytes a function of its content alone. fn read_timeline(&mut self, pts: Timestamp) -> Result { let waiter = kio::Waiter::noop(); let timescale = self.groups.info().timescale; @@ -767,9 +758,12 @@ impl Committer { let mut frames = Vec::new(); loop { match group.poll_read_frame(&waiter) { - Poll::Ready(Ok(Some(mut frame))) => { - frame.timestamp = pts; - frames.push(frame); + Poll::Ready(Ok(Some(frame))) => { + let timestamp = pts.convert(timescale).map_err(|_| Error::Overflow)?.value(); + frames.push(Frame { + timestamp: crate::path::check_id(timestamp)?, + payload: frame.payload, + }); } Poll::Ready(Ok(None)) => break, Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), @@ -777,13 +771,19 @@ impl Committer { Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), } } - let sequence = group.sequence.checked_add(self.sequence).ok_or(Error::Overflow)?; - groups.push(convert(sequence, frames, timescale)?); + let sequence = group.sequence.checked_add(self.offset).ok_or(Error::Overflow)?; + groups.push(Group { + sequence: crate::path::check_id(sequence)?, + frames, + }); } - Ok(Object { groups }) + Ok(Object::new(groups)) } } +/// The timescale of record `pts` and `duration`, as the catalog section advertises it. +const TIMESCALE: Timescale = Timescale::MILLI; + async fn delete(store: &Store, keys: Vec) { let deletes = keys.iter().map(|key| async move { (key, store.delete(key).await) }); for (key, result) in futures::future::join_all(deletes).await { @@ -812,7 +812,6 @@ fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { #[cfg(test)] mod tests { - use futures::TryStreamExt; use object_store::memory::InMemory; @@ -820,12 +819,29 @@ mod tests { use crate::mock::Mock; use crate::store::list::Query; - const TIMELINE: &str = hang::timeline::DEFAULT_NAME; - fn ms(v: u64) -> Timestamp { Timestamp::from_millis(v).unwrap() } + /// Media cut at 1s, so one-second test groups get a record each. + fn media() -> timeline::Config { + timeline::Config::default().with_duration_min(Duration::from_secs(1)) + } + + /// Sparse data: every group is its own record. + fn sparse() -> timeline::Config { + timeline::Config::default().with_duration_min(Duration::ZERO) + } + + /// An append-only log, split every three seconds. + fn append_only() -> timeline::Config { + sparse().with_duration_max(Duration::from_secs(3)) + } + + fn timeline(track: &str) -> String { + hang::timeline::default_name(track) + } + /// A millisecond track that keeps an hour of history, so enrollment sees every group. fn track(broadcast: &broadcast::Producer, name: &str) -> track::Producer { let info = track::Info::default() @@ -845,10 +861,11 @@ mod tests { group.finish().unwrap(); } - /// Replay every stored timeline object, returning the records still in the window. - async fn window(store: &Store) -> Vec { + /// Replay every stored timeline object of `track`, returning the records still in its window. + async fn window(store: &Store, track: &str) -> Vec { + let timeline = timeline(track); let entries: Vec<_> = store - .list(&Query::segments(TIMELINE).unwrap()) + .list(&Query::segments(&timeline).unwrap()) .try_collect() .await .unwrap(); @@ -860,17 +877,19 @@ mod tests { }) .collect(); segments.sort(); - assert_eq!( - segments, - (0..segments.len() as u64).collect::>(), - "timeline objects are consecutive" - ); + if let (Some(first), Some(last)) = (segments.first(), segments.last()) { + assert_eq!( + segments, + (*first..=*last).collect::>(), + "timeline objects are consecutive" + ); + } let config = moq_json::window::ConsumerConfig::default().with_compression(true); let mut decoder = moq_json::window::Decoder::::new(config); let mut records = BTreeMap::new(); for segment in segments { - let object = store.get_segments(TIMELINE, segment).await.unwrap(); + let object = store.get_segments(&timeline, segment).await.unwrap(); for stored in object.groups { let mut group = decoder.group(); for frame in stored.frames { @@ -880,6 +899,7 @@ mod tests { while let Some(event) = decoder.next_event() { match event { moq_json::window::Event::Push { index, value } => { + assert_eq!(index, value.sequence); records.insert(index, value); } moq_json::window::Event::Pop(range) | moq_json::window::Event::Skip(range) => { @@ -894,29 +914,48 @@ mod tests { records.into_values().collect() } - /// The groups each track contributes, from the records. - fn ranges(records: &[Record], name: &str) -> Vec<(u64, u64)> { + /// Each record's groups, as `(first, last)`. + fn groups(records: &[Record]) -> Vec<(u64, u64)> { records .iter() - .flat_map(|record| record.tracks.get(name).into_iter().flatten()) - .map(|range| (range.start, range.end)) + .map(|record| (*record.groups().start(), *record.groups().end())) .collect() } - /// Every range in the records resolves to one stored object with exactly those groups. - async fn check_objects(store: &Store, records: &[Record]) { + fn sequences(records: &[Record]) -> Vec { + records.iter().map(|record| record.sequence).collect() + } + + /// Every record resolves to one stored object holding exactly its frames. + async fn check_objects(store: &Store, track: &str, records: &[Record]) { for record in records { - for (name, ranges) in &record.tracks { - let first = ranges.first().unwrap().start; - let last = ranges.last().unwrap().end; - let object = store.get_groups(name, first..=last).await.unwrap(); - let advertised: Vec = ranges.iter().flat_map(|range| range.start..=range.end).collect(); - let stored: Vec = object.groups.iter().map(|group| group.sequence).collect(); - assert_eq!(stored, advertised, "{name} segment {}", record.segment); - } + let object = store.get_segments(track, record.sequence).await.unwrap(); + object.check_span(record.start, record.end).unwrap(); } } + /// Every stored media object, across all tracks. + async fn stored(store: &Store) -> HashSet { + store + .list(&Query::new()) + .try_filter_map(|entry| async move { + let media = + matches!(&entry.key, Key::Segments { track, .. } if !track.ends_with(hang::timeline::SUFFIX)); + Ok(media.then_some(entry.key)) + }) + .try_collect() + .await + .unwrap() + } + + /// The media objects the retained records reference. + fn referenced(track: &str, records: &[Record]) -> HashSet { + records + .iter() + .map(|record| Key::segments(track, record.sequence).unwrap()) + .collect() + } + #[tokio::test] async fn records_selected_tracks() { let source = broadcast::Info::new().produce(); @@ -929,8 +968,8 @@ mod tests { .await .unwrap(); let control = writer.control(); - control.pacing_track("video").await.unwrap(); - control.track("catalog.json").await.unwrap(); + control.track("video", media()).await.unwrap(); + control.track("catalog.json", sparse()).await.unwrap(); group(&catalog, 0, &[0]); for sequence in 0..6 { @@ -938,25 +977,105 @@ mod tests { } video.finish().unwrap(); catalog.finish().unwrap(); - source.finish(); + source.close(); tokio::spawn(writer.run()).await.unwrap().unwrap(); - let records = window(&store).await; - assert_eq!(records.len(), 6); - assert_eq!(ranges(&records, "video"), (0..6).map(|s| (s, s)).collect::>()); - assert_eq!(ranges(&records, "catalog.json"), vec![(0, 0)]); - assert!(records.iter().all(|record| !record.tracks.contains_key("ignored"))); - check_objects(&store, &records).await; + let records = window(&store, "video").await; + assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, "video", &records).await; + let catalog = window(&store, "catalog.json").await; + assert_eq!(groups(&catalog), vec![(0, 0)]); + check_objects(&store, "catalog.json", &catalog).await; + assert!(window(&store, "ignored").await.is_empty()); - let object = store.get_groups("video", 2..=2).await.unwrap(); + let object = store.get_segments("video", 2).await.unwrap(); assert_eq!(object.groups[0].frames[1].timestamp, 2500); assert_eq!(object.groups[0].frames[1].payload, "2@2500"); assert_eq!(store.get_info("video").await.unwrap(), Info::new(0, 1000).unwrap()); - store.get_info(TIMELINE).await.unwrap(); + store.get_info(&timeline("video")).await.unwrap(); assert!(store.get_info("ignored").await.is_err()); } + #[tokio::test] + async fn tracks_cut_independently() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", timeline::Config::default()).await.unwrap(); + control.track("audio", timeline::Config::default()).await.unwrap(); + + // Two second GOPs, 400ms audio groups: with the default 2s minimum, audio packs by its own + // boundaries, not video's GOPs. + for sequence in 0..3 { + group(&video, sequence, &[sequence * 2000, sequence * 2000 + 1000]); + } + for sequence in 0..15 { + group(&audio, sequence, &[sequence * 400]); + } + video.finish().unwrap(); + audio.finish().unwrap(); + source.close(); + writer.run().await.unwrap(); + + let video = window(&store, "video").await; + assert_eq!(groups(&video), vec![(0, 0), (1, 1), (2, 2)]); + let audio = window(&store, "audio").await; + assert_eq!(groups(&audio), vec![(0, 4), (5, 9), (10, 14)]); + check_objects(&store, "audio", &audio).await; + } + + #[tokio::test] + async fn an_append_only_group_spans_several_objects() { + let source = broadcast::Info::new().produce(); + let log = track(&source, "log"); + + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default(); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("log", append_only()).await.unwrap(); + let run = tokio::spawn(writer.run()); + + // One group that never closes, a frame a second. + let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); + for second in 0..7 { + open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); + } + // The group is still open, yet its first six seconds are already durable. + while window(&store, "log").await.len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + let records = window(&store, "log").await; + assert_eq!( + records + .iter() + .map(|record| (record.start, record.end)) + .collect::>(), + vec![ + (Position::new(0, 0), Position::new(0, 3)), + (Position::new(0, 3), Position::new(0, 6)), + ] + ); + check_objects(&store, "log", &records).await; + assert_eq!(store.get_segments("log", 1).await.unwrap().frame_start, 3); + + open.finish().unwrap(); + log.finish().unwrap(); + source.close(); + run.await.unwrap().unwrap(); + + let records = window(&store, "log").await; + assert_eq!(records.last().unwrap().start, Position::new(0, 6)); + assert_eq!(records.last().unwrap().end, Position::group(1)); + check_objects(&store, "log", &records).await; + } + #[tokio::test] async fn a_failed_put_omits_only_that_track() { let source = broadcast::Info::new().produce(); @@ -964,14 +1083,14 @@ mod tests { let audio = track(&source, "audio"); let mock = Mock::memory(); - mock.fail_puts("/audio/groups/"); + mock.fail_puts("/audio/segments/"); let store = Store::new(mock, "rec"); let writer = Writer::new(store.clone(), source.consume(), Config::default()) .await .unwrap(); let control = writer.control(); - control.pacing_track("video").await.unwrap(); - control.pacing_track("audio").await.unwrap(); + control.track("video", media()).await.unwrap(); + control.track("audio", media()).await.unwrap(); for sequence in 0..3 { group(&video, sequence, &[sequence * 1000]); @@ -979,51 +1098,51 @@ mod tests { } video.finish().unwrap(); audio.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); - let records = window(&store).await; - assert_eq!(records.len(), 3); - assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2)]); - assert!(ranges(&records, "audio").is_empty()); - check_objects(&store, &records).await; + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); + check_objects(&store, "video", &records).await; + assert!(window(&store, "audio").await.is_empty()); } #[tokio::test] async fn retention_pops_then_deletes_expired_objects() { let source = broadcast::Info::new().produce(); let video = track(&source, "video"); + let catalog = track(&source, "catalog.json"); let store = Store::new(InMemory::new(), "rec"); let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + writer.control().track("catalog.json", sparse()).await.unwrap(); + // A static catalog published once, then six seconds of video. + group(&catalog, 0, &[0]); for sequence in 0..6 { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } video.finish().unwrap(); - source.finish(); + catalog.finish().unwrap(); + source.close(); writer.run().await.unwrap(); - // Segments 3 and 4 hold two seconds; the partial final segment 5 is always kept. - let records = window(&store).await; - assert_eq!( - records.iter().map(|record| record.segment).collect::>(), - vec![3, 4, 5] - ); - check_objects(&store, &records).await; + // Records 3 and 4 hold two seconds; the partial final record 5 is always kept. + let records = window(&store, "video").await; + assert_eq!(sequences(&records), vec![3, 4, 5]); + check_objects(&store, "video", &records).await; - let stored: HashSet<_> = store - .list(&Query::groups("video").unwrap()) - .map_ok(|entry| entry.key) - .try_collect() - .await - .unwrap(); - let expected: HashSet<_> = (3..6).map(|s| Key::groups("video", s..=s).unwrap()).collect(); - assert_eq!(stored, expected); + // The catalog's newest record outlives the video it was published with. + let catalog = window(&store, "catalog.json").await; + assert_eq!(sequences(&catalog), vec![0]); + check_objects(&store, "catalog.json", &catalog).await; + + let expected = &referenced("video", &records) | &referenced("catalog.json", &catalog); + assert_eq!(stored(&store).await, expected); } #[tokio::test] @@ -1037,16 +1156,16 @@ mod tests { Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); let control = writer.control(); - control.pacing_track("video").await.unwrap(); + control.track("video", media()).await.unwrap(); for sequence in 0..6 { group(&video, sequence, &[sequence * 1000]); } video.finish().unwrap(); - source.finish(); + source.close(); let run = tokio::spawn(writer.run()); - while window(&store).await.last().map(|record| record.segment) != Some(5) { + while window(&store, "video").await.last().map(|record| record.sequence) != Some(5) { tokio::time::sleep(Duration::from_millis(10)).await; } assert!(!run.is_finished()); @@ -1063,7 +1182,7 @@ mod tests { let writer = Writer::new(store.clone(), source.consume(), Config::default()) .await .unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); let run = tokio::spawn(writer.run()); let mut first = video.create_group(group::Info { sequence: 0 }).unwrap(); @@ -1071,18 +1190,18 @@ mod tests { group(&video, 1, &[1000]); group(&video, 2, &[2000]); tokio::time::sleep(Duration::from_millis(50)).await; - assert!(window(&store).await.is_empty(), "group 1 waits for group 0"); + assert!(window(&store, "video").await.is_empty(), "group 1 waits for group 0"); first.write_frame(ms(500), "0@500").unwrap(); first.finish().unwrap(); video.finish().unwrap(); - source.finish(); + source.close(); run.await.unwrap().unwrap(); - let records = window(&store).await; - assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2)]); - check_objects(&store, &records).await; - let object = store.get_groups("video", 0..=0).await.unwrap(); + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); + check_objects(&store, "video", &records).await; + let object = store.get_segments("video", 0).await.unwrap(); let frames: Vec<_> = object.groups[0].frames.iter().map(|f| f.timestamp).collect(); assert_eq!(frames, vec![0, 500]); } @@ -1096,20 +1215,21 @@ mod tests { let writer = Writer::new(store.clone(), source.consume(), Config::default()) .await .unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); group(&video, 0, &[0]); group(&video, 2, &[1000]); group(&video, 1, &[500]); group(&video, 3, &[2000]); video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); - let records = window(&store).await; - assert_eq!(ranges(&records, "video"), vec![(0, 0), (2, 2), (3, 3)]); - check_objects(&store, &records).await; + // The skipped sequence closes a record, so no record holds a gap. + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (2, 2), (3, 3)]); + check_objects(&store, "video", &records).await; } #[tokio::test] @@ -1119,13 +1239,13 @@ mod tests { let store = Store::new(InMemory::new(), "rec"); let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); group(&video, 0, &[0]); // One past the recording's largest group ID. group(&video, 1 << 53, &[1000]); video.finish().unwrap(); - source.finish(); + source.close(); assert_eq!( writer.run().await, @@ -1144,12 +1264,12 @@ mod tests { let store = Store::new(InMemory::new(), "rec"); let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); group(&video, 0, &[1000]); group(&video, 1, &[500]); video.finish().unwrap(); - source.finish(); + source.close(); assert_eq!( writer.run().await, @@ -1158,7 +1278,7 @@ mod tests { } #[tokio::test] - async fn removing_a_stalled_pacing_track_releases_the_recording() { + async fn a_stalled_track_holds_only_itself() { let source = broadcast::Info::new().produce(); let video = track(&source, "video"); let audio = track(&source, "audio"); @@ -1168,49 +1288,61 @@ mod tests { .await .unwrap(); let control = writer.control(); - control.pacing_track("video").await.unwrap(); - control.pacing_track("audio").await.unwrap(); + control.track("video", media()).await.unwrap(); + control.track("audio", media()).await.unwrap(); let run = tokio::spawn(writer.run()); group(&audio, 0, &[0]); - // The audio group never completes, so audio never reports past it. + // The audio group never completes, so its record never closes on its own. let mut stalled = audio.create_group(group::Info { sequence: 1 }).unwrap(); stalled.write_frame(ms(1000), "stalled").unwrap(); + group(&audio, 2, &[2000]); for sequence in 0..4 { group(&video, sequence, &[sequence * 1000]); } video.finish().unwrap(); - source.finish(); + source.close(); - tokio::time::sleep(Duration::from_millis(50)).await; - assert!(!run.is_finished(), "a stalled pacing track holds the recording open"); - assert!(window(&store).await.is_empty(), "and holds every segment back"); + // Video commits without waiting for the stalled audio. + while window(&store, "video").await.len() < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!run.is_finished(), "a stalled track holds the recording open"); control.remove("audio").unwrap(); run.await.unwrap().unwrap(); - let records = window(&store).await; - assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2), (3, 3)]); + // What arrived of the stalled group is stored; the group queued behind it is dropped. + let records = window(&store, "audio").await; assert_eq!( - ranges(&records, "audio"), - vec![(0, 0)], - "the incomplete group is dropped" + records + .iter() + .map(|record| (record.start, record.end)) + .collect::>(), + vec![ + (Position::group(0), Position::group(1)), + (Position::group(1), Position::new(1, 1)) + ] ); - check_objects(&store, &records).await; + check_objects(&store, "audio", &records).await; drop(stalled); } #[tokio::test] - async fn enrollment_is_refused_for_duplicates_and_the_timeline() { + async fn enrollment_is_refused_for_duplicates_and_timelines() { let source = broadcast::Info::new().produce(); let _video = track(&source, "video"); let store = Store::new(InMemory::new(), "rec"); let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); let control = writer.control(); - control.pacing_track("video").await.unwrap(); - assert_eq!(control.track("video").await, Err(Error::Enrolled("video".into()))); - assert_eq!(control.track(TIMELINE).await, Err(Error::Enrolled(TIMELINE.into()))); + control.track("video", media()).await.unwrap(); + assert_eq!( + control.track("video", sparse()).await, + Err(Error::Enrolled("video".into())) + ); + let timeline = timeline("video"); + assert_eq!(control.track(&timeline, media()).await, Err(Error::Enrolled(timeline))); } #[tokio::test] @@ -1223,7 +1355,7 @@ mod tests { let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); let control = writer.control(); assert_eq!( - control.pacing_track("video").await, + control.track("video", media()).await, Err(Error::Priority { existing: 7, intended: 0 @@ -1236,44 +1368,23 @@ mod tests { let source = broadcast::Info::new().produce(); let video = track(&source, "video"); let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); for sequence in sequences { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); } - /// Every stored group object, across all tracks. - async fn stored_groups(store: &Store) -> HashSet { - store - .list(&Query::new()) - .try_filter_map(|entry| async move { Ok(matches!(entry.key, Key::Groups { .. }).then_some(entry.key)) }) - .try_collect() - .await - .unwrap() - } - - /// The group objects the retained records advertise. - fn referenced(records: &[Record]) -> HashSet { - records - .iter() - .flat_map(|record| &record.tracks) - .map(|(name, ranges)| Key::groups(name.clone(), ranges[0].start..=ranges.last().unwrap().end).unwrap()) - .collect() - } - fn orphan(sequence: u64) -> Object { - Object { - groups: vec![Group { - sequence, - frames: vec![Frame { - timestamp: sequence * 1000, - payload: "orphan".into(), - }], + Object::new(vec![Group { + sequence, + frames: vec![Frame { + timestamp: sequence * 1000, + payload: "orphan".into(), }], - } + }]) } #[tokio::test] @@ -1283,48 +1394,96 @@ mod tests { // The source's cache replays groups the recording already holds. record(&store, Config::default(), 0..6).await; - let records = window(&store).await; - assert_eq!( - records.iter().map(|record| record.segment).collect::>(), - (0..6).collect::>() - ); - assert_eq!(ranges(&records, "video"), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, &records).await; + let records = window(&store, "video").await; + assert_eq!(sequences(&records), (0..6).collect::>()); + assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, "video", &records).await; // The resumed timeline groups continue the stored numbering. - let mut sequences = Vec::new(); + let mut numbers = Vec::new(); for segment in 0..6 { - let object = store.get_segments(TIMELINE, segment).await.unwrap(); - sequences.extend(object.groups.iter().map(|group| group.sequence)); + let object = store.get_segments(&timeline("video"), segment).await.unwrap(); + numbers.extend(object.groups.iter().map(|group| group.sequence)); } - assert!(sequences.windows(2).all(|pair| pair[0] < pair[1]), "{sequences:?}"); + assert!(numbers.windows(2).all(|pair| pair[0] < pair[1]), "{numbers:?}"); } #[tokio::test] - async fn a_restarted_dvr_deletes_unreferenced_groups() { + async fn a_resumed_append_only_group_continues_mid_group() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default(); + + // The first run stores frames 0..6 of a group that never closes, then stops. + let source = broadcast::Info::new().produce(); + let log = track(&source, "log"); + let writer = Writer::new(store.clone(), source.consume(), config.clone()) + .await + .unwrap(); + writer.control().track("log", append_only()).await.unwrap(); + let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); + for second in 0..7 { + open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); + } + let run = tokio::spawn(writer.run()); + while window(&store, "log").await.len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + run.abort(); + let _ = run.await; + drop(open); + + // The restarted source replays the whole group; the stored head is skipped. + let source = broadcast::Info::new().produce(); + let log = track(&source, "log"); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("log", append_only()).await.unwrap(); + let mut replayed = log.create_group(group::Info { sequence: 0 }).unwrap(); + for second in 0..8 { + replayed + .write_frame(ms(second * 1000), format!("line {second}")) + .unwrap(); + } + replayed.finish().unwrap(); + log.finish().unwrap(); + source.close(); + writer.run().await.unwrap(); + + let records = window(&store, "log").await; + assert_eq!( + records + .iter() + .map(|record| (record.start, record.end)) + .collect::>(), + vec![ + (Position::new(0, 0), Position::new(0, 3)), + (Position::new(0, 3), Position::new(0, 6)), + (Position::new(0, 6), Position::group(1)), + ] + ); + check_objects(&store, "log", &records).await; + } + + #[tokio::test] + async fn a_restarted_dvr_deletes_unreferenced_objects() { let store = Store::new(InMemory::new(), "rec"); let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); record(&store, config.clone(), 0..6).await; - // An interrupted expiration, an uncommitted upload, and a track no retained record names. - store.put_groups("video", &orphan(1)).await.unwrap(); - store.put_groups("video", &orphan(7)).await.unwrap(); - store.put_groups("audio", &orphan(0)).await.unwrap(); + // An interrupted expiration, an uncommitted upload, and a track no timeline names. + store.put_segments("video", 1, &orphan(1)).await.unwrap(); + store.put_segments("video", 6, &orphan(7)).await.unwrap(); + store.put_segments("audio", 0, &orphan(0)).await.unwrap(); - // Groups at or below the uncommitted upload are refused, so nothing overlaps it. record(&store, config, 6..10).await; - let records = window(&store).await; - assert_eq!( - records.iter().map(|record| record.segment).collect::>(), - vec![5, 6, 7] - ); - assert_eq!(ranges(&records, "video"), vec![(5, 5), (8, 8), (9, 9)]); - check_objects(&store, &records).await; - assert_eq!(stored_groups(&store).await, referenced(&records)); + let records = window(&store, "video").await; + assert_eq!(sequences(&records), vec![7, 8, 9]); + assert_eq!(groups(&records), vec![(7, 7), (8, 8), (9, 9)]); + check_objects(&store, "video", &records).await; + assert_eq!(stored(&store).await, referenced("video", &records)); store.get_info("video").await.unwrap(); - store.get_info(TIMELINE).await.unwrap(); - store.get_segments(TIMELINE, 0).await.unwrap(); + store.get_info(&timeline("video")).await.unwrap(); + store.get_segments(&timeline("video"), 0).await.unwrap(); } #[tokio::test] @@ -1337,27 +1496,24 @@ mod tests { let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().pacing_track("video").await.unwrap(); + writer.control().track("video", media()).await.unwrap(); let run = tokio::spawn(writer.run()); for sequence in 0..6 { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } - // Segment 5 stays open, so the newest durable timeline object is segment 4. - while window(&store).await.last().map(|record| record.segment) != Some(4) { + // Record 5 stays open, so the newest durable timeline object is record 4. + while window(&store, "video").await.last().map(|record| record.sequence) != Some(4) { tokio::time::sleep(Duration::from_millis(10)).await; } run.abort(); let _ = run.await; - let retained = window(&store).await; - assert_eq!( - retained.iter().map(|record| record.segment).collect::>(), - vec![3, 4] - ); - let expired: HashSet<_> = (0..3).map(|s| Key::groups("video", s..=s).unwrap()).collect(); - assert_eq!(stored_groups(&store).await, &referenced(&retained) | &expired); + let retained = window(&store, "video").await; + assert_eq!(sequences(&retained), vec![3, 4]); + let expired: HashSet<_> = (0..3).map(|s| Key::segments("video", s).unwrap()).collect(); + assert_eq!(stored(&store).await, &referenced("video", &retained) | &expired); // An upload the crash left uncommitted. - store.put_groups("video", &orphan(5)).await.unwrap(); + store.put_segments("video", 5, &orphan(5)).await.unwrap(); let grace = Duration::from_millis(200); let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); @@ -1366,49 +1522,45 @@ mod tests { let started = Instant::now(); let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); assert!( - stored_groups(&store).await.is_superset(&expired), + stored(&store).await.is_superset(&expired), "expired objects outlive the grace, for readers holding the old timeline" ); + assert!( + !stored(&store).await.contains(&Key::segments("video", 5).unwrap()), + "the uncommitted upload is cleared before its key is reused" + ); - writer.control().pacing_track("video").await.unwrap(); - // The source replays the uncommitted group; it is refused rather than overwritten. - for sequence in 5..9 { + writer.control().track("video", media()).await.unwrap(); + // The source replays groups the recording already holds; they are refused. + for sequence in 4..9 { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); assert!(started.elapsed() >= grace); - let records = window(&store).await; - assert_eq!(ranges(&records, "video"), vec![(6, 6), (7, 7), (8, 8)]); - check_objects(&store, &records).await; - assert_eq!(stored_groups(&store).await, referenced(&records)); - store.get_info("video").await.unwrap(); - store.get_info(TIMELINE).await.unwrap(); + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(6, 6), (7, 7), (8, 8)]); + check_objects(&store, "video", &records).await; + assert_eq!(stored(&store).await, referenced("video", &records)); for segment in 0..=7 { - store.get_segments(TIMELINE, segment).await.unwrap(); + store.get_segments(&timeline("video"), segment).await.unwrap(); } } #[tokio::test] - async fn an_archive_restart_leaves_uncommitted_groups_unadvertised() { + async fn an_archive_restart_clears_uncommitted_objects() { let store = Store::new(InMemory::new(), "rec"); record(&store, Config::default(), 0..3).await; // Media stored after the last timeline commit: a crash before its record. - store.put_groups("video", &orphan(3)).await.unwrap(); + store.put_segments("video", 3, &orphan(3)).await.unwrap(); record(&store, Config::default(), 3..6).await; - let records = window(&store).await; - assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2), (4, 4), (5, 5)]); - check_objects(&store, &records).await; - // An archive deletes nothing; the orphan stays invisible. - assert!( - stored_groups(&store) - .await - .contains(&Key::groups("video", 3..=3).unwrap()) - ); - assert!(!referenced(&records).contains(&Key::groups("video", 3..=3).unwrap())); + let records = window(&store, "video").await; + assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, "video", &records).await; + assert_eq!(stored(&store).await, referenced("video", &records)); } #[tokio::test] @@ -1417,8 +1569,8 @@ mod tests { let store = Store::new(mock.clone(), "rec"); let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); record(&store, config.clone(), 0..6).await; - store.put_groups("video", &orphan(1)).await.unwrap(); - let before = stored_groups(&store).await; + store.put_segments("video", 1, &orphan(1)).await.unwrap(); + let before = stored(&store).await; let failing = mock.fork(); failing.fail_lists(); @@ -1427,11 +1579,14 @@ mod tests { assert!(matches!(result, Err(Error::Store(_)))); // A missing timeline object leaves the retained window unrecoverable. - store.delete(&Key::segments(TIMELINE, 3).unwrap()).await.unwrap(); + store + .delete(&Key::segments(timeline("video"), 3).unwrap()) + .await + .unwrap(); let result = Writer::new(store.clone(), source.consume(), config).await; assert!(matches!(result, Err(Error::Timeline(_)))); - assert_eq!(stored_groups(&store).await, before); + assert_eq!(stored(&store).await, before); } #[tokio::test] @@ -1440,27 +1595,31 @@ mod tests { let config = Config::default().with_retention(Retention::new(Duration::from_secs(280), Duration::ZERO)); record(&store, config, 0..300).await; - let recovery = recover(&store, TIMELINE, true).await.unwrap(); - let checkpoint = recovery.checkpoint.unwrap(); - let records = window(&store).await; + let recovery = recover(&store, true).await.unwrap(); + let resume = &recovery.tracks["video"]; + let records = window(&store, "video").await; assert!(records.len() > 256, "the window outgrows one checkpoint"); - assert_eq!(checkpoint.records, records); - assert_eq!(checkpoint.range.end, 300); + assert_eq!(resume.checkpoint.records, records); + assert_eq!(resume.checkpoint.range.end, 300); assert!(recovery.orphans.is_empty()); - assert_eq!(recovery.floors["video"], 299); + assert!(recovery.uncommitted.is_empty()); + assert_eq!(resume.floor(), Some(Position::group(300))); } #[tokio::test] async fn a_timeline_that_does_not_end_at_the_next_segment_fails_recovery() { let store = Store::new(InMemory::new(), "rec"); record(&store, Config::default(), 0..6).await; - let before = stored_groups(&store).await; + let before = stored(&store).await; - // `segments/5` still decodes, but it restates an earlier window, so resuming - // would write the next segment on top of it. - let older = store.get_segments(TIMELINE, 0).await.unwrap(); - store.delete(&Key::segments(TIMELINE, 5).unwrap()).await.unwrap(); - store.put_segments(TIMELINE, 5, &older).await.unwrap(); + // `segments/5` still decodes, but it restates an earlier window, so resuming would write the + // next record on top of it. + let older = store.get_segments(&timeline("video"), 0).await.unwrap(); + store + .delete(&Key::segments(timeline("video"), 5).unwrap()) + .await + .unwrap(); + store.put_segments(&timeline("video"), 5, &older).await.unwrap(); let source = broadcast::Info::new().produce(); match Writer::new(store.clone(), source.consume(), Config::default()).await { @@ -1468,6 +1627,6 @@ mod tests { Err(err) => panic!("expected a timeline error, got {err}"), Ok(_) => panic!("expected recovery to fail"), } - assert_eq!(stored_groups(&store).await, before); + assert_eq!(stored(&store).await, before); } } diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 33fc3f428f..15ba37bcd9 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2245,7 +2245,7 @@ mod tests { fixture.finish().await; } - /// A recording replays what the live edge published: the archive's segment records + /// A recording replays what the live edge published: the archive's records /// carry the live timestamps across an idle restart, with the idle gap left in. #[tokio::test] async fn retained_archive_playback_keeps_the_live_timestamps() { @@ -2270,19 +2270,19 @@ mod tests { .archive .expect("the audio track enrolls an archive"); timeline = Some( - moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion) + moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion, "audio") .await .unwrap(), ); } drop(track); wait_for(&mut fixture.publication, Status::Waiting).await; - // Idle past the minimum segment, so each run is archived as its own segment. + // Idle past the minimum segment, so each run is archived as its own record. tokio::time::sleep(moq_mux::timeline::DEFAULT_DURATION_MIN + Duration::from_millis(100)).await; } let (catalog, _consumer) = fixture.finish().await; - catalog.timeline().finish().unwrap(); + catalog.timeline().finish(); let mut timeline = timeline.unwrap(); let mut archived = Vec::new(); while let Some(event) = timeline.next().await.unwrap() { @@ -2292,16 +2292,15 @@ mod tests { } } - assert_eq!(archived.len(), live.len(), "one segment per capture run: {archived:?}"); + assert_eq!(archived.len(), live.len(), "one record per capture run: {archived:?}"); for (entry, live) in archived.iter().zip(&live) { // The archive keeps millisecond precision. assert_eq!(entry.pts.as_micros() / 1000, u128::from(*live / 1000), "{archived:?}"); - assert!(entry.tracks.contains_key("audio"), "{archived:?}"); } let first = &archived[0]; assert!( archived[1].pts.as_micros() >= first.pts.as_micros() + first.duration.as_micros(), - "the resumed segment overlaps the one before it: {archived:?}" + "the resumed record overlaps the one before it: {archived:?}" ); } } diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 2689096655..411e299542 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -24,7 +24,7 @@ pub struct ExportArgs { /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. pub store: Url, - /// Keep only this much recent content (a DVR), deleting older segments. Unset keeps everything. + /// Keep only this much recent content (a DVR), deleting older records. Unset keeps everything. #[usage(long)] pub retention: Option, @@ -49,8 +49,9 @@ pub struct ImportArgs { /// Record the broadcast `name` into `args.store` until the broadcast ends. /// -/// Reads the broadcast's own catalog: video and audio renditions pace the segments, while the -/// catalog and every other track are recorded alongside without pacing. +/// Reads the broadcast's own catalog: every track gets its own timeline, cut at group boundaries +/// between 2s and 10s, while the catalog and data tracks use a zero minimum and record each group +/// as it finishes. pub async fn export( origin: moq_net::origin::Consumer, name: String, @@ -77,7 +78,7 @@ pub async fn export( .await .with_context(|| format!("failed to start recording into {}", args.store))?; let control = writer.control(); - control.track(catalog_track).await?; + control.track(catalog_track, sparse()).await?; let catalog = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; tracing::info!(%name, store = %args.store, "recording"); @@ -92,11 +93,16 @@ pub async fn export( /// Republish the recording at `args.store` as the broadcast `name`. /// -/// The timeline replays as a live track and every other track's groups are served on request. +/// Each track's timeline replays as a live track and every other track's groups are served on +/// request. pub async fn import(origin: moq_net::origin::Producer, name: String, args: ImportArgs) -> anyhow::Result<()> { let store = open(&args.store)?; let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; - let config = moq_archive::reader::Config::new(hang::timeline::DEFAULT_NAME); + let timelines = store + .timelines() + .await + .with_context(|| format!("failed to list the recording at {}", args.store))?; + let config = moq_archive::reader::Config::new(timelines); let mut reader = moq_archive::Reader::open(store, &broadcast, config) .await .with_context(|| format!("no readable recording at {}", args.store))?; @@ -137,6 +143,11 @@ fn open(url: &Url) -> anyhow::Result>> { Ok(moq_archive::Store::new(store, prefix)) } +/// How sparse data, such as a catalog, is cut: every group is its own record. +fn sparse() -> moq_mux::timeline::Config { + moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO) +} + /// The track carrying the catalog `format` reads. fn catalog_track(format: CatalogFormat) -> anyhow::Result<&'static str> { Ok(match format { @@ -154,14 +165,10 @@ async fn enroll( ) -> anyhow::Result<()> { let mut tracks = Tracks::default(); while let Some(snapshot) = catalog.next().await? { - // One rendition's subscription can outrun the rest of this snapshot. Without the - // hold, the writer closes a segment from that rendition alone and the others never - // enter the record. - let _hold = control.reserve(); for change in tracks.update(&snapshot)? { match change { - Change::Pacing(name) => control.pacing_track(&name).await?, - Change::Track(name) => control.track(&name).await?, + Change::Media(name) => control.track(&name, moq_mux::timeline::Config::default()).await?, + Change::Sparse(name) => control.track(&name, sparse()).await?, Change::Remove(name) => control.remove(&name)?, } } @@ -172,10 +179,10 @@ async fn enroll( /// A change to the recorded track set. #[derive(Debug, PartialEq, Eq)] enum Change { - /// Record a rendition that paces the segments. - Pacing(String), - /// Record a track without letting it pace the segments. - Track(String), + /// Record a rendition, its records cut by duration. + Media(String), + /// Record a data track, each group its own record. + Sparse(String), /// Stop recording a track the catalog dropped. Remove(String), } @@ -219,7 +226,7 @@ impl Tracks { let mut listed = HashSet::new(); let mut changes = Vec::new(); - for (name, broadcast, pacing) in video.chain(audio).chain(text).chain(json).chain(binary) { + for (name, broadcast, media) in video.chain(audio).chain(text).chain(json).chain(binary) { if let Some(broadcast) = broadcast { anyhow::bail!( "track `{name}` is served from broadcast `{broadcast}`; an archive records one broadcast" @@ -232,9 +239,9 @@ impl Tracks { self.enrolled.insert(name.clone()), "track `{name}` returned to the catalog after it was dropped; the recording cannot resume it" ); - changes.push(match pacing { - true => Change::Pacing(name.clone()), - false => Change::Track(name.clone()), + changes.push(match media { + true => Change::Media(name.clone()), + false => Change::Sparse(name.clone()), }); } @@ -257,9 +264,9 @@ mod tests { AudioConfig::new(AudioCodec::Opus, 48_000, 2) } - /// Renditions pace, data tracks don't, and a later snapshot only adds what is new. + /// Renditions are media, data tracks are sparse, and a later snapshot only adds what is new. #[test] - fn renditions_pace_and_the_rest_follow() { + fn renditions_are_media_and_the_rest_sparse() { let mut tracks = Tracks::default(); let mut catalog = hang::Catalog::default(); catalog.audio.renditions.insert("audio".into(), audio()); @@ -270,15 +277,15 @@ mod tests { assert_eq!( tracks.update(&catalog).unwrap(), - [Change::Pacing("audio".into()), Change::Track("chat".into())] + [Change::Media("audio".into()), Change::Sparse("chat".into())] ); catalog.audio.renditions.insert("audio2".into(), audio()); - assert_eq!(tracks.update(&catalog).unwrap(), [Change::Pacing("audio2".into())]); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Media("audio2".into())]); assert_eq!(tracks.update(&catalog).unwrap(), []); } - /// A dropped rendition stops pacing, and coming back is refused rather than silently lost. + /// A dropped rendition stops recording, and coming back is refused rather than silently lost. #[test] fn a_dropped_track_is_removed_for_good() { let mut tracks = Tracks::default(); @@ -355,7 +362,7 @@ mod tests { .expect("the rendition is enrolled"); track.finish().unwrap(); catalog.finish().unwrap(); - broadcast.finish(); + broadcast.close(); tokio::time::timeout(Duration::from_secs(10), recording) .await @@ -385,7 +392,7 @@ mod tests { serving.abort(); } - /// Renditions already live in the opening catalog all land in its first segment. + /// Renditions already live in the opening catalog are all recorded from their first group. /// /// Groups are published before export starts, so the writer can drain the first rendition /// while the next subscription is still in flight. @@ -438,7 +445,7 @@ mod tests { first.finish().unwrap(); second.finish().unwrap(); catalog.finish().unwrap(); - broadcast.finish(); + broadcast.close(); tokio::time::timeout(Duration::from_secs(10), recording) .await @@ -446,29 +453,25 @@ mod tests { .unwrap() .expect("the recording succeeds"); + // Each rendition's own timeline opens at its first group. let store = super::open(&url).unwrap(); - let object = store - .get_segments(hang::timeline::DEFAULT_NAME, 0) - .await - .expect("segment 0"); - let config = moq_json::window::ConsumerConfig::default().with_compression(true); - let mut decoder = moq_json::window::Decoder::::new(config); - for stored in object.groups { - let mut group = decoder.group(); - for frame in stored.frames { - group.decode(&frame.payload).unwrap(); - } - } - let mut opening = None; - while let Some(event) = decoder.next_event() { - if let moq_json::window::Event::Push { value, .. } = event - && value.segment == 0 - { - opening = Some(value); + for track in ["audio", "audio2"] { + let object = store + .get_segments(&hang::timeline::default_name(track), 0) + .await + .expect("record 0"); + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } } + let Some(moq_json::window::Event::Push { value, .. }) = decoder.next_event() else { + panic!("{track} record 0 is pushed first"); + }; + assert_eq!(value.start, hang::timeline::Position::group(0), "{track}"); } - let opening = opening.expect("segment 0 has a record"); - assert!(opening.tracks.contains_key("audio"), "{opening:?}"); - assert!(opening.tracks.contains_key("audio2"), "{opening:?}"); } } diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 40f327c8f6..2981880576 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -1,17 +1,18 @@ //! Serving a `moq-archive` recording as HLS. //! -//! A [`moq_archive::Reader`] replays the recording's timeline onto a broadcast and answers FETCH -//! from range-named objects, so the exporter serves it exactly like a live broadcast. These tests -//! pin the storage traffic that composition produces: playlists read only the timeline, and a -//! segment GETs exactly one object of the requested rendition. +//! A [`moq_archive::Reader`] replays each track's timeline onto a broadcast and answers FETCH from +//! its record objects, so the exporter serves it exactly like a live broadcast. These tests pin +//! the storage traffic that composition produces: playlists read only the timelines, and a segment +//! GETs only objects of the requested rendition. +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; use bytes::{Bytes, BytesMut}; use futures::stream::BoxStream; -use hang::timeline::{Range, Record}; +use hang::timeline::{Position, Record}; use moq_archive::object_store::memory::InMemory; use moq_archive::object_store::path::Path; use moq_archive::object_store::{ @@ -23,8 +24,6 @@ use moq_json::window; use super::*; -const TIMELINE: &str = hang::timeline::DEFAULT_NAME; - /// In-memory store that records every GET path and counts listings. #[derive(Debug, Clone, Default)] struct Counting { @@ -108,35 +107,62 @@ impl ObjectStore for Counting { } } +/// One track's timeline encoder, as `moq_archive::Writer` drives it. +struct Encoder { + encoder: window::Encoder, + /// Next timeline group sequence. + group: u64, + /// Next record sequence. + record: u64, +} + /// Writes recording objects the way `moq_archive::Writer` lays them out, with full control over -/// each record's timing and ranges. +/// each record's timing and groups. struct Recording { store: Store, - encoder: window::Encoder, - /// Next timeline group sequence. - sequence: u64, + encoders: HashMap, } impl Recording { async fn new(tracks: &[&str]) -> Self { let store = Store::new(Counting::default(), "rec"); - store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); + let mut encoders = HashMap::new(); for track in tracks { // Legacy media is stamped in microseconds. store.put_info(track, &Info::new(1, 1_000_000).unwrap()).await.unwrap(); + store + .put_info(&timeline(track), &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(0); + let encoder = Encoder { + encoder: window::Encoder::new(config), + group: 0, + record: 0, + }; + encoders.insert(track.to_string(), encoder); } - let config = window::ProducerConfig::default() - .with_compression(true) - .with_op_ratio(0); - Self { - store, - encoder: window::Encoder::new(config), - sequence: 0, - } + Self { store, encoders } } - /// Store one object for `track` holding `groups`, each a list of frame timestamps in micros. - async fn media(&self, track: &str, groups: &[(u64, &[u64])]) { + /// Store and commit `track`'s next record: `groups`, each a list of frame timestamps in micros, + /// spanning `pts..pts + duration` in milliseconds, popping `pop` older records. + async fn add(&mut self, track: &str, pts: u64, duration: u64, groups: &[(u64, &[u64])], pop: u64) { + let encoder = self.encoders.get_mut(track).unwrap(); + let sequence = encoder.record; + encoder.record += 1; + let first = groups.first().unwrap().0; + let last = groups.last().unwrap().0; + let record = Record::new( + sequence, + pts, + duration, + Position::group(first), + Position::group(last + 1), + ); + let groups = groups .iter() .map(|&(sequence, frames)| Group { @@ -148,28 +174,27 @@ impl Recording { .collect(), }) .collect(); - self.store.put_groups(track, &Object { groups }).await.unwrap(); - } + self.store + .put_segments(track, sequence, &Object::new(groups)) + .await + .unwrap(); - /// Commit `record` as timeline segment `record.segment`, popping `pop` older records. - async fn commit(&mut self, record: &Record, pop: u64) { let mut payloads = Vec::new(); - let pending = self.encoder.push(record).unwrap(); + let pending = encoder.encoder.push(&record).unwrap(); payloads.push((pending.keyframe, pending.payload.clone())); pending.commit(); - if let Some(pending) = self.encoder.pop(pop).unwrap() { + if let Some(pending) = encoder.encoder.pop(pop).unwrap() { payloads.push((pending.keyframe, pending.payload.clone())); pending.commit(); } - let mut groups: Vec = Vec::new(); for (keyframe, payload) in payloads { if keyframe || groups.is_empty() { groups.push(Group { - sequence: self.sequence, + sequence: encoder.group, frames: Vec::new(), }); - self.sequence += 1; + encoder.group += 1; } let frame = Frame { timestamp: record.pts, @@ -178,7 +203,7 @@ impl Recording { groups.last_mut().unwrap().frames.push(frame); } self.store - .put_segments(TIMELINE, record.segment, &Object { groups }) + .put_segments(&timeline(track), sequence, &Object::new(groups)) .await .unwrap(); } @@ -187,6 +212,17 @@ impl Recording { fn gets(&self) -> Vec { self.store.inner().take() } + + fn timelines(&self) -> std::collections::BTreeMap { + self.encoders + .keys() + .map(|track| (track.clone(), timeline(track))) + .collect() + } +} + +fn timeline(track: &str) -> String { + hang::timeline::default_name(track) } /// One Legacy frame: a VP8 keyframe for video (geometry the muxer can parse), filler otherwise. @@ -208,17 +244,18 @@ fn legacy(track: &str, micros: u64, keyframe: bool) -> Frame { } } -fn record(segment: u64, pts: u64, duration: u64, tracks: &[(&str, u64, u64)]) -> Record { - let mut record = Record::new(segment, pts, duration); - for &(track, start, end) in tracks { - record.tracks.insert(track.to_string(), vec![Range::new(start, end)]); +/// A live-style archive entry naming every rendition's timeline. +fn live() -> hang::catalog::Archive { + let mut archive = hang::catalog::Archive::new(); + for track in ["360p", "1080p", "audio"] { + archive.timelines.insert(track.to_string(), timeline(track)); } - record + archive } -/// The archive entry a replay advertises: the recording's timeline, durable in its store. +/// The archive entry a replay advertises: the recording's timelines, durable in its store. fn durable() -> hang::catalog::Archive { - let mut archive = hang::catalog::Archive::new(TIMELINE); + let mut archive = live(); archive.store = Some("memory:///rec/".parse().unwrap()); archive.version = Some(hang::catalog::Archive::VERSION); archive @@ -265,7 +302,7 @@ impl Replay { let mut catalog = moq_json::snapshot::Producer::new(track, json); catalog.update(&self::catalog(archive)).unwrap(); - let config = reader::Config::new(TIMELINE).with_cache(cache); + let config = reader::Config::new(recording.timelines()).with_cache(cache); let reader = moq_archive::Reader::open(recording.store.clone(), &broadcast, config) .await .unwrap(); @@ -319,69 +356,69 @@ impl Replay { } fn is_media(path: &str) -> bool { - path.contains("/groups/") + path.contains("/segments/") && !path.contains("timeline") } -/// Three aligned 2s segments: one keyframe group per video rendition, four audio groups each. +/// Three 2s segments: one keyframe group per video record, four audio groups per audio record. +/// The first video rendition by name, 1080p, is the reference. async fn three_segments() -> Recording { segments(3).await } -/// `count` aligned 2s segments, laid out like [`three_segments`]. +/// `count` 2s segments, laid out like [`three_segments`]. async fn segments(count: u64) -> Recording { let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; for segment in 0..count { let pts = segment * 2_000_000; for video in ["360p", "1080p"] { - recording.media(video, &[(segment, &[pts, pts + 1_000_000])]).await; + recording + .add(video, segment * 2000, 2000, &[(segment, &[pts, pts + 1_000_000])], 0) + .await; } let audio: Vec<(u64, [u64; 1])> = (0..4).map(|i| (segment * 4 + i, [pts + i * 500_000])).collect(); let audio: Vec<(u64, &[u64])> = audio .iter() .map(|(sequence, frames)| (*sequence, &frames[..])) .collect(); - recording.media("audio", &audio).await; - - let tracks = [ - ("360p", segment, segment), - ("1080p", segment, segment), - ("audio", segment * 4, segment * 4 + 3), - ]; - recording - .commit(&record(segment, segment * 2000, 2000, &tracks), 0) - .await; + recording.add("audio", segment * 2000, 2000, &audio, 0).await; } recording } #[tokio::test] -async fn playlists_read_only_the_timeline_and_segments_one_object() { +async fn playlists_read_only_timelines_and_segments_their_own_objects() { let recording = three_segments().await; let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; let master = replay.broadcaster.master_playlist(None); assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); - // Render and reload every playlist with media GETs refused: aligned numbering, and not one - // media GET. + // Render and reload every playlist with media GETs refused: shared numbering, and not one + // media GET. The reference lists every segment; the others hold back the newest until their + // own timeline passes its end, or ends. recording.store.inner().reject_media(true); for _ in 0..2 { - for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { + for (kind, name, listed) in [ + (Kind::Video, "1080p", 3), + (Kind::Video, "360p", 2), + (Kind::Audio, "audio", 3), + ] { let playlist = replay.playlist(kind, name).await; for segment in 0..3 { - assert!(playlist.contains(&format!("seg/{segment}.m4s\n")), "{name}: {playlist}"); + let line = format!("seg/{segment}.m4s\n"); + assert_eq!(playlist.contains(&line), segment < listed, "{name}: {playlist}"); } assert!(!playlist.contains("#EXT-X-ENDLIST"), "no finality was supplied"); } } let gets = recording.gets(); - assert!(gets.iter().any(|path| path.contains("/segments/")), "{gets:?}"); + assert!(gets.iter().any(|path| path.contains("timeline")), "{gets:?}"); assert!( !gets.iter().any(|path| is_media(path)), "playlists must not GET media: {gets:?}" ); recording.store.inner().reject_media(false); - // A range-bearing segment URI resolves its object directly: no listing, no index object. + // A segment resolves its objects from the timelines: no listing, no index object. let lists = recording.store.inner().lists(); // Switching renditions downloads only the selected rendition's object. @@ -389,10 +426,7 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { assert_eq!(&low[4..8], b"moof"); assert_eq!( recording.gets(), - [ - "rec/360p/.info", - "rec/360p/groups/0000000000000000001.0000000000000000001" - ] + ["rec/360p/.info", "rec/360p/segments/0000000000000000001"] ); let high = replay .rendition(Kind::Video, "1080p") @@ -403,10 +437,7 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { assert_eq!(&high[4..8], b"moof"); assert_eq!( recording.gets(), - [ - "rec/1080p/.info", - "rec/1080p/groups/0000000000000000002.0000000000000000002" - ] + ["rec/1080p/.info", "rec/1080p/segments/0000000000000000002"] ); // An audio segment spans four groups but still costs one object GET. @@ -419,10 +450,7 @@ async fn playlists_read_only_the_timeline_and_segments_one_object() { assert_eq!(&audio[4..8], b"moof"); assert_eq!( recording.gets(), - [ - "rec/audio/.info", - "rec/audio/groups/0000000000000000007.0000000000000000004" - ] + ["rec/audio/.info", "rec/audio/segments/0000000000000000001"] ); // A repeated request hits the reader's cache, including the immutable `.info`. @@ -448,48 +476,42 @@ async fn a_bounded_cache_rereads_evicted_objects() { .unwrap(); assert_eq!(&audio[4..8], b"moof"); let gets: Vec = recording.gets().into_iter().filter(|path| is_media(path)).collect(); - assert_eq!( - gets, - vec!["rec/audio/groups/0000000000000000007.0000000000000000004"; 4] - ); + assert_eq!(gets, vec!["rec/audio/segments/0000000000000000001"; 4]); } #[tokio::test] -async fn missing_track_segments_are_gaps_and_time_jumps_are_discontinuities() { +async fn missing_rendition_segments_are_gaps_and_time_jumps_are_discontinuities() { let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; - recording.media("360p", &[(0, &[0])]).await; - recording.media("1080p", &[(0, &[0])]).await; - recording - .commit(&record(0, 0, 2000, &[("360p", 0, 0), ("1080p", 0, 0)]), 0) - .await; - // 1080p stored nothing for segment 1. - recording.media("360p", &[(1, &[2_000_000])]).await; - recording.commit(&record(1, 2000, 2000, &[("360p", 1, 1)]), 0).await; + recording.add("1080p", 0, 2000, &[(0, &[0])], 0).await; + recording.add("360p", 0, 8000, &[(0, &[0])], 0).await; + // 360p stored nothing for segment 1. + recording.add("1080p", 2000, 2000, &[(1, &[2_000_000])], 0).await; // Content time jumps from 4s to 10s. - recording.media("360p", &[(2, &[10_000_000])]).await; - recording.media("1080p", &[(1, &[10_000_000])]).await; - recording - .commit(&record(2, 10_000, 2000, &[("360p", 2, 2), ("1080p", 1, 1)]), 0) - .await; + recording.add("1080p", 10_000, 2000, &[(2, &[10_000_000])], 0).await; + recording.add("360p", 10_000, 2000, &[(1, &[10_000_000])], 0).await; - let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; - let high = replay.playlist(Kind::Video, "1080p").await; + let mut replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + // Finality resolves each rendition's newest segment. + replay.reader.take().unwrap().finish().unwrap(); + let low = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; let expected = concat!( "#EXTINF:2.00000,\nseg/0.m4s\n", "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/1.m4s\n", "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n", ); - assert!(high.contains(expected), "{high}"); - let low = replay.playlist(Kind::Video, "360p").await; - assert!(!low.contains("#EXT-X-GAP"), "{low}"); + assert!(low.contains(expected), "{low}"); + let high = replay.playlist(Kind::Video, "1080p").await; + assert!(!high.contains("#EXT-X-GAP"), "{high}"); assert!( - low.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n"), - "{low}" + high.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n"), + "{high}" ); // A gap is never fetched. recording.gets(); - let rendition = replay.rendition(Kind::Video, "1080p"); + let rendition = replay.rendition(Kind::Video, "360p"); assert!(rendition.segment(1).await.unwrap().is_none()); assert!(!recording.gets().iter().any(|path| is_media(path))); let after = rendition.segment(2).await.unwrap().unwrap(); @@ -500,16 +522,15 @@ async fn missing_track_segments_are_gaps_and_time_jumps_are_discontinuities() { async fn a_growing_recording_ends_only_on_caller_finality() { let mut recording = three_segments().await; let mut replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; - replay.playlist(Kind::Video, "360p").await; + replay.playlist(Kind::Video, "1080p").await; // A DVR commit: segment 3 arrives and segment 0 expires. - recording.media("360p", &[(3, &[6_000_000])]).await; - recording.commit(&record(3, 6000, 2000, &[("360p", 3, 3)]), 1).await; + recording.add("1080p", 6000, 2000, &[(3, &[6_000_000])], 1).await; recording.gets(); let mut reader = replay.reader.take().unwrap(); reader.refresh().await.unwrap(); let playlist = replay - .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/3.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/3.m4s\n")) .await; assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); assert!(!playlist.contains("seg/0.m4s"), "{playlist}"); @@ -520,13 +541,13 @@ async fn a_growing_recording_ends_only_on_caller_finality() { let gets = recording.gets(); assert!( !gets.iter().any(|path| is_media(path)), - "following reads only the timeline: {gets:?}" + "following reads only the timelines: {gets:?}" ); // The store holds no completion marker; the caller supplies finality. reader.finish().unwrap(); let playlist = replay - .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("#EXT-X-ENDLIST")) .await; assert!(playlist.contains("seg/3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); } @@ -539,7 +560,7 @@ async fn a_durable_timeline_lists_past_the_window() { let recording = segments(12).await; let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; let playlist = replay - .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/11.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/11.m4s\n")) .await; assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0\n"), "{playlist}"); assert!(playlist.contains("seg/0.m4s\n"), "{playlist}"); @@ -552,10 +573,10 @@ async fn a_durable_timeline_lists_past_the_window() { let mut elsewhere = durable(); elsewhere.replay = Some(moq_net::path::RelativeOwned::new("./recording")); - for archive in [hang::catalog::Archive::new(TIMELINE), elsewhere] { + for archive in [live(), elsewhere] { let live = Replay::open(&recording, 64 * 1024 * 1024, archive).await; let playlist = live - .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/11.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/11.m4s\n")) .await; assert!(!playlist.contains("seg/0.m4s\n"), "{playlist}"); } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 2e530a0337..c6131d0675 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -1,13 +1,17 @@ //! Export: serve a MoQ broadcast as HLS or DASH, fetching media on demand. //! -//! A [`Broadcaster`] subscribes to one broadcast's catalog and its single timeline track (see -//! [`hang::timeline`]). That is the *only* standing traffic: playlists and manifests are a -//! pure function of the timeline records (each names a complete aligned segment with -//! per-track group ranges), and media bytes move only when an HTTP client requests a segment, -//! which FETCHes exactly the groups that segment covers from the relay cache and transmuxes -//! them to CMAF. A broadcast whose catalog advertises no timeline can't be served this way and -//! is skipped. A `moq-archive` recording replayed onto a broadcast is served the same way: its -//! playlists read only the stored timeline, and each segment GETs one stored object. +//! A [`Broadcaster`] subscribes to one broadcast's catalog and each rendition's timeline track +//! (see [`hang::timeline`]). That is the *only* standing traffic: playlists and manifests are a +//! pure function of the timeline records, and media bytes move only when an HTTP client requests +//! a segment, which FETCHes exactly the frames that segment covers from the relay cache and +//! transmuxes them to CMAF. A broadcast whose catalog advertises no timelines can't be served this +//! way and is skipped. A `moq-archive` recording replayed onto a broadcast is served the same way: +//! its playlists read only the stored timelines, and a segment GETs only its rendition's objects. +//! +//! Segment boundaries come from one reference rendition's records (the first video rendition, or +//! the first audio one when there is no video), numbered by record sequence, so every edge and +//! every reload agree without the publisher cutting for HLS. Every other rendition resolves each +//! segment against its own timeline. //! An inline-parameter-set codec with no catalog `description` GETs one keyframe //! group on the first playlist render to build its init, then caches it. //! @@ -27,6 +31,7 @@ pub mod master; mod mpd; mod playlist; mod rendition; +mod spans; mod upstream; pub mod renditions; @@ -40,6 +45,7 @@ use rand::RngExt; pub(crate) use playlist::render_media; pub use rendition::{Kind, Rendition}; +use segments::Reference; pub(crate) use upstream::Upstream; /// Backoff bounds for the initial catalog subscription. @@ -93,9 +99,9 @@ pub struct Broadcaster { /// renditions themselves (see [`renditions::Producer::clear`]). Set once, right after /// construction. watcher: Mutex>>, - /// The broadcast's timeline watcher, spawned by the catalog watcher once the catalog - /// advertises the timeline track. On drop it is aborted only when no cursor still needs - /// its records (see `Drop`). + /// The reference rendition's timeline watcher, spawned by the catalog watcher once the + /// catalog advertises it. On drop it is aborted only when no cursor still needs its records + /// (see `Drop`). timeline_watcher: Arc>>>, } @@ -342,20 +348,35 @@ async fn watch_catalog( } }; - let mut timeline_started = false; + let mut unbound = false; + let mut reference: Option = None; loop { match kio::wait(|waiter| consumer.poll_next(waiter)).await { Ok(Some(catalog)) => { + // Decide the playlists' retention before any rendition starts following its timeline. + if !unbound && catalog.archive.as_ref().is_some_and(durable) { + unbound = true; + renditions.fanout().unbound(); + } renditions.sync(&upstream, &catalog); - // The broadcast has one timeline; subscribe once it's advertised and fan its - // records out to every rendition. - if !timeline_started && let Some(archive) = catalog.archive.clone() { - timeline_started = true; - if durable(&archive) { - renditions.fanout().unbound(); + + // Follow the reference rendition's timeline; a new reference renumbers every segment. + let next = self::reference(&catalog); + if next != reference { + if let Some(watcher) = timeline_watcher.lock().unwrap().take() { + watcher.abort(); + renditions.fanout().skip(); } - let watcher = tokio::spawn(watch_timeline(broadcast.clone(), archive, renditions.fanout())); - *timeline_watcher.lock().unwrap() = Some(watcher); + if let (Some(next), Some(archive)) = (&next, catalog.archive.clone()) { + let watcher = tokio::spawn(watch_timeline( + broadcast.clone(), + archive, + next.clone(), + renditions.fanout(), + )); + *timeline_watcher.lock().unwrap() = Some(watcher); + } + reference = next; } } Ok(None) => break, @@ -370,6 +391,29 @@ async fn watch_catalog( renditions.close(); } +/// The rendition every segment boundary comes from: the first video rendition with a timeline, +/// or the first audio one when the broadcast has no video. Renditions are ordered by name, so +/// every edge picks the same one. +fn reference(catalog: &moq_mux::catalog::hang::Catalog) -> Option { + let archive = catalog.archive.as_ref()?; + let indexed = |name: &&String| archive.timelines.contains_key(*name); + let video = catalog + .video + .renditions + .keys() + .find(indexed) + .map(|name| (Kind::Video, name)); + let audio = || { + catalog + .audio + .renditions + .keys() + .find(indexed) + .map(|name| (Kind::Audio, name)) + }; + video.or_else(audio).map(|(kind, name)| Arc::new((kind, name.clone()))) +} + /// Whether every range `archive` advertises stays FETCHable from this broadcast until the /// timeline pops it: a store makes the ranges durable, and no `replay` path means this /// broadcast serves them. The catalog states this, so the playlists follow the timeline's own @@ -378,21 +422,22 @@ fn durable(archive: &hang::catalog::Archive) -> bool { archive.store.is_some() && archive.replay.is_none() } -/// The broadcast's timeline watcher: read the single timeline track and fan each record out -/// to every rendition's window. +/// The segment watcher: read the reference rendition's timeline and fan each record out to every +/// rendition's window as a segment. async fn watch_timeline( broadcast: moq_net::broadcast::Consumer, section: hang::catalog::Archive, + reference: Reference, renditions: renditions::Fanout, ) { - match watch(&broadcast, §ion, &renditions).await { + match watch(&broadcast, §ion, &reference, &renditions).await { // The timeline finished cleanly: the publisher is done, so every window can end // (ENDLIST) and recording cursors drain to completion. Ok(()) => renditions.end_windows(), // A transient error (subscription reset, relay hiccup): don't mark the windows ended; // the serve path keeps serving the frozen windows. Err(err) => { - tracing::warn!(track = %section.track, %err, "timeline watcher error; leaving the playlists live") + tracing::warn!(track = %reference.1, %err, "timeline watcher error; leaving the playlists live") } } // The timeline stream is over either way: close so recording cursors terminate instead of @@ -403,12 +448,13 @@ async fn watch_timeline( async fn watch( broadcast: &moq_net::broadcast::Consumer, section: &hang::catalog::Archive, + reference: &Reference, renditions: &renditions::Fanout, ) -> crate::Result<()> { - let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(broadcast, section).await?; + let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(broadcast, section, &reference.1).await?; while let Some(event) = timeline.next().await? { match event { - moq_mux::timeline::Event::Push { index, entry } => renditions.push(index, entry), + moq_mux::timeline::Event::Push { index, entry } => renditions.push(index, entry, reference), moq_mux::timeline::Event::Pop(range) => renditions.pop(range), moq_mux::timeline::Event::Skip(_) => renditions.skip(), _ => unreachable!("unknown timeline event"), @@ -528,10 +574,11 @@ mod tests { impl CatalogTestExt for moq_mux::catalog::Producer { fn enroll_test(&self, track: &str) -> moq_mux::Result { - let recorder = self.timeline().pacing_track(track)?; + let recorder = self.timeline().track(track, moq_mux::timeline::Config::default())?; + let section = self.timeline().section(); let mut producer = self.clone(); let mut catalog = producer.modify()?; - catalog.archive.get_or_insert_with(|| self.timeline().section()); + catalog.archive = Some(section); catalog.commit()?; Ok(recorder) } @@ -549,6 +596,17 @@ mod tests { } } + /// A catalog `archive` entry indexing `tracks` under their conventional timeline names. + fn archive(tracks: &[&str]) -> hang::catalog::Archive { + let mut archive = hang::catalog::Archive::new(); + for track in tracks { + archive + .timelines + .insert(track.to_string(), hang::timeline::default_name(track)); + } + archive + } + fn frame(micros: u64, keyframe: bool) -> moq_mux::container::Frame { payload_frame(micros, keyframe, &[0xDE, 0xAD, 0xBE, 0xEF]) } @@ -606,7 +664,7 @@ mod tests { let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); config.broadcast = Some(moq_net::path::Relative::new("../../source").to_owned()); let mut catalog = moq_mux::catalog::hang::Catalog::default(); - catalog.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + catalog.archive = Some(archive(&["video0", "audio0"])); catalog.video.renditions.insert("video".to_string(), config); let renditions = renditions::Producer::new(Config::default().window); @@ -634,7 +692,7 @@ mod tests { audio.bitrate = Some(96_000); let mut catalog = moq_mux::catalog::hang::Catalog::default(); - catalog.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + catalog.archive = Some(archive(&["video0", "audio0"])); catalog.video.renditions.insert("video0".to_string(), video_config()); catalog.audio.renditions.insert("audio0".to_string(), audio); catalog @@ -1445,11 +1503,152 @@ mod tests { // The property HLS needs from the timeline rework: audio and video renditions share one // segment numbering, cut at the same boundaries. Video is one group per segment; an audio // segment packs every audio group inside the video segment's span. - // Most publishers declare no durationMax (a real-time GOP can be minutes long), so the - // exporter has to derive EXT-X-TARGETDURATION from the segments it has: a playlist whose - // EXTINF exceeds its target duration is invalid. + // Every rendition has its own timeline. The reference (the first video rendition by name) sets + // the segment boundaries; another rendition snaps them to its own keyframes, and a boundary with + // no keyframe nearby, such as before a rendition starts, is a gap. #[tokio::test] - async fn target_duration_covers_the_window_without_a_declared_bound() { + async fn renditions_snap_to_their_own_keyframes() { + let origin = produce_origin(); + let mut broadcast = origin.create_broadcast("live").expect("publish allowed"); + broadcast.announce(Default::default()).expect("publish allowed"); + settle().await; + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + + let reserved = catalog.reserve(); + let mut registrations = Vec::new(); + for name in ["video0", "video1", "video2"] { + let mut registration = catalog.test_video(name).unwrap(); + registration.set(video_config()).unwrap(); + registrations.push(registration); + } + drop(reserved); + + let mut producers = Vec::new(); + // The reference cuts every 2s; video1's GOPs sit 300ms later; video2 starts at 4s. + for (name, gops) in [ + ("video0", &[0u64, 2_000, 4_000, 6_000, 8_000][..]), + ("video1", &[300, 2_300, 4_300, 6_300, 8_300]), + ("video2", &[4_000, 6_000, 8_000]), + ] { + let track = broadcast.create_track(name, None).unwrap(); + let mut media = catalog + .media_producer( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), + ) + .unwrap(); + for ms in gops { + media.write(frame(ms * 1_000, true)).unwrap(); + } + producers.push(media); + } + + let source = moq_mux::Source::new(origin.consume(), "live"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + let listed = |name: &'static str, count: usize| { + let broadcaster = broadcaster.clone(); + async move { + for _ in 0..500 { + if let Some(rendition) = broadcaster.rendition(Kind::Video, name) { + let snapshot = rendition.snapshot(); + if snapshot.segments.len() >= count { + return snapshot; + } + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} never listed {count} segments"); + } + }; + + let reference = listed("video0", 4).await; + assert_eq!( + reference.segments.iter().map(|s| s.segment).collect::>(), + vec![0, 1, 2, 3] + ); + + // A segment is listed once the rendition's own timeline is a tolerance past its end. + let offset = listed("video1", 3).await; + assert_eq!(offset.media_sequence, reference.media_sequence, "one numbering"); + assert!(offset.segments.iter().all(|s| !s.gap), "every boundary snaps"); + let snapped = broadcaster + .rendition(Kind::Video, "video1") + .unwrap() + .segment(1) + .await + .unwrap() + .expect("the snapped segment is served"); + assert_eq!(&snapped[4..8], b"moof"); + + let late = listed("video2", 3).await; + assert_eq!( + late.segments.iter().map(|s| s.gap).collect::>(), + vec![true, true, false], + "nothing to serve before the rendition starts" + ); + let video2 = broadcaster.rendition(Kind::Video, "video2").unwrap(); + assert!(video2.segment(0).await.unwrap().is_none(), "a gap is never fetched"); + + drop((producers, registrations, broadcast)); + } + + // A broadcast without video takes its boundaries from its first audio rendition. + #[tokio::test] + async fn an_audio_only_broadcast_segments_on_its_audio() { + let origin = produce_origin(); + let mut broadcast = origin.create_broadcast("live").expect("publish allowed"); + broadcast.announce(Default::default()).expect("publish allowed"); + settle().await; + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + + let reserved = catalog.reserve(); + let mut registration = catalog.test_audio("audio0").unwrap(); + registration + .set(hang::catalog::AudioConfig::new( + hang::catalog::AudioCodec::Opus, + 48_000, + 2, + )) + .unwrap(); + drop(reserved); + + let track = broadcast.create_track("audio0", None).unwrap(); + let mut audio = catalog + .media_producer( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), + ) + .unwrap(); + // 500ms groups pack into records of the default 2s minimum. + for micros in (0..=6_000_000u64).step_by(500_000) { + let keyframe = audio.needs_keyframe(); + audio.write(frame(micros, keyframe)).unwrap(); + audio.cut(None).unwrap(); + } + + let source = moq_mux::Source::new(origin.consume(), "live"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; + let rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); + let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; + + let playlist = rendition.snapshot(); + assert_eq!( + playlist.segments.iter().map(|s| s.segment).collect::>(), + vec![0, 1, 2] + ); + assert_eq!(playlist.segments[0].duration, Duration::from_secs(2)); + let segment = rendition.segment(1).await.unwrap().expect("audio segment"); + assert_eq!(&segment[4..8], b"moof"); + + drop((audio, registration, broadcast)); + } + + // The catalog's durationMax is the record split ceiling, far above a GOP, so the exporter + // derives EXT-X-TARGETDURATION from the segments it has: a playlist whose EXTINF exceeds its + // target duration is invalid, and one far above them delays every player. + #[tokio::test] + async fn target_duration_follows_the_observed_segments() { let origin = produce_origin(); let mut broadcast = origin.create_broadcast("live").expect("publish allowed"); broadcast.announce(Default::default()).expect("publish allowed"); @@ -1485,7 +1684,7 @@ mod tests { assert_eq!(playlist.segments[0].duration, Duration::from_secs(3)); assert_eq!( playlist.target_duration, 3, - "no bound was declared, so the target duration must still cover the 3s segment" + "the target duration covers the 3s segment, not the 10s split ceiling" ); } @@ -1655,12 +1854,18 @@ mod tests { ) -> (Arc, tokio::task::JoinHandle<()>) { let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video0".to_string(), config.clone()); - let archive = hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME); + let archive = archive(&["video0", "audio0"]); catalog.archive = Some(archive.clone()); let renditions = renditions::Producer::new(Config::default().window); renditions.sync(upstream, &catalog); - let watcher = tokio::spawn(watch_timeline(upstream.broadcast.clone(), archive, renditions.fanout())); + let reference = Arc::new((Kind::Video, "video0".to_string())); + let watcher = tokio::spawn(watch_timeline( + upstream.broadcast.clone(), + archive, + reference, + renditions.fanout(), + )); let rendition = renditions.get(Kind::Video, "video0").expect("rendition synced"); (rendition, watcher) } @@ -2172,7 +2377,7 @@ mod tests { let media = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video".to_string(), media); - catalog.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + catalog.archive = Some(archive(&["video0", "audio0"])); renditions.sync(&upstream, &catalog); let rendition = renditions.get(Kind::Video, "video").expect("rendition synced"); @@ -2180,7 +2385,7 @@ mod tests { // The catalog drops the rendition: its cursor must run dry rather than park. let mut empty = moq_mux::catalog::hang::Catalog::default(); - empty.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + empty.archive = Some(archive(&["video0", "audio0"])); renditions.sync(&upstream, &empty); let ended = tokio::time::timeout(Duration::from_secs(5), segments.next()) .await diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 2107d5d548..34da1e0556 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -1,5 +1,5 @@ -//! One rendition: playlists from its view of the broadcast timeline, segments fetched on -//! demand. +//! One rendition: playlists from its view of the broadcast's segments, resolved against its own +//! timeline, and segments fetched on demand. use std::sync::{Arc, Mutex, RwLock}; use std::task::Poll; @@ -12,6 +12,7 @@ use moq_mux::timeline::Entry; use sha2::{Digest, Sha256}; use super::playlist::{Segment, Snapshot}; +use super::spans::{Content, Spans}; use super::upstream::Upstream; use super::{mpd, segments}; use crate::Result; @@ -231,9 +232,15 @@ pub struct Rendition { /// The catalog's root broadcast clock: the wall anchor timings map through, after /// timescale conversion. Absent when the publisher exposes none. clock: Option, - /// This rendition's window over the broadcast timeline, fed by the catalog watcher's + /// This rendition's window over the broadcast's segments, fed by the catalog watcher's /// fan-out. live: Arc, + /// This rendition's own timeline, which resolves each segment to its frames. + spans: Arc, + /// The broadcast carrying the catalog and every timeline. + catalog: moq_net::broadcast::Consumer, + /// Feeds [`spans`](Self::spans); aborted when the rendition drops. + watcher: Mutex>>, /// The broadcast serving this rendition's media. A sibling is bound when the rendition is /// created and rebound if that publisher is replaced (see [`Upstream::bind`]). media: Media, @@ -311,6 +318,9 @@ impl Rendition { section, clock, live: Arc::new(segments::Producer::new()), + spans: Arc::new(Spans::new()), + catalog: upstream.broadcast.clone(), + watcher: Mutex::new(None), media: Media::bind(upstream, config.broadcast.as_ref())?, run: Mutex::default(), building: tokio::sync::Mutex::new(()), @@ -337,31 +347,87 @@ impl Rendition { section, clock, live: Arc::new(segments::Producer::new()), + spans: Arc::new(Spans::new()), + catalog: upstream.broadcast.clone(), + watcher: Mutex::new(None), media: Media::bind(upstream, config.broadcast.as_ref())?, run: Mutex::default(), building: tokio::sync::Mutex::new(()), }) } - /// Feed one timeline record into this rendition's window: its own ranges (empty when the - /// record carries none for it, a gap), timed by the record. With no `window`, only source - /// timeline pops trim it. - pub(crate) fn push(&self, index: u64, entry: &Entry, discontinuity: u64, window: Option) { + /// Start following this rendition's own timeline, trimmed to `window` like the segments. + /// + /// A rendition the catalog indexes no timeline for resolves nothing but the reference's rows. + pub(crate) fn watch(&self, window: Option) { + let Some(_) = self.section.timelines.get(&self.name) else { + self.spans.end(); + return; + }; + let watcher = tokio::spawn(watch_spans( + self.catalog.clone(), + self.section.clone(), + self.name.clone(), + self.spans.clone(), + window, + )); + *self.watcher.lock().expect("watcher lock poisoned") = Some(watcher); + } + + /// Feed one reference timeline record into this rendition's window as a segment. With no + /// `window`, only source timeline pops trim it. + pub(crate) fn push( + &self, + index: u64, + entry: &Entry, + reference: &segments::Reference, + discontinuity: u64, + window: Option, + ) { if !self.media.admits(&self.live) { return; } let row = segments::Row { index, - segment: entry.segment, - ranges: entry.tracks.get(&self.name).cloned().unwrap_or_default(), + segment: entry.sequence, + reference: reference.clone(), + frames: entry.start..entry.end, + keyframe: entry.keyframe, duration: entry.duration, pts: entry.pts, - end: Duration::from(entry.pts) + entry.duration, + end: entry.end_time(), discontinuity, }; self.live.push(row, window); } + /// What `row` holds on this rendition: the reference serves its own record, and every other + /// rendition resolves the span against its own timeline. + fn resolve(&self, row: &segments::Row) -> Content { + if self.is(&row.reference) { + return Content::Frames { + ranges: vec![row.frames.clone()], + filter: None, + }; + } + self.spans + .resolve(self.kind == Kind::Video, Duration::from(row.pts)..row.end) + } + + /// Poll until `row` resolves on this rendition. + pub(crate) fn poll_resolved(&self, waiter: &kio::Waiter, row: &segments::Row) -> Poll<()> { + if self.is(&row.reference) { + return Poll::Ready(()); + } + self.spans + .poll_resolved(waiter, self.kind == Kind::Video, Duration::from(row.pts)..row.end) + } + + /// Whether this rendition is `reference`. + fn is(&self, reference: &(Kind, String)) -> bool { + reference.0 == self.kind && reference.1 == self.name + } + /// Remove source timeline records in `range` from this rendition's window. pub(crate) fn pop(&self, range: std::ops::Range) { self.live.pop(range); @@ -483,39 +549,28 @@ impl Rendition { self.media.sync(&self.live); let window = self.live.window(); - // EXT-X-TARGETDURATION starts from the catalog's declared bound when the publisher - // offered one: it knows its segment duration up front, so the first playlist already - // carries the real value instead of a guess grown from observation. Rounded up, since - // HLS wants whole seconds and the advertised value must never understate the bound. - let declared = self - .section - .duration_max - .map(|max| max.div_ceil((self.section.timescale as u64).max(1))) - .unwrap_or(0); - - // Most publishers can't promise a bound (a real-time GOP can be minutes long), and even - // a declared one is only a promise about content the publisher controls. HLS requires - // EXT-X-TARGETDURATION to be at least every EXTINF, so cover the window either way. - let observed = window + // HLS requires EXT-X-TARGETDURATION to be at least every EXTINF, rounded up to whole + // seconds. The catalog's `durationMax` is only the ceiling a record is split at, far above a + // typical GOP, and a target that large would push every player's live edge back by it. + let target_duration = window .segments .iter() .map(|s| s.duration.as_secs_f64().ceil() as u64) .max() - .unwrap_or(0); - let target_duration = declared.max(observed).max(1); + .unwrap_or(0) + .max(1); let program_date_time = window.segments.first().and_then(|first| self.wall_clock(first.pts)); let mut previous: Option = None; - let segments = window - .segments - .into_iter() - .map(|s| { + let segments = self + .resolved(&window.segments) + .map(|(s, content)| { let discontinuity = previous.replace(s.discontinuity).is_some_and(|p| p != s.discontinuity); Segment { segment: s.segment, duration: s.duration, - gap: s.ranges.is_empty(), + gap: content == Content::Gap, discontinuity, } }) @@ -531,6 +586,14 @@ impl Rendition { } } + /// `rows` with what each holds, up to the first one this rendition cannot resolve yet: a + /// segment is listed only once its content is final. + fn resolved<'a>(&'a self, rows: &'a [segments::Row]) -> impl Iterator { + rows.iter() + .map(|row| (row, self.resolve(row))) + .take_while(|(_, content)| *content != Content::Pending) + } + /// Whether the playlist has anything to serve yet (at least one segment, or the broadcast /// already ended). pub(crate) fn is_playable(&self) -> bool { @@ -549,11 +612,10 @@ impl Rendition { self.media.sync(&self.live); let window = self.live.window(); let timescale = self.timescale(); - let segments = window - .segments - .iter() - .filter(|row| !row.ranges.is_empty()) - .filter_map(|row| { + let segments = self + .resolved(&window.segments) + .filter(|(_, content)| *content != Content::Gap) + .filter_map(|(row, _)| { let t = row.pts.as_scale(timescale) as u64; let span = row.end.saturating_sub(row.pts.into()); let d = (span.as_nanos() * timescale.as_u64() as u128 / 1_000_000_000) as u64; @@ -696,13 +758,16 @@ impl Rendition { let bytes = match muxer.init()? { Some(bytes) => bytes, None => { - let Some(sequence) = self.live.latest_keyframe_group() else { + let reference = (self.kind, self.name.clone()); + let latest = self.spans.latest_keyframe_group(); + let Some(sequence) = latest.or_else(|| self.live.latest_keyframe_group(&reference)) else { return Ok(None); }; let Some((broadcast, track)) = self.track(&binding).await else { return Ok(None); }; - let Ok(mut group) = fetch(&track, sequence, &broadcast).await? else { + let Ok(mut group) = fetch(&track, hang::timeline::Position::group(sequence), None, &broadcast).await? + else { return Ok(None); }; // A cache eviction mid-read leaves the init unbuildable for now, not an error. @@ -720,26 +785,31 @@ impl Rendition { /// Fetch and transmux the segment numbered `segment`. /// - /// Fetches every group the segment's ranges cover (an audio segment packs many short - /// groups) and encodes them as a single CMAF fragment. `None` when the segment isn't in - /// the playlist window, is a gap for this rendition, or its groups already left the relay - /// cache. + /// Fetches every frame the segment resolves to on this rendition (an audio segment packs many + /// short groups) and encodes them as a single CMAF fragment. `None` when the segment isn't in + /// the playlist window, is a gap or not yet resolved for this rendition, or its groups already + /// left the relay cache. pub async fn segment(&self, segment: u64) -> Result> { - let binding = self.media.sync(&self.live); - let Some(ranges) = self.live.segment_ranges(segment) else { + self.media.sync(&self.live); + let Some(row) = self.live.row(segment) else { return Ok(None); }; - if ranges.is_empty() { - // A gap: the record says this rendition has no content for the span. + self.fetch(&row).await + } + + /// Fetch and transmux `row`'s frames on this rendition; see [`segment`](Self::segment). + pub(crate) async fn fetch(&self, row: &segments::Row) -> Result> { + let binding = self.media.sync(&self.live); + let Content::Frames { ranges, filter } = self.resolve(row) else { return Ok(None); - } + }; // A segment must be served whole. If its group span exceeds the safety cap (a // pathologically coarse timeline), 404 rather than silently truncating it to a segment // shorter than its advertised duration. let total: u64 = ranges .iter() - .map(|r| r.end.saturating_sub(r.start).saturating_add(1)) + .map(|range| last_group(range).saturating_sub(range.start.group).saturating_add(1)) .sum(); if total > MAX_SEGMENT_GROUPS { tracing::warn!( @@ -760,8 +830,13 @@ impl Rendition { // many groups. let mut frames = Vec::new(); for range in &ranges { - for sequence in range.start..=range.end { - let miss = match fetch(&track, sequence, &broadcast).await? { + for sequence in range.start.group..=last_group(range) { + let start = match sequence == range.start.group { + true => range.start, + false => hang::timeline::Position::group(sequence), + }; + let end = (sequence == range.end.group).then_some(range.end.frame); + let miss = match fetch(&track, start, end, &broadcast).await? { Ok(mut group) => match read_group(&mut muxer, &mut group, &broadcast).await? { Ok(mut group_frames) => { frames.append(&mut group_frames); @@ -772,7 +847,7 @@ impl Rendition { Err(err) => err, }; tracing::warn!( - segment, + segment = row.segment, group = sequence, err = %miss, "abandoning an advertised segment: a group is unavailable" @@ -781,22 +856,78 @@ impl Rendition { } } + // A rendition resolved by time keeps only the frames inside the segment's span. + if let Some(filter) = filter { + frames.retain(|frame| filter.contains(&Duration::from(frame.timestamp))); + } if frames.is_empty() { return Ok(None); } - Ok(Some(muxer.fragment(segment as u32, &frames)?)) + Ok(Some(muxer.fragment(row.segment as u32, &frames)?)) + } +} + +/// The last group holding a frame of `range`. +fn last_group(range: &std::ops::Range) -> u64 { + match range.end.frame { + 0 => range.end.group.saturating_sub(1), + _ => range.end.group, + } +} + +/// Follow `track`'s own timeline into `spans` until it ends, then mark nothing pending. +async fn watch_spans( + broadcast: moq_net::broadcast::Consumer, + section: Archive, + track: String, + spans: Arc, + window: Option, +) { + let result: Result<()> = async { + let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&broadcast, §ion, &track).await?; + while let Some(event) = timeline.next().await? { + match event { + moq_mux::timeline::Event::Push { entry, .. } => spans.push(entry, window), + moq_mux::timeline::Event::Pop(range) => spans.pop(range), + moq_mux::timeline::Event::Skip(_) => spans.clear(), + _ => unreachable!("unknown timeline event"), + } + } + Ok(()) + } + .await; + if let Err(err) = result { + tracing::warn!(%track, %err, "rendition timeline ended with an error"); + } + spans.end(); +} + +impl Drop for Rendition { + fn drop(&mut self) { + if let Some(watcher) = self.watcher.lock().expect("watcher lock poisoned").take() { + watcher.abort(); + } } } /// Fetch one group, mapping "no longer (or not yet) servable" to the error that says so /// rather than a bare `None`: a caller abandoning a segment can only explain itself with it. +/// +/// Reads from `start` and, when `end` is set, stops before that frame of the group. async fn fetch( track: &moq_net::track::Consumer, - sequence: u64, + start: hang::timeline::Position, + end: Option, broadcast: &moq_net::broadcast::Consumer, ) -> Result> { - match track.fetch_group(sequence, None).await { - Ok(group) => Ok(Ok(group)), + let options = moq_net::group::Fetch::default().with_frame_start(start.frame); + match track.fetch_group(start.group, options).await { + Ok(mut group) => { + if let Some(end) = end { + group.set_frames(..end); + } + Ok(Ok(group)) + } Err(err) if is_unservable(&err, broadcast) => Ok(Err(err)), Err(err) => Err(err.into()), } diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index ab66923500..8c5900b809 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -2,8 +2,9 @@ //! //! The `Producer` holds the current renditions, reconciled from the catalog by its `sync`; //! the HTTP serve path reads them synchronously (look one up, render the master playlist). It -//! is also the fan-out point for the broadcast's single timeline: the timeline watcher's -//! `Fanout` handle hands each record to every rendition, which keeps its own window over it. +//! is also the fan-out point for the broadcast's segments: the reference rendition's timeline +//! watcher hands each record to every rendition through the `Fanout` handle, and each rendition +//! keeps its own window over them. //! A [`Consumer`] is a cursor for a recorder that mirrors the *whole* broadcast: [`next`](Consumer::next) //! yields one [`Event`] at a time as renditions are added or removed, replaying the current //! set as [`Added`](Event::Added) when first consumed, and returning `None` once the source @@ -19,7 +20,7 @@ use moq_mux::timeline::Entry; use super::Upstream; use super::rendition::{Kind, Rendition}; -use super::segments::Discontinuities; +use super::segments::{Discontinuities, Reference}; /// The `(kind, name)` identity of a rendition. Video and audio are separate axes, so a video /// and an audio rendition may share a name without colliding. @@ -47,10 +48,10 @@ pub enum Event { struct Feed { /// Every rendition ever created and still alive, pruned as they drop. targets: Vec>, - /// Recent records with their discontinuity sequence, replayed into a rendition created - /// mid-broadcast so its playlist window isn't empty until the next record. Evicted with the - /// same policy as the windows. - history: VecDeque<(u64, Entry, u64)>, + /// Recent reference records with their discontinuity sequence, replayed into a rendition + /// created mid-broadcast so its playlist window isn't empty until the next record. Evicted with + /// the same policy as the windows. + history: VecDeque<(u64, Entry, Reference, u64)>, /// Stamps each record once, so every rendition agrees on where the timeline breaks. discontinuities: Discontinuities, /// The timeline ended cleanly; late-created renditions start ended (`EXT-X-ENDLIST`). @@ -193,14 +194,15 @@ impl Fanout { self.feed.lock().unwrap().window = None; } - /// Fan one timeline record out to every living rendition, and into the replay history. - pub fn push(&self, index: u64, entry: Entry) { + /// Fan one reference timeline record out to every living rendition, and into the replay + /// history. + pub fn push(&self, index: u64, entry: Entry, reference: &Reference) { let mut feed = self.feed.lock().unwrap(); // Same eviction policy as the per-rendition windows, so a replay reconstructs the // same window a live rendition would have. - if let Some((_, back, _)) = feed.history.back() - && (Duration::from(entry.pts) < Duration::from(back.pts) || entry.segment <= back.segment) + if let Some((_, back, _, _)) = feed.history.back() + && (Duration::from(entry.pts) < Duration::from(back.pts) || entry.sequence <= back.sequence) { feed.history.clear(); feed.anchor = None; @@ -213,7 +215,8 @@ impl Fanout { } let pts = Duration::from(entry.pts); let discontinuity = feed.discontinuities.stamp(pts, pts + entry.duration); - feed.history.push_back((index, entry.clone(), discontinuity)); + feed.history + .push_back((index, entry.clone(), reference.clone(), discontinuity)); let window = feed.window; while let Some(window) = window && feed.history.len() >= 2 @@ -231,7 +234,7 @@ impl Fanout { let Some(rendition) = target.upgrade() else { return false; }; - rendition.push(index, &entry, discontinuity, window); + rendition.push(index, &entry, reference, discontinuity, window); true }); } @@ -239,7 +242,7 @@ impl Fanout { /// Remove records that left the source timeline window from every rendition window. pub fn pop(&self, range: std::ops::Range) { let mut feed = self.feed.lock().unwrap(); - feed.history.retain(|(index, _, _)| !range.contains(index)); + feed.history.retain(|(index, ..)| !range.contains(index)); feed.targets.retain(|target| { let Some(rendition) = target.upgrade() else { return false; @@ -341,12 +344,14 @@ impl Producer { } /// Enroll a freshly-created rendition in the timeline feed, replaying the recent history - /// (and the ended/closed markers) so its window matches its siblings'. + /// (and the ended/closed markers) so its window matches its siblings', and start following its + /// own timeline. fn register(&self, rendition: &Arc) { let mut feed = self.fanout.feed.lock().unwrap(); rendition.label(feed.generation.clone()); - for (index, entry, discontinuity) in &feed.history { - rendition.push(*index, entry, *discontinuity, feed.window); + rendition.watch(feed.window); + for (index, entry, reference, discontinuity) in &feed.history { + rendition.push(*index, entry, reference, *discontinuity, feed.window); } if feed.ended { rendition.end(); @@ -365,9 +370,9 @@ impl Producer { /// new advertised bitrate, since the publisher republishes the catalog every time its /// measured bitrate or jitter moves. /// - /// Renditions are only servable when the catalog advertises the broadcast's timeline (its - /// root `archive` entry): without one there is nothing to render playlists from, so the - /// whole catalog is skipped with a warning. + /// Renditions are only servable when the catalog advertises its timelines (its root `archive` + /// entry): without one there is nothing to render playlists from, so the whole catalog is + /// skipped with a warning. /// /// `upstream` carries the broadcast the snapshot was read from, so each rendition serves media /// from that same broadcast rather than whatever is at the path by the time it is asked. A diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 6b8a92a0b6..1437090c0e 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -1,21 +1,20 @@ -//! One rendition's view of the broadcast timeline, as a `Producer`/[`Consumer`] pair. +//! One rendition's view of the broadcast's segments, as a `Producer`/[`Consumer`] pair. //! -//! The broadcast has a single timeline track; the catalog watcher reads it and fans each -//! record out to every rendition as a row: the segment's number, timing, and this -//! rendition's group ranges (empty when the record carries no content for it, a gap). Records -//! are self-contained (the timeline only publishes a segment once its content is final on -//! every track), so every row is immediately listable and fetchable. Two things read the -//! window: +//! Segments are derived from one reference rendition's timeline: each of its records is a +//! segment, numbered by the record's sequence. The catalog watcher reads that timeline and fans +//! each record out to every rendition as a row: the segment's number and timing, plus the +//! reference record's frames. Each rendition resolves a row against its own timeline. Two things +//! read the window: //! -//! * the HTTP serve path, synchronously, to render a media playlist and look up a segment's -//! group ranges (nothing here touches media bytes on that path); and +//! * the HTTP serve path, synchronously, to render a media playlist and resolve a segment's +//! frames (nothing here touches media bytes on that path); and //! * a [`Consumer`] cursor, for a recorder that wants every segment *with its media*, in -//! order, exactly once. `next()` waits for the next row, FETCHes and transmuxes its groups -//! (via [`Rendition`]), and yields the CMAF bytes. +//! order, exactly once. `next()` waits for the next resolved row, FETCHes and transmuxes its +//! frames (via [`Rendition`]), and yields the CMAF bytes. //! -//! Rows carry the broadcast's aligned segment numbers, so a segment is addressed by that -//! number everywhere (the `seg/{segment}.m4s` URI, `EXT-X-MEDIA-SEQUENCE`, the recorder -//! cursor), and the same number names the same span of content time on every rendition. +//! A segment is addressed by its number everywhere (the `seg/{segment}.m4s` URI, +//! `EXT-X-MEDIA-SEQUENCE`, the recorder cursor), and the same number names the same span of +//! content time on every rendition, on every edge, and after every reload. use std::collections::VecDeque; use std::sync::Arc; @@ -23,9 +22,8 @@ use std::task::Poll; use std::time::{Duration, SystemTime}; use bytes::Bytes; -use hang::timeline::Range; -use super::Rendition; +use super::{Kind, Rendition}; use crate::Result; /// The producing side of a rendition's timeline window. @@ -81,16 +79,22 @@ impl Discontinuities { } } -/// One playlist segment: its aligned number, timing, and this rendition's group ranges. +/// The rendition a broadcast's segment boundaries come from. +pub(crate) type Reference = Arc<(Kind, String)>; + +/// One playlist segment: its number, timing, and the reference record's frames. #[derive(Clone, Debug, PartialEq)] pub(crate) struct Row { - /// The source timeline record index, used to mirror exact window trims. + /// The reference timeline's record index, used to mirror exact window trims. pub index: u64, - /// The aligned segment number (its URI: `seg/{segment}.m4s`), shared across renditions. + /// The segment number (its URI: `seg/{segment}.m4s`), shared across renditions. pub segment: u64, - /// This rendition's group ranges within the segment. Empty means the rendition has no - /// content for the span (`EXT-X-GAP`). - pub ranges: Vec, + /// The rendition the boundaries come from. + pub reference: Reference, + /// The reference record's frames, which the reference rendition serves as is. + pub frames: std::ops::Range, + /// Whether the reference record starts on a keyframe. + pub keyframe: bool, /// Presentation duration. pub duration: Duration, /// The segment's starting presentation timestamp. @@ -247,12 +251,10 @@ impl Producer { self.state.read().window() } - /// The group ranges segment `segment` covers for this rendition, or `None` if it isn't in - /// the window. An empty vec means the segment is a gap for this rendition. - pub fn segment_ranges(&self, segment: u64) -> Option> { + /// Segment `segment`'s row, or `None` if it isn't in the window. + pub fn row(&self, segment: u64) -> Option { let state = self.state.read(); - let row = state.rows.iter().find(|r| r.segment == segment)?; - Some(row.ranges.clone()) + state.rows.iter().find(|r| r.segment == segment).cloned() } /// The number of the segment whose `pts` is exactly `time` in the timeline's timescale @@ -269,17 +271,17 @@ impl Producer { .map(|row| row.segment) } - /// The newest group known to start with a keyframe, used to bootstrap an init segment for - /// inline-parameter-set codecs. - pub fn latest_keyframe_group(&self) -> Option { + /// The newest group of `reference` a row starts on a keyframe, used to bootstrap an init + /// segment for inline-parameter-set codecs. + pub fn latest_keyframe_group(&self, reference: &(Kind, String)) -> Option { let state = self.state.read(); state .rows .iter() .rev() - .flat_map(|row| row.ranges.iter().rev()) - .find(|range| range.keyframe) - .map(|range| range.start) + .filter(|row| *row.reference == *reference) + .find(|row| row.keyframe && row.frames.start.frame == 0) + .map(|row| row.frames.start.group) } /// Whether the playlist has anything to serve yet (at least one segment, or the broadcast @@ -381,16 +383,17 @@ impl Consumer { /// The next segment, with its media; `None` once the rendition ends. /// - /// Waits for the next segment, then FETCHes and transmuxes its groups. A segment whose - /// groups already left the relay cache (or that is a gap for this rendition) is skipped, - /// resuming at the next one, rather than surfaced as an error; a real fetch/transmux - /// failure is returned, leaving the cursor to retry it on the next call. + /// Waits for the next segment to resolve on this rendition, then FETCHes and transmuxes its + /// frames. A segment whose groups already left the relay cache (or that is a gap for this + /// rendition) is skipped, resuming at the next one, rather than surfaced as an error; a real + /// fetch/transmux failure is returned, leaving the cursor to retry it on the next call. pub async fn next(&mut self) -> Result> { loop { let Some(row) = kio::wait(|waiter| self.poll_next(waiter)).await else { return Ok(None); }; - match self.rendition.segment(row.segment).await? { + kio::wait(|waiter| self.rendition.poll_resolved(waiter, &row)).await; + match self.rendition.fetch(&row).await? { Some(media) => { return Ok(Some(Segment { segment: row.segment, @@ -431,7 +434,9 @@ mod tests { Row { index: segment, segment, - ranges: vec![Range::new(group, group)], + reference: Arc::new((Kind::Video, "video".to_string())), + frames: hang::timeline::Position::group(group)..hang::timeline::Position::group(group + 1), + keyframe: true, duration: Duration::from_millis(duration_ms), pts, end: Duration::from(pts) + Duration::from_millis(duration_ms), @@ -568,31 +573,30 @@ mod tests { } #[test] - fn segment_ranges_and_gaps() { + fn rows_and_the_keyframe_group() { let live = Producer::new(); let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 1_000), window); - // Segment 1 is a gap for this rendition: no ranges. live.push( Row { - index: 1, - segment: 1, - ranges: Vec::new(), - duration: Duration::from_secs(1), - pts: moq_net::Timestamp::from_millis(1_000).unwrap(), - end: Duration::from_millis(2_000), - discontinuity: 0, + keyframe: false, + ..row(1, 1, 1_000, 1_000) }, window, ); - live.push(row(2, 100, 2_000, 1_000), window); - - assert_eq!(live.segment_ranges(0), Some(vec![Range::new(0, 0)])); - assert_eq!(live.segment_ranges(1), Some(vec![]), "a gap is present but empty"); - assert_eq!(live.segment_ranges(7), None, "unknown segments miss"); - // The gap row carries no keyframe group; the bootstrap group comes from segment 2. - assert_eq!(live.latest_keyframe_group(), Some(100)); + assert_eq!( + live.row(0).unwrap().frames, + hang::timeline::Position::group(0)..hang::timeline::Position::group(1) + ); + assert_eq!(live.row(7), None, "unknown segments miss"); + let video = (Kind::Video, "video".to_string()); + assert_eq!( + live.latest_keyframe_group(&video), + Some(0), + "row 1 does not start on a keyframe" + ); + assert_eq!(live.latest_keyframe_group(&(Kind::Audio, "video".to_string())), None); } #[test] diff --git a/rs/moq-hls/src/export/spans.rs b/rs/moq-hls/src/export/spans.rs new file mode 100644 index 0000000000..769128a5e8 --- /dev/null +++ b/rs/moq-hls/src/export/spans.rs @@ -0,0 +1,359 @@ +//! One rendition's own timeline, which resolves the broadcast's segments to that rendition's frames. +//! +//! Segment boundaries come from the reference rendition's records (see [`super::segments`]). +//! Every other rendition maps each boundary onto its own timeline: +//! +//! * a video rendition snaps each boundary to its nearest record starting on a keyframe within +//! [`TOLERANCE`]; a segment with no start in range is a gap (HLS `EXT-X-GAP`), so a player +//! switching renditions lands on the next real segment; +//! * an audio rendition takes every frame whose timestamp falls in the segment's span, possibly +//! from more than one record. +//! +//! A segment resolves only once the rendition's timeline has reached past its end (or ended), so +//! every edge and every reload agree on its content. + +use std::collections::VecDeque; +use std::ops::Range; +use std::task::Poll; +use std::time::Duration; + +use hang::timeline::Position; +use moq_mux::timeline::Entry; + +/// How far a video boundary may move to land on a keyframe. +pub(crate) const TOLERANCE: Duration = Duration::from_secs(1); + +/// What one segment holds on one rendition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Content { + /// The rendition's timeline has not reached the segment's end yet. + Pending, + /// The rendition has no content for the segment. + Gap, + /// The frames to fetch, as position ranges, keeping only those whose timestamps fall in + /// `filter` when set. + Frames { + ranges: Vec>, + filter: Option>, + }, +} + +struct State { + /// Retained records, oldest first. + records: VecDeque, + /// No more records will arrive, so nothing is pending. + ended: bool, +} + +impl State { + fn resolve(&self, is_video: bool, span: Range) -> Content { + let records = self.records.iter().collect(); + match is_video { + true => video(records, self.ended, span), + false => audio(records, self.ended, span), + } + } +} + +/// A rendition's own record window, fed by its timeline watcher. +pub(crate) struct Spans { + state: kio::Producer, +} + +impl Spans { + pub fn new() -> Self { + Self { + state: kio::Producer::new(State { + records: VecDeque::new(), + ended: false, + }), + } + } + + /// Append a record, evicting records that end before `window` (plus the snapping tolerance) + /// of the newest content. With no `window`, only source pops remove records. + pub fn push(&self, entry: Entry, window: Option) { + let Ok(mut state) = self.state.write() else { + return; + }; + if state + .records + .back() + .is_some_and(|back| entry.sequence <= back.sequence || entry.pts < back.pts) + { + // The publisher restarted its timeline; the old records describe other media. + state.records.clear(); + } + state.records.push_back(entry); + let Some(window) = window else { + return; + }; + let newest = state.records.back().expect("just pushed").end_time(); + while state.records.len() >= 2 && state.records[0].end_time() + window + 2 * TOLERANCE < newest { + state.records.pop_front(); + } + } + + /// Remove the records with these sequences. + pub fn pop(&self, sequences: Range) { + if let Ok(mut state) = self.state.write() { + state.records.retain(|record| !sequences.contains(&record.sequence)); + } + } + + /// Forget every record after the source skipped some. + pub fn clear(&self) { + if let Ok(mut state) = self.state.write() { + state.records.clear(); + } + } + + /// No more records will arrive. + pub fn end(&self) { + if let Ok(mut state) = self.state.write() { + state.ended = true; + } + } + + /// Resolve the segment spanning `span`, snapping to keyframes when `video`. + pub fn resolve(&self, video: bool, span: Range) -> Content { + self.state.read().resolve(video, span) + } + + /// Poll until the segment spanning `span` stops being [`Content::Pending`]. + pub fn poll_resolved(&self, waiter: &kio::Waiter, video: bool, span: Range) -> Poll<()> { + let poll = self + .state + .poll_ref(waiter, |state| match state.resolve(video, span.clone()) { + Content::Pending => Poll::Pending, + _ => Poll::Ready(()), + }); + match poll { + Poll::Ready(_) => Poll::Ready(()), + Poll::Pending => Poll::Pending, + } + } + + /// The newest group known to start with a keyframe, used to bootstrap an init segment for + /// inline-parameter-set codecs. + pub fn latest_keyframe_group(&self) -> Option { + let state = self.state.read(); + state + .records + .iter() + .rev() + .find(|record| record.keyframe && record.start.frame == 0) + .map(|record| record.start.group) + } +} + +/// Snap `boundary` to the nearest keyframe record start within [`TOLERANCE`], preferring the +/// earlier one on a tie. +fn snap<'a>(candidates: &[&'a Entry], boundary: Duration) -> Option<&'a Entry> { + candidates + .iter() + .copied() + .filter(|record| Duration::from(record.pts).abs_diff(boundary) <= TOLERANCE) + .min_by_key(|record| Duration::from(record.pts).abs_diff(boundary)) +} + +/// Every record between two positions, merged into contiguous ranges. +fn ranges(records: &[&Entry], frames: Range) -> Vec> { + let mut out: Vec> = Vec::new(); + for record in records { + let start = record.start.max(frames.start); + let end = record.end.min(frames.end); + if start >= end { + continue; + } + match out.last_mut() { + Some(last) if last.end == start => last.end = end, + _ => out.push(start..end), + } + } + out +} + +fn video(records: Vec<&Entry>, ended: bool, span: Range) -> Content { + // A nearer keyframe can still arrive until the timeline passes the boundary plus the tolerance. + let reached = |time: Duration| ended || records.last().is_some_and(|last| last.end_time() >= time + TOLERANCE); + if !reached(span.end) { + return Content::Pending; + } + + let candidates: Vec<&Entry> = records + .iter() + .copied() + .filter(|record| record.keyframe && record.start.frame == 0) + .collect(); + let Some(first) = snap(&candidates, span.start) else { + return Content::Gap; + }; + // Without a keyframe near the end, run to the next one, so no frame falls between segments. + let end = match snap(&candidates, span.end) { + Some(record) => record.start, + None => match candidates.iter().find(|record| Duration::from(record.pts) > span.end) { + Some(record) => record.start, + None if ended => records.last().expect("a candidate exists").end, + None => return Content::Pending, + }, + }; + if first.start >= end { + return Content::Gap; + } + Content::Frames { + ranges: ranges(&records, first.start..end), + filter: None, + } +} + +fn audio(records: Vec<&Entry>, ended: bool, span: Range) -> Content { + if !ended && records.last().is_none_or(|last| last.end_time() < span.end) { + return Content::Pending; + } + let overlapping: Vec<&Entry> = records + .iter() + .copied() + .filter(|record| Duration::from(record.pts) < span.end && record.end_time() > span.start) + .collect(); + let (Some(first), Some(last)) = (overlapping.first(), overlapping.last()) else { + return Content::Gap; + }; + Content::Frames { + ranges: ranges(&overlapping, first.start..last.end), + filter: Some(span), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secs(s: f64) -> Duration { + Duration::from_secs_f64(s) + } + + fn record(sequence: u64, pts_ms: u64, duration_ms: u64, start: Position, end: Position) -> Entry { + Entry { + sequence, + pts: moq_net::Timestamp::from_millis(pts_ms).unwrap(), + duration: Duration::from_millis(duration_ms), + start, + end, + keyframe: start.frame == 0, + ext: (), + } + } + + /// One whole-group record per GOP, starting at `starts` (ms), the last one ending at `end`. + fn gops(starts: &[u64], end: u64) -> Vec { + starts + .iter() + .enumerate() + .map(|(i, &pts)| { + let next = starts.get(i + 1).copied().unwrap_or(end); + let group = i as u64; + record( + i as u64, + pts, + next - pts, + Position::group(group), + Position::group(group + 1), + ) + }) + .collect() + } + + fn frames(ranges: &[(Position, Position)], filter: Option>) -> Content { + Content::Frames { + ranges: ranges.iter().map(|&(start, end)| start..end).collect(), + filter, + } + } + + #[test] + fn video_snaps_each_boundary_to_the_nearest_gop() { + // GOPs 300ms off the reference's 2s boundaries. + let records = gops(&[300, 2_300, 4_300, 6_300], 8_300); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!( + video(refs.clone(), false, secs(2.0)..secs(4.0)), + frames(&[(Position::group(1), Position::group(2))], None) + ); + // The end boundary is not yet a tolerance behind the newest record. + assert_eq!(video(refs.clone(), false, secs(6.0)..secs(8.0)), Content::Pending); + assert_eq!( + video(refs, true, secs(6.0)..secs(8.0)), + frames(&[(Position::group(3), Position::group(4))], None) + ); + } + + #[test] + fn video_without_a_nearby_start_is_a_gap() { + // GOPs never coincide with the reference: 3.5s cadence against 2s boundaries. + let records = gops(&[0, 3_500, 7_000], 10_500); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!(video(refs.clone(), true, secs(2.0)..secs(4.0)), Content::Gap); + // The segment before the gap runs to the next GOP, so no frame is lost between them. + assert_eq!( + video(refs.clone(), true, secs(0.0)..secs(2.0)), + frames(&[(Position::group(0), Position::group(1))], None) + ); + assert_eq!( + video(refs, true, secs(4.0)..secs(6.0)), + frames(&[(Position::group(1), Position::group(2))], None) + ); + } + + #[test] + fn a_rendition_that_starts_later_is_a_gap_until_it_starts() { + let records = gops(&[4_000, 6_000], 8_000); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!(video(refs.clone(), false, secs(0.0)..secs(2.0)), Content::Gap); + assert_eq!( + video(refs, false, secs(4.0)..secs(6.0)), + frames(&[(Position::group(0), Position::group(1))], None) + ); + } + + #[test] + fn audio_takes_every_overlapping_record() { + // 1s audio records against 2.5s segments. + let records: Vec = (0..6) + .map(|i| record(i, i * 1_000, 1_000, Position::group(i * 2), Position::group(i * 2 + 2))) + .collect(); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!( + audio(refs.clone(), false, secs(2.5)..secs(5.0)), + frames(&[(Position::group(4), Position::group(10))], Some(secs(2.5)..secs(5.0))) + ); + assert_eq!(audio(refs.clone(), false, secs(5.0)..secs(7.5)), Content::Pending); + assert_eq!(audio(refs, true, secs(7.0)..secs(9.0)), Content::Gap); + } + + #[test] + fn a_frame_split_record_is_not_a_video_boundary() { + let records = [ + record(0, 0, 10_000, Position::group(0), Position::new(0, 300)), + record(1, 10_000, 2_000, Position::new(0, 300), Position::group(1)), + record(2, 12_000, 2_000, Position::group(1), Position::group(2)), + ]; + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!(video(refs.clone(), true, secs(10.0)..secs(12.0)), Content::Gap); + assert_eq!( + video(refs, true, secs(0.0)..secs(12.0)), + frames(&[(Position::group(0), Position::group(1))], None), + "the split's halves merge back into one range" + ); + } + + #[test] + fn the_window_keeps_records_a_snap_may_need() { + let spans = Spans::new(); + for entry in gops(&[0, 2_000, 4_000, 6_000, 8_000, 10_000], 12_000) { + spans.push(entry, Some(secs(4.0))); + } + let state = spans.state.read(); + assert_eq!(state.records.front().unwrap().sequence, 2); + } +} diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 165872e63d..6b9cb09c16 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -717,13 +717,6 @@ impl Import { async fn step(&mut self, on_error: OnError) -> Result { self.ensure_tracks().await?; - // Reserve the timeline for the whole pass, the way each importer reserves the catalog. - // Renditions are ingested one at a time, and a record is immutable once published - // against the tracks enrolled at that moment, so a record flushed mid-pass would omit - // every rendition that hasn't loaded its init segment yet (a permanent EXT-X-GAP) and - // fold that rendition's first groups into whichever segment flushes next. - let _reserved = self.sink.catalog.timeline().reserve(); - let mut wrote_segments = 0; let mut target_duration = None; let mut failed = None; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 0811704b1b..4d0281cddf 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -62,16 +62,18 @@ struct Reservations { published: bool, } -/// The built-in catalog's enrollment in the broadcast timeline. +/// The built-in catalog's own timeline, enrolled with the first media track. +#[derive(Default)] struct CatalogTimeline { - recorder: crate::timeline::Recorder, + recorder: Option, last_sequence: Option, } impl CatalogTimeline { - /// Report a newly published plaintext catalog group once. + /// Report a newly published plaintext catalog group once. Each catalog group is one complete + /// snapshot, so it finishes as soon as it is reported. fn record(&mut self, track: &moq_net::track::Producer) { - let Some(sequence) = track.latest() else { + let (Some(recorder), Some(sequence)) = (self.recorder.as_mut(), track.latest()) else { return; }; if self.last_sequence == Some(sequence) { @@ -79,7 +81,12 @@ impl CatalogTimeline { } self.last_sequence = Some(sequence); - self.recorder.record(sequence, moq_net::Timestamp::now(), true); + recorder.frame( + hang::timeline::Position::group(sequence), + moq_net::Timestamp::now(), + true, + ); + recorder.finish_group(sequence); } } @@ -170,10 +177,11 @@ pub struct Producer { /// `clock: { wall, timescale }` independently of any archive timeline. clock: crate::Clock, - /// The broadcast's timeline: the shared boundary list every enrolled track's groups map - /// onto, and the track those segment records are published on. See - /// [`timeline`](Self::timeline). - timeline: crate::timeline::Producer, + /// The broadcast's per-track timelines. See [`timeline`](Self::timeline). + timeline: crate::timeline::Timelines, + /// How each media track's timeline is cut. The catalog's own uses a zero minimum. + /// See [`Config::with_timeline`]. + cutting: crate::timeline::Config, /// Retention override for the media tracks minted under this catalog, or `None` to keep /// hang's default. Fixed at construction, so every clone and every /// [`Reserved`](super::Reserved) mints tracks under one policy. See @@ -195,6 +203,7 @@ impl Clone for Producer { current: self.current.clone(), clock: self.clock, timeline: self.timeline.clone(), + cutting: self.cutting.clone(), max_age: self.max_age, bandwidth: self.bandwidth.clone(), baseline: self.baseline.clone(), @@ -282,7 +291,7 @@ impl Config { self } - /// Pace the broadcast's timeline with `timeline`. + /// Cut each media track's timeline with `timeline`; the catalog's own always uses a zero minimum. pub fn with_timeline(mut self, timeline: crate::timeline::Config) -> Self { self.timeline = timeline; self @@ -311,12 +320,9 @@ impl Producer { json_config.compression = moq_json::Compression::Deflate; let hangz = moq_json::snapshot::Producer::new(hangz_track, json_config); - let timeline = crate::timeline::Producer::new(broadcast, config.timeline); + let timeline = crate::timeline::Timelines::new(broadcast); #[allow(clippy::arc_with_non_send_sync)] - let catalog_timeline = Arc::new(Mutex::new(CatalogTimeline { - recorder: timeline.track(hang::Catalog::DEFAULT_NAME), - last_sequence: None, - })); + let catalog_timeline = Arc::new(Mutex::new(CatalogTimeline::default())); // The broadcast clock is advertised at the catalog root from the first snapshot, // independently of any archive timeline: a live-only publisher exposes its mapping @@ -344,6 +350,7 @@ impl Producer { })), clock: config.clock, timeline, + cutting: config.timeline, max_age: config.max_age, bandwidth: config.bandwidth, baseline: Default::default(), @@ -575,11 +582,10 @@ impl Producer { } /// Build the media [`container::Producer`](crate::container::Producer) for `track`, - /// enrolling it in the broadcast's timeline so its groups are indexed into the aligned - /// segments. + /// enrolling it in the broadcast's timelines so its groups are indexed. /// - /// The broadcast's one timeline track is created (and advertised in the catalog's root - /// `archive` entry) on first use; see [`timeline`](crate::timeline) for the whole model. + /// The track's timeline track is created and advertised in the catalog's root `archive` + /// entry; see [`timeline`](crate::timeline) for the whole model. pub(super) fn media( &self, track: moq_net::track::Producer, @@ -630,33 +636,43 @@ impl Producer { self.bandwidth.clone() } - /// Enroll `track` in the broadcast's timeline, advertising the timeline in the catalog's - /// root section the first time. + /// Enroll `track` in the broadcast's timelines, advertising its timeline in the catalog's + /// root section with the next catalog change. The first enrollment also enrolls the catalog + /// itself. /// /// The role-specific track constructors call this for you. fMP4 passthrough calls it directly /// because it writes groups by hand instead of using a [`container::Producer`](crate::container::Producer). pub(crate) fn enroll(&mut self, track: &str) -> crate::Result { - let recorder = self.timeline.pacing_track(track)?; + let recorder = self.timeline.track(track, self.cutting.clone())?; + { + let mut catalog = self.outputs.catalog_timeline.lock().unwrap(); + if catalog.recorder.is_none() { + // Sparse data: a catalog may not change again, so each group is its own record. + let cutting = self.cutting.clone().with_duration_min(std::time::Duration::ZERO); + catalog.recorder = Some(self.timeline.track(hang::Catalog::DEFAULT_NAME, cutting)?); + } + } + // Staged without publishing: the entry the track belongs to commits it, so no consumer sees a + // catalog advertising a timeline before its rendition. let section = self.timeline.section(); - let mut catalog = self.modify()?; - if catalog.archive.is_none() { - catalog.archive = Some(section); + let mut state = take(&self.current); + if let Some(err) = &state.closed { + return Err(err.clone()); + } + match &mut state.catalog.archive { + Some(archive) => archive.timelines = section.timelines, + None => state.catalog.archive = Some(section), } - catalog.commit()?; Ok(recorder) } - /// The broadcast's [`timeline::Producer`](crate::timeline::Producer): its segment index. + /// The broadcast's [`Timelines`](crate::timeline::Timelines): each enrolled track's index. /// - /// The MoQ track behind it is created (and advertised in the catalog's root `archive` - /// entry) when the first media track enrolls, so reading this costs nothing on a - /// broadcast that never segments. Use it to declare boundaries - /// ([`cut`](crate::timeline::Producer::cut)) or to hold publishing back while tracks - /// enroll ([`reserve`](crate::timeline::Producer::reserve), the timeline's counterpart to - /// this producer's own [`reserve`](Self::reserve)). - pub fn timeline(&self) -> crate::timeline::Producer { + /// A track's timeline is created (and advertised in the catalog's root `archive` entry) when + /// it enrolls, so reading this costs nothing on a broadcast that never enrolls one. + pub fn timeline(&self) -> crate::timeline::Timelines { self.timeline.clone() } @@ -749,7 +765,7 @@ impl Producer { self.outputs.hang.finish()?; self.outputs.hangz.finish()?; self.outputs.msf_track.finish()?; - self.timeline.finish()?; + self.timeline.finish(); Ok(()) } } @@ -1011,15 +1027,14 @@ mod test { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = Producer::new(&mut broadcast, Config::default()).unwrap(); - // Pacing enrollment is what mints the timeline track; a passive one publishes none. - catalog.timeline().pacing_track("video").unwrap(); + let _recorder = catalog.timeline().track("video", Default::default()).unwrap(); let consumer = broadcast.consume(); for name in [ hang::Catalog::DEFAULT_NAME, hang::Catalog::COMPRESSED_NAME, moq_msf::DEFAULT_NAME, - hang::timeline::DEFAULT_NAME, + "video.timeline.z", ] { let track = consumer.track(name).expect("track"); let info = track.query().await.expect("info"); @@ -1236,7 +1251,7 @@ mod test { let mut catalog = Producer::new(&mut broadcast, Config::default()).unwrap(); // Something else already took the name the timeline track wants. - let _taken = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _taken = broadcast.create_track("video0.timeline.z", None).unwrap(); assert!(catalog.enroll("video0").is_err()); } @@ -1248,10 +1263,15 @@ mod test { // A broadcast that never segments never advertises an archive. assert_eq!(catalog.snapshot().archive, None); let _recorder = catalog.enroll("video0").unwrap(); + let archive = catalog + .snapshot() + .archive + .expect("the root archive advertises the timelines"); + assert_eq!(archive, catalog.timeline().section()); assert_eq!( - catalog.snapshot().archive, - Some(catalog.timeline().section()), - "the root archive should advertise the timeline track" + archive.timelines.keys().collect::>(), + vec!["catalog.json", "video0"], + "the catalog indexes itself alongside its first media track" ); } diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 91c0670fcc..1f05e2e967 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -1111,21 +1111,18 @@ mod tests { ); } - /// The broadcast has one timeline: every rendition's groups index into the same track, so - /// an aligned ladder (source + rung) shares it by construction. + /// Every rendition gets its own timeline, advertised together at the catalog root. #[test] - fn renditions_share_the_broadcast_timeline() { + fn renditions_get_their_own_timelines() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let mut catalog = super::super::Producer::new(&mut broadcast, super::super::Config::default()).unwrap(); - let _recorder = catalog.enroll("video0").unwrap(); - let timeline = catalog.timeline(); - assert_eq!(timeline.section().track, hang::timeline::DEFAULT_NAME); - assert_eq!( - catalog.snapshot().archive, - Some(timeline.section()), - "the one timeline is advertised at the catalog root" - ); + let _video0 = catalog.enroll("video0").unwrap(); + let _video1 = catalog.enroll("video1").unwrap(); + let section = catalog.timeline().section(); + assert_eq!(section.timelines["video0"], "video0.timeline.z"); + assert_eq!(section.timelines["video1"], "video1.timeline.z"); + assert_eq!(catalog.snapshot().archive, Some(section)); } mod custom { diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index ceb6a78897..46897ebf5f 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -114,10 +114,12 @@ async fn rendition_is_not_published_when_the_media_track_fails() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - // Squat the broadcast's timeline track, so enrolling the first rendition (and with it building - // its media producer) fails. The handle must stay alive: the broadcast tracks names weakly, so + // Squat the catalog's timeline track, so enrolling the first rendition (which enrolls the catalog + // too, and with it building its media producer) fails. The handle must stay alive: the broadcast tracks names weakly, so // dropping it frees the name. - let _squat = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _squat = broadcast + .create_track(hang::timeline::default_name(hang::Catalog::DEFAULT_NAME), None) + .unwrap(); let mut importer = Import::new(broadcast, catalog.reserve()); // A track it cannot build surfaces in the catalog rather than in this result. diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 15d2cf26ea..2a13b73bd3 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -66,29 +66,9 @@ pub struct Import { // open group belongs to an older one rolls, so every track segments identically. segment: u64, - // A boundary the source declared (rather than one inferred from a keyframe) still has to - // reach the timeline. - pending_timeline_cut: bool, - - // Where the current segment starts: the timestamp the FIRST track to open a group for it - // reported, which every other track then reports too. - // - // Tracks do not start a segment at the same instant. The boundary is nominal and each begins - // at its own nearest sample, so audio's first sample routinely sits a frame either side of - // video's IDR. The timeline files a group under a segment by the pts reported for it, so - // reporting each track's own first sample would file whichever track ran early under the - // previous record. That was a rounding error when an audio group was one fragment; now a - // group is a whole segment, so a 20ms skew moves a segment's entire audio into its - // predecessor and leaves the next record with none. - // - // Anchoring on the first roll rather than the earliest also keeps the declared cut on the - // source's own cadence: the timeline drops a cut landing within `duration_min` of the last - // one, so a boundary pulled 20ms earlier than a 1 second source's would be discarded and two - // segments would merge. - // - // Only the timeline report is anchored. Each fragment still carries its own timestamp on the - // wire, and `Recorder::end` still reports real content time. - segment_start: Option, + // The current segment's boundary was declared by the source (rather than inferred from a + // keyframe), so each track cuts its timeline where it opens its group for the segment. + declared: bool, // The source's mapping onto the broadcast clock, set by `live`. `None` publishes the source's // decode times verbatim. @@ -122,9 +102,8 @@ struct Fmp4Track { track: moq_net::track::Producer, group: Option, - // Reports this track's group opens into the broadcast's timeline. Passthrough writes - // groups by hand (no `container::Producer`), so the recorder is fed directly at each - // keyframe fragment rather than through `with_recorder`. + // Reports this track's fragments into its timeline. Passthrough writes groups by hand (no + // `container::Producer`), so the recorder is fed directly rather than through `with_recorder`. recorder: Option, // The decode time of the last fragment, which the next one has to advance past. @@ -179,8 +158,7 @@ impl Import { buffer: BytesMut::new(), pending_cut: false, segment: 0, - pending_timeline_cut: false, - segment_start: None, + declared: false, anchor: None, } } @@ -201,9 +179,9 @@ impl Import { /// segmentation out of band (e.g. an HLS import following its playlist). /// /// A `styp` atom (a CMAF segment on disk) does this on its own, so an importer reading a - /// segmented file needs no help. Boundaries are broadcast-wide, but redundant ones cost - /// nothing: the timeline ignores a cut that would land inside its minimum segment duration, - /// so several renditions of one source may all declare the same boundaries. + /// segmented file needs no help. Each track's timeline then closes a record where that track + /// opens its group for the new segment, following the source's segmentation instead of the + /// minimum-duration pacing. /// /// This is where groups are drawn too: every track rolls its group at a segment boundary, so /// a group is a segment and each fragment inside it is a frame. @@ -298,16 +276,6 @@ impl Import { // Held from construction until the track set is declared, so a second moov would be // re-declaring a track set the catalog already published. let reserved = self.initial_reservation.clone().ok_or(Error::DuplicateMoov)?; - let timeline = self.catalog.timeline(); - - // The tracks below enroll in the timeline, so advertise it in the same catalog update - // rather than publishing a second snapshot for it. - { - let mut catalog = self.catalog.modify()?; - if catalog.archive.is_none() && !moov.trak.is_empty() { - catalog.archive = Some(timeline.section()); - } - } for trak in &moov.trak { let track_id = trak.tkhd.track_id; @@ -333,10 +301,10 @@ impl Import { self.catalog.track_info(kind.priority()).with_timescale(timescale), )?; - // Enroll every track in the broadcast's timeline: passthrough writes groups by hand - // (no `container::Producer`), so the recorder is fed directly at each group open. - // The root archive entry is advertised before any rendition releases its reservation. - let recorder = timeline.pacing_track(track.name())?; + // Enroll every track in the broadcast's timelines: passthrough writes groups by hand + // (no `container::Producer`), so the recorder is fed directly at each fragment. The + // reservation holds the archive entry back until the whole track set is declared. + let recorder = self.catalog.enroll(track.name())?; // Whatever the descriptor declared (a bitrate) is authoritative; the rest is filled by // `estimate` as fragments arrive. @@ -685,9 +653,8 @@ impl Import { self.segment += 1; // A boundary the source declared is also a timeline boundary; one merely inferred // from a keyframe is not, since the default pacing already sees that group open. - self.pending_timeline_cut |= self.pending_cut; + self.declared = self.pending_cut; self.pending_cut = false; - self.segment_start = None; } let moov = self.moov.as_ref().ok_or(Error::NoMoov)?; @@ -938,6 +905,9 @@ impl Import { let mut g = if start_group { if let Some(prev) = track.group.take() { prev.finish()?; + if let Some(recorder) = track.recorder.as_mut() { + recorder.finish_group(prev.sequence); + } } track.segment = Some(self.segment); match track.pending_sequence.take() { @@ -952,29 +922,16 @@ impl Import { // in the track's native timescale. The relay reads it off the wire; the // consumer still drives playback from the fragment's internal timing. let timestamp = min_timestamp.ok_or(Error::MissingTrun)?; + let position = hang::timeline::Position::new(g.sequence, g.frame_count() as u64); if start_group { - // A group just opened; report it so the broadcast's timeline can index the segment - // (the timeline absorbs publish failures). Audio is always independently decodable; - // video says whether this segment really begins on an IDR, which is what an HLS - // export reads to bootstrap an init segment. A group reopened mid-segment (after a - // `seek`) reports its own timestamp: it does not start a segment. - let reported = match new_segment { - false => timestamp, - true => match self.segment_start { - Some(start) => start, - None => { - self.segment_start = Some(timestamp); - if self.pending_timeline_cut { - self.pending_timeline_cut = false; - self.catalog.timeline().cut(timestamp)?; - } - timestamp - } - }, - }; - if let Some(recorder) = track.recorder.as_mut() { - recorder.record(g.sequence, reported, contains_keyframe); + // A group opening for a declared segment is a boundary on this track's timeline. A + // group reopened mid-segment (after a `seek`) is not. + if new_segment + && self.declared + && let Some(recorder) = track.recorder.as_mut() + { + recorder.cut(timestamp); } // Close the previous group for the bitrate estimator (used only when the @@ -982,7 +939,6 @@ impl Import { track.estimator.cut(Some(timestamp)); } let fragment_len = fragment_bytes.len(); - let mut frame = g.create_frame(moq_net::frame::Info { size: fragment_bytes.len() as u64, timestamp, @@ -990,6 +946,13 @@ impl Import { frame.write(fragment_bytes)?; frame.finish()?; + // Only once published, so a rejected fragment is never indexed. Audio is always + // independently decodable; video says whether this fragment really begins on an IDR, + // which is what an HLS export reads to bootstrap an init segment. + if let Some(recorder) = track.recorder.as_mut() { + recorder.frame(position, timestamp, contains_keyframe); + } + track.group = Some(g); track.estimator.write(timestamp, fragment_len); @@ -1018,6 +981,9 @@ impl Import { track.publish_estimate()?; if let Some(g) = track.group.take() { g.finish()?; + if let Some(recorder) = track.recorder.as_mut() { + recorder.finish_group(g.sequence); + } } track.track.finish()?; } @@ -1046,6 +1012,9 @@ impl Import { track.publish_estimate()?; if let Some(g) = track.group.take() { g.finish()?; + if let Some(recorder) = track.recorder.as_mut() { + recorder.finish_group(g.sequence); + } } track.pending_sequence = Some(sequence); track.last_decode_time = None; diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 9d54992e80..b9cded60c2 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -419,10 +419,34 @@ async fn test_msf_catalog_roundtrip() { assert!(matches!(audio.container, Container::Cmaf { .. })); } -/// Passthrough writes groups by hand (no `container::Producer`), so the timeline recorder is fed -/// directly. This guards that the import advertises the broadcast's timeline at the catalog root -/// and actually records both renditions' group opens into it, the index the VOD/playlist export -/// reads. Regression for the missing-timeline break that skipped VOD renditions. +/// Subscribe to `track`'s timeline as `section` advertises it. +async fn subscribe( + consumer: &moq_net::broadcast::Consumer, + section: &hang::catalog::Archive, + track: &str, +) -> crate::timeline::Consumer { + crate::timeline::Consumer::<()>::subscribe(consumer, section, track) + .await + .unwrap() +} + +/// Every record a finished timeline published, as `(start group, end group)`. +async fn groups(timeline: &mut crate::timeline::Consumer) -> Vec<(u64, u64)> { + let mut out = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let crate::timeline::Event::Push { entry, .. } = event { + assert_eq!(entry.start.frame, 0, "fragments never split a group here"); + assert_eq!(entry.end.frame, 0, "fragments never split a group here"); + out.push((entry.start.group, entry.end.group)); + } + } + out +} + +/// Passthrough writes groups by hand (no `container::Producer`), so the timeline recorders are fed +/// directly. This guards that the import advertises both renditions' timelines at the catalog root +/// and actually records their groups, the index the VOD/playlist export reads. Regression for the +/// missing-timeline break that skipped VOD renditions. #[tokio::test] async fn import_populates_the_broadcast_timeline() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -442,31 +466,21 @@ async fn import_populates_the_broadcast_timeline() { let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - // The one timeline is advertised at the catalog root, named by convention. - let section = snapshot.archive.clone().expect("the import advertises a timeline"); - assert_eq!(section.track, hang::timeline::DEFAULT_NAME); + // Each rendition's timeline is advertised at the catalog root. + let section = snapshot.archive.clone().expect("the import advertises its timelines"); + assert_eq!( + section.timelines[&video_name], + hang::timeline::default_name(&video_name) + ); - // Subscribe while the producer is alive, then finish so the timeline group closes and the - // reader terminates rather than blocking. - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + // Subscribe while the producer is alive, then finish so the timeline groups close and the + // readers terminate rather than blocking. + let mut video = subscribe(&consumer, §ion, &video_name).await; + let mut audio = subscribe(&consumer, §ion, &audio_name).await; catalog.finish().unwrap(); - // The first record indexes both renditions from their first group (sequence 0). - let event = timeline - .next() - .await - .unwrap() - .expect("a segment should be recorded in the timeline"); - let crate::timeline::Event::Push { entry: first, .. } = event else { - panic!("the first timeline event was not a segment"); - }; - assert_eq!(first.segment, 0); - let video_ranges = first.tracks.get(&video_name).expect("video is indexed"); - assert_eq!(video_ranges[0].start, 0, "the first video group is sequence 0"); - let audio_ranges = first.tracks.get(&audio_name).expect("audio is indexed"); - assert_eq!(audio_ranges[0].start, 0, "the first audio group is sequence 0"); + assert_eq!(groups(&mut video).await[0].0, 0, "the first video group is sequence 0"); + assert_eq!(groups(&mut audio).await[0].0, 0, "the first audio group is sequence 0"); } // ---- Sample-duration handling in decode() ---- @@ -705,12 +719,11 @@ async fn segmented_source_groups_per_segment() { ); } -/// The timeline indexes each segment as one group range per track, which is what makes the -/// segment servable: an HLS/DASH export fetches `start..=end` and concatenates it into one CMAF -/// segment. With audio grouping per fragment those ranges spanned many short groups, one FETCH -/// round-trip each, and the two tracks' boundaries no longer coincided. +/// Each declared segment is one group per track, and each track's timeline indexes it as one +/// record. With audio grouping per fragment those records spanned many short groups, one FETCH +/// round-trip each. #[tokio::test] -async fn segmented_source_indexes_one_group_range_per_track() { +async fn segmented_source_indexes_one_group_per_record() { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -722,12 +735,11 @@ async fn segmented_source_indexes_one_group_range_per_track() { let snapshot = catalog.snapshot(); let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - let section = snapshot.archive.clone().expect("the import advertises a timeline"); + let section = snapshot.archive.clone().expect("the import advertises its timelines"); - // Subscribe while the producer is alive so the reader terminates on finish rather than blocking. - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + // Subscribe while the producer is alive so the readers terminate on finish rather than blocking. + let mut video = subscribe(&consumer, §ion, &video_name).await; + let mut audio = subscribe(&consumer, §ion, &audio_name).await; for segment in 0..3u64 { let base = segment * 1_000_000; @@ -752,47 +764,22 @@ async fn segmented_source_indexes_one_group_range_per_track() { fmp4.finish().unwrap(); catalog.finish().unwrap(); - let mut records = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let crate::timeline::Event::Push { entry, .. } = event { - records.push(entry); - } - } - assert_eq!(records.len(), 3, "one record per declared segment"); - - for (i, record) in records.iter().enumerate() { - assert_eq!(record.segment, i as u64); - let video = record.tracks.get(&video_name).expect("video is indexed"); - let audio = record.tracks.get(&audio_name).expect("audio is indexed"); - assert_eq!(video.len(), 1, "segment {i}: video is one contiguous group range"); - assert_eq!(audio.len(), 1, "segment {i}: audio is one contiguous group range"); - assert_eq!( - (video[0].start, video[0].end), - (i as u64, i as u64), - "segment {i}: one video group" - ); - assert_eq!( - (audio[0].start, audio[0].end), - (i as u64, i as u64), - "segment {i}: one audio group, not one per fragment" - ); - } + let expected = vec![(0, 1), (1, 2), (2, 3)]; + assert_eq!(groups(&mut video).await, expected, "one video group per record"); + assert_eq!( + groups(&mut audio).await, + expected, + "one audio group per record, not one per fragment" + ); } /// Audio and video rarely start a segment at exactly the same timestamp: the boundary is nominal -/// and each track begins at its own nearest sample. The timeline files a group under a segment by -/// the group's start, so a boundary declared later than any track's first group files that whole -/// group under the previous segment. -/// -/// That was survivable when audio groups were one fragment. Now a group is a whole segment, so a -/// 20ms skew moves an entire segment of audio into the wrong record: the previous segment gets a -/// second copy's worth and the next one opens with no audio at all. +/// and each track begins at its own nearest sample. Each track cuts its own timeline where it opens +/// its group for a declared segment, so a skew between them moves no group into another record. /// /// `video_offset_us` shifts video relative to audio, so the same source is exercised with audio -/// leading and lagging. -async fn segment_ranges_with_skew( - video_offset_us: i64, -) -> Vec<(Vec, Vec)> { +/// leading and lagging. Returns each track's records as `(start group, end group)`. +async fn records_with_skew(video_offset_us: i64) -> (Vec<(u64, u64)>, Vec<(u64, u64)>) { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -805,9 +792,8 @@ async fn segment_ranges_with_skew( let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); let section = snapshot.archive.clone().unwrap(); - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + let mut video = subscribe(&consumer, §ion, &video_name).await; + let mut audio = subscribe(&consumer, §ion, &audio_name).await; for segment in 0..3u64 { let base = segment * 1_000_000; @@ -815,7 +801,7 @@ async fn segment_ranges_with_skew( fmp4.decode(&styp()).unwrap(); // Audio first, so the segment transition is detected on the audio fragment and video's - // IDR lands after it. Two fragments each keeps the ranges unambiguous. + // IDR lands after it. Two fragments each keeps the records unambiguous. for offset in [0u64, 500_000] { let frag = super::encode_fragment( info(audio_id, audio_scale, segment as u32), @@ -836,58 +822,21 @@ async fn segment_ranges_with_skew( fmp4.finish().unwrap(); catalog.finish().unwrap(); - let mut out = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let crate::timeline::Event::Push { entry, .. } = event { - out.push(( - entry.tracks.get(&video_name).cloned().unwrap_or_default(), - entry.tracks.get(&audio_name).cloned().unwrap_or_default(), - )); - } - } - out + (groups(&mut video).await, groups(&mut audio).await) } /// Video's IDR lands 20ms AFTER audio's first sample of the segment, the normal interleave. #[tokio::test] async fn audio_leading_video_still_indexes_one_group_each() { - let records = segment_ranges_with_skew(20_000).await; - assert_eq!(records.len(), 3, "one record per segment"); - for (i, (video, audio)) in records.iter().enumerate() { - assert_eq!(video.len(), 1, "segment {i}: one video range, got {video:?}"); - assert_eq!(audio.len(), 1, "segment {i}: one audio range, got {audio:?}"); - assert_eq!( - (audio[0].start, audio[0].end), - (i as u64, i as u64), - "segment {i} audio" - ); - assert_eq!( - (video[0].start, video[0].end), - (i as u64, i as u64), - "segment {i} video" - ); - } + let expected = vec![(0, 1), (1, 2), (2, 3)]; + assert_eq!(records_with_skew(20_000).await, (expected.clone(), expected)); } /// Video's IDR lands 20ms BEFORE audio's first sample, the other side of the same skew. #[tokio::test] async fn audio_lagging_video_still_indexes_one_group_each() { - let records = segment_ranges_with_skew(-20_000).await; - assert_eq!(records.len(), 3, "one record per segment"); - for (i, (video, audio)) in records.iter().enumerate() { - assert_eq!(video.len(), 1, "segment {i}: one video range, got {video:?}"); - assert_eq!(audio.len(), 1, "segment {i}: one audio range, got {audio:?}"); - assert_eq!( - (audio[0].start, audio[0].end), - (i as u64, i as u64), - "segment {i} audio" - ); - assert_eq!( - (video[0].start, video[0].end), - (i as u64, i as u64), - "segment {i} video" - ); - } + let expected = vec![(0, 1), (1, 2), (2, 3)]; + assert_eq!(records_with_skew(-20_000).await, (expected.clone(), expected)); } /// A source that emits one leading `styp` and then never another still segments on its video @@ -1216,7 +1165,7 @@ struct LiveImport { clock: crate::Clock, before: u128, after: u128, - /// The first segment the broadcast timeline recorded, the index an archive replays from. + /// The first record the video timeline published, the index an archive replays from. record: moq_net::Timestamp, } @@ -1241,10 +1190,9 @@ async fn live_import( fmp4.decode(&init).unwrap(); let snapshot = catalog.snapshot(); - let section = snapshot.archive.clone().expect("the import advertises a timeline"); - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + let video = snapshot.video.renditions.keys().next().unwrap().clone(); + let section = snapshot.archive.clone().expect("the import advertises its timelines"); + let mut timeline = subscribe(&consumer, §ion, &video).await; let before = clock.now().as_micros(); let mut after = before; @@ -1260,14 +1208,14 @@ async fn live_import( fmp4.finish().unwrap(); catalog.finish().unwrap(); - let event = timeline.next().await.unwrap().expect("a recorded segment"); + let event = timeline.next().await.unwrap().expect("a recorded span"); let crate::timeline::Event::Push { entry, .. } = event else { - panic!("the first timeline event was not a segment"); + panic!("the first timeline event was not a record"); }; LiveImport { published: crate::container::test_util::published(&consumer, &snapshot).await, - video: snapshot.video.renditions.keys().next().unwrap().clone(), + video, clock, before, after, diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index e244513c68..92da3b1796 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -434,10 +434,12 @@ fn rendition_is_not_published_when_the_media_track_fails() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - // Squat the broadcast's timeline track, so building the media producer (which creates it - // on first use) fails. The handle must stay alive: the broadcast tracks names weakly, so + // Squat the catalog's timeline track, so building the media producer (whose enrollment + // enrolls the catalog too) fails. The handle must stay alive: the broadcast tracks names weakly, so // dropping it frees the name. - let _squat = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _squat = broadcast + .create_track(hang::timeline::default_name(hang::Catalog::DEFAULT_NAME), None) + .unwrap(); let mut mkv = crate::container::mkv::Import::new(broadcast, catalog.reserve()); let buf = bytes::BytesMut::from(&data[..]); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index abdd85fd72..71fd64d83f 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -58,13 +58,13 @@ pub struct Producer { /// Set by [`Self::seek`] and consumed on the next group creation. pending_sequence: Option, - /// Records each group open (sequence + keyframe timestamp) into this rendition's - /// timeline track, when the producer was built with one. + /// Reports each written frame and finished group into this rendition's timeline, when the + /// producer was built with one. recorder: Option, /// The furthest presentation point written, i.e. `max(timestamp + duration)`. Reported to /// `recorder` on each [`cut`](Self::cut), since the last group of a track has no successor - /// to bound it and its segment would otherwise be published a group short. Also the base + /// to bound it and its record would otherwise end at its last frame's start. Also the base /// for a duration marker when the caller does not pass a bound. end: Option, @@ -340,11 +340,10 @@ where self } - /// Report each group open (sequence, timestamp, keyframe) through `recorder`, enrolling - /// this track in the broadcast's timeline so consumers can index the media without - /// downloading it. + /// Report each written frame and finished group through `recorder`, enrolling this track in + /// its timeline so consumers can index the media without downloading it. /// - /// Mint the recorder from the broadcast's [`timeline::Producer`](crate::timeline::Producer). + /// Mint the recorder from the broadcast's [`Timelines`](crate::timeline::Timelines). pub fn with_recorder(mut self, recorder: crate::timeline::Recorder) -> Self { self.recorder = Some(recorder); self @@ -433,14 +432,6 @@ where Some(sequence) => self.inner.create_group(moq_net::group::Info { sequence })?, None => self.inner.append_group()?, }; - - // Report the group the moment it opens: its start is this frame's timestamp. The - // timeline absorbs publish failures itself (it is an optional sidecar), - // so reporting can't abort the media write. - if let Some(recorder) = self.recorder.as_mut() { - recorder.record(group.sequence, frame.timestamp, frame.keyframe); - } - self.group = Some(group); } @@ -448,7 +439,13 @@ where if self.buffer_duration.is_zero() { let group = self.group.as_mut().unwrap(); let (timestamp, duration, bytes) = (frame.timestamp, frame.duration, frame.payload.len()); + let (position, keyframe) = (position(group), frame.keyframe); self.container.write(group, &[frame])?; + // The timeline absorbs publish failures itself (it is an optional sidecar), so + // reporting can't abort the media write. + if let Some(recorder) = self.recorder.as_mut() { + recorder.frame(position, timestamp, keyframe); + } // Only what the container accepted is measured. A rejected frame (too large for the // group, a timestamp that won't convert) leaves the producer usable, and the estimate's @@ -509,21 +506,25 @@ where self.estimator.cut(end); self.claim(); + let tail_end = marker_at.filter(|_| !self.reordered); + self.flush_buffer(tail_end)?; + // Tell the timeline where this group's content stops: the duration marker when we - // write one, else the caller's bound, else the furthest point we wrote. + // write one, else the caller's bound, else the furthest point we wrote. After the flush, + // so a group still wholly buffered has reported its first frame and opened a record. if let Some(recorder) = self.recorder.as_mut() && let Some(end) = marker_at.or(end).max(self.end) { recorder.end(end); } - - let tail_end = marker_at.filter(|_| !self.reordered); - self.flush_buffer(tail_end)?; if let Some(group) = self.group.as_mut() { self.container.finish_group(group, tail_end)?; } if let Some(group) = self.group.take() { group.finish()?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.finish_group(group.sequence); + } } if let Some(end) = self.end { self.raise_live_edge(end); @@ -632,7 +633,7 @@ where }; let timestamp = self.live_edge.unwrap_or(moq_net::Timestamp::ZERO); if let Some(recorder) = self.recorder.as_mut() { - recorder.record(group.sequence, timestamp, false); + recorder.frame(hang::timeline::Position::group(group.sequence), timestamp, false); recorder.end(timestamp); } self.container.write( @@ -645,6 +646,11 @@ where }], )?; group.finish()?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.finish_group(group.sequence); + // The marker is contiguous, so without this the next record boundary would absorb the pause. + recorder.flush(); + } Ok(()) } @@ -686,7 +692,11 @@ where None => return Ok(()), }; + let position = position(group); self.container.write(group, &self.buffer)?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.frame(position, self.buffer[0].timestamp, self.buffer[0].keyframe); + } self.buffer.clear(); Ok(()) @@ -728,6 +738,11 @@ impl std::ops::Deref for Producer { } } +/// Where the next frame written to `group` lands. +fn position(group: &moq_net::group::Producer) -> hang::timeline::Position { + hang::timeline::Position::new(group.sequence, group.frame_count() as u64) +} + #[cfg(test)] mod tests { use super::*; @@ -1344,6 +1359,66 @@ mod tests { crate::container::fmp4::encode_fragment(info, group).unwrap(); } + /// A group shorter than the buffer reaches the timeline only when it closes, and its record + /// still runs to the group's end rather than stopping at its first frame. + #[tokio::test] + async fn a_wholly_buffered_group_records_its_end() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = crate::timeline::Timelines::new(&broadcast); + let recorder = timelines.track("video", crate::timeline::Config::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)) + .with_buffer(std::time::Duration::from_secs(10)) + .with_recorder(recorder); + + producer.write(frame(0, true)).unwrap(); + producer.write(frame(33_000, false)).unwrap(); + producer.cut(Some(Timestamp::from_micros(66_000).unwrap())).unwrap(); + drop(producer); + timelines.finish(); + + let mut timeline = + crate::timeline::Consumer::<()>::subscribe(&broadcast.consume(), &timelines.section(), "video") + .await + .unwrap(); + let Some(crate::timeline::Event::Push { entry, .. }) = timeline.next().await.unwrap() else { + panic!("expected a record"); + }; + assert_eq!(entry.duration, std::time::Duration::from_millis(66)); + } + + /// The record before a discontinuity ends at the marker, not where content resumes. + #[tokio::test] + async fn a_discontinuity_ends_the_record_at_the_marker() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = crate::timeline::Timelines::new(&broadcast); + let recorder = timelines.track("video", crate::timeline::Config::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let mut producer = + Producer::new(track, Container::Legacy(crate::container::Kind::Video)).with_recorder(recorder); + + producer.write(frame(0, true)).unwrap(); + producer.write(frame(33_000, false)).unwrap(); + producer.cut(Some(Timestamp::from_micros(66_000).unwrap())).unwrap(); + producer.discontinuity().unwrap(); + // Resumed 40 minutes later. + producer.write(frame(2_400_000_000, true)).unwrap(); + drop(producer); + timelines.finish(); + + let mut timeline = + crate::timeline::Consumer::<()>::subscribe(&broadcast.consume(), &timelines.section(), "video") + .await + .unwrap(); + let mut spans = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let crate::timeline::Event::Push { entry, .. } = event { + spans.push((entry.pts.as_micros() / 1_000, entry.duration.as_millis())); + } + } + assert_eq!(spans, vec![(0, 66), (2_400_000, 0)]); + } + #[tokio::test] async fn duration_marker_uses_cadence_not_batching_delay() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index a6f09c8821..d928ef7e02 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -180,34 +180,32 @@ pub enum Error { composition_time_ms: i32, }, - /// A segment ran past the [`duration_max`](crate::timeline::Config::duration_max) the - /// catalog advertised, so the timeline stopped publishing rather than contradict it. The - /// publisher declared a bound its media can't honor. - #[error("timeline segment {segment} lasted {duration:?}, over the declared maximum {duration_max:?}")] - TimelineOverrun { - /// The segment that broke the bound. - segment: u64, - /// How long it actually ran. - duration: std::time::Duration, - /// The bound the catalog advertised. - duration_max: std::time::Duration, - }, + /// [`timeline::Producer::resume`](crate::timeline::Producer::resume) received a checkpoint + /// whose record at this window index has a different sequence. + #[error("timeline checkpoint record at index {0} has a different sequence")] + TimelineCheckpoint(u64), - /// [`timeline::Producer::finish`](crate::timeline::Producer::finish) was called before its - /// deferred [`timeline::Segmenter`](crate::timeline::Segmenter) completed and every record - /// was committed. - #[error("finish and commit every deferred timeline record before closing the Producer")] - TimelineDeferredPending, + /// [`timeline::Producer::push`](crate::timeline::Producer::push) received a record out of order. + #[error("timeline record {actual} pushed where {expected} was next")] + TimelineSequence { + /// The window's next index. + expected: u64, + /// The pushed record's sequence. + actual: u64, + }, - /// [`timeline::Producer::push`](crate::timeline::Producer::push) received a pending record that - /// its [`timeline::Deferred`](crate::timeline::Deferred) did not yield. - #[error("timeline segment {0} was not yielded for deferred publication")] - TimelineDeferredRecord(u64), + /// A timeline frame report did not advance past the previous one. + #[error("timeline report at {position:?} does not advance past {last:?}")] + TimelinePosition { + /// The rejected report's position. + position: hang::timeline::Position, + /// The previous report's position. + last: hang::timeline::Position, + }, - /// [`timeline::Producer::resume`](crate::timeline::Producer::resume) received a checkpoint - /// whose record at this window index is a different segment. - #[error("timeline checkpoint record at index {0} is a different segment")] - TimelineCheckpoint(u64), + /// The catalog's `archive` entry indexes no timeline for this track. + #[error("no timeline for track {0}")] + TimelineMissing(String), /// Error from a muxer/demuxer that reports via `anyhow` (currently MPEG-TS). /// Boxed in an `Arc` so the enum stays `Clone` (`anyhow::Error` is not). diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs index 669df15f49..dc13b55b63 100644 --- a/rs/moq-mux/src/timeline.rs +++ b/rs/moq-mux/src/timeline.rs @@ -1,1240 +1,670 @@ -//! Timeline publish/subscribe: the broadcast's segment index HLS/DASH export is built from. +//! Per-track timelines: each track's index of spans, which HLS/DASH export and recordings use. //! -//! A broadcast has one timeline track, carrying one [`hang::timeline::Record`] per *segment*: -//! a span of content time shared by every media track, mapped to the group ranges that carry -//! it on each track. A consumer can answer "which groups cover segment N on track X" and -//! "where is the live edge" from a few bytes per segment without subscribing to media, which -//! is the primitive a playlist server (HLS/DASH), a seek bar, or a recorder index needs. +//! Every indexed track gets its own timeline track carrying one [`hang::timeline::Record`] per +//! *span*: a run of the track's frames mapped to their content time. A consumer can answer "which +//! groups cover time T on track X" and "where is the live edge" from a few bytes per span +//! without subscribing to media, which is the primitive a playlist server (HLS/DASH), a seek bar, +//! or a recorder index needs. //! -//! ## Facts up, policy down +//! ## Cutting //! -//! The write side splits into facts, policy, and publication: +//! Tracks cut independently; nothing waits for another track. //! -//! - **Tracks report facts.** Each media track enrolls via [`Producer::pacing_track`], and its -//! [`Recorder`] reports every group open (sequence, timestamp, keyframe) plus where its -//! content ends. A container import never decides where segments fall; it only states what -//! it published. -//! - **The timeline sets policy.** A segment ends at the first group boundary that gives it at -//! least [`Config::duration_min`] on every enrolled pacing track (see [`Config`] for the exact -//! rule). An application that knows its own boundaries overrides that with [`Producer::cut`]. -//! - **Pacing is explicit.** [`Producer::track`] safely enrolls a catalog or metadata track -//! without giving it control over segmentation. Continuous media opts in through -//! [`Producer::pacing_track`], which lets it vote on boundaries and gate completeness. -//! - **Records close on completeness.** A segment's record is ready only once every -//! enrolled pacing track has reported a group at or past the segment's end (or closed), proving -//! the segment's group ranges are final on every track that paces. The record is then -//! self-contained. [`Producer::reserve`] extends that across a batch -//! of enrollments, the way the catalog's own reservation does. -//! - **Publication is a commit.** [`Deferred`] yields complete records without publishing them. -//! A recorder can first persist the media groups they name, then pass the [`Pending`] handle to -//! [`Producer::push`] to make the record visible. The convenience methods on [`Producer`] commit -//! immediately for live publishers that need no asynchronous work between those steps. +//! Every track follows one rule, parameterized by its [`Config`]: //! -//! Alignment falls out of construction: every track maps its groups onto the same boundary -//! list, so segment N covers the same span of content time on every track, which is what HLS -//! requires of switchable renditions. +//! - A record ends at the first group boundary at least [`Config::duration_min`] (2s by default) +//! past its own start, so short groups (audio) pack into one record and long ones (video GOPs) +//! get one each. Only sparse data, such as a catalog that may not publish again for the rest of +//! the broadcast, uses a zero minimum: every group is then its own record, ended as soon as the +//! group finishes. +//! - A group still open [`Config::duration_max`] (10s by default) past the record's start is split +//! at the next reported frame, so an append-only group that never closes is still indexed as it +//! grows. Frames farther apart than the maximum leave nothing to split at, so the record's +//! duration stops at the maximum and the rest of the gap is uncovered. +//! - A skipped group sequence always closes the record at its newest reported content time, so a +//! record's groups are contiguous and a pause is not absorbed into its duration. +//! - [`Segmenter::cut`] adds an application boundary, such as a video keyframe cutting audio so a +//! derived segment needs fewer objects. The first cut takes over from minimum-duration pacing. //! //! ## Wiring //! -//! [`catalog::Producer`](crate::catalog::Producer) owns the broadcast's timeline (the catalog -//! is what owns the broadcast's shape) and wires all of this up: -//! The catalog's role-specific track constructors enroll the track, which -//! creates the timeline track on first use and advertises it in the catalog's root -//! [`hang::catalog::Archive`] entry. A broadcast that never enrolls a track publishes no -//! timeline at all: segmentation is opt-in per broadcast, never per track. +//! [`catalog::Producer`](crate::catalog::Producer) owns the broadcast's [`Timelines`] and advertises +//! them in the catalog's root [`hang::catalog::Archive`] entry. Its role-specific track +//! constructors enroll each media track, and the catalog enrolls itself with a zero minimum. //! -//! On the read side, [`Consumer::subscribe`] reads the timeline straight from the catalog's -//! [`hang::catalog::Archive`] entry (so the track name and timescale can't be mismatched) -//! and yields decoded [`Event`]s. On the wire the track is a DEFLATE-compressed -//! [`moq_json::window`], so a DVR can trim old records while an unbounded timeline simply never -//! pops them (see [`hang::timeline`] for the record schema). - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::ops::Deref; +//! A [`Segmenter`] builds records from a track's frame reports without publishing them, and a +//! [`Producer`] publishes records onto a timeline track. A [`Recorder`] pairs the two for live +//! publishing; a recording writer drives them separately so it can store the media a record names +//! before publishing it. On the read side, [`Consumer::subscribe`] reads one track's timeline from +//! the catalog's [`Archive`] entry and yields decoded [`Event`]s. On the wire each timeline is a +//! DEFLATE-compressed [`moq_json::window`], so a DVR can trim old records while an unbounded +//! timeline simply never pops them. + +use std::collections::{BTreeMap, VecDeque}; use std::sync::{Arc, Mutex}; use std::task::Poll; use std::time::Duration; use hang::catalog::Archive; -use hang::timeline::{DEFAULT_NAME, Range, Record, RecordExt}; +use hang::timeline::{Position, Record, RecordExt}; use moq_json::window::Checkpoint; use moq_net::{Timescale, Timestamp}; -/// The conventional [`Config::duration_min`] (1 second), for callers with no opinion. -pub const DEFAULT_DURATION_MIN: Duration = Duration::from_secs(1); +/// The default [`Config::duration_min`] (2 seconds). Only sparse data, such as a catalog, uses zero. +pub const DEFAULT_DURATION_MIN: Duration = Duration::from_secs(2); -/// Recent segment records repeated when the Window track rolls to a new group. +/// The default [`Config::duration_max`] (10 seconds). +pub const DEFAULT_DURATION_MAX: Duration = Duration::from_secs(10); + +/// Recent records repeated when the window track rolls to a new group. const CHECKPOINT_RECORDS: usize = 256; -/// How a [`Producer`] paces its segments. +/// How a track's records are cut. +/// +/// Every track follows one rule: a record ends at the first group boundary at least +/// [`duration_min`](Self::duration_min) past its start, and a group still open +/// [`duration_max`](Self::duration_max) past it is split between frames. #[derive(Clone, Debug)] #[non_exhaustive] pub struct Config { - /// The shortest a segment may be. + /// The shortest a record may be before a group boundary ends it. /// - /// A segment ends at the first point that is a group boundary on every enrolled track and - /// at least this far past the segment's start, so the track with the coarsest groups paces - /// the broadcast. A 2 second GOP against a 1 second minimum yields 2 second segments; a - /// real-time encoder with a 10 minute GOP yields one 10 minute segment, because there is - /// nowhere else a segment could start and stay decodable. - /// - /// A floor rather than a target on purpose: a floor is always satisfiable (wait longer), - /// while a ceiling is not (a single group longer than it can't be split). + /// A floor rather than a target on purpose: a floor is always satisfiable (wait for the next + /// group), so no group is split to honor it. Zero, for sparse data such as a catalog, makes + /// every group its own record, ended as soon as the group finishes. pub duration_min: Duration, - /// The longest a segment may be, advertised in the catalog when set. + /// The longest a record may run before a frame inside a group splits it. /// - /// Set it only when the publisher can actually promise one, i.e. it controls the encoder's - /// keyframe cadence. Consumers that need a bound up front use it (an HLS exporter's - /// `EXT-X-TARGETDURATION`), so it is a contract rather than a hint: a segment that would - /// exceed it fails the timeline instead of publishing a record that contradicts the - /// catalog. Leave it `None` when the media decides, which is the common case for real-time - /// and for anything importing a source it doesn't control. - pub duration_max: Option, + /// The longest enrolled maximum is advertised in the catalog as `durationMax`. + pub duration_max: Duration, +} + +impl Config { + /// Set [`duration_min`](Self::duration_min). + pub fn with_duration_min(mut self, duration: Duration) -> Self { + self.duration_min = duration; + self + } + + /// Set [`duration_max`](Self::duration_max). + pub fn with_duration_max(mut self, duration: Duration) -> Self { + self.duration_max = duration; + self + } } impl Default for Config { fn default() -> Self { Self { duration_min: DEFAULT_DURATION_MIN, - duration_max: None, + duration_max: DEFAULT_DURATION_MAX, } } } -/// One enrolled track's report state. -#[derive(Default)] -struct TrackState { - /// Group opens reported and not yet flushed into a record: (sequence, pts, keyframe). - pending: VecDeque<(u64, Timestamp, bool)>, - /// The newest reported timestamp: everything earlier is known, which is what lets a - /// segment ending at or before it flush. Advanced by a group open (the group starts there) - /// and by [`Recorder::end`] (the content stops there). - frontier: Option, - /// The recorder was dropped; this track no longer paces boundaries or gates completeness. - closed: bool, - /// Whether this track paces boundaries and gates completeness while open. - pacing: bool, +/// The wire timescale for record `pts`/`duration`: the catalog section's default, milliseconds. +const TIMESCALE: Timescale = Timescale::MILLI; + +/// A duration in wire units, rounded up so a bound never understates itself. +fn units(duration: Duration) -> u64 { + (duration.as_micros() * TIMESCALE.as_u64() as u128).div_ceil(1_000_000) as u64 } -impl TrackState { - /// The first unflushed group starting at or after `threshold` micros: this track's vote for - /// where the open segment can end. - fn candidate(&self, threshold: u128) -> Option { - self.pending - .iter() - .map(|&(_, pts, _)| pts) - .find(|pts| pts.as_micros() >= threshold) - } +fn wire(pts: Timestamp) -> u64 { + pts.as_scale(TIMESCALE) as u64 +} + +/// The record being built. +struct Open { + start: Position, + pts: Timestamp, + keyframe: bool, } -/// The state one [`Segmenter`] guards. -struct State { +/// Builds one track's records from reports of the frames it published. +/// +/// Reports arrive in position order: [`frame`](Self::frame) for each frame that may start a record +/// (every frame, or at least the first of each group), [`finish_group`](Self::finish_group) once a +/// group can gain no more frames, and [`end`](Self::end) wherever the content is known to stop. +/// Closed records queue until [`Iterator::next`] takes them. +pub struct Segmenter { config: Config, - /// Complete records waiting for the caller to commit. - ready: VecDeque, - /// Deferred records handed to the caller but not yet committed to the visible timeline. - committing: BTreeSet, - /// Persisted deferred records waiting for every earlier segment to be persisted too. - committed: BTreeMap, - /// Where the open (unflushed) segment starts; `None` until the first report. - start: Option, - /// Explicit [`Producer::cut`] boundaries not yet reached, in order. + /// The number the next closed record gets. + sequence: u64, + open: Option, + /// The newest reported frame position. + last: Option, + /// Whether the group of `last` has finished. + finished: bool, + /// The newest reported content time in the open record. + frontier: Option, + /// Application boundaries not yet reached, in order. cuts: VecDeque, - /// A [`Producer::cut`] arrived, so the application owns the boundaries from here on and the - /// `duration_min` pacing stops. Without this the pacing races ahead of a source whose - /// segments are longer than the minimum, closing one before its real boundary is declared. + /// A cut arrived, so the application owns the group boundaries from here on. manual: bool, - /// The number the next flushed record gets. - next_segment: u64, - /// Every enrolled track, keyed by media track name. - tracks: BTreeMap, - /// Live [`Reserved`] handles: while any exists, no record flushes (more tracks are still - /// enrolling). Mirrors the catalog's own reservation gate. - reservers: usize, - /// A segment overran [`Config::duration_max`]: `(segment, duration)`. The timeline stops - /// publishing, since the catalog's promise is already broken. - overrun: Option<(u64, Duration)>, - /// The terminal flush ran; surviving clones and recorders can no longer mutate segmentation. - finished: bool, - /// The owning [`Producer`] stopped auto-publishing ready records. - deferred: bool, - /// The wire timescale for `pts`/`duration` (the catalog section's default: milliseconds). - timescale: Timescale, + ready: VecDeque, + closed: bool, } -impl State { - /// Return an abandoned deferred record to its segment-ordered ready queue. - fn requeue(&mut self, record: Record) { - self.committing.remove(&record.segment); - let index = self - .ready - .iter() - .position(|ready| ready.segment > record.segment) - .unwrap_or(self.ready.len()); - self.ready.insert(index, record); +impl Segmenter { + /// A segmenter numbering its records from zero. + pub fn new(config: Config) -> Self { + Self { + config, + sequence: 0, + open: None, + last: None, + finished: false, + frontier: None, + cuts: VecDeque::new(), + manual: false, + ready: VecDeque::new(), + closed: false, + } } - /// A group open reported by `name`: record the fact and close whatever became complete. - fn report(&mut self, name: &str, sequence: u64, pts: Timestamp, keyframe: bool) { - if self.finished { - return; - } - let Some(track) = self.tracks.get_mut(name) else { - return; - }; - track.pending.push_back((sequence, pts, keyframe)); - self.advance(name, pts); + /// Number the next record `sequence`, continuing a resumed timeline. + pub fn with_sequence(mut self, sequence: u64) -> Self { + self.sequence = sequence; + self } - /// `name`'s content ends at `pts`, without a group opening there. This is what gives the - /// final segment an honest duration, since its end is not a boundary anybody cut. - fn report_end(&mut self, name: &str, pts: Timestamp) { - if self.finished { - return; - } - if self.tracks.contains_key(name) { - self.advance(name, pts); - } + /// The number the next closed record gets. + pub fn sequence(&self) -> u64 { + self.sequence } - /// Raise `name`'s frontier to `pts`, then close whatever that completed. - fn advance(&mut self, name: &str, pts: Timestamp) { - if let Some(track) = self.tracks.get_mut(name) - && track.frontier.is_none_or(|f| pts.as_micros() > f.as_micros()) + /// Report the frame at `position`, presented at `pts`. + /// + /// Fails with [`TimelinePosition`](crate::Error::TimelinePosition) when `position` is at or + /// before the previous report: records never overlap, so that frame could never be indexed. + pub fn frame(&mut self, position: Position, pts: Timestamp, keyframe: bool) -> crate::Result<()> { + if self.closed { + return Ok(()); + } + if let Some(last) = self.last + && position <= last { - track.frontier = Some(pts); + return Err(crate::Error::TimelinePosition { position, last }); + } + + if let (Some(start), Some(last)) = (self.open.as_ref().map(|open| open.pts), self.last) { + let elapsed = pts.as_micros().saturating_sub(start.as_micros()); + let end = if position.group != last.group { + let contiguous = position.group == last.group + 1 && position.frame == 0; + if !contiguous { + // A skipped sequence or a group joined mid-way always closes, so a record's groups are + // contiguous and the reader never has to guess what lies between. The skipped time is + // not content, so the record ends at its newest report rather than at this frame. + let frontier = self.frontier.expect("an open record has a frontier"); + Some((Position::group(last.group + 1), frontier)) + } else if elapsed >= self.config.duration_max.as_micros() || self.boundary(start, pts, elapsed) { + Some((Position::group(last.group + 1), pts)) + } else { + None + } + } else { + (elapsed >= self.config.duration_max.as_micros()).then_some((position, pts)) + }; + if let Some((end, pts)) = end { + self.emit(end, pts); + } } - // The first thing a pacing track reports anchors the first segment, so content produced - // before any boundary exists belongs to the oldest segment rather than to nowhere. A - // non-pacing track must not anchor it: a catalog published while the encoder is still warming - // up would otherwise stretch segment 0 across the whole startup gap. - if self.start.is_none() && self.tracks.get(name).is_some_and(|t| t.pacing) { - self.start = Some(pts); + if self.open.is_none() { + self.open = Some(Open { + start: position, + pts, + keyframe, + }); + self.frontier = None; } - - self.pump(false); + self.last = Some(position); + self.finished = false; + self.advance(pts); + Ok(()) } - /// A track's recorder was dropped: stop gating completeness on it. - fn close(&mut self, name: &str) { - if self.finished { - return; + /// Whether a group starting at `pts` ends the record that started at `start`. + fn boundary(&mut self, start: Timestamp, pts: Timestamp, elapsed: u128) -> bool { + while self + .cuts + .front() + .is_some_and(|cut| cut.as_micros() <= start.as_micros()) + { + self.cuts.pop_front(); } - if let Some(track) = self.tracks.get_mut(name) { - track.closed = true; + if self.cuts.front().is_some_and(|cut| cut.as_micros() <= pts.as_micros()) { + while self.cuts.front().is_some_and(|cut| cut.as_micros() <= pts.as_micros()) { + self.cuts.pop_front(); + } + return true; } - self.pump(false); + !self.manual && elapsed >= self.config.duration_min.as_micros() } - /// Close every segment the media has finalized. + /// Report that group `group` can gain no more frames. /// - /// `finished` is the terminal pass: no track will report again, so a track that never - /// reached a boundary stops voting instead of holding the timeline open forever. - fn pump(&mut self, finished: bool) { - // With nothing enrolled there is nothing to describe. A record is immutable once - // published, so a caller that knows more tracks are still enrolling withholds it (see - // [`Producer::reserve`]), and an overrun has already broken the catalog's promise. - if self.finished || self.tracks.is_empty() || self.reservers > 0 || self.overrun.is_some() { + /// With a zero [`Config::duration_min`] this closes the record, so a sparse track's newest + /// group is indexed without waiting for the next one. + pub fn finish_group(&mut self, group: u64) { + if self.closed || self.last.is_none_or(|last| last.group != group) { return; } - - while self.close_segment(finished) { - // A segment that broke the declared maximum ends the timeline; every later record - // would inherit the same broken promise. - if self.overrun.is_some() { - break; - } + self.finished = true; + if self.config.duration_min.is_zero() && self.open.is_some() { + let pts = self.frontier.expect("an open record has a frontier"); + self.emit(Position::group(group + 1), pts); } } - /// Close the open segment if the media has finalized it, returning whether it did. - fn close_segment(&mut self, finished: bool) -> bool { - let Some(start) = self.start else { - return false; - }; - - // Discard boundaries the timeline has already reached. Only the front is ever consulted, - // so leaving a spent one there would block every later cut behind it and silently drop - // the caller back to `duration_min` pacing. - while self.cuts.front().is_some_and(|c| c.as_micros() <= start.as_micros()) { - self.cuts.pop_front(); - } - - let Some((end, cut)) = self.boundary(start, finished) else { - return false; - }; - - // Every open pacing track has to have reported at or past the boundary, proving its ranges - // for this segment are final. The track that voted for `end` has by construction; a track - // with shorter groups can still be behind it. A non-pacing track is excluded: it may publish - // rarely or never again, so waiting on it would stall the timeline for good. - let complete = finished - || self - .tracks - .values() - .all(|t| !t.pacing || t.closed || t.frontier.is_some_and(|f| f.as_micros() >= end.as_micros())); - if !complete { - return false; - } - - self.flush_segment(start, Some(end)); - if cut { - self.cuts.pop_front(); + /// Report that the content extends to `pts`, without a frame starting there. + /// + /// A frame report says where content starts; the last record has no successor to bound it, so + /// its duration would otherwise stop at its last frame's start. + pub fn end(&mut self, pts: Timestamp) { + if !self.closed && self.open.is_some() { + self.advance(pts); } - self.start = Some(end); - true } - /// Where the segment starting at `start` ends, once the media says so: an explicit cut when - /// one is registered, otherwise the first group boundary shared by every track that gives - /// the segment its minimum duration. The bool reports which. - fn boundary(&self, start: Timestamp, finished: bool) -> Option<(Timestamp, bool)> { - if let Some(&cut) = self.cuts.front() - && cut.as_micros() > start.as_micros() + fn advance(&mut self, pts: Timestamp) { + if self + .frontier + .is_none_or(|frontier| pts.as_micros() > frontier.as_micros()) { - return Some((cut, true)); + self.frontier = Some(pts); } + } - // The application declared a boundary at some point, so it owns them all: pacing here - // would close a segment the caller is about to cut somewhere else. - if self.manual { - return None; + /// Declare a boundary at `pts`: the record closes at the first group starting at or after it. + /// + /// The first cut takes over from [`Config::duration_min`] pacing for good, since pacing would + /// otherwise close a record just before the caller declares where it really ends. A cut at or + /// before an earlier one is ignored, so several producers declaring the same boundaries (the + /// renditions of one import) cost nothing. + pub fn cut(&mut self, pts: Timestamp) { + if self.closed { + return; } - - let threshold = start.as_micros() + self.config.duration_min.as_micros(); - - let mut end: Option = None; - for track in self.tracks.values().filter(|t| t.pacing) { - match track.candidate(threshold) { - // The latest vote wins: it is a group boundary on the coarsest track, and every - // finer track assigns its groups by start, so no group is split. A closed track - // still votes: it can't report more, but the groups it did report are boundaries - // like any other, and without them a backlog would collapse into one segment. - Some(pts) => { - if end.is_none_or(|e| pts.as_micros() > e.as_micros()) { - end = Some(pts); - } - } - // This track has produced nothing past the minimum yet, so ending the segment - // would strand it. A closed track never will, and neither does anything on the - // terminal pass, so neither one blocks. - None if finished || track.closed => continue, - None => return None, - } + self.manual = true; + if self.cuts.back().is_none_or(|back| pts.as_micros() > back.as_micros()) { + self.cuts.push_back(pts); } + } - end.map(|end| (end, false)) - } - - /// Build the record for the segment starting at `start`: drain every track's groups before - /// `end` (all of them for the final, unbounded segment) into ranges. - fn flush_segment(&mut self, start: Timestamp, end: Option) { - let pts = start.as_scale(self.timescale) as u64; - let duration = match end { - Some(end) => (end.as_scale(self.timescale) as u64).saturating_sub(pts), - // The final segment has no end boundary, so it runs to the newest thing any track - // reported: its end of content when the track reported one (a finished - // `container::Producer` does), otherwise the last group it opened, which - // undercounts that group's tail. - None => self - .tracks - .values() - .filter(|t| t.pacing) - .filter_map(|t| t.frontier) - .map(|f| f.as_scale(self.timescale) as u64) - .max() - .unwrap_or(pts) - .saturating_sub(pts), + /// Close the open record now, at the end of the newest reported frame. + /// + /// The record ends after the newest group when that group finished, otherwise after the newest + /// reported frame. Later reports start a new record. + pub fn flush(&mut self) { + let (Some(last), Some(frontier)) = (self.last, self.frontier) else { + return; }; - - // The catalog promised a bound and this segment breaks it, so the record would - // contradict what consumers were told. Fail the timeline rather than publish it: a - // declared maximum the media can't honor is a bug in the publisher, and nothing a - // consumer reading the catalog can work around. - if let Some(max) = self.config.duration_max - && duration > self.units(max) - { - let observed = Duration::from_nanos(duration * (1_000_000_000 / self.timescale.as_u64())); - tracing::error!( - segment = self.next_segment, - duration = ?observed, - duration_max = ?max, - "segment exceeded the declared duration_max; dropping the timeline track" - ); - self.overrun = Some((self.next_segment, observed)); + if self.open.is_none() { return; } - - let mut record = Record::new(self.next_segment, pts, duration); - self.next_segment += 1; - - for (name, track) in &mut self.tracks { - let mut ranges: Vec = Vec::new(); - while let Some(&(sequence, group_pts, keyframe)) = track.pending.front() { - // A pacing track is assigned by content time. A non-pacing track is assigned by - // arrival, so every group pending when the segment closes belongs to it regardless - // of the timestamp basis carried by that track. - if track.pacing && end.is_some_and(|end| group_pts.as_micros() >= end.as_micros()) { - break; - } - track.pending.pop_front(); - match ranges.last_mut() { - // Contiguous sequences extend the run; a skip starts a new range (a gap: - // groups that never existed). - Some(last) if last.end + 1 == sequence => last.end = sequence, - _ => { - let mut range = Range::new(sequence, sequence); - range.keyframe = keyframe; - ranges.push(range); - } - } - } - if !ranges.is_empty() { - record.tracks.insert(name.clone(), ranges); - } - } - - self.ready.push_back(record); + let end = match self.finished { + true => Position::group(last.group + 1), + false => Position::new(last.group, last.frame + 1), + }; + self.emit(end, frontier); } - /// A duration in the wire timescale's units, rounded up so a bound never understates itself. - fn units(&self, duration: Duration) -> u64 { - (duration.as_micros() * self.timescale.as_u64() as u128).div_ceil(1_000_000) as u64 + /// Flush, then ignore every later report. + pub fn close(&mut self) { + self.flush(); + self.closed = true; } - /// The terminal flush: close what the media finalized, then the open tail. - fn finish(&mut self) { - if self.finished { + /// Close the open record at `end`, its content running to `pts`. + fn emit(&mut self, end: Position, pts: Timestamp) { + let Some(open) = self.open.take() else { return; - } - - // Nothing more will enroll, so an outstanding reservation has nothing left to wait for. - self.reservers = 0; - self.pump(true); - - // Skip an empty tail: a boundary with no content after it describes nothing. - if self.overrun.is_none() - && let Some(start) = self.start.take() - && self.tracks.values().any(|t| !t.pending.is_empty()) - { - self.flush_segment(start, None); - } - - self.finished = true; + }; + let start = wire(open.pts); + // Frames farther apart than the maximum leave nothing to split at, so the record stops at the + // advertised bound and the rest of the gap is uncovered. + let duration = wire(pts).saturating_sub(start).min(units(self.config.duration_max)); + let mut record = Record::new(self.sequence, start, duration, open.start, end); + record.keyframe = open.keyframe; + self.sequence += 1; + self.ready.push_back(record); } +} - /// The error a segment overrunning [`Config::duration_max`] left behind, if any. - fn failure(&self) -> Option { - let (segment, duration) = self.overrun?; - Some(crate::Error::TimelineOverrun { - segment, - duration, - duration_max: self.config.duration_max.unwrap_or_default(), - }) - } +/// Yields each closed record once; `None` means none is ready yet, not that none ever will be. +impl Iterator for Segmenter { + type Item = Record; - /// The catalog section advertising this timeline. - /// - /// Carries the track name, timescale, and duration bound. Wall-clock mapping is the - /// catalog root clock's job ([`Clock`](crate::Clock)), not this section's. - fn section(&self) -> Archive { - let mut section = Archive::new(DEFAULT_NAME); - section.timescale = self.timescale.as_u64() as u32; - section.duration_max = self.config.duration_max.map(|max| self.units(max)); - section + fn next(&mut self) -> Option { + self.ready.pop_front() } } -/// Builds complete segment records without publishing them. +/// Publishes one track's records onto its timeline track. /// -/// Use this directly to build records without a broadcast. An archive writer instead obtains a -/// cancellation-safe [`Deferred`] from [`Producer::deferred`], stores the groups named by each -/// [`Pending`] record, and passes that handle to [`Producer::push`]. `Clone` shares the -/// segmentation state. -#[derive(Clone)] -pub struct Segmenter { - state: Arc>, +/// Records must be pushed in sequence order, starting at the window's next index. +pub struct Producer { + sink: moq_json::window::Producer, } -impl Segmenter { - /// Create a segmenter paced by `config`. - pub fn new(config: Config) -> Self { - // The contents are `Send + Sync` natively; on wasm moq-net's handles are - // `Rc`-backed, so clippy sees a pointlessly atomic `Arc`. Keeping one type for - // both targets is worth the unused atomics on the single-threaded one. - #[allow(clippy::arc_with_non_send_sync)] +impl Producer { + /// The properties a timeline track is created with. + pub fn info() -> moq_net::track::Info { + moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog) + } + + /// Publish onto `track`, starting at record zero. + pub fn new(track: moq_net::track::Producer) -> Self { Self { - state: Arc::new(Mutex::new(State { - config, - ready: VecDeque::new(), - committing: BTreeSet::new(), - committed: BTreeMap::new(), - start: None, - cuts: VecDeque::new(), - manual: false, - next_segment: 0, - tracks: BTreeMap::new(), - reservers: 0, - overrun: None, - finished: false, - deferred: false, - timescale: Timescale::MILLI, - })), + sink: moq_json::window::Producer::new(track, Self::config()), } } - /// Enroll `name` without letting it influence segmentation. - pub fn track(&self, name: &str) -> Recorder { - self.enroll(name, false, None) + /// Publish onto `track`, continuing `checkpoint`, such as a window recovered from storage. + /// + /// The track's first group restates the checkpoint, and the next record is sequence + /// `checkpoint.range.end`. Fails when a checkpoint record is not the sequence at its index or the + /// checkpoint is malformed. + pub fn resume(track: moq_net::track::Producer, checkpoint: &Checkpoint) -> crate::Result { + let sink = moq_json::window::Producer::resume(track, Self::config(), checkpoint).map_err(json)?; + // The encoder accepted the checkpoint, so the records fit before `range.end`. + let start = checkpoint.range.end - checkpoint.records.len() as u64; + for (index, record) in (start..).zip(&checkpoint.records) { + if record.sequence != index { + return Err(crate::Error::TimelineCheckpoint(index)); + } + } + Ok(Self { sink }) } - /// Enroll `name` as a pacing track. - pub fn pacing_track(&self, name: &str) -> Recorder { - self.enroll(name, true, None) + fn config() -> moq_json::window::ProducerConfig { + moq_json::window::ProducerConfig::default() + .with_compression(true) + .with_checkpoint_records(CHECKPOINT_RECORDS) } - fn enroll(&self, name: &str, pacing: bool, output: Option>>) -> Recorder { - let mut state = self.state.lock().unwrap(); - let output = if state.finished { - None - } else { - state.tracks.insert( - name.to_string(), - TrackState { - pacing, - ..Default::default() - }, - ); - output - }; - drop(state); - - Recorder { - segmenter: self.clone(), - output, - name: name.to_string(), - } + /// The window indices currently retained. + pub fn range(&self) -> std::ops::Range { + self.sink.range() } - /// Declare a segment boundary at `pts`, overriding automatic minimum-duration pacing. - pub fn cut(&self, pts: Timestamp) -> crate::Result<()> { - let mut state = self.state.lock().unwrap(); - if let Some(err) = state.failure() { - return Err(err); - } - if state.finished { - return Err(moq_net::Error::Closed.into()); - } - - state.manual = true; - let floor = state - .cuts - .back() - .copied() - .or(state.start) - .map(|since| since.as_micros() + state.config.duration_min.as_micros()); - if floor.is_none_or(|floor| pts.as_micros() >= floor) { - state.cuts.push_back(pts); - state.pump(false); + /// Append `record`, which must be the window's next sequence. + pub fn push(&mut self, record: &Record) -> crate::Result<()> { + let next = self.sink.range().end; + if record.sequence != next { + return Err(crate::Error::TimelineSequence { + expected: next, + actual: record.sequence, + }); } - - Ok(()) + self.sink.push(record).map_err(json) } - /// Withhold complete records while a batch of tracks enrolls. - pub fn reserve(&self) -> Reserved { - let mut state = self.state.lock().unwrap(); - if !state.finished { - state.reservers += 1; - } - Reserved { - segmenter: self.clone(), - output: None, - } + /// Remove up to `count` oldest records from the window. + pub fn pop(&mut self, count: u64) -> crate::Result<()> { + self.sink.pop(count).map_err(json) } - /// The catalog section describing records built by this segmenter. - pub fn section(&self) -> Archive { - self.state.lock().unwrap().section() + /// Close the track's open group, so every push and pop so far sits in a complete group. + /// + /// A recorder stores those groups before committing the next record. The next edit opens a + /// new group restating the window. + pub fn flush(&mut self) -> crate::Result<()> { + self.sink.cut().map_err(json) } - /// Flush the final open segment and return a handle that can only drain records. - pub fn finish(self) -> Drain { - let mut state = self.state.lock().unwrap(); - state.finish(); - drop(state); - Drain { segmenter: self } + /// Finish the timeline track. + pub fn finish(&mut self) -> crate::Result<()> { + self.sink.finish().map_err(json) } +} - /// Take the next complete segment record, if one is ready. - pub fn next(&self) -> Option { - self.state.lock().unwrap().ready.pop_front() +/// Surface a transport failure as one, rather than wrapped in the JSON layer. +fn json(err: moq_json::Error) -> crate::Error { + match err { + moq_json::Error::Net(err) => err.into(), + err => err.into(), } } -/// A finished segmenter that can only drain its final complete records. -pub struct Drain { +/// A live track's segmenter and the timeline it publishes to. +struct Live { segmenter: Segmenter, + /// `None` once a publish failed: the timeline is an optional sidecar, so it stops instead. + output: Option, } -impl Drain { - /// Report whether terminal segmentation exceeded the declared duration bound. - pub fn result(&self) -> crate::Result<()> { - result(&self.segmenter) +impl Live { + fn publish(&mut self) { + for record in self.segmenter.by_ref() { + let Some(output) = self.output.as_mut() else { + continue; + }; + if let Err(err) = output.push(&record) { + tracing::warn!(%err, "timeline publish failed; dropping the timeline track"); + let _ = output.finish(); + self.output = None; + } + } } - /// Take the next complete segment record, if one is ready. - pub fn next(&self) -> Option { - self.segmenter.next() + fn finish(&mut self) { + self.segmenter.close(); + self.publish(); + if let Some(output) = self.output.as_mut() { + let _ = output.finish(); + } } } -/// A deferred segmenter whose records must be persisted before publication. +struct Registry { + /// Each enrolled track's timeline, by track name. Kept after its recorder drops, so a finished + /// span stays discoverable and a re-enrolled track continues its numbering. + tracks: BTreeMap>)>, + finished: bool, +} + +/// A broadcast's timelines, one per enrolled track. /// -/// [`next`](Self::next) returns a cancellation-safe [`Pending`] handle. Dropping the handle puts -/// the record back at the front of the ready queue so another storage task can retry it. +/// Each enrollment creates the track's timeline track and returns the [`Recorder`] its producer +/// reports through. `Clone` shares the set. #[derive(Clone)] -pub struct Deferred { - segmenter: Segmenter, +pub struct Timelines { + broadcast: moq_net::broadcast::Producer, + registry: Arc>, } -impl Deferred { - /// Enroll `name` without letting it influence segmentation. - pub fn track(&self, name: &str) -> Recorder { - self.segmenter.track(name) - } - - /// Enroll `name` as a pacing track. - pub fn pacing_track(&self, name: &str) -> Recorder { - self.segmenter.pacing_track(name) +impl Timelines { + /// Timelines for tracks of `broadcast`. + pub fn new(broadcast: &moq_net::broadcast::Producer) -> Self { + // The contents are `Send + Sync` natively; on wasm moq-net's handles are `Rc`-backed, so + // clippy sees a pointlessly atomic `Arc`. One type for both targets is worth it. + #[allow(clippy::arc_with_non_send_sync)] + let registry = Arc::new(Mutex::new(Registry { + tracks: BTreeMap::new(), + finished: false, + })); + Self { + broadcast: broadcast.clone(), + registry, + } } - /// Declare a segment boundary at `pts`, overriding automatic minimum-duration pacing. - pub fn cut(&self, pts: Timestamp) -> crate::Result<()> { - self.segmenter.cut(pts) - } + /// Enroll the track `name`, its records cut by `config`. + /// + /// Creates its timeline track, named by [`hang::timeline::default_name`], on first enrollment. + /// Enrolling a name again continues its timeline's numbering for a new producer, cut by the new + /// `config`. Errors when the timeline track name is taken or the timelines finished. + pub fn track(&self, name: &str, config: Config) -> crate::Result { + let mut registry = self.registry.lock().unwrap(); + if registry.finished { + return Err(moq_net::Error::Closed.into()); + } + if let Some((_, existing)) = registry.tracks.get(name) { + // A re-enrolled track is a new producer whose group sequences may restart, so it gets a + // fresh segmenter continuing the record numbering. + let mut live = existing.lock().unwrap(); + live.segmenter.flush(); + live.publish(); + let sequence = live.segmenter.sequence(); + live.segmenter = Segmenter::new(config).with_sequence(sequence); + drop(live); + return Ok(Recorder { live: existing.clone() }); + } - /// Withhold complete records while a batch of tracks enrolls. - pub fn reserve(&self) -> Reserved { - self.segmenter.reserve() + let timeline = hang::timeline::default_name(name); + let track = self.broadcast.create_track(timeline.as_str(), Producer::info())?; + #[allow(clippy::arc_with_non_send_sync)] + let live = Arc::new(Mutex::new(Live { + segmenter: Segmenter::new(config), + output: Some(Producer::new(track)), + })); + registry.tracks.insert(name.to_string(), (timeline, live.clone())); + Ok(Recorder { live }) } - /// The catalog's root `archive` entry describing records built by this segmenter. + /// The catalog's root `archive` entry advertising every enrolled track's timeline, bounded by + /// the longest maximum any of them is cut by. pub fn section(&self) -> Archive { - self.segmenter.section() + let registry = self.registry.lock().unwrap(); + let mut section = Archive::new(); + section.timescale = TIMESCALE.as_u64() as u32; + section.duration_max = registry + .tracks + .values() + .map(|(_, live)| units(live.lock().unwrap().segmenter.config.duration_max)) + .max(); + section.timelines = registry + .tracks + .iter() + .map(|(name, (timeline, _))| (name.clone(), timeline.clone())) + .collect(); + section } - /// Flush the final open segment and return a handle that can only drain pending records. - pub fn finish(self) -> DeferredDrain { - let drain = self.segmenter.finish(); - DeferredDrain { - segmenter: drain.segmenter, + /// Flush every open record and finish every timeline track. Later enrollments fail. + pub fn finish(&self) { + let mut registry = self.registry.lock().unwrap(); + registry.finished = true; + for (_, live) in registry.tracks.values() { + live.lock().unwrap().finish(); } } - - /// Take the next complete record for storage, if one is ready. - pub fn next(&self) -> Option { - pending(&self.segmenter) - } } -/// A finished deferred segmenter that can only drain its final records for storage. -pub struct DeferredDrain { - segmenter: Segmenter, +/// Reports one track's frames into its live timeline. +/// +/// Move-only: it is the track's reporting handle, minted by [`Timelines::track`] and held by a +/// rendition's [`container::Producer`](crate::container::Producer). Dropping it flushes the open +/// record; the timeline track stays open until [`Timelines::finish`]. +pub struct Recorder { + live: Arc>, } -impl DeferredDrain { - /// Report whether terminal segmentation exceeded the declared duration bound. - pub fn result(&self) -> crate::Result<()> { - result(&self.segmenter) +impl Recorder { + fn report(&mut self, f: impl FnOnce(&mut Segmenter)) { + let mut live = self.live.lock().unwrap(); + f(&mut live.segmenter); + live.publish(); } - /// Take the next complete record for storage, if one is ready. - pub fn next(&self) -> Option { - pending(&self.segmenter) + /// Report the frame at `position`, presented at `pts`; see [`Segmenter::frame`]. + /// + /// A report the timeline cannot place ends the timeline track with an error rather than + /// publishing a timeline that silently omits frames. Media publishing is unaffected. + pub fn frame(&mut self, position: Position, pts: Timestamp, keyframe: bool) { + let mut live = self.live.lock().unwrap(); + if let Err(err) = live.segmenter.frame(position, pts, keyframe) { + tracing::warn!(%err, "timeline report rejected; dropping the timeline track"); + live.segmenter.close(); + // Dropped without a finish, so subscribers see the timeline fail rather than end. + live.output = None; + } + live.publish(); } -} -fn result(segmenter: &Segmenter) -> crate::Result<()> { - match segmenter.state.lock().unwrap().failure() { - Some(err) => Err(err), - None => Ok(()), + /// Report that group `group` can gain no more frames; see [`Segmenter::finish_group`]. + pub fn finish_group(&mut self, group: u64) { + self.report(|segmenter| segmenter.finish_group(group)); } -} - -fn pending(segmenter: &Segmenter) -> Option { - let mut state = segmenter.state.lock().unwrap(); - let record = state.ready.pop_front()?; - state.committing.insert(record.segment); - drop(state); - Some(Pending { - segmenter: segmenter.clone(), - record: Some(record), - }) -} -/// A complete deferred record being stored before timeline publication. -/// -/// Dereference it to inspect or serialize the [`Record`]. Pass ownership to [`Producer::push`] -/// only after storage succeeds. Dropping it requeues the record for another attempt. -pub struct Pending { - segmenter: Segmenter, - record: Option, -} - -impl Pending { - /// Convert a record whose media could not be stored into a gap. - /// - /// Its segment number and timing stay intact, so committing it unblocks later segments without - /// advertising media objects that are not durable. - pub fn gap(mut self) -> Self { - self.record - .as_mut() - .expect("a pending record is present until commit") - .tracks - .clear(); - self + /// Report that the content extends to `pts`; see [`Segmenter::end`]. + pub fn end(&mut self, pts: Timestamp) { + self.report(|segmenter| segmenter.end(pts)); } - /// Drop one track whose media could not be stored, keeping every other track's ranges. - /// - /// Like [`gap`](Self::gap) for a single track: the record stays committable without advertising - /// objects that are not durable. - pub fn omit(mut self, track: &str) -> Self { - self.record - .as_mut() - .expect("a pending record is present until commit") - .tracks - .remove(track); - self + /// Declare a boundary at `pts` on this track; see [`Segmenter::cut`]. + pub fn cut(&mut self, pts: Timestamp) { + self.report(|segmenter| segmenter.cut(pts)); } -} -impl Deref for Pending { - type Target = Record; - - fn deref(&self) -> &Self::Target { - self.record.as_ref().expect("a pending record is present until commit") + /// Close the open record now; see [`Segmenter::flush`]. + pub(crate) fn flush(&mut self) { + self.report(|segmenter| segmenter.flush()); } } -impl Drop for Pending { +impl Drop for Recorder { fn drop(&mut self) { - let Some(record) = self.record.take() else { - return; - }; - let mut state = self.segmenter.state.lock().unwrap(); - state.requeue(record); + self.report(Segmenter::flush); } } -struct Output { - broadcast: moq_net::broadcast::Producer, - sink: Option>, - closed: bool, -} - -impl Output { - fn prepare(&mut self) -> crate::Result<()> { - if self.sink.is_none() && !self.closed { - let net = self.create_track()?; - self.sink = Some(moq_json::window::Producer::new(net, Self::config())); - } - Ok(()) - } +/// One decoded timeline record, with real [`Timestamp`]s. +#[derive(Debug, Clone, PartialEq)] +pub struct Entry { + /// The record's number, consecutive within its track's timeline. + pub sequence: u64, - /// Create the timeline track now, continuing `checkpoint`. - fn resume(&mut self, checkpoint: &Checkpoint) -> crate::Result<()> { - let net = self.create_track()?; - self.sink = Some(moq_json::window::Producer::resume(net, Self::config(), checkpoint)?); - Ok(()) - } + /// The span's start. + pub pts: Timestamp, - fn create_track(&self) -> crate::Result { - let info = moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog); - Ok(self.broadcast.create_track(DEFAULT_NAME, info)?) - } + /// The span's duration. The next entry starts at `pts + duration` unless content time itself + /// jumped (a discontinuity). + pub duration: Duration, - fn config() -> moq_json::window::ProducerConfig { - moq_json::window::ProducerConfig::default() - .with_compression(true) - .with_checkpoint_records(CHECKPOINT_RECORDS) - } + /// The first frame of the span, inclusive. + pub start: Position, - fn push(&mut self, record: &Record) -> crate::Result<()> { - self.prepare()?; - let Some(sink) = self.sink.as_mut() else { - return Err(moq_net::Error::Closed.into()); - }; - match sink.push(record) { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), - } - } + /// The end of the span, exclusive. + pub end: Position, - fn pop(&mut self, count: u64) -> crate::Result<()> { - let Some(sink) = self.sink.as_mut() else { - return Ok(()); - }; - match sink.pop(count) { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), - } - } + /// Whether the span's first frame is a keyframe. + pub keyframe: bool, - fn flush(&mut self) -> crate::Result<()> { - let Some(sink) = self.sink.as_mut() else { - return Ok(()); - }; - match sink.cut() { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), - } - } + /// The record's application extension (nothing for the default `()`). + pub ext: E, +} - fn finish(&mut self) -> crate::Result<()> { - self.closed = true; - let Some(sink) = self.sink.as_mut() else { - return Ok(()); - }; - // Keep this handle alive after closing so the finished track remains discoverable for - // subscribers. - match sink.finish() { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), - } +impl Entry { + /// The span's exclusive end time. + pub fn end_time(&self) -> Duration { + Duration::from(self.pts) + self.duration } } -/// The broadcast's timeline: a segmenter plus the track committed records are published on. -/// -/// Its enrollment methods commit complete records immediately. Use [`deferred`](Self::deferred) -/// when media must be stored before the corresponding record becomes visible. -#[derive(Clone)] -pub struct Producer { - segmenter: Segmenter, - output: Arc>, +/// One change to the visible timeline window. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum Event { + /// A record became visible at this window index, which equals its sequence. + Push { + /// Absolute window index assigned to the record. + index: u64, + /// The decoded record. + entry: Entry, + }, + /// These previously visible window indices were removed. + Pop(std::ops::Range), + /// These indices were removed before this consumer saw them. + Skip(std::ops::Range), } -impl Producer { - /// A timeline for `broadcast`, paced by `config`. - pub fn new(broadcast: &moq_net::broadcast::Producer, config: Config) -> Self { - // Like Segmenter, the output is Send + Sync natively and Rc-backed on single-threaded wasm. - #[allow(clippy::arc_with_non_send_sync)] - let output = Arc::new(Mutex::new(Output { - broadcast: broadcast.clone(), - sink: None, - closed: false, - })); - Self { - segmenter: Segmenter::new(config), - output, - } - } - - /// A timeline for `broadcast` continuing `checkpoint`, such as a window recovered from storage. - /// - /// A timeline's window index is its segment number, so the next record is segment - /// `checkpoint.range.end`. Creates the timeline track immediately; its first group restates the - /// checkpoint. Fails when a checkpoint record is not the segment at its index, the checkpoint - /// is malformed, or the track cannot be created. - pub fn resume( - broadcast: &moq_net::broadcast::Producer, - config: Config, - checkpoint: &Checkpoint, - ) -> crate::Result { - let producer = Self::new(broadcast, config); - producer.output.lock().unwrap().resume(checkpoint)?; - - // The encoder accepted the checkpoint, so the records fit before `range.end`. - let start = checkpoint.range.end - checkpoint.records.len() as u64; - for (index, record) in (start..).zip(&checkpoint.records) { - if record.segment != index { - return Err(crate::Error::TimelineCheckpoint(index)); - } - } - producer.segmenter.state.lock().unwrap().next_segment = checkpoint.range.end; - Ok(producer) - } - - /// Enroll `name` without letting it influence segmentation, returning its [`Recorder`]. - /// - /// Its groups are recorded into whichever segment is open when they arrive, but the track - /// never votes on where a boundary falls and never holds a record back. This safe default is - /// for a catalog that emits only when renditions change, or an application's metadata track. - /// - /// Placement is by arrival rather than by content time: nothing waits for the track, so a group - /// that arrives after its segment flushed is recorded in the next one. Frames still carry - /// their own timestamps. A group that never closes (a `moq_json::stream` log) is recorded once, - /// in the segment its group opened in. - /// - /// This does not create the timeline track, so enrolling only non-pacing tracks publishes no - /// timeline. One recorder per track: enrolling the same name again resets its state. - pub fn track(&self, name: &str) -> Recorder { - self.segmenter.enroll(name, false, Some(self.output.clone())) - } - - /// Enroll the media track `name` as a pacing track, returning the [`Recorder`] it reports - /// through. - /// - /// The segment records key ranges by this name, and the track votes on boundaries and gates - /// completeness until its recorder drops. Enroll it when it is about to produce: an enrolled - /// but silent pacing track holds every record back, by design, since a segment is not complete - /// until every pacing track's content is known. - /// - /// This opt-in is for continuously publishing media. A sparse track belongs in - /// [`track`](Self::track), otherwise it can stall the timeline when it goes quiet. - /// - /// Creates the timeline track on first use, which errors if the broadcast cannot because - /// something else already took the name. - pub fn pacing_track(&self, name: &str) -> crate::Result { - self.output.lock().unwrap().prepare()?; - Ok(self.segmenter.enroll(name, true, Some(self.output.clone()))) - } - - /// Declare a segment boundary at `pts`, overriding the [`Config::duration_min`] pacing. - /// - /// For applications that know their own boundaries (an HLS import following the source - /// playlist, CMAF segments on disk, an encoder placing keyframes). Cutting ahead of the - /// media is fine: the segment's record still waits for every track's groups. A cut that - /// would make a segment shorter than [`Config::duration_min`] is ignored, so several - /// producers declaring the same boundaries (the renditions of one import) cost nothing. - /// - /// The first call takes over for good: [`Config::duration_min`] pacing stops, since it would - /// otherwise close a segment just before the caller declares where it really ends. Segments - /// then last exactly as long as the caller says, and the final one runs to the end of the - /// media. - /// - /// Errors if a segment already overran [`Config::duration_max`]. - pub fn cut(&self, pts: Timestamp) -> crate::Result<()> { - self.segmenter.cut(pts)?; - self.publish_ready(); - Ok(()) - } - - /// Begin reserving the track set, returning a clonable [`Reserved`]. - /// - /// The counterpart to [`catalog::Producer::reserve`](crate::catalog::Producer::reserve), for - /// the same reason: while any `Reserved` clone is alive the track set may still grow, so - /// records are withheld from the broadcast. A record is immutable once published and its - /// completeness is judged against the tracks enrolled *at that moment*, so a segment that - /// flushes while a sibling rendition is still enrolling omits it for good, and that - /// rendition's earlier groups then land in whatever segment flushes next. - /// - /// Hand it (or clones) to whatever brings the tracks up, so an importer that enrolls its - /// renditions one at a time publishes nothing until they are all in. Unlike the catalog's, - /// this gate is not one-shot: the catalog is a snapshot, so only its *first* publish needs - /// protecting, while every timeline record is an immutable log entry. Take a fresh - /// reservation around every batch. - pub fn reserve(&self) -> Reserved { - let mut state = self.segmenter.state.lock().unwrap(); - let output = if state.finished { - None - } else { - state.reservers += 1; - Some(self.output.clone()) - }; - Reserved { - segmenter: self.segmenter.clone(), - output, - } - } - - /// The catalog's root `archive` entry advertising this timeline. - pub fn section(&self) -> Archive { - self.segmenter.section() - } - - /// Create a handle that closes segments without publishing them automatically. - /// - /// Drain it with [`Deferred::next`], perform any asynchronous storage, then commit the final - /// record with [`push`](Self::push). Preparing the timeline track here makes that later commit - /// infallible with respect to track-name ownership. This switches the shared producer for good: - /// recorders and reservations minted before this call also stop auto-publishing. - pub fn deferred(&self) -> crate::Result { - // Match publish_ready's output-then-state lock order. Once this returns, no auto-publisher can - // still be between dequeuing a record and making it visible. - let mut output = self.output.lock().unwrap(); - let mut state = self.segmenter.state.lock().unwrap(); - if state.finished { - return Err(moq_net::Error::Closed.into()); - } - output.prepare()?; - state.deferred = true; - Ok(Deferred { - segmenter: self.segmenter.clone(), - }) - } - - /// Reacquire the drain for a finished deferred segmenter. - /// - /// This is cancellation recovery for the task that owned [`DeferredDrain`]. It is rejected - /// until deferred segmentation has completed. - pub fn drain(&self) -> crate::Result { - let state = self.segmenter.state.lock().unwrap(); - if !state.deferred || !state.finished { - return Err(crate::Error::TimelineDeferredPending); - } - drop(state); - Ok(DeferredDrain { - segmenter: self.segmenter.clone(), - }) - } - - /// Mark one complete record committed, publishing it once every earlier record is committed. - pub fn push(&self, mut pending: Pending) -> crate::Result<()> { - if !Arc::ptr_eq(&self.segmenter.state, &pending.segmenter.state) { - return Err(crate::Error::TimelineDeferredRecord(pending.segment)); - } - - let mut output = self.output.lock().unwrap(); - let mut state = self.segmenter.state.lock().unwrap(); - if !state.deferred || !state.committing.contains(&pending.segment) { - return Err(crate::Error::TimelineDeferredRecord(pending.segment)); - } - - let record = pending.record.take().expect("a pending record is present until commit"); - state.committing.remove(&record.segment); - state.committed.insert(record.segment, record); - - while let Some((&segment, record)) = state.committed.first_key_value() { - let earlier_ready = state.ready.front().is_some_and(|record| record.segment < segment); - let earlier_committing = state.committing.first().is_some_and(|pending| *pending < segment); - if earlier_ready || earlier_committing { - break; - } - - let record = record.clone(); - if let Err(err) = output.push(&record) { - let record = state.committed.remove(&segment).unwrap(); - state.requeue(record); - return Err(err); - } - state.committed.remove(&segment); - } - - Ok(()) - } - - /// Remove up to `count` oldest records from the visible timeline window. - pub fn pop(&self, count: u64) -> crate::Result<()> { - self.output.lock().unwrap().pop(count) - } - - /// Close the timeline track's open group, so every push and pop so far sits in a complete group. - /// - /// A recorder stores those groups before committing the next segment. The next edit opens a - /// new group restating the window. - pub fn flush(&self) -> crate::Result<()> { - self.output.lock().unwrap().flush() - } - - fn publish_ready(&self) { - // Serialize the dequeue and publish steps together. Recorders can report from different - // tasks, and taking one record before locking the output would let a later record win the - // output lock and become visible first. - let mut output = self.output.lock().unwrap(); - if self.segmenter.state.lock().unwrap().deferred { - return; - } - while let Some(record) = self.segmenter.next() { - if let Err(err) = output.push(&record) { - tracing::warn!(%err, "timeline publish failed; dropping the timeline track"); - let _ = output.finish(); - return; - } - } - if self.segmenter.state.lock().unwrap().failure().is_some() { - let _ = output.finish(); - } - } - - /// Flush the final (still open) segment and finish the track. - /// - /// In deferred mode, first call [`Deferred::finish`] and use its returned drain handle to store - /// and [`push`](Self::push) every remaining record, then call this to close the timeline track. - /// Calling this before the deferred segmenter finishes or while any record remains uncommitted - /// is rejected without changing state. - /// - /// Also errors if a segment overran [`Config::duration_max`], or if the track can't be finished. - pub fn finish(&mut self) -> crate::Result<()> { - // Serialize both the mode decision and terminal flush with deferred(), which takes these - // locks in the same order. Either deferred mode wins before finishing starts, or finishing - // closes the output and a later deferred() is rejected. - let mut output = self.output.lock().unwrap(); - let mut state = self.segmenter.state.lock().unwrap(); - if state.deferred { - if !state.finished || !state.ready.is_empty() || !state.committing.is_empty() || !state.committed.is_empty() - { - return Err(crate::Error::TimelineDeferredPending); - } - if let Some(err) = state.failure() { - drop(state); - let _ = output.finish(); - return Err(err); - } - drop(state); - return output.finish(); - } - - state.finish(); - if let Some(err) = state.failure() { - drop(state); - let _ = output.finish(); - return Err(err); - } - while let Some(record) = state.ready.pop_front() { - if let Err(err) = output.push(&record) { - drop(state); - let _ = output.finish(); - return Err(err); - } - } - drop(state); - output.finish() - } -} - -/// A clonable reservation withholding [`Producer`] records while tracks are still enrolling. -/// -/// Made via [`Producer::reserve`], mirroring [`catalog::Reserved`](crate::catalog::Reserved). -/// Whatever became complete meanwhile flushes once the last clone drops. -pub struct Reserved { - segmenter: Segmenter, - output: Option>>, -} - -impl Clone for Reserved { - fn clone(&self) -> Self { - let mut state = self.segmenter.state.lock().unwrap(); - if !state.finished { - state.reservers += 1; - } - Self { - segmenter: self.segmenter.clone(), - output: self.output.clone(), - } - } -} - -impl Drop for Reserved { - fn drop(&mut self) { - let mut state = self.segmenter.state.lock().unwrap(); - state.reservers = state.reservers.saturating_sub(1); - state.pump(false); - drop(state); - if let Some(output) = &self.output { - Producer { - segmenter: self.segmenter.clone(), - output: output.clone(), - } - .publish_ready(); - } - } -} - -/// Reports one media track's group opens into the shared [`Producer`]. -/// -/// Move-only: it is the track's single reporting handle, and dropping it closes the track's -/// enrollment (segments stop waiting on it). Minted by [`Producer::track`] and held by a -/// rendition's [`container::Producer`](crate::container::Producer). -pub struct Recorder { - segmenter: Segmenter, - output: Option>>, - name: String, -} - -impl Recorder { - /// Report that group `sequence` opened at presentation time `pts`, `keyframe` stating - /// whether its first frame is one (i.e. whether a player could join here). - /// - /// Reports must be in group order with monotonic timestamps; this is the fact the timeline - /// builds ranges, boundaries and completeness from. - pub fn record(&mut self, sequence: u64, pts: Timestamp, keyframe: bool) { - self.segmenter - .state - .lock() - .unwrap() - .report(&self.name, sequence, pts, keyframe); - self.publish_ready(); - } - - /// Report that this track's content extends to `pts`, without a group opening there. - /// - /// A group open says where content *starts*; the last group of a broadcast has no successor - /// to bound it, so its segment would otherwise be published a group short (zero for a - /// segment that is a single group). Report the end whenever you know it: closing a group, - /// finishing a track. - pub fn end(&mut self, pts: Timestamp) { - self.segmenter.state.lock().unwrap().report_end(&self.name, pts); - self.publish_ready(); - } - - fn publish_ready(&self) { - if let Some(output) = &self.output { - Producer { - segmenter: self.segmenter.clone(), - output: output.clone(), - } - .publish_ready(); - } - } -} - -impl Drop for Recorder { - fn drop(&mut self) { - self.segmenter.state.lock().unwrap().close(&self.name); - self.publish_ready(); - } -} - -/// One decoded timeline entry: a complete aligned segment with real [`Timestamp`]s and the -/// group ranges each track contributes. -#[derive(Debug, Clone, PartialEq)] -pub struct Entry { - /// The segment's number, consecutive within the broadcast. - pub segment: u64, - - /// The segment's start. - pub pts: Timestamp, - - /// The segment's duration. The next entry starts at `pts + duration` unless content time - /// itself jumped (a discontinuity). - pub duration: Duration, - - /// The group ranges each participating media track contributes, keyed by track name. A - /// track absent from the map has no content in this span (HLS `EXT-X-GAP`). - pub tracks: BTreeMap>, - - /// The record's application extension (nothing for the default `()`). - pub ext: E, -} - -/// One change to the visible timeline window. -#[derive(Debug, Clone, PartialEq)] -#[non_exhaustive] -pub enum Event { - /// A complete segment record became visible at this window index. - Push { - /// Absolute window index assigned to the record. - index: u64, - /// The decoded segment entry. - entry: Entry, - }, - /// These previously visible window indices were removed. - Pop(std::ops::Range), - /// These indices were removed before this consumer saw them. - Skip(std::ops::Range), -} - -/// Reads a broadcast's timeline, yielding decoded [`Event`]s in window order. +/// Reads one track's timeline, yielding decoded [`Event`]s in window order. /// /// Generic over the record extension `E` (see [`RecordExt`]). pub struct Consumer { @@ -1243,20 +673,27 @@ pub struct Consumer { } impl Consumer { - /// Subscribe to the timeline advertised by the catalog's [`Archive`] entry. + /// Subscribe to `track`'s timeline as advertised by the catalog's [`Archive`] entry. /// - /// The section supplies both the track name and the timescale, so a reader can't pair the - /// wrong scale with the track. Errors if the section declares a timescale that isn't - /// representable. - pub async fn subscribe(broadcast: &moq_net::broadcast::Consumer, section: &Archive) -> crate::Result { - let track = broadcast.track(§ion.track)?.subscribe(None).await?; - + /// The section supplies both the timeline name and the timescale, so a reader can't pair the + /// wrong scale with the track. Errors if the section indexes no such track or declares a + /// timescale that isn't representable. + pub async fn subscribe( + broadcast: &moq_net::broadcast::Consumer, + section: &Archive, + track: &str, + ) -> crate::Result { + let name = section + .timelines + .get(track) + .ok_or_else(|| crate::Error::TimelineMissing(track.to_string()))?; + let timescale = + Timescale::new(section.timescale as u64).map_err(|_| crate::Error::InvalidTimescale(section.timescale))?; + let track = broadcast.track(name)?.subscribe(None).await?; let config = moq_json::window::ConsumerConfig::default().with_compression(true); - Ok(Self { inner: moq_json::window::Consumer::new(track, config), - timescale: Timescale::new(section.timescale as u64) - .map_err(|_| crate::Error::InvalidTimescale(section.timescale))?, + timescale, }) } @@ -1268,10 +705,12 @@ impl Consumer { let scale = self.timescale.as_u64() as u128; let nanos = (record.duration as u128) * 1_000_000_000 / scale; Ok(Entry { - segment: record.segment, + sequence: record.sequence, pts: Timestamp::new(record.pts, self.timescale)?, duration: Duration::from_nanos(nanos as u64), - tracks: record.tracks, + start: record.start, + end: record.end, + keyframe: record.keyframe, ext: record.ext, }) } @@ -1310,968 +749,436 @@ impl Consumer { mod test { use super::*; + /// Sparse data: every group is its own record. + fn sparse() -> Config { + Config::default().with_duration_min(Duration::ZERO) + } + fn ms(v: u64) -> Timestamp { Timestamp::from_millis(v).unwrap() } - /// Build an Entry the tests compare against. - fn entry(segment: u64, pts_ms: u64, duration_ms: u64, tracks: &[(&str, &[(u64, u64)])]) -> Entry { - Entry { - segment, - pts: ms(pts_ms), - duration: Duration::from_millis(duration_ms), - tracks: tracks - .iter() - .map(|(name, ranges)| { - ( - name.to_string(), - ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(), - ) - }) - .collect(), - ext: (), - } + fn at(group: u64) -> Position { + Position::group(group) } - /// Drain a finished timeline track by subscribing to the producer's advertised section. - async fn drain(broadcast: &moq_net::broadcast::Producer, producer: &Producer) -> Vec { - let mut consumer = Consumer::subscribe(&broadcast.consume(), &producer.section()) - .await - .unwrap(); - drain_consumer(&mut consumer) + /// A record's `(pts, duration, start, end)` in wire units, for compact comparisons. + fn span(record: &Record) -> (u64, u64, Position, Position) { + (record.pts, record.duration, record.start, record.end) } - fn drain_consumer(consumer: &mut Consumer<()>) -> Vec { - let waiter = kio::Waiter::noop(); + fn drain(segmenter: &mut Segmenter) -> Vec<(u64, u64, Position, Position)> { let mut out = Vec::new(); - while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { - if let Event::Push { entry, .. } = event { - out.push(entry); + let mut sequence = None; + for record in segmenter.by_ref() { + if let Some(previous) = sequence { + assert_eq!(record.sequence, previous + 1, "records are numbered consecutively"); } + sequence = Some(record.sequence); + out.push(span(&record)); } out } - fn setup() -> (moq_net::broadcast::Producer, Producer) { - setup_with(Config::default()) - } - - fn setup_with(config: Config) -> (moq_net::broadcast::Producer, Producer) { - let broadcast = moq_net::broadcast::Info::new().produce(); - let timeline = Producer::new(&broadcast, config); - (broadcast, timeline) + /// One frame per group, one group every `step` ms. + fn groups(segmenter: &mut Segmenter, groups: std::ops::Range, step: u64) { + for group in groups { + segmenter.frame(at(group), ms(group * step), true).unwrap(); + segmenter.finish_group(group); + } } - #[tokio::test] - async fn a_resumed_timeline_continues_the_checkpoint() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let checkpoint = Checkpoint { - range: 1..3, - records: vec![Record::new(1, 1_000, 1_000), Record::new(2, 2_000, 1_000)], - }; - let mut timeline = Producer::resume(&broadcast, Config::default(), &checkpoint).unwrap(); - - let mut video = timeline.pacing_track("video0").unwrap(); - video.record(10, ms(3_000), true); - video.record(11, ms(4_000), true); - video.end(ms(5_000)); - drop(video); - timeline.finish().unwrap(); + #[test] + fn long_groups_get_a_record_each() { + let mut video = Segmenter::new(Config::default()); + groups(&mut video, 0..3, 2_000); + video.end(ms(6_000)); + video.close(); assert_eq!( - drain(&broadcast, &timeline).await, + drain(&mut video), vec![ - entry(1, 1_000, 1_000, &[]), - entry(2, 2_000, 1_000, &[]), - entry(3, 3_000, 1_000, &[("video0", &[(10, 10)])]), - entry(4, 4_000, 1_000, &[("video0", &[(11, 11)])]), + (0, 2_000, at(0), at(1)), + (2_000, 2_000, at(1), at(2)), + (4_000, 2_000, at(2), at(3)), ] ); } #[test] - fn a_checkpoint_record_must_be_the_segment_at_its_index() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let checkpoint = Checkpoint { - range: 0..1, - records: vec![Record::new(5, 0, 1_000)], - }; - assert!(matches!( - Producer::resume(&broadcast, Config::default(), &checkpoint), - Err(crate::Error::TimelineCheckpoint(0)) - )); - } + fn short_groups_pack_up_to_the_minimum() { + let mut audio = Segmenter::new(Config::default().with_duration_min(Duration::from_millis(1_500))); + groups(&mut audio, 0..8, 500); + audio.close(); - #[tokio::test] - async fn deferred_records_are_invisible_until_committed() { - let (broadcast, timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - let record = segmenter.next().expect("the complete segment is ready for storage"); - assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - - timeline.push(record).unwrap(); - let Poll::Ready(Ok(Some(Event::Push { index, entry: actual }))) = consumer.poll_next(&waiter) else { - panic!("the committed segment was not visible"); - }; - assert_eq!(index, 0); - assert_eq!(actual, entry(0, 0, 2_000, &[("video0", &[(0, 0)])])); - - timeline.pop(1).unwrap(); - assert!(matches!( - consumer.poll_next(&waiter), - Poll::Ready(Ok(Some(Event::Pop(range)))) if range == (0..1) - )); - - video.end(ms(4_000)); - drop(video); - segmenter.finish().result().unwrap(); - } - - #[tokio::test] - async fn deferred_disables_existing_auto_publishing_recorders() { - let (broadcast, timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let segmenter = timeline.deferred().unwrap(); - let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - let record = segmenter.next().expect("the complete segment is ready for storage"); - assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - - timeline.push(record).unwrap(); - assert!(matches!( - consumer.poll_next(&waiter), - Poll::Ready(Ok(Some(Event::Push { index: 0, .. }))) - )); + assert_eq!( + drain(&mut audio), + vec![ + (0, 1_500, at(0), at(3)), + (1_500, 1_500, at(3), at(6)), + (3_000, 500, at(6), at(8)), + ] + ); } - #[tokio::test] - async fn omitting_a_track_keeps_the_others() { - let (broadcast, mut timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - let mut audio = segmenter.pacing_track("audio0"); - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - video.record(1, ms(2_000), true); - audio.record(1, ms(2_000), true); - - let pending = segmenter.next().expect("the complete segment is ready for storage"); - timeline.push(pending.omit("audio0")).unwrap(); - drop((video, audio)); - let tail = segmenter.finish(); - while let Some(pending) = tail.next() { - timeline.push(pending).unwrap(); + #[test] + fn a_long_group_splits_by_frame_at_the_maximum() { + let mut video = Segmenter::new(Config::default().with_duration_max(Duration::from_secs(3))); + for frame in 0..8 { + video + .frame(Position::new(0, frame), ms(frame * 1_000), frame == 0) + .unwrap(); } - timeline.finish().unwrap(); + video.close(); + let records: Vec = std::iter::from_fn(|| video.next()).collect(); assert_eq!( - drain(&broadcast, &timeline).await, + records.iter().map(span).collect::>(), vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), - entry(1, 2_000, 0, &[("audio0", &[(1, 1)]), ("video0", &[(1, 1)])]), + (0, 3_000, Position::new(0, 0), Position::new(0, 3)), + (3_000, 3_000, Position::new(0, 3), Position::new(0, 6)), + (6_000, 1_000, Position::new(0, 6), Position::new(0, 8)), ] ); + assert!(records[0].keyframe); + assert!( + !records[1].keyframe, + "a split inside a group does not start on a keyframe" + ); } #[test] - fn flush_closes_the_timeline_group() { - let (broadcast, timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let replay = moq_net::track::Subscription::default().with_max_age(Duration::from_secs(30)); - let groups = broadcast.consume().track(DEFAULT_NAME).unwrap().subscribe(replay); - let waiter = kio::Waiter::noop(); - let Poll::Ready(Ok(mut groups)) = groups.poll_ok(&waiter) else { - panic!("the deferred timeline track exists"); - }; + fn a_sparse_track_records_each_group_when_it_finishes() { + let mut catalog = Segmenter::new(sparse()); + catalog.frame(at(0), ms(0), true).unwrap(); + assert!(catalog.next().is_none(), "the group may still grow"); + catalog.finish_group(0); + assert_eq!(drain(&mut catalog), vec![(0, 0, at(0), at(1))]); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - timeline.push(segmenter.next().unwrap()).unwrap(); - timeline.pop(1).unwrap(); - timeline.flush().unwrap(); - - let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { - panic!("the pushed record opened a group"); - }; - assert_eq!(group.frame_count(), 2, "the push header and the pop share one group"); - assert!(group.poll_finished(&waiter).is_ready(), "flush closes it"); - assert!(groups.poll_recv_group(&waiter).is_pending()); + catalog.frame(at(1), ms(60_000), true).unwrap(); + catalog.finish_group(1); + assert_eq!(drain(&mut catalog), vec![(60_000, 0, at(1), at(2))]); } #[test] - fn dropping_a_pending_record_requeues_it() { - let (_broadcast, timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - - let pending = segmenter.next().expect("the complete segment is ready for storage"); - assert_eq!(pending.segment, 0); - drop(pending); - - let retry = segmenter.next().expect("the abandoned segment is ready to retry"); - assert_eq!(retry.segment, 0); - timeline.push(retry).unwrap(); - } + fn an_append_only_group_is_indexed_as_it_grows() { + // A never-closing log: one frame a second, split every three seconds. + let mut log = Segmenter::new(sparse().with_duration_max(Duration::from_secs(3))); + for frame in 0..7 { + log.frame(Position::new(0, frame), ms(frame * 1_000), true).unwrap(); + } + assert_eq!( + drain(&mut log), + vec![ + (0, 3_000, Position::new(0, 0), Position::new(0, 3)), + (3_000, 3_000, Position::new(0, 3), Position::new(0, 6)), + ] + ); - #[test] - fn ordinary_producer_rejects_a_deferred_record() { - let (_ordinary_broadcast, ordinary) = setup(); - let (_deferred_broadcast, deferred) = setup(); - let segmenter = deferred.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - - let pending = segmenter.next().expect("the complete segment is ready for storage"); - assert!(matches!( - ordinary.push(pending), - Err(crate::Error::TimelineDeferredRecord(0)) - )); + // The tail ends after the newest frame, since the group never finished. + log.close(); assert_eq!( - segmenter - .next() - .expect("the rejected segment returns to its producer") - .segment, - 0 + drain(&mut log), + vec![(6_000, 0, Position::new(0, 6), Position::new(0, 7))] ); } #[test] - fn deferred_mode_is_rejected_after_finishing_starts() { - let (_broadcast, mut timeline) = setup(); - timeline.finish().unwrap(); - assert!(matches!( - timeline.deferred(), - Err(crate::Error::Moq(moq_net::Error::Closed)) - )); - } - - #[tokio::test] - async fn deferred_records_publish_in_segment_order_before_finish() { - let (broadcast, mut timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - drop(video); - let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - let drain = segmenter.finish(); - drain.result().unwrap(); - drop(drain); - let drain = timeline.drain().expect("a cancelled terminal drain is recoverable"); - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - // Storage failed for the first segment, so publish its timing as a gap. It must still - // unblock the successfully stored segment behind it. - let first = drain.next().expect("the first record is ready for storage").gap(); - let second = drain.next().expect("the final record is ready for storage"); - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - - timeline.push(second).unwrap(); - assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - - timeline.push(first).unwrap(); - timeline.finish().unwrap(); + fn a_skipped_sequence_closes_the_record() { + let mut audio = Segmenter::new(Config::default().with_duration_min(Duration::from_secs(1))); + audio.frame(at(0), ms(0), true).unwrap(); + audio.frame(at(1), ms(300), true).unwrap(); + audio.frame(at(5), ms(600), true).unwrap(); + audio.frame(at(6), ms(1_700), true).unwrap(); + audio.close(); assert_eq!( - drain_consumer(&mut consumer), + drain(&mut audio), vec![ - entry(0, 0, 2_000, &[]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), + (0, 300, at(0), at(2)), + (600, 1_100, at(5), at(6)), + (1_700, 0, at(6), Position::new(6, 1)) ] ); } #[test] - fn segmenter_finish_is_terminal() { - let segmenter = Segmenter::new(Config::default()); - let late_segmenter = segmenter.clone(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.end(ms(2_000)); - - let drain = segmenter.finish(); - drain.result().unwrap(); - assert!(drain.next().is_some(), "the terminal flush keeps its final record"); - - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - let mut late = late_segmenter.pacing_track("late"); - late.record(0, ms(2_000), true); - late.end(ms(4_000)); - - assert!(drain.next().is_none(), "surviving recorders cannot reopen the timeline"); - assert!(matches!( - late_segmenter.cut(ms(4_000)), - Err(crate::Error::Moq(moq_net::Error::Closed)) - )); - } - - // The coarsest track paces: video GOPs are longer than duration_min, so segments are GOPs. - #[tokio::test] - async fn the_coarsest_track_paces_the_segments() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - // Video keyframes every 2s, audio groups every 500ms, minimum 1s. - video.record(0, ms(0), true); - for (seq, t) in [(0u64, 0u64), (1, 500), (2, 1_000), (3, 1_500)] { - audio.record(seq, ms(t), true); - } - video.record(1, ms(2_000), true); - for (seq, t) in [(4u64, 2_000u64), (5, 2_500), (6, 3_000), (7, 3_500)] { - audio.record(seq, ms(t), true); - } - video.record(2, ms(4_000), true); - audio.record(8, ms(4_000), true); - drop(video); - drop(audio); - timeline.finish().unwrap(); + fn a_skipped_run_ends_at_the_previous_frontier() { + // A data track paused for 40 minutes, skipping a sequence as its discontinuity marker. + let mut data = Segmenter::new(Config::default()); + groups(&mut data, 0..2, 1_000); + data.end(ms(2_000)); + data.frame(at(3), ms(2_400_000), true).unwrap(); + data.close(); - // Audio's own candidate (1s) loses to video's (2s), so no segment splits a GOP. Every - // record is self-contained: start AND end groups, explicit duration. assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 3)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)]), ("audio0", &[(4, 7)])]), - entry(2, 4_000, 0, &[("video0", &[(2, 2)]), ("audio0", &[(8, 8)])]), - ] + drain(&mut data), + vec![(0, 2_000, at(0), at(2)), (2_400_000, 0, at(3), Position::new(3, 1))], + "the pause is not absorbed into the record before it" ); } - // A real-time encoder's GOP can dwarf the minimum. Nothing is violated: there is nowhere - // else a segment could start and stay decodable, so the segment is simply long. - #[tokio::test] - async fn a_gop_longer_than_the_minimum_is_one_segment() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - video.record(0, ms(0), true); - video.record(1, ms(30_000), true); - video.end(ms(60_000)); - drop(video); - timeline.finish().unwrap(); + #[test] + fn a_record_never_exceeds_the_maximum() { + // A quiet log: nothing between its frames to split at. + let mut log = Segmenter::new(sparse().with_duration_max(Duration::from_secs(10))); + log.frame(Position::new(0, 0), ms(0), true).unwrap(); + log.frame(Position::new(0, 1), ms(20_000), true).unwrap(); + log.frame(at(1), ms(45_000), true).unwrap(); + log.close(); assert_eq!( - drain(&broadcast, &timeline).await, + drain(&mut log), vec![ - entry(0, 0, 30_000, &[("video0", &[(0, 0)])]), - entry(1, 30_000, 30_000, &[("video0", &[(1, 1)])]), + (0, 10_000, Position::new(0, 0), Position::new(0, 1)), + (20_000, 10_000, Position::new(0, 1), at(1)), + (45_000, 0, at(1), Position::new(1, 1)), ] ); } - // Groups shorter than the minimum pack into one segment rather than each becoming one. - #[tokio::test] - async fn short_groups_pack_up_to_the_minimum() { - let (broadcast, mut timeline) = setup_with(Config { - duration_min: Duration::from_millis(1_500), - ..Default::default() - }); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - for seq in 0..8u64 { - audio.record(seq, ms(seq * 500), true); - } - drop(audio); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - // The first group at or past 1500ms ends segment 0, so it holds groups 0..=2. - assert_eq!(entries[0], entry(0, 0, 1_500, &[("audio0", &[(0, 2)])])); - assert_eq!(entries[1], entry(1, 1_500, 1_500, &[("audio0", &[(3, 5)])])); - } + #[test] + fn explicit_cuts_override_the_pacing() { + let mut video = Segmenter::new(Config::default()); + // Keyframes every second, cut every three: the records follow the cuts, not the GOPs. + video.cut(ms(3_000)); + video.cut(ms(3_000)); + video.cut(ms(6_000)); + groups(&mut video, 0..7, 1_000); - // A track that hasn't reached the minimum yet holds the segment open, so a record never - // omits a rendition that was about to contribute to it. - #[tokio::test] - async fn a_segment_waits_for_every_track() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - video.record(1, ms(2_000), true); - // Video alone would close segment 0 here, but audio hasn't crossed 2s. - assert!(drain(&broadcast, &timeline).await.is_empty()); - - audio.record(1, ms(2_000), true); - let entries = drain(&broadcast, &timeline).await; assert_eq!( - entries, - vec![entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 0)])])] + drain(&mut video), + vec![(0, 3_000, at(0), at(3)), (3_000, 3_000, at(3), at(6))] ); - drop(video); - drop(audio); - timeline.finish().unwrap(); } - // An application that knows its own boundaries overrides the pacing. - #[tokio::test] - async fn explicit_cuts_override_the_pacing() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - // Keyframes every second, cut every three: the segments follow the cuts, not the GOPs. - timeline.cut(ms(3_000)).unwrap(); - timeline.cut(ms(6_000)).unwrap(); - for seq in 0..7u64 { - video.record(seq, ms(seq * 1_000), true); + #[test] + fn a_cut_on_the_first_group_does_not_poison_later_cuts() { + let mut video = Segmenter::new(Config::default()); + video.cut(ms(0)); + for group in 0..10u64 { + if group % 3 == 0 { + video.cut(ms(group * 1_000)); + } + video.frame(at(group), ms(group * 1_000), true).unwrap(); } - drop(video); - timeline.finish().unwrap(); - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries[0], entry(0, 0, 3_000, &[("video0", &[(0, 2)])])); - assert_eq!(entries[1], entry(1, 3_000, 3_000, &[("video0", &[(3, 5)])])); + assert_eq!(drain(&mut video)[0], (0, 3_000, at(0), at(3))); } - // Every rendition of one import declares the same boundaries, so a cut that would make a - // segment shorter than the minimum is dropped rather than producing a stray segment. - #[tokio::test] - async fn a_cut_below_the_minimum_is_ignored() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - video.record(0, ms(0), true); - timeline.cut(ms(2_000)).unwrap(); - // A sibling rendition's duplicate, and a boundary too close to be a segment. - timeline.cut(ms(2_000)).unwrap(); - timeline.cut(ms(2_500)).unwrap(); - timeline.cut(ms(4_000)).unwrap(); - video.record(1, ms(2_000), true); - video.record(2, ms(4_000), true); - drop(video); - timeline.finish().unwrap(); + #[test] + fn a_non_keyframe_start_is_flagged() { + let mut video = Segmenter::new(Config::default()); + video.frame(at(0), ms(0), true).unwrap(); + // A mid-stream join: the group doesn't open on an IDR. + video.frame(at(1), ms(2_000), false).unwrap(); + video.frame(at(2), ms(4_000), true).unwrap(); - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries.len(), 3, "the duplicate and the 500ms cut are both dropped"); - assert_eq!(entries[0], entry(0, 0, 2_000, &[("video0", &[(0, 0)])])); - assert_eq!(entries[1], entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])])); + let records: Vec = std::iter::from_fn(|| video.next()).collect(); + assert!(records[0].keyframe); + assert!(!records[1].keyframe); } - // The declared maximum is a contract with consumers (an HLS EXT-X-TARGETDURATION), so a - // segment that breaks it fails the timeline rather than publishing a record that - // contradicts the catalog. - #[tokio::test] - async fn exceeding_the_declared_maximum_fails_the_timeline() { - let (broadcast, mut timeline) = setup_with(Config { - duration_min: Duration::from_secs(1), - duration_max: Some(Duration::from_secs(3)), - ..Default::default() - }); - let mut video = timeline.pacing_track("video0").unwrap(); - - let mut consumer = { - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap() - }; - - // A 4s GOP against a declared 3s maximum. - video.record(2, ms(6_000), true); - drop(video); - - let err = timeline.finish().unwrap_err(); - assert!( - matches!(err, crate::Error::TimelineOverrun { segment: 1, .. }), - "unexpected error: {err}" - ); - - // The record that was still true published; the one that would have contradicted the - // catalog did not, and the track ended there. - let waiter = kio::Waiter::noop(); - let mut entries = Vec::new(); - while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { - if let Event::Push { entry, .. } = event { - entries.push(entry); - } + #[test] + fn a_report_that_does_not_advance_is_refused() { + let mut video = Segmenter::new(Config::default()); + video.frame(at(3), ms(0), true).unwrap(); + for position in [at(2), at(3)] { + assert!(matches!( + video.frame(position, ms(2_000), true), + Err(crate::Error::TimelinePosition { last, .. }) if last == at(3) + )); } - assert_eq!(entries, vec![entry(0, 0, 2_000, &[("video0", &[(0, 0)])])]); + video.frame(at(4), ms(2_000), true).unwrap(); + assert_eq!(drain(&mut video), vec![(0, 2_000, at(3), at(4))]); } #[test] - fn a_deferred_overrun_is_reported_when_the_output_finishes() { - let (_broadcast, mut timeline) = setup_with(Config { - duration_min: Duration::from_secs(1), - duration_max: Some(Duration::from_secs(3)), - ..Default::default() - }); - let deferred = timeline.deferred().unwrap(); - let mut video = deferred.pacing_track("video0"); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - - video.record(2, ms(6_000), true); - drop(video); - let drain = deferred.finish(); - assert!(matches!( - drain.result(), - Err(crate::Error::TimelineOverrun { segment: 1, .. }) - )); - // The overrun does not consume the only path to the valid prefix. - timeline.push(drain.next().unwrap()).unwrap(); - assert!(matches!( - timeline.finish(), - Err(crate::Error::TimelineOverrun { segment: 1, .. }) - )); + fn a_resumed_segmenter_continues_the_numbering() { + let mut video = Segmenter::new(Config::default()).with_sequence(7); + groups(&mut video, 10..12, 2_000); + assert_eq!(video.next().unwrap().sequence, 7); } - #[tokio::test] - async fn an_undeclared_maximum_is_omitted_from_the_catalog() { - let (_broadcast, timeline) = setup(); - assert_eq!(timeline.section().duration_max, None); - - let (_broadcast, timeline) = setup_with(Config { - duration_max: Some(Duration::from_millis(2_500)), - ..Default::default() - }); - assert_eq!(timeline.section().duration_max, Some(2_500)); - } - - // The last group of a broadcast has no successor to bound it, so without a reported end the - // final segment's duration collapses to zero (an HLS EXTINF:0). - #[tokio::test] - async fn the_final_segment_runs_to_the_reported_end() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - // A finished `container::Producer` reports where its content stops. - video.end(ms(4_000)); - drop(video); - timeline.finish().unwrap(); - - assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] - ); - } - - // A record is immutable and judged against the tracks enrolled when it flushes, so a - // producer that enrolls its tracks one batch at a time (an HLS import walking its - // renditions) holds flushing back until they are all in. - #[tokio::test] - async fn a_reservation_defers_flushing_until_every_track_enrolls() { - let (broadcast, mut timeline) = setup(); - let reserved = timeline.reserve(); - - // The primary rendition runs a whole batch of segments through before its sibling has - // even loaded an init segment. - let mut first = timeline.pacing_track("video0").unwrap(); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - first.record(seq, ms(t), true); - } - - let mut second = timeline.pacing_track("video1").unwrap(); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - second.record(seq, ms(t), true); - } - - drop(reserved); - drop(first); - drop(second); - timeline.finish().unwrap(); - - // Both renditions are indexed from segment 0. Without the reservation, segments 0 and 1 would - // have flushed knowing only video0 (a permanent gap for video1), and video1's first two - // groups would have folded into segment 2. - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries[0], - entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("video1", &[(0, 0)])]) - ); - assert_eq!( - entries[1], - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)]), ("video1", &[(1, 1)])]) - ); - } - - // A track that races ahead of the others still lands in the segment its content falls in. - #[tokio::test] - async fn groups_before_the_first_boundary_join_the_first_segment() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - // Audio races ahead of video's first keyframe (the startup race): its early group - // belongs to segment 0, which starts where the earliest content does. - audio.record(0, ms(0), true); - video.record(0, ms(30), true); - for (seq, t) in [(1u64, 500u64), (2, 1_000), (3, 1_500), (4, 2_000)] { - audio.record(seq, ms(t), true); + /// Drain a finished timeline track. + async fn read(broadcast: &moq_net::broadcast::Producer, section: &Archive, track: &str) -> Vec { + let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), section, track) + .await + .unwrap(); + let waiter = kio::Waiter::noop(); + let mut out = Vec::new(); + while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { + if let Event::Push { index, entry } = event { + assert_eq!(index, entry.sequence); + out.push(entry); + } } - video.record(1, ms(2_030), true); - audio.record(5, ms(2_500), true); - drop(video); - drop(audio); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries[0], - entry(0, 0, 2_030, &[("video0", &[(0, 0)]), ("audio0", &[(0, 4)])]), - ); - } - - #[tokio::test] - async fn sequence_gaps_split_ranges() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - // Elemental-style gap: audio groups 2..=4 never existed inside segment 0. - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - audio.record(1, ms(300), true); - audio.record(5, ms(1_500), true); - video.record(1, ms(2_000), true); - audio.record(6, ms(2_100), true); - drop(video); - drop(audio); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries[0], - entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 1), (5, 5)])]), - ); - } - - // A track with nothing to contribute is a whole-segment gap: the record simply omits it - // (an HLS exporter renders EXT-X-GAP). Only a *closed* track produces one, since a live - // track that has merely gone quiet still gets to say where the boundary is. - #[tokio::test] - async fn a_closed_track_leaves_a_whole_segment_gap() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - drop(audio); - video.record(1, ms(2_000), true); - video.record(2, ms(4_000), true); - drop(video); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries[1], entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])])); + out } #[tokio::test] - async fn a_non_keyframe_range_start_is_flagged() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); + async fn tracks_publish_independent_timelines() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut video = timelines.track("video0", Config::default()).unwrap(); + let mut catalog = timelines.track("catalog.json", sparse()).unwrap(); - video.record(0, ms(0), true); - // A mid-stream join: the group doesn't open on an IDR. - video.record(1, ms(2_000), false); - video.record(2, ms(4_000), true); + catalog.frame(at(0), ms(0), true); + catalog.finish_group(0); + for group in 0..3 { + video.frame(at(group), ms(group * 2_000), true); + video.finish_group(group); + } + video.end(ms(6_000)); drop(video); - timeline.finish().unwrap(); + timelines.finish(); - let entries = drain(&broadcast, &timeline).await; - let range = entries[1].tracks["video0"][0]; - assert!(!range.keyframe, "a range whose first group isn't an IDR says so"); - } + let section = timelines.section(); + assert_eq!(section.timelines["video0"], "video0.timeline.z"); + assert_eq!(section.timelines["catalog.json"], "catalog.json.timeline.z"); + assert_eq!(section.duration_max, Some(10_000)); - // A dead track would otherwise hold every later segment: dropping its recorder is what - // says it will never report again. - #[tokio::test] - async fn a_closed_track_stops_gating() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - video.record(1, ms(2_000), true); - assert!(drain(&broadcast, &timeline).await.is_empty(), "audio still gates"); - - drop(audio); - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries, - vec![entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 0)])])] - ); - drop(video); - timeline.finish().unwrap(); + let video = read(&broadcast, §ion, "video0").await; + assert_eq!(video.len(), 3); + assert_eq!(video[2].end, at(3), "the terminal flush ends after the finished group"); + let catalog = read(&broadcast, §ion, "catalog.json").await; + assert_eq!(catalog.len(), 1, "the catalog does not wait for the video"); } - // The timeline track (and its catalog section) exist only once a media track enrolls. #[tokio::test] - async fn the_track_is_created_on_first_enrollment() { + async fn a_sparse_record_publishes_before_the_track_ends() { let broadcast = moq_net::broadcast::Info::new().produce(); - let timeline = Producer::new(&broadcast, Config::default()); - assert!( - broadcast.create_track(DEFAULT_NAME, None).is_ok(), - "nothing took the name yet" - ); + let timelines = Timelines::new(&broadcast); + let mut catalog = timelines.track("catalog.json", sparse()).unwrap(); + catalog.frame(at(0), ms(0), true); + catalog.finish_group(0); - let broadcast = moq_net::broadcast::Info::new().produce(); - let timeline2 = Producer::new(&broadcast, Config::default()); - let _recorder = timeline2.pacing_track("video0").unwrap(); - assert!( - broadcast.create_track(DEFAULT_NAME, None).is_err(), - "the timeline took the name once a track enrolled" - ); - drop(timeline); + assert_eq!(read(&broadcast, &timelines.section(), "catalog.json").await.len(), 1); + drop(catalog); } #[tokio::test] - async fn section_advertises_track_without_wall() { - // Wall mapping moved to the catalog root clock: the timeline section carries the track, - // timescale, and duration bound, and nothing else time-anchoring. - let (_broadcast, timeline) = setup(); - let section = timeline.section(); - assert_eq!(section.track, DEFAULT_NAME); - assert_eq!(section.timescale, 1000); - assert_eq!(section.duration_max, None); - } - - // Clones nest like the catalog's, so the first batch to finish doesn't publish records the - // others are still filling in. - #[tokio::test] - async fn reservations_nest() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - let outer = timeline.reserve(); - let inner = outer.clone(); + async fn a_rejected_report_fails_the_timeline() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut video = timelines.track("video0", Config::default()).unwrap(); + let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timelines.section(), "video0") + .await + .unwrap(); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - video.record(seq, ms(t), true); + video.frame(at(1), ms(0), true); + video.frame(at(0), ms(2_000), true); + loop { + match consumer.next().await { + Ok(Some(_)) => continue, + Ok(None) => panic!("the timeline ended cleanly despite omitting a frame"), + Err(_) => break, + } } - - drop(inner); - assert!( - drain(&broadcast, &timeline).await.is_empty(), - "the other clone still gates" - ); - - drop(outer); - assert_eq!( - drain(&broadcast, &timeline).await.len(), - 2, - "the last clone dropped flushes" - ); - - drop(video); - timeline.finish().unwrap(); } - // A reservation outstanding when the broadcast ends must not strand the terminal flush. #[tokio::test] - async fn finish_overrides_an_outstanding_reservation() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let reserved = timeline.reserve(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - drop(video); + async fn re_enrolling_continues_the_timeline() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut first = timelines.track("video0", Config::default()).unwrap(); + first.frame(at(0), ms(0), true); + drop(first); + let mut second = timelines.track("video0", Config::default()).unwrap(); + // The new producer restarts its group sequences. + second.frame(at(0), ms(2_000), true); + drop(second); + timelines.finish(); - timeline.finish().unwrap(); + let entries = read(&broadcast, &timelines.section(), "video0").await; assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] + entries.iter().map(|entry| entry.sequence).collect::>(), + vec![0, 1] ); - - // Dropping it afterwards is a no-op rather than an underflow. - drop(reserved); } - #[tokio::test] - async fn rejects_an_invalid_timescale() { - let (broadcast, timeline) = setup(); - let _recorder = timeline.pacing_track("video0").unwrap(); - let mut section = timeline.section(); - section.timescale = 0; - let err = Consumer::<()>::subscribe(&broadcast.consume(), §ion).await; - assert!(matches!(err, Err(crate::Error::InvalidTimescale(0)))); + #[test] + fn a_taken_timeline_name_fails_enrollment() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _squat = broadcast.create_track("video0.timeline.z", None).unwrap(); + let timelines = Timelines::new(&broadcast); + assert!(timelines.track("video0", Config::default()).is_err()); + assert!(timelines.section().timelines.is_empty()); } - // A cut registered before the media, landing exactly on the first reported group (what the - // fMP4 importer does: it cuts at the keyframe fragment's timestamp, then records that group). - #[tokio::test] - async fn a_cut_on_the_first_group_does_not_poison_later_cuts() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - // Source segments every 3s, keyframes every 1s: 3s segments, not the 1s the - // duration_min pacing would produce on its own. - timeline.cut(ms(0)).unwrap(); - video.record(0, ms(0), true); - for seq in 1..10u64 { - if seq % 3 == 0 { - timeline.cut(ms(seq * 1_000)).unwrap(); - } - video.record(seq, ms(seq * 1_000), true); - } - drop(video); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries[0], - entry(0, 0, 3_000, &[("video0", &[(0, 2)])]), - "the source's 3s boundaries should be reproduced, not duration_min pacing" - ); + #[test] + fn enrollment_after_finish_is_refused() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + timelines.finish(); + assert!(matches!( + timelines.track("video0", Config::default()), + Err(crate::Error::Moq(moq_net::Error::Closed)) + )); } - // A catalog publishes a group only when the renditions change, so it can go quiet for the - // rest of the broadcast. Enrolled as a pacing track it would stall the timeline for good; - // non-pacing, it rides along in whichever segment is open. #[tokio::test] - async fn a_non_pacing_track_neither_paces_nor_gates() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut catalog = timeline.track("catalog.json"); - - catalog.record(0, ms(0), true); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - video.record(seq, ms(t), true); - } - video.end(ms(6_000)); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries, - vec![ - entry(0, 0, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - entry(2, 4_000, 2_000, &[("video0", &[(2, 2)])]), + async fn a_resumed_producer_continues_the_checkpoint() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video0.timeline.z", Producer::info()).unwrap(); + let checkpoint = Checkpoint { + range: 1..3, + records: vec![ + Record::new(1, 1_000, 1_000, at(1), at(2)), + Record::new(2, 2_000, 1_000, at(2), at(3)), ], - "the silent catalog should neither hold records back nor move a boundary" - ); - } - - // Nothing waits for a non-pacing track, so a group arriving after its segment already flushed - // is recorded in the next one. Placement is by arrival; the frames still carry their own - // timestamps. - #[tokio::test] - async fn a_late_non_pacing_group_lands_in_the_next_segment() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut catalog = timeline.track("catalog.json"); - - video.record(0, ms(0), true); - // Flushes segment 0, which the catalog has contributed nothing to. - video.record(1, ms(2_000), true); - - // The update happened a second in, but only reaches the timeline now. - catalog.record(0, ms(1_000), true); - - video.record(2, ms(4_000), true); - video.end(ms(6_000)); - timeline.finish().unwrap(); + }; + let mut producer = Producer::resume(track, &checkpoint).unwrap(); + assert!(matches!( + producer.push(&Record::new(4, 3_000, 1_000, at(3), at(4))), + Err(crate::Error::TimelineSequence { expected: 3, actual: 4 }) + )); + producer.push(&Record::new(3, 3_000, 1_000, at(3), at(4))).unwrap(); + producer.finish().unwrap(); - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries[0], entry(0, 0, 2_000, &[("video0", &[(0, 0)])])); + let mut section = Archive::new(); + section + .timelines + .insert("video0".to_string(), "video0.timeline.z".to_string()); + let entries = read(&broadcast, §ion, "video0").await; assert_eq!( - entries[1], - entry(1, 2_000, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(1, 1)])]), - "the late group belongs to the segment that was open when it arrived" + entries.iter().map(|entry| entry.sequence).collect::>(), + vec![1, 2, 3] ); } - // Non-pacing placement is by arrival rather than timestamp. A catalog can stamp snapshots from a - // wall-clock basis while imported media starts at PTS zero, so its frontier must neither hold - // the group for a much later segment nor stretch the final media segment to that timestamp. - #[tokio::test] - async fn a_non_pacing_track_uses_arrival_and_does_not_extend_the_tail() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut catalog = timeline.track("catalog.json"); - - video.record(0, ms(0), true); - catalog.record(0, ms(10_000), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - timeline.finish().unwrap(); - - assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] - ); + #[test] + fn a_checkpoint_record_must_be_the_sequence_at_its_index() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video0.timeline.z", Producer::info()).unwrap(); + let checkpoint = Checkpoint { + range: 0..1, + records: vec![Record::new(5, 0, 1_000, at(0), at(1))], + }; + assert!(matches!( + Producer::resume(track, &checkpoint), + Err(crate::Error::TimelineCheckpoint(0)) + )); } #[tokio::test] - async fn the_built_in_catalog_is_recorded_without_pacing() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let timeline = catalog.timeline(); - let mut video = catalog.enroll("video0").unwrap(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - drop(video); - catalog.finish().unwrap(); - - assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] - ); + async fn a_missing_timeline_is_an_error() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let section = Archive::new(); + assert!(matches!( + Consumer::<()>::subscribe(&broadcast.consume(), §ion, "video0").await, + Err(crate::Error::TimelineMissing(_)) + )); } - // Segmentation stays opt-in by pacing track: non-pacing tracks alone describe nothing, so the - // timeline track is never created and its name stays free. #[tokio::test] - async fn non_pacing_tracks_alone_publish_no_timeline() { - let (broadcast, mut timeline) = setup(); - let mut catalog = timeline.track("catalog.json"); - - catalog.record(0, ms(0), true); - timeline.finish().unwrap(); - - assert!( - broadcast.create_track(DEFAULT_NAME, None).is_ok(), - "the timeline track should never have been created" - ); + async fn rejects_an_invalid_timescale() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let _recorder = timelines.track("video0", Config::default()).unwrap(); + let mut section = timelines.section(); + section.timescale = 0; + let err = Consumer::<()>::subscribe(&broadcast.consume(), §ion, "video0").await; + assert!(matches!(err, Err(crate::Error::InvalidTimescale(0)))); } } diff --git a/rs/moq-rtc/src/codec/vp8.rs b/rs/moq-rtc/src/codec/vp8.rs index 4cede5be7e..9f458a7e0c 100644 --- a/rs/moq-rtc/src/codec/vp8.rs +++ b/rs/moq-rtc/src/codec/vp8.rs @@ -67,7 +67,9 @@ mod tests { async fn importer_creation_failure_preserves_abort_error() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let _collision = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _collision = broadcast + .create_track(hang::timeline::default_name(hang::Catalog::DEFAULT_NAME), None) + .unwrap(); let consumer = broadcast.consume(); let mut bridge = super::Bridge::new(broadcast, catalog).unwrap(); let mut track = consumer.track("0.vp8").unwrap().subscribe(None).await.unwrap(); diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index 5568b15626..22190b6ad4 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -857,7 +857,10 @@ mod tests { .video .insert("video", source(1920, 1080, Some(6_000_000))) .unwrap(); - let mut archive = hang::catalog::Archive::new("timeline.z"); + let mut archive = hang::catalog::Archive::new(); + archive + .timelines + .insert("video".to_string(), "video.timeline.z".to_string()); archive.replay = Some(RelativeOwned::from("./recordings/clip".to_string())); archive.version = Some(hang::catalog::Archive::VERSION); child.archive = Some(archive.clone()); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 998e7dbf1c..cd6715ed35 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1181,7 +1181,7 @@ mod tests { .await } - /// A recording replays what the live edge published: the archive's segment records + /// A recording replays what the live edge published: the archive's records /// carry the live timestamps across an idle restart, with the idle gap left in. #[tokio::test] async fn retained_archive_playback_keeps_the_live_timestamps() { @@ -1193,9 +1193,10 @@ mod tests { .snapshot() .archive .expect("the video track enrolls an archive"); - let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion) - .await - .unwrap(); + let mut timeline = + moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion, "video") + .await + .unwrap(); let mut live = Vec::new(); for _ in 0..2 { @@ -1207,12 +1208,12 @@ mod tests { fixture.assert_acquired(published, captured); live.push(published); drop(track); - // Idle past the minimum segment, so each run is archived as its own segment. + // Idle past the minimum segment, so each run is archived as its own record. tokio::time::sleep(moq_mux::timeline::DEFAULT_DURATION_MIN + Duration::from_millis(100)).await; } let (catalog, _consumer) = fixture.finish().await; - catalog.timeline().finish().unwrap(); + catalog.timeline().finish(); let mut archived = Vec::new(); while let Some(event) = timeline.next().await.unwrap() { match event { @@ -1221,16 +1222,15 @@ mod tests { } } - assert_eq!(archived.len(), live.len(), "one segment per capture run: {archived:?}"); + assert_eq!(archived.len(), live.len(), "one record per capture run: {archived:?}"); for (entry, live) in archived.iter().zip(&live) { // The archive keeps millisecond precision. assert_eq!(entry.pts.as_micros() / 1000, u128::from(*live / 1000), "{archived:?}"); - assert!(entry.tracks.contains_key("video"), "{archived:?}"); } let first = &archived[0]; assert!( archived[1].pts.as_micros() >= first.pts.as_micros() + first.duration.as_micros(), - "the resumed segment overlaps the one before it: {archived:?}" + "the resumed record overlaps the one before it: {archived:?}" ); }) .await From 6e319ada27d7f561658ef0cb838ad02c6b693eee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 10:39:27 -0700 Subject: [PATCH 16/49] quest(archive/track-timeline): block the line on the #4280 HLS findings Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/README.md | 12 +++++++ .../archive/track-timeline/hls-resubscribe.md | 35 ++++++++++++++++++ quest/m1/archive/track-timeline/hls-target.md | 36 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 quest/m1/archive/track-timeline/hls-resubscribe.md create mode 100644 quest/m1/archive/track-timeline/hls-target.md diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index fc9fc5e645..b78885ba6c 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -38,10 +38,22 @@ Decisions: - HLS and DASH segments are derived at the edge from group timestamps, not from storage objects. Fetching extra objects is fine when they land in the reader's cache for the next request. +- HLS `EXT-X-TARGETDURATION` is fixed for the run, not the observed maximum + #4280 shipped. This reverses that PR's decision 4 (09-28 merged-PR audit). + +For triage, not blocking: two Codex P2s arrived after #4280 merged and are +unanswered. A timeline record whose `sequence` differs from its window index +is passed through unvalidated +([review](https://github.com/moq-dev/moq/pull/4280#pullrequestreview-5332725290)), +and `Timelines::track` re-enrolling a name while its old `Recorder` is alive +leaves two handles on one segmenter +([r4117516035](https://github.com/moq-dev/moq/pull/4280#discussion_r4117516035)). ## Quests - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust +- [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment longer than it +- [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of turning a rendition's segments into gaps ## Related diff --git a/quest/m1/archive/track-timeline/hls-resubscribe.md b/quest/m1/archive/track-timeline/hls-resubscribe.md new file mode 100644 index 0000000000..8991e24448 --- /dev/null +++ b/quest/m1/archive/track-timeline/hls-resubscribe.md @@ -0,0 +1,35 @@ +# [S] HLS timeline resubscribe + +## Goal + +A transient error on a rendition's timeline subscription does not turn that +rendition's remaining segments into `EXT-X-GAP`. The watcher subscribes again +and keeps resolving segments; only a clean end of the timeline ends its spans. + +## Plan + +In [#4280](https://github.com/moq-dev/moq/pull/4280), `watch_spans` in +`rs/moq-hls/src/export/rendition.rs` logs any error and then calls +`spans.end()`, so rows past the last record resolve as gaps for good. Codex +flagged it as a P1 +([r4112645622](https://github.com/moq-dev/moq/pull/4280#discussion_r4112645622)). +The agent declined it because the watcher never retries, and named +re-subscribing as the real fix. The maintainer's decision in the 09-28 +merged-PR audit is to do that re-subscribe. + +- Re-subscribe on an error while the broadcast is still live, resuming from + the records the spans already hold rather than rebuilding them. +- Mark the spans ended only on a clean end of the timeline or when the + broadcast itself goes away. The concern the decline raised still holds: + nothing may park forever (`poll_resolved`, a recording + `segments::Consumer`), so a rendition whose timeline never returns must + still resolve. +- Re-subscribe on the event that makes it possible (the broadcast or track + being available again), not on a timer. + +Add a regression test where a non-reference rendition's timeline errors and +comes back, and its later segments resolve to media, not gaps. + +## Related + +- [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - the line this blocks diff --git a/quest/m1/archive/track-timeline/hls-target.md b/quest/m1/archive/track-timeline/hls-target.md new file mode 100644 index 0000000000..8becbeec5d --- /dev/null +++ b/quest/m1/archive/track-timeline/hls-target.md @@ -0,0 +1,36 @@ +# [S] Fixed HLS target duration + +## Goal + +A `moq-hls` media playlist advertises one `EXT-X-TARGETDURATION` for the +whole run, as RFC 8216 requires, and never lists a segment longer than it. + +## Plan + +[#4280](https://github.com/moq-dev/moq/pull/4280) derives the target from the +current window's longest segment (`snapshot_as` in +`rs/moq-hls/src/export/rendition.rs`), so it rises when a long record arrives +and falls when that record is evicted. Codex flagged it +([r4113921580](https://github.com/moq-dev/moq/pull/4280#discussion_r4113921580)), +and the PR kept the observed maximum as recorded decision 4. The maintainer +has reversed that decision in the 09-28 merged-PR audit: + +- Set the target once, from what the publisher declares: the catalog, or the + GOP (keyframe interval) configuration where one is known. Not the catalog's + `durationMax` blindly: it is a 10s split ceiling, and advertising it would + push every player's live edge back by that much. +- A segment longer than the target is refused or split, never listed as is. + Which one fits depends on where the long segment comes from (a reference + record, or a non-reference rendition's snapped keyframe), so decide per + case and say why. +- If the catalog declares nothing usable, decide whether to refuse the + rendition or fix a target from the first segments and hold it; ask the + maintainer if neither is clearly right. + +Replace `target_duration_follows_the_observed_segments` with tests that pin a +constant target across a window whose segment durations vary, and that cover +the over-long segment path. + +## Related + +- [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - the line this blocks From 0d0346d35f18344531d459c7f7fde519567683a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 16:38:57 -0700 Subject: [PATCH 17/49] fix(archive): end broadcasts with close() after main deprecated finish() A broadcast end carries no cause now, so the writer treats Dropped as a clean end and only the deprecated abort reports an error. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/proof.rs | 4 ++-- rs/moq-archive/src/writer.rs | 32 ++++++++++++++++---------------- rs/moq-cli/src/archive.rs | 4 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs index 4acd6238ab..904db71df5 100644 --- a/rs/moq-archive/src/proof.rs +++ b/rs/moq-archive/src/proof.rs @@ -128,7 +128,7 @@ async fn record(store: &Store) { for track in tracks.values() { track.finish().unwrap(); } - source.finish(); + source.close(); run.await.unwrap().unwrap(); } @@ -390,6 +390,6 @@ async fn an_offline_reader_follows_dvr_expiry() { } video.finish().unwrap(); - source.finish(); + source.close(); run.await.unwrap().unwrap(); } diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 9ed25d86b7..3f09decdc0 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -230,7 +230,7 @@ impl Writer { /// Then flush the final segment and finish the timeline. Fails when the timeline cannot be /// committed or stored, or an enrolled track delivers a group the recording cannot represent: /// the recording stops at its last durable timeline object. Returns the source's error, after - /// finishing, when the broadcast aborted. + /// finishing, when the broadcast was aborted with one. pub async fn run(self) -> Result<()> { let Self { control, @@ -336,10 +336,10 @@ impl Writer { delete(&shared.store, keys).await; } - if source.is_finished() { - Ok(()) - } else { - Err(source_error(source.closed().await)) + // A broadcast end carries no cause; only the deprecated `abort` still reports one. + match source.closed().await { + moq_net::Error::Dropped => Ok(()), + err => Err(source_error(err)), } } } @@ -938,7 +938,7 @@ mod tests { } video.finish().unwrap(); catalog.finish().unwrap(); - source.finish(); + source.close(); tokio::spawn(writer.run()).await.unwrap().unwrap(); @@ -979,7 +979,7 @@ mod tests { } video.finish().unwrap(); audio.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); @@ -1004,7 +1004,7 @@ mod tests { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); @@ -1043,7 +1043,7 @@ mod tests { group(&video, sequence, &[sequence * 1000]); } video.finish().unwrap(); - source.finish(); + source.close(); let run = tokio::spawn(writer.run()); while window(&store).await.last().map(|record| record.segment) != Some(5) { @@ -1076,7 +1076,7 @@ mod tests { first.write_frame(ms(500), "0@500").unwrap(); first.finish().unwrap(); video.finish().unwrap(); - source.finish(); + source.close(); run.await.unwrap().unwrap(); let records = window(&store).await; @@ -1103,7 +1103,7 @@ mod tests { group(&video, 1, &[500]); group(&video, 3, &[2000]); video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); @@ -1125,7 +1125,7 @@ mod tests { // One past the recording's largest group ID. group(&video, 1 << 53, &[1000]); video.finish().unwrap(); - source.finish(); + source.close(); assert_eq!( writer.run().await, @@ -1149,7 +1149,7 @@ mod tests { group(&video, 0, &[1000]); group(&video, 1, &[500]); video.finish().unwrap(); - source.finish(); + source.close(); assert_eq!( writer.run().await, @@ -1180,7 +1180,7 @@ mod tests { group(&video, sequence, &[sequence * 1000]); } video.finish().unwrap(); - source.finish(); + source.close(); tokio::time::sleep(Duration::from_millis(50)).await; assert!(!run.is_finished(), "a stalled pacing track holds the recording open"); @@ -1241,7 +1241,7 @@ mod tests { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); } @@ -1376,7 +1376,7 @@ mod tests { group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); } video.finish().unwrap(); - source.finish(); + source.close(); writer.run().await.unwrap(); assert!(started.elapsed() >= grace); diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 2689096655..e04ab0f885 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -355,7 +355,7 @@ mod tests { .expect("the rendition is enrolled"); track.finish().unwrap(); catalog.finish().unwrap(); - broadcast.finish(); + broadcast.close(); tokio::time::timeout(Duration::from_secs(10), recording) .await @@ -438,7 +438,7 @@ mod tests { first.finish().unwrap(); second.finish().unwrap(); catalog.finish().unwrap(); - broadcast.finish(); + broadcast.close(); tokio::time::timeout(Duration::from_secs(10), recording) .await From ca88b75c4f6809538db36ba8a704cbac2846f51d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 21:18:01 -0700 Subject: [PATCH 18/49] chore(quest): claim enrollment-flake Co-Authored-By: Claude Sonnet 5.5 From c9db4323a0a77e7357c64e748e0645574b620d69 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 21:29:24 -0700 Subject: [PATCH 19/49] test(cli): wait for the export to enroll, not for a rendition's .info Co-Authored-By: Claude Sonnet 5.5 --- quest/m1/archive/README.md | 1 - quest/m1/archive/enrollment-flake.md | 26 --------------------- quest/m1/test-flakes-2.md | 5 ---- rs/moq-cli/src/archive.rs | 34 ++++++++++++++++------------ 4 files changed, 19 insertions(+), 47 deletions(-) delete mode 100644 quest/m1/archive/enrollment-flake.md diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index c617a50a79..996849c80a 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -148,7 +148,6 @@ owned by that prerequisite, not duplicated in archive storage. - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback - [DVR timeline pruning](/quest/m1/archive/pruning.md) - a DVR deletes timeline objects no retained checkpoint needs -- [Enrollment flake](/quest/m1/archive/enrollment-flake.md) - the opening-snapshot test waits for real enrollment, not the `.info` file ## Related diff --git a/quest/m1/archive/enrollment-flake.md b/quest/m1/archive/enrollment-flake.md deleted file mode 100644 index 4395024703..0000000000 --- a/quest/m1/archive/enrollment-flake.md +++ /dev/null @@ -1,26 +0,0 @@ -# [XS] The opening-snapshot test waits for real enrollment - -## Goal - -`archive::tests::an_opening_snapshot_records_every_rendition` in -`rs/moq-cli/src/archive.rs` passes under load. It fails with "writer closed" -because it treats a rendition's `.info` file as proof the export enrolled -that track, then finishes the tracks and the catalog; under load the writer -can still be subscribing when the tracks end. Seen while landing -[#4169](https://github.com/moq-dev/moq/pull/4169). - -## Plan - -- Wait on the signal enrollment actually produces (the subscription reaching - the publisher, or an explicit event from the exporter), not a file that - appears earlier. No longer timeout and no retry. -- If the exporter can genuinely lose a rendition that ends right after it - appears in the catalog, that is a bug in the exporter, not the test; fix it - there. - -Public API: none. Wire: none. - -## Related - -- [More tests hold up under load](/quest/m1/test-flakes-2.md) - the same - round of load-only failures on `main` diff --git a/quest/m1/test-flakes-2.md b/quest/m1/test-flakes-2.md index 8cc9502a74..a4162acbd9 100644 --- a/quest/m1/test-flakes-2.md +++ b/quest/m1/test-flakes-2.md @@ -53,8 +53,3 @@ retry. as the first round did. Public API: none. Wire: none. - -## Related - -- [Archive enrollment](/quest/m1/archive/enrollment-flake.md) - the same - kind of flake on the archive line, where its test lives diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index e04ab0f885..2e3f040e48 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -306,6 +306,23 @@ mod tests { assert!(err.contains("broadcast `source`"), "{err}"); } + /// Wait until the export has enrolled every rendition the catalog lists. + /// + /// A rendition's `.info` only shows its subscription landed: the writer takes the track + /// a moment later, and one that ends first is refused. The first segment is the proof, + /// since the writer holds it back until every pacing track has reported its groups. + async fn enrolled(url: &Url, recording: &tokio::task::JoinHandle>) { + let store = super::open(url).unwrap(); + // The store is real disk I/O, so this polls on the wall clock; nextest kills a hang. + while store.get_segments(hang::timeline::DEFAULT_NAME, 0).await.is_err() { + assert!( + !recording.is_finished(), + "the export ended before it committed a segment" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + /// What `export archive` records, `import archive` serves back group for group, and the /// export ends cleanly with its broadcast. #[tokio::test] @@ -345,14 +362,7 @@ mod tests { group.finish().unwrap(); } - // Finish only once the rendition is enrolled, which writes its `.info`. - tokio::time::timeout(Duration::from_secs(10), async { - while !dir.path().join("audio/.info").exists() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("the rendition is enrolled"); + enrolled(&url, &recording).await; track.finish().unwrap(); catalog.finish().unwrap(); broadcast.close(); @@ -428,13 +438,7 @@ mod tests { }; let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); - tokio::time::timeout(Duration::from_secs(10), async { - while !dir.path().join("audio/.info").exists() || !dir.path().join("audio2/.info").exists() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("both renditions are enrolled"); + enrolled(&url, &recording).await; first.finish().unwrap(); second.finish().unwrap(); catalog.finish().unwrap(); From 44f06458b62cd4f3b414074b9b81143fa7a39a79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 10:48:21 -0700 Subject: [PATCH 20/49] quest(archive/track-timeline): retry only recoverable timeline errors, define the terminal state Co-Authored-By: Claude Opus 5.5 --- .../archive/track-timeline/hls-resubscribe.md | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/quest/m1/archive/track-timeline/hls-resubscribe.md b/quest/m1/archive/track-timeline/hls-resubscribe.md index 8991e24448..5bb81ddd03 100644 --- a/quest/m1/archive/track-timeline/hls-resubscribe.md +++ b/quest/m1/archive/track-timeline/hls-resubscribe.md @@ -4,7 +4,8 @@ A transient error on a rendition's timeline subscription does not turn that rendition's remaining segments into `EXT-X-GAP`. The watcher subscribes again -and keeps resolving segments; only a clean end of the timeline ends its spans. +and keeps resolving segments. Only a clean end, a malformed timeline, or the +track or broadcast going away ends its spans. ## Plan @@ -17,15 +18,22 @@ The agent declined it because the watcher never retries, and named re-subscribing as the real fix. The maintainer's decision in the 09-28 merged-PR audit is to do that re-subscribe. -- Re-subscribe on an error while the broadcast is still live, resuming from - the records the spans already hold rather than rebuilding them. -- Mark the spans ended only on a clean end of the timeline or when the - broadcast itself goes away. The concern the decline raised still holds: - nothing may park forever (`poll_resolved`, a recording - `segments::Consumer`), so a rendition whose timeline never returns must - still resolve. +- Re-subscribe only on a recoverable error (a transport reset, a lost + session, the publisher's track aborting) while the broadcast is still live, + resuming from the records the spans already hold rather than rebuilding + them. A malformed or unsupported timeline (a bad archive timescale, broken + JSON or DEFLATE) would replay the same retained record forever, so it fails + loud and ends the spans instead. - Re-subscribe on the event that makes it possible (the broadcast or track being available again), not on a timer. +- The spans end on a clean end of the timeline, when the broadcast goes + away, or when a re-subscribe is refused because the track is no longer + published. That last case is the terminal state for a timeline that never + returns while the broadcast stays live on other tracks: its remaining rows + resolve as gaps. The concern the decline raised still holds, so nothing may + park forever (`poll_resolved`, a recording `segments::Consumer`); a + re-subscribe is either served, refused, or cut short by the broadcast + ending. Add a regression test where a non-reference rendition's timeline errors and comes back, and its later segments resolve to media, not gaps. From a367ee83f358db1970115aa38b9f62d1bef5d5e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 11:21:52 -0700 Subject: [PATCH 21/49] quest(archive/track-timeline): bound the target by rounded EXTINF Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/hls-target.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/quest/m1/archive/track-timeline/hls-target.md b/quest/m1/archive/track-timeline/hls-target.md index 8becbeec5d..81cbbd1b70 100644 --- a/quest/m1/archive/track-timeline/hls-target.md +++ b/quest/m1/archive/track-timeline/hls-target.md @@ -3,7 +3,8 @@ ## Goal A `moq-hls` media playlist advertises one `EXT-X-TARGETDURATION` for the -whole run, as RFC 8216 requires, and never lists a segment longer than it. +whole run, as RFC 8216 requires, and never lists a segment whose `EXTINF`, +rounded to the nearest integer, exceeds it (RFC 8216 4.3.3.1). ## Plan @@ -19,7 +20,8 @@ has reversed that decision in the 09-28 merged-PR audit: GOP (keyframe interval) configuration where one is known. Not the catalog's `durationMax` blindly: it is a 10s split ceiling, and advertising it would push every player's live edge back by that much. -- A segment longer than the target is refused or split, never listed as is. +- A segment whose rounded `EXTINF` exceeds the target is refused or split, + never listed as is. A 2.4s segment under a 2s target is valid and listed. Which one fits depends on where the long segment comes from (a reference record, or a non-reference rendition's snapped keyframe), so decide per case and say why. @@ -29,7 +31,7 @@ has reversed that decision in the 09-28 merged-PR audit: Replace `target_duration_follows_the_observed_segments` with tests that pin a constant target across a window whose segment durations vary, and that cover -the over-long segment path. +the over-long segment path at the rounding boundary. ## Related From 0bfdc978b9a8e0454351e8f06c257bc18539ab80 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 11:55:43 -0700 Subject: [PATCH 22/49] quest(archive/track-timeline): align the README with rounded EXTINF Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index 0b2fe1cc74..483ffc88d0 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -52,7 +52,7 @@ leaves two handles on one segmenter ## Required - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust -- [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment longer than it +- [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment whose rounded `EXTINF` exceeds it - [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of turning a rendition's segments into gaps ## Related From d2ce3e1dc6dfaae5c0cbd9162a4a0689d1fb31cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:07:33 -0700 Subject: [PATCH 23/49] quest(archive/track-timeline): the reference watcher resubscribes too Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/README.md | 2 +- .../m1/archive/track-timeline/hls-resubscribe.md | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index 483ffc88d0..b42a1e31e5 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -53,7 +53,7 @@ leaves two handles on one segmenter - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust - [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment whose rounded `EXTINF` exceeds it -- [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of turning a rendition's segments into gaps +- [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of freezing the playlists or turning a rendition's segments into gaps ## Related diff --git a/quest/m1/archive/track-timeline/hls-resubscribe.md b/quest/m1/archive/track-timeline/hls-resubscribe.md index 5bb81ddd03..bd32d93e48 100644 --- a/quest/m1/archive/track-timeline/hls-resubscribe.md +++ b/quest/m1/archive/track-timeline/hls-resubscribe.md @@ -1,9 +1,9 @@ -# [S] HLS timeline resubscribe +# [M] HLS timeline resubscribe ## Goal -A transient error on a rendition's timeline subscription does not turn that -rendition's remaining segments into `EXT-X-GAP`. The watcher subscribes again +A transient error on a timeline subscription does not freeze the playlists +or turn a rendition's remaining segments into `EXT-X-GAP`. The watcher subscribes again and keeps resolving segments. Only a clean end, a malformed timeline, or the track or broadcast going away ends its spans. @@ -18,6 +18,11 @@ The agent declined it because the watcher never retries, and named re-subscribing as the real fix. The maintainer's decision in the 09-28 merged-PR audit is to do that re-subscribe. +The reference rendition has the same flaw on a separate path, and a worse +one: `watch_timeline` in `rs/moq-hls/src/export/mod.rs` warns on the error, +then closes every window, so all playlists freeze and every recording cursor +ends. It follows the same rules below. + - Re-subscribe only on a recoverable error (a transport reset, a lost session, the publisher's track aborting) while the broadcast is still live, resuming from the records the spans already hold rather than rebuilding @@ -35,8 +40,9 @@ merged-PR audit is to do that re-subscribe. re-subscribe is either served, refused, or cut short by the broadcast ending. -Add a regression test where a non-reference rendition's timeline errors and -comes back, and its later segments resolve to media, not gaps. +Add regression tests where a non-reference rendition's timeline errors and +comes back, and its later segments resolve to media, not gaps; and where the +reference timeline does the same, and every playlist keeps advancing. ## Related From 3de0dd67530cca7425747c077b0fd2d9ff9ef21c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:18:08 -0700 Subject: [PATCH 24/49] quest(archive/track-timeline): keep history across resubscribe, target across reconfigure Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/hls-resubscribe.md | 5 ++++- quest/m1/archive/track-timeline/hls-target.md | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/quest/m1/archive/track-timeline/hls-resubscribe.md b/quest/m1/archive/track-timeline/hls-resubscribe.md index bd32d93e48..ebcea7528b 100644 --- a/quest/m1/archive/track-timeline/hls-resubscribe.md +++ b/quest/m1/archive/track-timeline/hls-resubscribe.md @@ -42,7 +42,10 @@ ends. It follows the same rules below. Add regression tests where a non-reference rendition's timeline errors and comes back, and its later segments resolve to media, not gaps; and where the -reference timeline does the same, and every playlist keeps advancing. +reference timeline does the same, and every playlist keeps advancing. One +reconnects past `timeline::CHECKPOINT_RECORDS` on a durable timeline: the +fresh decoder's leading `Skip` for the omitted prefix must not clear the +rows and spans already held, so older retained segments stay listed. ## Related diff --git a/quest/m1/archive/track-timeline/hls-target.md b/quest/m1/archive/track-timeline/hls-target.md index 81cbbd1b70..d75cc1652b 100644 --- a/quest/m1/archive/track-timeline/hls-target.md +++ b/quest/m1/archive/track-timeline/hls-target.md @@ -31,7 +31,10 @@ has reversed that decision in the 09-28 merged-PR audit: Replace `target_duration_follows_the_observed_segments` with tests that pin a constant target across a window whose segment durations vary, and that cover -the over-long segment path at the rounding boundary. +the over-long segment path at the rounding boundary. Hold the target per +playlist URI, not per `Rendition`: `renditions::Producer::sync` replaces the +`Rendition` on a catalog reconfigure, so test that a reconfigure mid-run +keeps the original target. ## Related From 772f03170d55898fb17b6aca97321f05f8ffc249 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:47:21 -0700 Subject: [PATCH 25/49] quest(archive/track-timeline): drop popped rows on a reconnect Skip Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/hls-resubscribe.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/quest/m1/archive/track-timeline/hls-resubscribe.md b/quest/m1/archive/track-timeline/hls-resubscribe.md index ebcea7528b..b901b83bdb 100644 --- a/quest/m1/archive/track-timeline/hls-resubscribe.md +++ b/quest/m1/archive/track-timeline/hls-resubscribe.md @@ -44,8 +44,10 @@ Add regression tests where a non-reference rendition's timeline errors and comes back, and its later segments resolve to media, not gaps; and where the reference timeline does the same, and every playlist keeps advancing. One reconnects past `timeline::CHECKPOINT_RECORDS` on a durable timeline: the -fresh decoder's leading `Skip` for the omitted prefix must not clear the -rows and spans already held, so older retained segments stay listed. +fresh decoder's leading `Skip` covers both records the source popped during +the outage and retained records the checkpoint omitted. Drop only the rows +before the new logical offset and keep the retained ones, so older segments +stay listed and popped ones go; test it with pops during the outage. ## Related From 1bf32fac049341802bddbf8288cb80dd2a352342 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:30:47 -0700 Subject: [PATCH 26/49] chore(quest): claim hls-resubscribe Co-Authored-By: Claude Opus 5.5 From 5ee1c0a6a0dbc3896516015ca4185d81ca6b63e3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:47:33 -0700 Subject: [PATCH 27/49] quest(archive/track-timeline): timelines declare their segment duration; HLS target follows it Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/README.md | 3 + .../track-timeline/declared-duration.md | 53 ++++++++++++++++++ quest/m1/archive/track-timeline/hls-target.md | 55 +++++++++++-------- quest/m1/archive/track-timeline/js.md | 4 ++ 4 files changed, 93 insertions(+), 22 deletions(-) create mode 100644 quest/m1/archive/track-timeline/declared-duration.md diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index b42a1e31e5..6b991dbf8d 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -40,6 +40,8 @@ Decisions: reader's cache for the next request. - HLS `EXT-X-TARGETDURATION` is fixed for the run, not the observed maximum #4280 shipped. This reverses that PR's decision 4 (09-28 merged-PR audit). + The target comes from each timeline's declared duration, which replaces the + broadcast-wide `durationMax` (09-29 planning). For triage, not blocking: two Codex P2s arrived after #4280 merged and are unanswered. A timeline record whose `sequence` differs from its window index @@ -51,6 +53,7 @@ leaves two handles on one segmenter ## Required +- [Timelines declare their segment duration](/quest/m1/archive/track-timeline/declared-duration.md) - each timeline entry declares its segment duration (reported or estimated by the publisher), replacing the root `durationMax` - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust - [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment whose rounded `EXTINF` exceeds it - [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of freezing the playlists or turning a rendition's segments into gaps diff --git a/quest/m1/archive/track-timeline/declared-duration.md b/quest/m1/archive/track-timeline/declared-duration.md new file mode 100644 index 0000000000..7a92549d01 --- /dev/null +++ b/quest/m1/archive/track-timeline/declared-duration.md @@ -0,0 +1,53 @@ +# [M] Timelines declare their segment duration + +## Goal + +Each timeline in the catalog `archive` map declares its own maximum segment +duration, reported by the application like bitrate or framerate, or estimated +by the publisher when the application doesn't know it. The broadcast-wide +`durationMax`, filled from moq-mux's 10s default, is deleted. Every edge reads +the same declared value, so an HLS edge can fix its target duration from it. + +## Plan + +Decided in planning (09-29), after the hls-target agent found that nothing in +the catalog declares a usable target: + +- **Shape.** Each `timelines` entry becomes an object: + `timelines: { video: { track: "video.timeline.z", duration: 2000 } }`, in + the archive `timescale`. The root `durationMax` goes away. This is a break in + place on this line, with no compatibility path. Update `rs/hang`, + `drafts/draft-lcurley-moq-hang.md`, and every `durationMax` reference + (grep it, including `doc/`). The JS port happens in + [JS per-track timelines](/quest/m1/archive/track-timeline/js.md). +- **Meaning: a nominal target, not a hard max.** The segmenter cuts at the + group boundary nearest the declared duration, so records start on keyframes. + A GOP longer than the target overruns, and moq-hls lists the overrun with a + warning. A mid-group split remains only as a safety ceiling at a multiple of + the target (pick one, for example 3x), which is internal and not declared. + A hard max that splits at the target was rejected: jitter makes slivers, and + segments would start without a keyframe. +- **Reported by the application.** moq-mux takes an optional per-track + duration hint. moq-video's encoder sets it from its `encode::Gop`. Importers + pass nothing and estimate. +- **Estimated by the publisher when not reported.** moq-mux holds that + timeline's catalog entry until the first complete group, then declares its + duration rounded up to the timescale. If no second keyframe arrives within + the safety ceiling, it declares the ceiling. Estimating at the edge was + rejected because edges and restarts would disagree. A rolling estimate was + rejected because it conflicts with a fixed target. The user chose the first + group over the longer of the first two groups, accepting that an irregular + first GOP sets the value. +- **Fixed per timeline.** The declared value never changes for a timeline + track's life. A reconfigure that makes a new timeline track declares its own. +- Sparse timelines (the catalog's own, zero minimum) keep their current + cutting. Decide what, if anything, they declare, and keep it out of HLS. + +Tests: a hinted track declares its hint. An unhinted track holds its entry +until the first group completes, then declares that group's duration. A first +group that never closes declares the ceiling. The cut lands on the boundary +nearest the target. The declared value survives later, longer groups. + +## Related + +- [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - consumes the declared duration diff --git a/quest/m1/archive/track-timeline/hls-target.md b/quest/m1/archive/track-timeline/hls-target.md index d75cc1652b..acc50ead9e 100644 --- a/quest/m1/archive/track-timeline/hls-target.md +++ b/quest/m1/archive/track-timeline/hls-target.md @@ -3,8 +3,9 @@ ## Goal A `moq-hls` media playlist advertises one `EXT-X-TARGETDURATION` for the -whole run, as RFC 8216 requires, and never lists a segment whose `EXTINF`, -rounded to the nearest integer, exceeds it (RFC 8216 4.3.3.1). +whole run, taken from the reference timeline's declared duration, so every +edge and every reload agree on it. Segments stay one-to-one with timeline +records: the edge never skips, splits, or renumbers one. ## Plan @@ -12,30 +13,40 @@ rounded to the nearest integer, exceeds it (RFC 8216 4.3.3.1). current window's longest segment (`snapshot_as` in `rs/moq-hls/src/export/rendition.rs`), so it rises when a long record arrives and falls when that record is evicted. Codex flagged it -([r4113921580](https://github.com/moq-dev/moq/pull/4280#discussion_r4113921580)), -and the PR kept the observed maximum as recorded decision 4. The maintainer -has reversed that decision in the 09-28 merged-PR audit: - -- Set the target once, from what the publisher declares: the catalog, or the - GOP (keyframe interval) configuration where one is known. Not the catalog's - `durationMax` blindly: it is a 10s split ceiling, and advertising it would - push every player's live edge back by that much. -- A segment whose rounded `EXTINF` exceeds the target is refused or split, - never listed as is. A 2.4s segment under a 2s target is valid and listed. - Which one fits depends on where the long segment comes from (a reference - record, or a non-reference rendition's snapped keyframe), so decide per - case and say why. -- If the catalog declares nothing usable, decide whether to refuse the - rendition or fix a target from the first segments and hold it; ask the - maintainer if neither is clearly right. - -Replace `target_duration_follows_the_observed_segments` with tests that pin a -constant target across a window whose segment durations vary, and that cover -the over-long segment path at the rounding boundary. Hold the target per +([r4113921580](https://github.com/moq-dev/moq/pull/4280#discussion_r4113921580)). +The 09-28 audit reversed #4280's decision 4 (the observed maximum), and the +09-29 planning settled the rest: + +- The target is `ceil` of the reference timeline's declared duration from + [Timelines declare their segment duration](/quest/m1/archive/track-timeline/declared-duration.md). + Until that timeline's entry is declared, the rendition has no playlist yet. +- A segment whose rounded `EXTINF` exceeds the target (a GOP that overran the + publisher's declared value) is listed anyway, with a rate-limited warning. + This deliberately deviates from RFC 8216 4.3.3.1's MUST, because refusing + the segment loses content, and splitting or skipping it at the edge breaks + the stable media sequence numbers of RFC 8216 6.2.2. Document the + deviation. hls.js tolerates it; Apple's mediastreamvalidator flags it. +- A record the publisher split mid-group at its safety ceiling is listed as + its own segment. Stop merging split halves back into one range + (`a_frame_split_record_is_not_a_video_boundary` in `spans.rs`). This is legal + because moq-hls never emits `EXT-X-INDEPENDENT-SEGMENTS`, so a segment may + start without a keyframe. +- A non-reference rendition keeps the reference record's `EXTINF`, even though + its keyframe-snapped content can run longer. The user decided this needs no + change. + +Replace `target_duration_follows_the_observed_segments` with tests that pin +the declared target across a window whose segment durations vary, and that +cover an overrun at the rounding boundary (listed, warned). Fix its stale +comment ("default 1s minimum"; the minimum is 2s). Hold the target per playlist URI, not per `Rendition`: `renditions::Producer::sync` replaces the `Rendition` on a catalog reconfigure, so test that a reconfigure mid-run keeps the original target. +## Required + +- [Timelines declare their segment duration](/quest/m1/archive/track-timeline/declared-duration.md) - the declared value the target is taken from + ## Related - [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - the line this blocks diff --git a/quest/m1/archive/track-timeline/js.md b/quest/m1/archive/track-timeline/js.md index 30bc460798..3a76ff5cff 100644 --- a/quest/m1/archive/track-timeline/js.md +++ b/quest/m1/archive/track-timeline/js.md @@ -11,3 +11,7 @@ Port the landed Rust shape to `js/hang/src/timeline.ts` and its catalog schema, replacing the aligned timeline and cross-track pacing rather than keeping both. Cover the same cut rules and a static catalog outliving other tracks' records, and check the records against Rust output in the interop suite. + +## Required + +- [Timelines declare their segment duration](/quest/m1/archive/track-timeline/declared-duration.md) - the final `timelines` entry shape to port, so JS ports it once From 9c1780e4572bfa92c7c3f8b8bcf4f8d189fc0af3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 14:04:34 -0700 Subject: [PATCH 28/49] quest(archive/track-timeline): 4s record minimum; estimate from the first record, not group Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/README.md | 8 +-- .../track-timeline/declared-duration.md | 61 +++++++++++-------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index 6b991dbf8d..c8e1981001 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -30,9 +30,9 @@ Decisions: - The catalog's root `archive` entry maps each track to its timeline, including the catalog track itself. `replay`, `store`, and `version` stay beside it. - Each track cuts on its own by one rule: at a group boundary between a - minimum and maximum duration (2s and 10s by default, a zero minimum only for - sparse data such as the catalog), splitting a long-lived group by frame at - the maximum. Manual cuts stay as an optimization, such as a + minimum and maximum duration (a 4s minimum and a multiple of the declared + duration, a zero minimum only for sparse data such as the catalog), + splitting a long-lived group by frame at the maximum. Manual cuts stay as an optimization, such as a video keyframe cutting audio so derived segments need fewer objects. - A stored object may hold a frame range of a group, not only whole groups. - HLS and DASH segments are derived at the edge from group timestamps, not @@ -55,7 +55,7 @@ leaves two handles on one segmenter - [Timelines declare their segment duration](/quest/m1/archive/track-timeline/declared-duration.md) - each timeline entry declares its segment duration (reported or estimated by the publisher), replacing the root `durationMax` - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust -- [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment whose rounded `EXTINF` exceeds it +- [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, from the reference timeline's declared duration; an overrun is listed with a warning - [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of freezing the playlists or turning a rendition's segments into gaps ## Related diff --git a/quest/m1/archive/track-timeline/declared-duration.md b/quest/m1/archive/track-timeline/declared-duration.md index 7a92549d01..e86ccfca39 100644 --- a/quest/m1/archive/track-timeline/declared-duration.md +++ b/quest/m1/archive/track-timeline/declared-duration.md @@ -2,10 +2,10 @@ ## Goal -Each timeline in the catalog `archive` map declares its own maximum segment -duration, reported by the application like bitrate or framerate, or estimated -by the publisher when the application doesn't know it. The broadcast-wide -`durationMax`, filled from moq-mux's 10s default, is deleted. Every edge reads +Each timeline in the catalog `archive` map declares its own nominal segment +duration, derived from what the application reports (like bitrate or +framerate), or estimated by the publisher when the application doesn't know +it. The broadcast-wide `durationMax`, filled from moq-mux's 10s default, is deleted. Every edge reads the same declared value, so an HLS edge can fix its target duration from it. ## Plan @@ -14,39 +14,48 @@ Decided in planning (09-29), after the hls-target agent found that nothing in the catalog declares a usable target: - **Shape.** Each `timelines` entry becomes an object: - `timelines: { video: { track: "video.timeline.z", duration: 2000 } }`, in + `timelines: { video: { track: "video.timeline.z", duration: 4000 } }`, in the archive `timescale`. The root `durationMax` goes away. This is a break in place on this line, with no compatibility path. Update `rs/hang`, `drafts/draft-lcurley-moq-hang.md`, and every `durationMax` reference (grep it, including `doc/`). The JS port happens in [JS per-track timelines](/quest/m1/archive/track-timeline/js.md). -- **Meaning: a nominal target, not a hard max.** The segmenter cuts at the - group boundary nearest the declared duration, so records start on keyframes. - A GOP longer than the target overruns, and moq-hls lists the overrun with a - warning. A mid-group split remains only as a safety ceiling at a multiple of - the target (pick one, for example 3x), which is internal and not declared. - A hard max that splits at the target was rejected: jitter makes slivers, and - segments would start without a keyframe. -- **Reported by the application.** moq-mux takes an optional per-track - duration hint. moq-video's encoder sets it from its `encode::Gop`. Importers - pass nothing and estimate. +- **A 4s minimum.** `duration_min` defaults to 4s (from 2s), so an archive + doesn't write a stream of tiny objects to S3. A record still ends at the + first group boundary at or past the minimum. This also packs audio, where + moq-audio publishes each ~20 ms codec packet as its own group (Codex on + this plan's PR). +- **Meaning: a nominal target, not a hard max.** Records start on keyframes. A + GOP longer than the declared duration overruns, and moq-hls lists the overrun + with a warning. A mid-group split remains only as a safety ceiling at a + multiple of the declared duration (pick one, for example 3x), which is + internal and not declared. Before an unhinted track declares, the ceiling + is a fixed default (today's 10s). A hard max that splits at the target was + rejected: jitter makes slivers, and segments would start without a keyframe. +- **Reported by the application.** moq-mux takes an optional per-track GOP + hint. moq-video's encoder sets it from its `encode::Gop`. Importers pass + nothing and estimate. With a hint, the declared duration is the smallest + multiple of the hint that is at least `duration_min`, the record length a + regular GOP produces. - **Estimated by the publisher when not reported.** moq-mux holds that - timeline's catalog entry until the first complete group, then declares its - duration rounded up to the timescale. If no second keyframe arrives within - the safety ceiling, it declares the ceiling. Estimating at the edge was - rejected because edges and restarts would disagree. A rolling estimate was - rejected because it conflicts with a fixed target. The user chose the first - group over the longer of the first two groups, accepting that an irregular - first GOP sets the value. + timeline's catalog entry until its first record closes, then declares that + record's duration, rounded up to the timescale. If the first record hits the + safety ceiling, it declares the ceiling. Estimating at the edge was rejected + because edges and restarts would disagree. A rolling estimate was rejected + because it conflicts with a fixed target. The user chose a single sample + over the longer of two, accepting that an irregular first GOP sets the value. + The sample is the first record, not the first group, because a group can be + a single audio packet. - **Fixed per timeline.** The declared value never changes for a timeline track's life. A reconfigure that makes a new timeline track declares its own. - Sparse timelines (the catalog's own, zero minimum) keep their current cutting. Decide what, if anything, they declare, and keep it out of HLS. -Tests: a hinted track declares its hint. An unhinted track holds its entry -until the first group completes, then declares that group's duration. A first -group that never closes declares the ceiling. The cut lands on the boundary -nearest the target. The declared value survives later, longer groups. +Tests: a hinted track declares the smallest multiple of its hint at or past +the minimum. An unhinted track holds its entry until its first record closes, +then declares that record's duration. A first record that hits the ceiling +declares the ceiling. Audio made of one packet per group declares about 4s, +not 20 ms. The declared value survives later, longer records. ## Related From 8542548cdf9a6544c1ed52dbee92c91c208f59d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:05:28 -0700 Subject: [PATCH 29/49] quest(archive/pruning): claim Co-Authored-By: Claude Opus 5.5 From 9b763ac88be43a4a1352f2185d76dd75c1046719 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:41:31 -0700 Subject: [PATCH 30/49] fix(hls): re-subscribe a timeline after a transport error A transient error on a timeline subscription no longer freezes the playlists or turns a rendition's remaining segments into gaps. The watcher subscribes again and reconciles the fresh window: rows the source popped go, retained rows the checkpoint omitted stay, restated records aren't repeated. A malformed timeline, a refused re-subscribe, or one that fails before delivering anything still ends it. Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/track-timeline/README.md | 1 - .../archive/track-timeline/hls-resubscribe.md | 54 --- rs/moq-hls/Cargo.toml | 2 +- rs/moq-hls/src/export/mod.rs | 205 +++++++++- rs/moq-hls/src/export/rendition.rs | 4 +- rs/moq-hls/src/export/timeline.rs | 366 ++++++++++++++++++ 6 files changed, 570 insertions(+), 62 deletions(-) delete mode 100644 quest/m1/archive/track-timeline/hls-resubscribe.md create mode 100644 rs/moq-hls/src/export/timeline.rs diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index b42a1e31e5..00e1958bcc 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -53,7 +53,6 @@ leaves two handles on one segmenter - [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust - [Fixed HLS target duration](/quest/m1/archive/track-timeline/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, and no segment whose rounded `EXTINF` exceeds it -- [HLS timeline resubscribe](/quest/m1/archive/track-timeline/hls-resubscribe.md) - a transient timeline error re-subscribes instead of freezing the playlists or turning a rendition's segments into gaps ## Related diff --git a/quest/m1/archive/track-timeline/hls-resubscribe.md b/quest/m1/archive/track-timeline/hls-resubscribe.md deleted file mode 100644 index b901b83bdb..0000000000 --- a/quest/m1/archive/track-timeline/hls-resubscribe.md +++ /dev/null @@ -1,54 +0,0 @@ -# [M] HLS timeline resubscribe - -## Goal - -A transient error on a timeline subscription does not freeze the playlists -or turn a rendition's remaining segments into `EXT-X-GAP`. The watcher subscribes again -and keeps resolving segments. Only a clean end, a malformed timeline, or the -track or broadcast going away ends its spans. - -## Plan - -In [#4280](https://github.com/moq-dev/moq/pull/4280), `watch_spans` in -`rs/moq-hls/src/export/rendition.rs` logs any error and then calls -`spans.end()`, so rows past the last record resolve as gaps for good. Codex -flagged it as a P1 -([r4112645622](https://github.com/moq-dev/moq/pull/4280#discussion_r4112645622)). -The agent declined it because the watcher never retries, and named -re-subscribing as the real fix. The maintainer's decision in the 09-28 -merged-PR audit is to do that re-subscribe. - -The reference rendition has the same flaw on a separate path, and a worse -one: `watch_timeline` in `rs/moq-hls/src/export/mod.rs` warns on the error, -then closes every window, so all playlists freeze and every recording cursor -ends. It follows the same rules below. - -- Re-subscribe only on a recoverable error (a transport reset, a lost - session, the publisher's track aborting) while the broadcast is still live, - resuming from the records the spans already hold rather than rebuilding - them. A malformed or unsupported timeline (a bad archive timescale, broken - JSON or DEFLATE) would replay the same retained record forever, so it fails - loud and ends the spans instead. -- Re-subscribe on the event that makes it possible (the broadcast or track - being available again), not on a timer. -- The spans end on a clean end of the timeline, when the broadcast goes - away, or when a re-subscribe is refused because the track is no longer - published. That last case is the terminal state for a timeline that never - returns while the broadcast stays live on other tracks: its remaining rows - resolve as gaps. The concern the decline raised still holds, so nothing may - park forever (`poll_resolved`, a recording `segments::Consumer`); a - re-subscribe is either served, refused, or cut short by the broadcast - ending. - -Add regression tests where a non-reference rendition's timeline errors and -comes back, and its later segments resolve to media, not gaps; and where the -reference timeline does the same, and every playlist keeps advancing. One -reconnects past `timeline::CHECKPOINT_RECORDS` on a durable timeline: the -fresh decoder's leading `Skip` covers both records the source popped during -the outage and retained records the checkpoint omitted. Drop only the rows -before the new logical offset and keep the retained ones, so older segments -stay listed and popped ones go; test it with pops during the outage. - -## Related - -- [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - the line this blocks diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index c69d02a7cb..1ab723caee 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -24,6 +24,7 @@ hang = { workspace = true } humantime = { workspace = true } kio = { workspace = true } m3u8-rs = "6" +moq-json = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } percent-encoding = { workspace = true } @@ -39,7 +40,6 @@ url = { workspace = true } async-trait = "0.1" futures = { workspace = true } moq-archive = { path = "../moq-archive" } -moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index c6131d0675..c7305ced50 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -32,6 +32,7 @@ mod mpd; mod playlist; mod rendition; mod spans; +mod timeline; mod upstream; pub mod renditions; @@ -434,10 +435,10 @@ async fn watch_timeline( // The timeline finished cleanly: the publisher is done, so every window can end // (ENDLIST) and recording cursors drain to completion. Ok(()) => renditions.end_windows(), - // A transient error (subscription reset, relay hiccup): don't mark the windows ended; - // the serve path keeps serving the frozen windows. + // A malformed timeline, or one that can't be subscribed again: don't mark the windows + // ended; the serve path keeps serving the frozen windows. Err(err) => { - tracing::warn!(track = %reference.1, %err, "timeline watcher error; leaving the playlists live") + tracing::error!(track = %reference.1, %err, "timeline watcher failed; freezing the playlists") } } // The timeline stream is over either way: close so recording cursors terminate instead of @@ -451,7 +452,7 @@ async fn watch( reference: &Reference, renditions: &renditions::Fanout, ) -> crate::Result<()> { - let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(broadcast, section, &reference.1).await?; + let mut timeline = timeline::Follower::subscribe(broadcast, section, &reference.1).await?; while let Some(event) = timeline.next().await? { match event { moq_mux::timeline::Event::Push { index, entry } => renditions.push(index, entry, reference), @@ -2539,4 +2540,200 @@ mod tests { .expect("renditions cursor resolves"); assert!(ended.is_none(), "renditions cursor ends when the broadcast closes"); } + + /// A two-second GOP record: record `n` covers group `n` from `2n` seconds. + fn gop(sequence: u64) -> hang::timeline::Record { + hang::timeline::Record::new( + sequence, + sequence * 2_000, + 2_000, + hang::timeline::Position::group(sequence), + hang::timeline::Position::group(sequence + 1), + ) + } + + /// A `video0` reference and a `video1` rendition over hand-published timelines, reconciled and + /// watched the way `watch_catalog` does. The broadcast's handler keeps a timeline's re-subscribe + /// waiting until the test publishes it again. + struct Resubscribe { + broadcast: moq_net::broadcast::Producer, + _handler: moq_net::broadcast::Dynamic, + renditions: renditions::Producer, + watcher: tokio::task::JoinHandle<()>, + } + + impl Resubscribe { + fn new(mut archive: hang::catalog::Archive) -> Self { + let broadcast = moq_net::broadcast::Info::new().produce(); + let handler = broadcast.dynamic(); + let upstream = Upstream { + source: moq_mux::Source::new(produce_origin().consume(), "live"), + broadcast: broadcast.consume(), + }; + + let mut catalog = moq_mux::catalog::hang::Catalog::default(); + for name in ["video0", "video1"] { + catalog.video.renditions.insert(name.to_string(), video_config()); + archive + .timelines + .insert(name.to_string(), hang::timeline::default_name(name)); + } + catalog.archive = Some(archive.clone()); + + let renditions = renditions::Producer::new(Config::default().window); + if durable(&archive) { + renditions.fanout().unbound(); + } + renditions.sync(&upstream, &catalog); + let watcher = tokio::spawn(watch_timeline( + upstream.broadcast.clone(), + archive, + reference(&catalog).unwrap(), + renditions.fanout(), + )); + Self { + broadcast, + _handler: handler, + renditions, + watcher, + } + } + + /// Publish `track`'s timeline, continuing `records` when set. Returns the track, to cut + /// it off, with the timeline writing to it. + fn publish( + &self, + track: &str, + records: Option>, + ) -> (moq_net::track::Producer, moq_mux::timeline::Producer) { + let track = self + .broadcast + .create_track(hang::timeline::default_name(track), moq_mux::timeline::Producer::info()) + .unwrap(); + let timeline = match records { + Some(records) => { + let range = records.first().unwrap().sequence..records.last().unwrap().sequence + 1; + let checkpoint = moq_json::window::Checkpoint { range, records }; + moq_mux::timeline::Producer::resume(track.clone(), &checkpoint).unwrap() + } + None => moq_mux::timeline::Producer::new(track.clone()), + }; + (track, timeline) + } + + fn snapshot(&self, name: &str) -> playlist::Snapshot { + self.renditions.get(Kind::Video, name).unwrap().snapshot() + } + + /// Wait until `name` lists segment `last`. + async fn listed(&self, name: &str, last: u64) -> playlist::Snapshot { + for _ in 0..500 { + let snapshot = self.snapshot(name); + if snapshot.segments.last().is_some_and(|s| s.segment >= last) { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} never listed segment {last}"); + } + } + + impl Drop for Resubscribe { + fn drop(&mut self) { + self.watcher.abort(); + } + } + + /// The timeline's publisher loses its session: the track and its open group fail. + fn cut(track: moq_net::track::Producer, timeline: moq_mux::timeline::Producer) { + drop(timeline); + track.abort(moq_net::Error::SessionClosed).unwrap(); + } + + fn numbers(snapshot: &playlist::Snapshot) -> Vec { + snapshot.segments.iter().map(|s| s.segment).collect() + } + + fn gaps(snapshot: &playlist::Snapshot) -> Vec { + snapshot.segments.iter().filter(|s| s.gap).map(|s| s.segment).collect() + } + + // A transient error on a rendition's own timeline subscribes again. Its segments stay pending + // through the outage and resolve to media afterwards, rather than every row past its last + // record turning into an `EXT-X-GAP`. + #[tokio::test(start_paused = true)] + async fn a_rendition_timeline_error_resubscribes_instead_of_listing_gaps() { + let test = Resubscribe::new(hang::catalog::Archive::new()); + let (_reference, mut video0) = test.publish("video0", None); + let (track, mut video1) = test.publish("video1", None); + for sequence in 0..4 { + video0.push(&gop(sequence)).unwrap(); + video1.push(&gop(sequence)).unwrap(); + } + // A segment resolves once the rendition's timeline is a snap tolerance past its end. + assert_eq!(numbers(&test.listed("video1", 2).await), [0, 1, 2]); + + cut(track, video1); + for sequence in 4..7 { + video0.push(&gop(sequence)).unwrap(); + } + tokio::time::sleep(Duration::from_secs(60)).await; + let outage = test.snapshot("video1"); + assert_eq!(numbers(&outage), [0, 1, 2], "the rest stays pending through the outage"); + assert_eq!(gaps(&outage), [] as [u64; 0]); + + let (_track, mut video1) = test.publish("video1", Some((0..4).map(gop).collect())); + for sequence in 4..7 { + video1.push(&gop(sequence)).unwrap(); + } + let resumed = test.listed("video1", 5).await; + assert_eq!(numbers(&resumed), [0, 1, 2, 3, 4, 5]); + assert_eq!( + gaps(&resumed), + [] as [u64; 0], + "the segments after the outage resolve to media" + ); + } + + // A transient error on the reference timeline subscribes again, and every playlist keeps + // advancing. The timeline is durable and longer than a group header restates + // (`CHECKPOINT_RECORDS`), and the source pops records during the outage: those rows go, + // while the retained rows the header omits stay listed. + #[tokio::test(start_paused = true)] + async fn a_reference_timeline_error_resubscribes_and_every_playlist_advances() { + let mut archive = hang::catalog::Archive::new(); + archive.store = Some("https://objects.example/rec/".parse().unwrap()); + let test = Resubscribe::new(archive); + let (track, mut video0) = test.publish("video0", None); + let (_track, mut video1) = test.publish("video1", None); + // Let the watchers read each record as it lands, so none falls a whole header behind. + for sequence in 0..300 { + video0.push(&gop(sequence)).unwrap(); + video1.push(&gop(sequence)).unwrap(); + settle().await; + } + let before = test.listed("video0", 299).await; + assert_eq!(numbers(&before), (0..300).collect::>()); + + cut(track, video0); + tokio::time::sleep(Duration::from_secs(60)).await; + assert_eq!( + numbers(&test.snapshot("video0")), + numbers(&before), + "the playlist holds" + ); + for sequence in 300..303 { + video1.push(&gop(sequence)).unwrap(); + } + // The source popped ten records while the watcher was away. + let (_track, mut video0) = test.publish("video0", Some((10..300).map(gop).collect())); + video0.push(&gop(300)).unwrap(); + + let reference = test.listed("video0", 300).await; + assert_eq!(numbers(&reference), (10..=300).collect::>()); + assert_eq!(reference.media_sequence, 10); + let rendition = test.listed("video1", 300).await; + assert_eq!(numbers(&rendition), (10..=300).collect::>()); + assert_eq!(gaps(&rendition), [] as [u64; 0]); + } } diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 34da1e0556..8c97ca9f90 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -884,7 +884,7 @@ async fn watch_spans( window: Option, ) { let result: Result<()> = async { - let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&broadcast, §ion, &track).await?; + let mut timeline = super::timeline::Follower::subscribe(&broadcast, §ion, &track).await?; while let Some(event) = timeline.next().await? { match event { moq_mux::timeline::Event::Push { entry, .. } => spans.push(entry, window), @@ -897,7 +897,7 @@ async fn watch_spans( } .await; if let Err(err) = result { - tracing::warn!(%track, %err, "rendition timeline ended with an error"); + tracing::error!(%track, %err, "rendition timeline failed; its remaining segments are gaps"); } spans.end(); } diff --git a/rs/moq-hls/src/export/timeline.rs b/rs/moq-hls/src/export/timeline.rs new file mode 100644 index 0000000000..ddc4610e3b --- /dev/null +++ b/rs/moq-hls/src/export/timeline.rs @@ -0,0 +1,366 @@ +//! One track's timeline, followed across transient subscription failures. +//! +//! A transport error on a timeline subscription (a reset stream, a lost session, the publisher's +//! track aborting) subscribes again rather than ending the timeline. The fresh subscription opens +//! on a group header that restates the retained window, so its events are reconciled against the +//! ones already delivered: rows the source popped during the outage are popped, retained rows the +//! header's checkpoint omitted are kept, and restated records are not delivered twice. +//! +//! Nothing waits on a timer. A re-subscribe is either served, refused (the track is no longer +//! published, or the broadcast went away), or fails before delivering anything; the last two end +//! the timeline with an error. A malformed timeline fails the same way on the first try, since a +//! fresh subscription would only decode the same retained record again. + +use std::collections::VecDeque; +use std::ops::Range; + +use hang::catalog::Archive; +use moq_mux::timeline::{Consumer, Entry, Event}; + +/// Reads one track's timeline, re-subscribing after a transport error. +pub(crate) struct Follower { + broadcast: moq_net::broadcast::Consumer, + section: Archive, + track: String, + consumer: Consumer, + /// Whether `consumer` has not yielded an event yet. + fresh: bool, + /// The window indices delivered and not since popped or skipped, once anything was. + held: Option>, + /// The newest delivered record, to tell a restated timeline from a restarted one. + last: Option, + /// Events reconciliation produced ahead of the one being handled. + queued: VecDeque, +} + +impl Follower { + /// Subscribe to `track`'s timeline as advertised by `section`. + pub async fn subscribe( + broadcast: &moq_net::broadcast::Consumer, + section: &Archive, + track: &str, + ) -> moq_mux::Result { + Ok(Self { + consumer: Consumer::subscribe(broadcast, section, track).await?, + broadcast: broadcast.clone(), + section: section.clone(), + track: track.to_string(), + fresh: true, + held: None, + last: None, + queued: VecDeque::new(), + }) + } + + /// The next window event, or `None` once the timeline ends cleanly. + pub async fn next(&mut self) -> moq_mux::Result> { + loop { + if let Some(event) = self.queued.pop_front() { + return Ok(Some(event)); + } + match self.consumer.next().await { + Ok(Some(event)) => self.reconcile(event), + Ok(None) => return Ok(None), + // A subscription that failed before delivering anything is not retried: it would + // fail the same way again, as fast as the source can refuse it. + Err(err) if is_transport(&err) && !self.fresh => { + tracing::warn!(track = %self.track, %err, "timeline subscription failed; subscribing again"); + self.consumer = Consumer::subscribe(&self.broadcast, &self.section, &self.track).await?; + self.fresh = true; + } + Err(err) => return Err(err), + } + } + } + + /// Queue what `event` means to a reader holding [`held`](Self::held). + fn reconcile(&mut self, event: Event) { + let fresh = std::mem::replace(&mut self.fresh, false); + let Some(held) = self.held.clone() else { + return self.deliver(event); + }; + + if fresh { + // A fresh subscription's first event names the window's front: its header's offset. + let front = match &event { + Event::Push { index, .. } => *index, + Event::Pop(range) | Event::Skip(range) => range.start, + _ => unreachable!("unknown timeline event"), + }; + // The source popped these while this reader was away. + let popped = held.start..front.min(held.end); + if !popped.is_empty() { + self.deliver(Event::Pop(popped)); + } + } + + let held = self.held.clone().expect("held survives a pop"); + match event { + // Records the header restated: this reader holds them unless there's a hole before the + // next one it needs. + Event::Skip(range) if range.end <= held.end => {} + Event::Skip(range) => self.deliver(Event::Skip(held.end..range.end)), + Event::Push { index, entry } if index < held.end => { + // A restated record. The newest one this reader holds must match, or the source + // restarted its timeline under the same track: start over from there. + if index + 1 == held.end && self.last.as_ref().is_some_and(|last| *last != entry) { + self.deliver(Event::Skip(held)); + self.deliver(Event::Push { index, entry }); + } + } + Event::Push { index, entry } if index > held.end => { + // Records pushed and popped while this reader was away. + self.deliver(Event::Skip(held.end..index)); + self.deliver(Event::Push { index, entry }); + } + event => self.deliver(event), + } + } + + /// Queue `event` for the reader and track what it now holds. + fn deliver(&mut self, event: Event) { + let held = self.held.get_or_insert(0..0); + match &event { + Event::Push { index, entry } => { + if held.is_empty() { + held.start = *index; + } + held.end = index + 1; + self.last = Some(entry.clone()); + } + Event::Pop(range) => held.start = held.start.max(range.end).min(held.end), + // A reader forgets everything on a skip, since the next record can't follow its last. + Event::Skip(range) => { + *held = range.end..range.end; + self.last = None; + } + _ => unreachable!("unknown timeline event"), + } + self.queued.push_back(event); + } +} + +/// Whether `err` came from the transport rather than the timeline's content. +fn is_transport(err: &moq_mux::Error) -> bool { + matches!( + err, + moq_mux::Error::Moq(_) | moq_mux::Error::Json(moq_json::Error::Net(_)) + ) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use hang::timeline::{Position, Record}; + use moq_json::window::Checkpoint; + + use super::*; + + const TRACK: &str = "video0"; + + fn section() -> Archive { + let mut section = Archive::new(); + section + .timelines + .insert(TRACK.to_string(), hang::timeline::default_name(TRACK)); + section + } + + fn record(sequence: u64, pts: u64) -> Record { + Record::new( + sequence, + pts, + 2_000, + Position::group(sequence), + Position::group(sequence + 1), + ) + } + + /// Publish the timeline track, continuing `records` when set, and keep a handle to abort it. + fn publish( + broadcast: &moq_net::broadcast::Producer, + records: Option>, + ) -> (moq_net::track::Producer, moq_mux::timeline::Producer) { + let track = broadcast + .create_track(hang::timeline::default_name(TRACK), moq_mux::timeline::Producer::info()) + .unwrap(); + let timeline = match records { + Some(records) => { + let start = records.first().map_or(0, |record| record.sequence); + let end = records.last().map_or(0, |record| record.sequence + 1); + let checkpoint = Checkpoint { + range: start..end, + records, + }; + moq_mux::timeline::Producer::resume(track.clone(), &checkpoint).unwrap() + } + None => moq_mux::timeline::Producer::new(track.clone()), + }; + (track, timeline) + } + + /// The publisher's session is lost: its track and open group end with an error. + fn abort(track: moq_net::track::Producer, timeline: moq_mux::timeline::Producer) { + drop(timeline); + track.abort(moq_net::Error::SessionClosed).unwrap(); + } + + #[derive(Debug, PartialEq)] + enum Seen { + Push(u64), + Pop(Range), + Skip(Range), + } + + fn seen(event: moq_mux::Result>) -> Seen { + match event.expect("the timeline is readable").expect("the timeline is live") { + Event::Push { index, .. } => Seen::Push(index), + Event::Pop(range) => Seen::Pop(range), + Event::Skip(range) => Seen::Skip(range), + _ => unreachable!("unknown timeline event"), + } + } + + async fn next(follower: &mut Follower) -> Seen { + seen( + tokio::time::timeout(Duration::from_secs(5), follower.next()) + .await + .expect("an event arrives"), + ) + } + + /// Subscribe, then read `count` records as they're pushed, so the reader keeps up. + async fn follow( + broadcast: &moq_net::broadcast::Producer, + timeline: &mut moq_mux::timeline::Producer, + count: u64, + ) -> Follower { + let mut follower = Follower::subscribe(&broadcast.consume(), §ion(), TRACK) + .await + .unwrap(); + for sequence in 0..count { + timeline.push(&record(sequence, sequence * 2_000)).unwrap(); + assert_eq!(next(&mut follower).await, Seen::Push(sequence)); + } + follower + } + + /// Read across an outage: the re-subscribe waits on the track, not a timer, until `restore`. + async fn across(follower: &mut Follower, restore: impl FnOnce() -> T) -> (moq_mux::Result>, T) { + let mut next = std::pin::pin!(follower.next()); + let waited = tokio::time::timeout(Duration::from_secs(60), &mut next).await; + assert!(waited.is_err(), "the re-subscribe did not wait: {waited:?}"); + let restored = restore(); + let next = tokio::time::timeout(Duration::from_secs(5), next) + .await + .expect("nothing parks once the source answers"); + (next, restored) + } + + #[tokio::test(start_paused = true)] + async fn a_reconnect_past_the_checkpoint_keeps_the_retained_records() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _handler = broadcast.dynamic(); + let (track, mut timeline) = publish(&broadcast, None); + // More records than a group header restates (`CHECKPOINT_RECORDS`, 256). + let mut follower = follow(&broadcast, &mut timeline, 300).await; + abort(track, timeline); + + // The source popped ten records during the outage. Its first header restates only the + // newest 256 of the rest, so the fresh subscription skips 10..44, which the reader holds. + let (first, _source) = across(&mut follower, || { + let retained = (10..300).map(|sequence| record(sequence, sequence * 2_000)).collect(); + let (track, mut timeline) = publish(&broadcast, Some(retained)); + timeline.push(&record(300, 600_000)).unwrap(); + (track, timeline) + }) + .await; + assert_eq!(seen(first), Seen::Pop(0..10)); + assert_eq!(next(&mut follower).await, Seen::Push(300)); + } + + #[tokio::test(start_paused = true)] + async fn a_restarted_timeline_starts_over() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _handler = broadcast.dynamic(); + let (track, mut timeline) = publish(&broadcast, None); + let mut follower = follow(&broadcast, &mut timeline, 3).await; + abort(track, timeline); + + // A new run numbers from zero again, describing other media. + let (first, _source) = across(&mut follower, || { + let (track, mut timeline) = publish(&broadcast, None); + for sequence in 0..4 { + timeline.push(&record(sequence, 100_000 + sequence * 2_000)).unwrap(); + } + (track, timeline) + }) + .await; + assert_eq!(seen(first), Seen::Skip(0..3)); + assert_eq!(next(&mut follower).await, Seen::Push(2)); + assert_eq!(next(&mut follower).await, Seen::Push(3)); + } + + #[tokio::test(start_paused = true)] + async fn a_refused_resubscribe_ends_the_timeline() { + // No handler, so a track that isn't published is refused rather than awaited. + let broadcast = moq_net::broadcast::Info::new().produce(); + let (track, mut timeline) = publish(&broadcast, None); + let mut follower = follow(&broadcast, &mut timeline, 1).await; + abort(track, timeline); + + let err = follower.next().await.expect_err("the re-subscribe is refused"); + assert!(matches!(err, moq_mux::Error::Moq(moq_net::Error::NotFound)), "{err}"); + } + + #[tokio::test(start_paused = true)] + async fn a_pending_resubscribe_ends_with_the_broadcast() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _handler = broadcast.dynamic(); + let (track, mut timeline) = publish(&broadcast, None); + let mut follower = follow(&broadcast, &mut timeline, 1).await; + abort(track, timeline); + + let (next, ()) = across(&mut follower, || broadcast.close()).await; + let err = next.expect_err("the broadcast is gone"); + assert!(matches!(err, moq_mux::Error::Moq(moq_net::Error::Unroutable)), "{err}"); + } + + #[tokio::test(start_paused = true)] + async fn a_resubscribe_that_fails_before_any_event_is_not_retried() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _handler = broadcast.dynamic(); + let (track, mut timeline) = publish(&broadcast, None); + let mut follower = follow(&broadcast, &mut timeline, 1).await; + abort(track, timeline); + + let mut next = std::pin::pin!(follower.next()); + assert!(tokio::time::timeout(Duration::from_secs(60), &mut next).await.is_err()); + // Serve the re-subscribe, then fail it before it delivers a single event. + let (track, timeline) = publish(&broadcast, None); + assert!(tokio::time::timeout(Duration::from_secs(1), &mut next).await.is_err()); + abort(track, timeline); + tokio::time::timeout(Duration::from_secs(5), next) + .await + .expect("a failing source is not subscribed again") + .expect_err("the timeline ends"); + } + + #[tokio::test(start_paused = true)] + async fn a_malformed_timeline_is_not_retried() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _handler = broadcast.dynamic(); + let (track, mut timeline) = publish(&broadcast, None); + let mut follower = follow(&broadcast, &mut timeline, 1).await; + + // A group that doesn't decode: a fresh subscription would read the same one again. + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::now(), &b"garbage"[..]).unwrap(); + let err = tokio::time::timeout(Duration::from_secs(5), follower.next()) + .await + .expect("a malformed timeline fails at once") + .expect_err("the timeline is malformed"); + assert!(!is_transport(&err), "{err}"); + } +} From 1e80e88521eaffd518bfa77f2dcff33072314b43 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:41:46 -0700 Subject: [PATCH 31/49] quest(archive): DVR resolves the live-end handover through the wildcard claim Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/dvr.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index e379b55192..a58861ea52 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -27,9 +27,18 @@ served stays in `moq_net`'s track cache until the pool reclaims it. Decide whether expiry during a seek needs a group eviction API in `moq-net`. `moq-hls` reads a timeline from the catalog's own broadcast, and an -`archive.replay` path only marks it non-durable. Decide whether a live -broadcast's exporter follows `replay` to the recording for rewind, or whether -viewers address the replay broadcast directly. +`archive.replay` path only marks it non-durable. Decided (09-29): the exporter +does not follow `replay`, and viewers don't address a separate replay +broadcast. A recording is the broadcast. Under the +[wildcard](/quest/m0/wildcard/README.md) plan the archive serves the source +path through the root claim, and a live announcement shadows it. So when live +ends, `moq-hls` resolves the same name and falls through to the recording: +playlists keep serving for rewind and for players finishing the last +segments. `moq-hls` gains no linger (the `hls-linger` quest was dropped, +because an unannounced broadcast can't be FETCHed and a linger would only +serve the cache). Test the handover: a live HLS session keeps its playlist +URIs and media sequence numbers when the name moves from the live publisher to +the archive. ## Required From b608c04b26b02e437297c6bf184cfda57ac5987a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:54:10 -0700 Subject: [PATCH 32/49] feat(archive): prune DVR timeline objects no recovery needs A DVR writer now deletes timeline segment objects older than the checkpoint recovery would start from, one grace period after they stop being needed. Deletes go oldest first and stop at the first failure, so the stored keys stay consecutive. A restart schedules the same cleanup for stale objects. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 4 +- drafts/draft-lcurley-moq-hang.md | 2 + quest/m1/archive/README.md | 7 +- quest/m1/archive/pruning.md | 17 --- rs/moq-archive/src/mock.rs | 18 ++- rs/moq-archive/src/proof.rs | 13 +- rs/moq-archive/src/recover.rs | 41 ++++-- rs/moq-archive/src/writer.rs | 225 ++++++++++++++++++++++++++----- 8 files changed, 253 insertions(+), 74 deletions(-) delete mode 100644 quest/m1/archive/pruning.md diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 1af39b4402..53b051b6ca 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -223,8 +223,8 @@ JSON, and binary track are recorded alongside. It refuses a rendition served from another broadcast, and one that returns after the catalog dropped it. The stage ends once the broadcast does, and it refuses a store URL that already holds a recording. `--retention 1h` keeps only the last hour (a DVR), -deleting expired objects `--retention-grace` (default 30s) after the timeline -stops advertising them. +deleting expired objects, and timeline objects no longer needed to recover it, +`--retention-grace` (default 30s) after the timeline stops needing them. `import archive` republishes a recording: the timeline replays as a live track and every other track's groups are served on request, one object GET per group diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 8384c75140..cabaf7388b 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -994,6 +994,7 @@ Trimming occurs only as part of committing a new segment, before that segment's A recording MUST NOT trim after its final segment is committed. Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired segments' objects. The writer MUST retain the latest timeline object, even for an all-gap segment, and enough earlier timeline groups to recover the retained window from a checkpoint. +It MAY delete earlier timeline objects, oldest first, so the remaining segment IDs stay consecutive. The index therefore never advertises media the retention process has already deleted, although media objects MAY temporarily outlive the index. Relay cache eviction does not change the recording timeline. @@ -1097,6 +1098,7 @@ A publisher MAY estimate an unknown final duration from the frame cadence, but M - Specified version 1 recording objects: JSON track properties and binary group/frame tables with ascending, delta-encoded group sequences. - Addressed track objects by inclusive group bounds and timeline objects by consecutive segment IDs, with incremental discovery and per-track omission on storage failure. - Restricted retention updates to segment commits and removed completion markers. +- Allowed a DVR to delete timeline objects no checkpoint recovery needs, oldest first. - Limited recorded group and segment IDs and frame timestamps to JSON-safe integers, including delta reconstruction. - Compared existing track properties by parsed values rather than JSON serialization. - Required exclusive DVR restart recovery to remove unreferenced group objects left by interrupted expiration. diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 996849c80a..c42520b47a 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -61,10 +61,12 @@ percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list `moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) records enrolled tracks through `Deferred`, omits failed tracks with `Pending::omit`, stores each segment's timeline groups after `Producer::flush`, and expires DVR segments -with a deletion grace. On a prefix that already holds a recording, it replays the +with a deletion grace, pruning the oldest timeline objects no checkpoint +recovery reads. On a prefix that already holds a recording, it replays the retained timeline from a checkpoint through `timeline::Producer::resume`, refuses groups at or below each track's largest stored group, and a DVR deletes -unreferenced group objects one grace period after recovery. +unreferenced group objects and unneeded timeline objects one grace period +after recovery. `moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays the timeline onto a supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` with a byte-bounded object LRU. `Reader::refresh` follows by listing timeline keys after its cursor, so gaps and @@ -147,7 +149,6 @@ owned by that prerequisite, not duplicated in archive storage. - [Paced replay](/quest/m1/archive/paced-replay.md) - a replay pushes its groups to live subscribers on one shared clock, so any live player plays it - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback -- [DVR timeline pruning](/quest/m1/archive/pruning.md) - a DVR deletes timeline objects no retained checkpoint needs ## Related diff --git a/quest/m1/archive/pruning.md b/quest/m1/archive/pruning.md deleted file mode 100644 index 3e2dfcfbc6..0000000000 --- a/quest/m1/archive/pruning.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] DVR timeline pruning - -## Goal - -A DVR writer deletes timeline objects that no retained checkpoint needs, so a -long-running DVR stores a bounded number of objects. - -## Plan - -`moq_archive::Writer` deletes expired group objects but never a -`segments/` timeline object, so a 24/7 DVR with 2s segments adds -about 43,000 objects a day, and every restart lists all of them. The draft -already allows it: the writer keeps the latest timeline object and enough -earlier groups to recover the retained window from a checkpoint. Delete the -oldest timeline objects no longer needed, one grace period after they stop -being needed, keeping the remaining keys contiguous, as recovery requires. -Extend the restart cleanup and its tests in `rs/moq-archive/src/writer.rs`. diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs index 2d7275badf..c35c9e12c8 100644 --- a/rs/moq-archive/src/mock.rs +++ b/rs/moq-archive/src/mock.rs @@ -36,6 +36,8 @@ struct State { fail_puts: Vec, /// GETs whose path contains any of these return Not Found. hide_gets: Vec, + /// Deletes whose path contains any of these fail. + fail_deletes: Vec, /// Streaming listings end with an error after every entry. fail_lists: bool, /// Streaming listings yield entries in descending order, like a backend that promises none. @@ -94,6 +96,11 @@ impl Mock { self } + pub fn fail_deletes(&self, pattern: &str) -> &Self { + self.state().fail_deletes.push(pattern.to_string()); + self + } + pub fn fail_lists(&self) -> &Self { self.state().fail_lists = true; self @@ -109,6 +116,7 @@ impl Mock { let mut state = self.state(); state.fail_puts.clear(); state.hide_gets.clear(); + state.fail_deletes.clear(); state.fail_lists = false; } @@ -206,7 +214,15 @@ impl ObjectStore for Mock { ) -> BoxStream<'static, object_store::Result> { let state = self.state.clone(); let locations = locations - .inspect_ok(move |path| state.lock().unwrap().ops.push(Op::Delete(path.to_string()))) + .and_then(move |path| { + let mut state = state.lock().unwrap(); + state.ops.push(Op::Delete(path.to_string())); + let failed = state.fail_deletes.iter().any(|p| path.as_ref().contains(p)); + futures::future::ready(match failed { + true => Err(unsupported("delete")), + false => Ok(path), + }) + }) .boxed(); self.inner.delete_stream(locations) } diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs index 904db71df5..0177a0c4bd 100644 --- a/rs/moq-archive/src/proof.rs +++ b/rs/moq-archive/src/proof.rs @@ -352,8 +352,8 @@ async fn an_offline_reader_follows_dvr_expiry() { write(&video, sequence, &frames(sequence)); } committed(8).await; - // Segments 7 and 8 hold the 2s window. - while store.get_groups("video", 6..=6).await.is_ok() { + // Segments 7 and 8 hold the 2s window, and the newest timeline object restates both. + while store.get_groups("video", 6..=6).await.is_ok() || store.get_segments(TIMELINE, 7).await.is_ok() { tokio::time::sleep(Duration::from_millis(5)).await; } @@ -368,10 +368,11 @@ async fn an_offline_reader_follows_dvr_expiry() { offset: Some(format!("rec/timeline%2Ez/segments/{}", id(1))), } ); - let segments: Vec<_> = (2..=8) - .map(|segment| Op::Get(format!("rec/timeline%2Ez/segments/{}", id(segment)))) - .collect(); - assert_eq!(gets, segments, "only the new timeline keys are read"); + assert_eq!( + gets, + &[Op::Get(format!("rec/timeline%2Ez/segments/{}", id(8)))], + "only the retained timeline key is read" + ); // Expired groups are gone from the index, so they cost no media GET. for sequence in 0..7 { diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs index f7ccfd5ace..350990729b 100644 --- a/rs/moq-archive/src/recover.rs +++ b/rs/moq-archive/src/recover.rs @@ -1,7 +1,7 @@ //! Recover a recording so a restarted [`Writer`](crate::Writer) continues it. use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::ops::RangeInclusive; +use std::ops::{Range, RangeInclusive}; use futures::TryStreamExt; use hang::timeline::Record; @@ -21,6 +21,11 @@ pub(crate) struct Recovery { pub floors: HashMap, /// Stored group objects no retained record references. Only collected for a DVR. pub orphans: Vec, + /// The timeline objects recovery replayed, oldest first, with the first record index each + /// one's opening checkpoint restates. Only collected for a DVR. + pub checkpoints: VecDeque<(u64, u64)>, + /// Stored timeline segments older than any recovery reads. Only collected for a DVR. + pub stale: Range, } /// List the whole recording, then replay its timeline from a retained checkpoint. @@ -45,14 +50,14 @@ pub(crate) async fn recover(store: &Store, timeline: &str, co } segments.sort_unstable(); - let (checkpoint, sequence) = match (segments.first(), segments.last()) { + let (checkpoint, sequence, checkpoints) = match (segments.first(), segments.last()) { (Some(&first), Some(&last)) => { if segments.len() as u64 != last - first + 1 { return Err(Error::Timeline(format!( "timeline segments {first}..={last} are not contiguous" ))); } - let (checkpoint, sequence) = replay(store, timeline, first..=last, complete).await?; + let (checkpoint, sequence, checkpoints) = replay(store, timeline, first..=last, complete).await?; // The newest object is segment `last`, so the window must end on the next one. // A shorter window would resume onto that segment and collide with it. let next = last.checked_add(1).ok_or(Error::Overflow)?; @@ -62,9 +67,9 @@ pub(crate) async fn recover(store: &Store, timeline: &str, co checkpoint.range.end ))); } - (Some(checkpoint), sequence) + (Some(checkpoint), sequence, checkpoints) } - _ => (None, 0), + _ => (None, 0, VecDeque::new()), }; let mut referenced = HashSet::new(); @@ -76,9 +81,14 @@ pub(crate) async fn recover(store: &Store, timeline: &str, co } } } - let orphans = match complete { - true => groups.into_iter().filter(|key| !referenced.contains(key)).collect(), - false => Vec::new(), + let (orphans, checkpoints, stale) = match complete { + true => { + let orphans = groups.into_iter().filter(|key| !referenced.contains(key)).collect(); + let first = segments.first().copied().unwrap_or_default(); + let needed = checkpoints.front().map_or(first, |(segment, _)| *segment); + (orphans, checkpoints, first..needed) + } + false => (Vec::new(), VecDeque::new(), 0..0), }; Ok(Recovery { @@ -86,6 +96,8 @@ pub(crate) async fn recover(store: &Store, timeline: &str, co sequence, floors, orphans, + checkpoints, + stale, }) } @@ -96,22 +108,25 @@ fn raise(floors: &mut HashMap, track: &str, group: u64) { /// Replay `segments` from the newest checkpoint that restates every record `complete` needs. /// -/// Returns the retained window and the next timeline group sequence. +/// Returns the retained window, the next timeline group sequence, and each replayed segment with +/// its checkpoint's first restated index. async fn replay( store: &Store, timeline: &str, segments: RangeInclusive, complete: bool, -) -> Result<(Checkpoint, u64)> { +) -> Result<(Checkpoint, u64, VecDeque<(u64, u64)>)> { // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, - // so walk back until a checkpoint restates from there. + // so walk back until a checkpoint restates from there. The writer prunes by the same rule. let mut objects = VecDeque::new(); + let mut checkpoints = VecDeque::new(); let mut needed = None; for segment in segments.rev() { let object = store.get_segments(timeline, segment).await?; let (offset, start) = checkpoint(&object)?; let needed = *needed.get_or_insert(offset); objects.push_front(object); + checkpoints.push_front((segment, start)); if !complete || start <= needed { break; } @@ -156,11 +171,11 @@ async fn replay( range, records: records.into_values().collect(), }; - Ok((checkpoint, sequence)) + Ok((checkpoint, sequence, checkpoints)) } /// The retained offset and first restated index of the checkpoint opening `object`. -fn checkpoint(object: &Object) -> Result<(u64, u64)> { +pub(crate) fn checkpoint(object: &Object) -> Result<(u64, u64)> { let frame = object .groups .first() diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 3f09decdc0..abc3a05e71 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -25,6 +25,7 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::future::Future; +use std::ops::Range; use std::sync::{Arc, Mutex}; use std::task::Poll; use std::time::Duration; @@ -39,7 +40,7 @@ use object_store::ObjectStore; use tokio::sync::{mpsc, watch}; use tokio::time::Instant; -use crate::recover::recover; +use crate::recover::{checkpoint, recover}; use crate::segment::{Frame, Group, Object}; use crate::{Error, Info, Key, Result, Store}; @@ -98,7 +99,9 @@ pub struct Writer { committer: Committer, grace: Option, /// Deadlines for deleting expired or orphaned objects, oldest first. - deletions: VecDeque<(Instant, Vec)>, + deletions: VecDeque<(Instant, Expired)>, + /// The oldest timeline segment that may still be stored; pruning deletes upward from it. + pruned: u64, // Owns the recording's timeline track. _timeline: broadcast::Producer, } @@ -156,8 +159,9 @@ impl Writer { /// any group at or below the largest one stored for it, so a source whose group sequences /// restarted needs a new prefix. A DVR also deletes, one grace period after recovery, every /// group object its retained records do not reference, such as interrupted expirations and - /// uploads. The writer must own the prefix exclusively. Fails, deleting nothing, when the - /// recording cannot be listed or its timeline cannot be replayed. + /// uploads, and every timeline object older than the checkpoint it recovered from. The writer + /// must own the prefix exclusively. Fails, deleting nothing, when the recording cannot be + /// listed or its timeline cannot be replayed. pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { let section = timeline::Segmenter::new(config.timeline.clone()).section(); let recovery = recover(&store, §ion.track, config.retention.is_some()).await?; @@ -193,10 +197,14 @@ impl Writer { let (commands, receiver) = mpsc::unbounded_channel(); let retention = config.retention; let mut deletions = VecDeque::new(); + let stale = Expired { + groups: recovery.orphans, + segments: recovery.stale.clone(), + }; if let Some(retention) = &retention - && !recovery.orphans.is_empty() + && !stale.is_empty() { - deletions.push_back((Instant::now() + retention.grace, recovery.orphans)); + deletions.push_back((Instant::now() + retention.grace, stale)); } Ok(Self { control: Control { @@ -212,10 +220,12 @@ impl Writer { timescale: section.timescale.into(), retention: retention.as_ref().map(|r| r.window), window: recovery.checkpoint.map(|c| c.records.into()).unwrap_or_default(), + checkpoints: recovery.checkpoints, sequence: recovery.sequence, }, grace: retention.map(|r| r.grace), deletions, + pruned: recovery.stale.start, _timeline: broadcast, }) } @@ -238,6 +248,7 @@ impl Writer { committer, grace, mut deletions, + mut pruned, _timeline, } = self; let shared = control.shared.clone(); @@ -316,8 +327,8 @@ impl Writer { } _ = source.closed(), if !closed => closed = true, _ = tokio::time::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { - let (_, keys) = deletions.pop_front().unwrap(); - delete(&shared.store, keys).await; + let (_, expired) = deletions.pop_front().unwrap(); + delete(&shared.store, &shared.timeline, &mut pruned, expired).await; } } } @@ -331,9 +342,9 @@ impl Writer { // Complete expirations already committed to the timeline; nothing new expires after the // final segment. - for (deadline, keys) in deletions { + for (deadline, expired) in deletions { tokio::time::sleep_until(deadline).await; - delete(&shared.store, keys).await; + delete(&shared.store, &shared.timeline, &mut pruned, expired).await; } // A broadcast end carries no cause; only the deprecated `abort` still reports one. @@ -647,7 +658,21 @@ fn take_objects( } /// An in-flight segment commit, returning the committer and the expired objects to delete. -type Commit = BoxFuture<'static, (Committer, Result>)>; +type Commit = BoxFuture<'static, (Committer, Result)>; + +/// Objects the timeline stopped needing, deleted once the grace elapses. +struct Expired { + /// Group objects no retained record references. + groups: Vec, + /// Timeline segments no recovery reads. + segments: Range, +} + +impl Expired { + fn is_empty(&self) -> bool { + self.groups.is_empty() && self.segments.is_empty() + } +} /// Deferred records to commit: live, then the terminal drain. enum Segments { @@ -675,18 +700,21 @@ struct Committer { retention: Option, /// Committed records still in the timeline window, oldest first. window: VecDeque, + /// A DVR's stored timeline segments that recovery still reads, oldest first, each with the + /// first record index its opening checkpoint restates. + checkpoints: VecDeque<(u64, u64)>, /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. sequence: u64, } impl Committer { /// Commit `pending`, returning the committer and the expired objects to delete after the grace. - async fn commit(mut self, pending: Pending, objects: Vec<(String, Option)>) -> (Self, Result>) { + async fn commit(mut self, pending: Pending, objects: Vec<(String, Option)>) -> (Self, Result) { let result = self.commit_inner(pending, objects).await; (self, result) } - async fn commit_inner(&mut self, mut pending: Pending, objects: Vec<(String, Option)>) -> Result> { + async fn commit_inner(&mut self, mut pending: Pending, objects: Vec<(String, Option)>) -> Result { let shared = self.shared.clone(); let store = &shared.store; let puts = objects.iter().map(|(name, object)| async move { @@ -722,15 +750,36 @@ impl Committer { let object = self.read_timeline(pts)?; store.put_segments(&shared.timeline, segment, &object).await?; - let mut keys = Vec::new(); + let mut groups = Vec::new(); for record in expired { for (name, ranges) in &record.tracks { if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { - keys.push(Key::groups(name.clone(), first.start..=last.end)?); + groups.push(Key::groups(name.clone(), first.start..=last.end)?); } } } - Ok(keys) + let segments = self.prune(segment, &object)?; + Ok(Expired { groups, segments }) + } + + /// Track the newly stored timeline `object`, returning the older segments no recovery reads. + /// + /// Recovery walks back from the newest object to the first whose checkpoint restates the + /// newest one's offset, so every object before that one is no longer needed. + fn prune(&mut self, segment: u64, object: &Object) -> Result> { + if self.retention.is_none() { + return Ok(segment..segment); + } + let (offset, start) = checkpoint(object)?; + self.checkpoints.push_back((segment, start)); + let keep = self + .checkpoints + .iter() + .rposition(|&(_, start)| start <= offset) + .unwrap_or(0); + let first = self.checkpoints[0].0; + self.checkpoints.drain(..keep); + Ok(first..self.checkpoints[0].0) } /// Pop the oldest records while the rest still cover the retention window. @@ -784,14 +833,39 @@ impl Committer { } } -async fn delete(store: &Store, keys: Vec) { - let deletes = keys.iter().map(|key| async move { (key, store.delete(key).await) }); - for (key, result) in futures::future::join_all(deletes).await { - // An object that outlives its expiry advertises nothing; recovery cleans it up. - if let Err(err) = result { - tracing::warn!(?key, %err, "failed to delete an expired object"); +/// Delete `expired`'s group objects, and the timeline objects from `pruned` up to its segments. +/// +/// Timeline objects go oldest first and stop at the first failure, so the stored segments stay +/// consecutive, as recovery requires, and a later deletion retries from there. +async fn delete(store: &Store, timeline: &str, pruned: &mut u64, expired: Expired) { + let groups = async { + let deletes = expired + .groups + .iter() + .map(|key| async move { (key, store.delete(key).await) }); + for (key, result) in futures::future::join_all(deletes).await { + // An object that outlives its expiry advertises nothing; recovery cleans it up. + if let Err(err) = result { + tracing::warn!(?key, %err, "failed to delete an expired object"); + } } - } + }; + let segments = async { + while *pruned < expired.segments.end { + let result = match Key::segments(timeline, *pruned) { + Ok(key) => store.delete(&key).await, + Err(err) => Err(err), + }; + match result { + Ok(()) | Err(Error::NotFound(_)) => *pruned += 1, + Err(err) => { + tracing::warn!(segment = *pruned, %err, "failed to delete a pruned timeline object"); + break; + } + } + } + }; + futures::join!(groups, segments); } fn timeline_error(err: moq_mux::Error) -> Error { @@ -845,8 +919,8 @@ mod tests { group.finish().unwrap(); } - /// Replay every stored timeline object, returning the records still in the window. - async fn window(store: &Store) -> Vec { + /// The stored timeline segments, ascending. + async fn segments(store: &Store) -> Vec { let entries: Vec<_> = store .list(&Query::segments(TIMELINE).unwrap()) .try_collect() @@ -860,9 +934,16 @@ mod tests { }) .collect(); segments.sort(); + segments + } + + /// Replay every stored timeline object, returning the records still in the window. + async fn window(store: &Store) -> Vec { + let segments = segments(store).await; + let first = segments.first().copied().unwrap_or_default(); assert_eq!( segments, - (0..segments.len() as u64).collect::>(), + (first..first + segments.len() as u64).collect::>(), "timeline objects are consecutive" ); @@ -1024,6 +1105,8 @@ mod tests { .unwrap(); let expected: HashSet<_> = (3..6).map(|s| Key::groups("video", s..=s).unwrap()).collect(); assert_eq!(stored, expected); + // Every checkpoint restates the whole window, so only the newest timeline object is needed. + assert_eq!(segments(&store).await, vec![5]); } #[tokio::test] @@ -1324,7 +1407,7 @@ mod tests { assert_eq!(stored_groups(&store).await, referenced(&records)); store.get_info("video").await.unwrap(); store.get_info(TIMELINE).await.unwrap(); - store.get_segments(TIMELINE, 0).await.unwrap(); + assert_eq!(segments(&store).await, vec![7]); } #[tokio::test] @@ -1356,6 +1439,7 @@ mod tests { ); let expired: HashSet<_> = (0..3).map(|s| Key::groups("video", s..=s).unwrap()).collect(); assert_eq!(stored_groups(&store).await, &referenced(&retained) | &expired); + assert_eq!(segments(&store).await, (0..=4).collect::>()); // An upload the crash left uncommitted. store.put_groups("video", &orphan(5)).await.unwrap(); @@ -1369,6 +1453,7 @@ mod tests { stored_groups(&store).await.is_superset(&expired), "expired objects outlive the grace, for readers holding the old timeline" ); + assert_eq!(segments(&store).await, (0..=4).collect::>()); writer.control().pacing_track("video").await.unwrap(); // The source replays the uncommitted group; it is refused rather than overwritten. @@ -1386,9 +1471,7 @@ mod tests { assert_eq!(stored_groups(&store).await, referenced(&records)); store.get_info("video").await.unwrap(); store.get_info(TIMELINE).await.unwrap(); - for segment in 0..=7 { - store.get_segments(TIMELINE, segment).await.unwrap(); - } + assert_eq!(segments(&store).await, vec![7]); } #[tokio::test] @@ -1426,10 +1509,12 @@ mod tests { let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; assert!(matches!(result, Err(Error::Store(_)))); - // A missing timeline object leaves the retained window unrecoverable. - store.delete(&Key::segments(TIMELINE, 3).unwrap()).await.unwrap(); + // A gap in the timeline keys leaves the retained window unrecoverable. + let newest = store.get_segments(TIMELINE, 5).await.unwrap(); + store.put_segments(TIMELINE, 3, &newest).await.unwrap(); let result = Writer::new(store.clone(), source.consume(), config).await; assert!(matches!(result, Err(Error::Timeline(_)))); + assert_eq!(segments(&store).await, vec![3, 5]); assert_eq!(stored_groups(&store).await, before); } @@ -1448,6 +1533,82 @@ mod tests { assert_eq!(checkpoint.range.end, 300); assert!(recovery.orphans.is_empty()); assert_eq!(recovery.floors["video"], 299); + + // The oldest stored timeline object is exactly the one recovery starts from. + let stored = segments(&store).await; + assert!(stored.len() > 1, "the newest checkpoint omits part of the window"); + assert!(stored[0] > 0, "earlier timeline objects are pruned"); + assert_eq!(recovery.checkpoints.front().unwrap().0, stored[0]); + assert!(recovery.stale.is_empty()); + } + + /// Yield to the writer until `check` holds. + async fn until>(mut check: impl FnMut() -> F) { + while !check().await { + tokio::task::yield_now().await; + } + } + + #[tokio::test(start_paused = true)] + async fn a_dvr_prunes_timeline_objects_one_grace_after_they_are_unneeded() { + let store = Store::new(InMemory::new(), "rec"); + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let grace = Duration::from_secs(10); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + + for sequence in 0..4 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + // Segment 3 stays open, so segment 2 is the newest timeline object. + until(|| async { segments(&store).await.last() == Some(&2) }).await; + assert_eq!( + segments(&store).await, + vec![0, 1, 2], + "unneeded timeline objects outlive the grace, for readers that listed them" + ); + + tokio::time::sleep(grace).await; + until(|| async { segments(&store).await == [2] }).await; + + for sequence in 4..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.close(); + run.await.unwrap().unwrap(); + + assert_eq!(segments(&store).await, vec![5]); + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(3, 3), (4, 4), (5, 5)]); + check_objects(&store, &records).await; + let recovery = recover(&store, TIMELINE, true).await.unwrap(); + assert_eq!(recovery.checkpoint.unwrap().records, records); + } + + #[tokio::test] + async fn a_failed_timeline_delete_keeps_the_keys_consecutive() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let failed = Key::segments(TIMELINE, 1).unwrap(); + mock.fail_deletes(store.path(&failed).unwrap().as_ref()); + record(&store, config.clone(), 0..6).await; + + // Pruning stops at the failed object instead of deleting past it. + assert_eq!(segments(&store).await, (1..=5).collect::>()); + window(&store).await; + + // A restart retries from the oldest stored object. + mock.heal(); + record(&store, config, 6..8).await; + assert_eq!(segments(&store).await, vec![7]); + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(5, 5), (6, 6), (7, 7)]); + check_objects(&store, &records).await; } #[tokio::test] From 2c29bdd6cf6b8fbcd81747024b52a8ceec61959f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:56:29 -0700 Subject: [PATCH 33/49] quest(archive): DVR handover requires the replay catalog Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/dvr.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index a58861ea52..eacb06a8ff 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -34,15 +34,17 @@ broadcast. A recording is the broadcast. Under the path through the root claim, and a live announcement shadows it. So when live ends, `moq-hls` resolves the same name and falls through to the recording: playlists keep serving for rewind and for players finishing the last -segments. `moq-hls` gains no linger (the `hls-linger` quest was dropped, -because an unannounced broadcast can't be FETCHed and a linger would only -serve the cache). Test the handover: a live HLS session keeps its playlist -URIs and media sequence numbers when the name moves from the live publisher to -the archive. +segments. The fall-through needs the recording to publish its catalog live, +since `moq-hls` subscribes to it rather than FETCHing it. `moq-hls` gains +no linger (the `hls-linger` quest was dropped, because an unannounced +broadcast can't be FETCHed and a linger would only serve the cache). Test +the handover: a live HLS session keeps its playlist URIs and media sequence +numbers when the name moves from the live publisher to the archive. ## Required - [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - seeks through per-track timelines +- [Replay catalog](/quest/m1/archive/replay-catalog.md) - the recording publishes its catalog live, so `moq-hls` finds it after the handover ## Closes From 2c1d0abc73f894ad210a1fcdb026b25d6a11610c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 16:05:14 -0700 Subject: [PATCH 34/49] fix(hls): fail loud on a timeline error instead of re-subscribing The origin already reconnects transient source failures, so moq-hls no longer retries. A failed reference timeline ends every playlist with EXT-X-ENDLIST; a failed rendition timeline ends that playlist at its last covered segment instead of listing gaps. Cursors end rather than park. Co-Authored-By: Claude Opus 5.5 --- doc/bin/hls.md | 6 +- quest/m1/archive/track-timeline/README.md | 8 + rs/moq-hls/Cargo.toml | 2 +- rs/moq-hls/src/export/mod.rs | 204 ++++++------ rs/moq-hls/src/export/rendition.rs | 31 +- rs/moq-hls/src/export/renditions.rs | 2 +- rs/moq-hls/src/export/segments.rs | 11 +- rs/moq-hls/src/export/spans.rs | 46 ++- rs/moq-hls/src/export/timeline.rs | 366 ---------------------- 9 files changed, 185 insertions(+), 491 deletions(-) delete mode 100644 rs/moq-hls/src/export/timeline.rs diff --git a/doc/bin/hls.md b/doc/bin/hls.md index d73c9a2009..1121ff8fd9 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -30,7 +30,11 @@ snaps each boundary to its nearest keyframe within about a second, and a segment with none in range renders as `EXT-X-GAP`; audio takes every frame inside the segment's span. A jump in content time renders as `EXT-X-DISCONTINUITY`. Gaps are a fallback: a publisher wanting clean HLS -export should align video GOPs across renditions. A broadcast +export should align video GOPs across renditions. A timeline that fails +(malformed, refused, or lost) is not retried, since the relay already rides out +transient source failures. The reference timeline failing ends every playlist +with `EXT-X-ENDLIST`, and any other rendition's timeline failing ends that +rendition's playlist at the last segment it covers. A broadcast whose catalog advertises no timelines is skipped. One server exposes every broadcast by path: diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md index 00e1958bcc..9dc11be1da 100644 --- a/quest/m1/archive/track-timeline/README.md +++ b/quest/m1/archive/track-timeline/README.md @@ -38,6 +38,14 @@ Decisions: - HLS and DASH segments are derived at the edge from group timestamps, not from storage objects. Fetching extra objects is fine when they land in the reader's cache for the next request. +- `moq-hls` never retries a failed timeline, because the origin already reconnects + transient source failures. A replacement publisher that restarts group + numbering is a publisher bug: a broadcast, track, or group name always means + the same content, and MoQ has no ETag-style invalidation + ([#4556](https://github.com/moq-dev/moq/pull/4556)). A failed timeline + fails loud instead: the reference ends every playlist with `EXT-X-ENDLIST`, + and another rendition's playlist ends at its last covered segment rather than + listing gaps. - HLS `EXT-X-TARGETDURATION` is fixed for the run, not the observed maximum #4280 shipped. This reverses that PR's decision 4 (09-28 merged-PR audit). diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 1ab723caee..c69d02a7cb 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -24,7 +24,6 @@ hang = { workspace = true } humantime = { workspace = true } kio = { workspace = true } m3u8-rs = "6" -moq-json = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } percent-encoding = { workspace = true } @@ -40,6 +39,7 @@ url = { workspace = true } async-trait = "0.1" futures = { workspace = true } moq-archive = { path = "../moq-archive" } +moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index c7305ced50..333ff348a0 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -32,7 +32,6 @@ mod mpd; mod playlist; mod rendition; mod spans; -mod timeline; mod upstream; pub mod renditions; @@ -431,18 +430,14 @@ async fn watch_timeline( reference: Reference, renditions: renditions::Fanout, ) { - match watch(&broadcast, §ion, &reference, &renditions).await { - // The timeline finished cleanly: the publisher is done, so every window can end - // (ENDLIST) and recording cursors drain to completion. - Ok(()) => renditions.end_windows(), - // A malformed timeline, or one that can't be subscribed again: don't mark the windows - // ended; the serve path keeps serving the frozen windows. - Err(err) => { - tracing::error!(track = %reference.1, %err, "timeline watcher failed; freezing the playlists") - } - } - // The timeline stream is over either way: close so recording cursors terminate instead of - // parking forever (the serve path still reads the last windows). + // A failed timeline is never retried: the origin already rides out transient source + // failures, so an error here is final. It ends every playlist like a clean finish + // (`EXT-X-ENDLIST`), since a window left live would freeze players with no signal. + if let Err(err) = watch(&broadcast, §ion, &reference, &renditions).await { + tracing::error!(track = %reference.1, %err, "timeline failed; ending every playlist"); + } + // Recording cursors drain what is listed and end instead of parking. + renditions.end_windows(); renditions.close_windows(); } @@ -452,7 +447,7 @@ async fn watch( reference: &Reference, renditions: &renditions::Fanout, ) -> crate::Result<()> { - let mut timeline = timeline::Follower::subscribe(broadcast, section, &reference.1).await?; + let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(broadcast, section, &reference.1).await?; while let Some(event) = timeline.next().await? { match event { moq_mux::timeline::Event::Push { index, entry } => renditions.push(index, entry, reference), @@ -2553,37 +2548,28 @@ mod tests { } /// A `video0` reference and a `video1` rendition over hand-published timelines, reconciled and - /// watched the way `watch_catalog` does. The broadcast's handler keeps a timeline's re-subscribe - /// waiting until the test publishes it again. - struct Resubscribe { + /// watched the way `watch_catalog` does. Nothing serves the media, so a segment fetch is a miss. + struct Failing { broadcast: moq_net::broadcast::Producer, - _handler: moq_net::broadcast::Dynamic, renditions: renditions::Producer, watcher: tokio::task::JoinHandle<()>, } - impl Resubscribe { - fn new(mut archive: hang::catalog::Archive) -> Self { + impl Failing { + fn new() -> Self { let broadcast = moq_net::broadcast::Info::new().produce(); - let handler = broadcast.dynamic(); let upstream = Upstream { source: moq_mux::Source::new(produce_origin().consume(), "live"), broadcast: broadcast.consume(), }; - let mut catalog = moq_mux::catalog::hang::Catalog::default(); for name in ["video0", "video1"] { catalog.video.renditions.insert(name.to_string(), video_config()); - archive - .timelines - .insert(name.to_string(), hang::timeline::default_name(name)); } + let archive = archive(&["video0", "video1"]); catalog.archive = Some(archive.clone()); let renditions = renditions::Producer::new(Config::default().window); - if durable(&archive) { - renditions.fanout().unbound(); - } renditions.sync(&upstream, &catalog); let watcher = tokio::spawn(watch_timeline( upstream.broadcast.clone(), @@ -2593,42 +2579,29 @@ mod tests { )); Self { broadcast, - _handler: handler, renditions, watcher, } } - /// Publish `track`'s timeline, continuing `records` when set. Returns the track, to cut - /// it off, with the timeline writing to it. - fn publish( - &self, - track: &str, - records: Option>, - ) -> (moq_net::track::Producer, moq_mux::timeline::Producer) { + /// Publish `track`'s timeline. Returns the track, to fail it, with the timeline writing to it. + fn publish(&self, track: &str) -> (moq_net::track::Producer, moq_mux::timeline::Producer) { let track = self .broadcast .create_track(hang::timeline::default_name(track), moq_mux::timeline::Producer::info()) .unwrap(); - let timeline = match records { - Some(records) => { - let range = records.first().unwrap().sequence..records.last().unwrap().sequence + 1; - let checkpoint = moq_json::window::Checkpoint { range, records }; - moq_mux::timeline::Producer::resume(track.clone(), &checkpoint).unwrap() - } - None => moq_mux::timeline::Producer::new(track.clone()), - }; + let timeline = moq_mux::timeline::Producer::new(track.clone()); (track, timeline) } - fn snapshot(&self, name: &str) -> playlist::Snapshot { - self.renditions.get(Kind::Video, name).unwrap().snapshot() + fn rendition(&self, name: &str) -> Arc { + self.renditions.get(Kind::Video, name).unwrap() } /// Wait until `name` lists segment `last`. async fn listed(&self, name: &str, last: u64) -> playlist::Snapshot { for _ in 0..500 { - let snapshot = self.snapshot(name); + let snapshot = self.rendition(name).snapshot(); if snapshot.segments.last().is_some_and(|s| s.segment >= last) { return snapshot; } @@ -2636,9 +2609,31 @@ mod tests { } panic!("{name} never listed segment {last}"); } + + /// Wait until `name`'s playlist ends. + async fn finished(&self, name: &str) -> playlist::Snapshot { + for _ in 0..500 { + let snapshot = self.rendition(name).snapshot(); + if snapshot.finished { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} never ended"); + } + + /// Drain `name`'s recording cursor, which must end rather than park. + async fn drain(&self, name: &str) { + let mut segments = self.rendition(name).segments(); + let drained = tokio::time::timeout(Duration::from_secs(5), async { + while segments.next().await.unwrap().is_some() {} + }) + .await; + assert!(drained.is_ok(), "{name}'s cursor parked"); + } } - impl Drop for Resubscribe { + impl Drop for Failing { fn drop(&mut self) { self.watcher.abort(); } @@ -2654,18 +2649,13 @@ mod tests { snapshot.segments.iter().map(|s| s.segment).collect() } - fn gaps(snapshot: &playlist::Snapshot) -> Vec { - snapshot.segments.iter().filter(|s| s.gap).map(|s| s.segment).collect() - } - - // A transient error on a rendition's own timeline subscribes again. Its segments stay pending - // through the outage and resolve to media afterwards, rather than every row past its last - // record turning into an `EXT-X-GAP`. + // A failed rendition timeline is not retried. Its playlist ends at the last segment its + // records cover, rather than listing the rest as gaps, while the other renditions go on. #[tokio::test(start_paused = true)] - async fn a_rendition_timeline_error_resubscribes_instead_of_listing_gaps() { - let test = Resubscribe::new(hang::catalog::Archive::new()); - let (_reference, mut video0) = test.publish("video0", None); - let (track, mut video1) = test.publish("video1", None); + async fn a_rendition_timeline_error_ends_its_playlist() { + let test = Failing::new(); + let (_reference, mut video0) = test.publish("video0"); + let (track, mut video1) = test.publish("video1"); for sequence in 0..4 { video0.push(&gop(sequence)).unwrap(); video1.push(&gop(sequence)).unwrap(); @@ -2677,63 +2667,63 @@ mod tests { for sequence in 4..7 { video0.push(&gop(sequence)).unwrap(); } - tokio::time::sleep(Duration::from_secs(60)).await; - let outage = test.snapshot("video1"); - assert_eq!(numbers(&outage), [0, 1, 2], "the rest stays pending through the outage"); - assert_eq!(gaps(&outage), [] as [u64; 0]); - - let (_track, mut video1) = test.publish("video1", Some((0..4).map(gop).collect())); - for sequence in 4..7 { - video1.push(&gop(sequence)).unwrap(); - } - let resumed = test.listed("video1", 5).await; - assert_eq!(numbers(&resumed), [0, 1, 2, 3, 4, 5]); - assert_eq!( - gaps(&resumed), - [] as [u64; 0], - "the segments after the outage resolve to media" + let failed = test.finished("video1").await; + assert_eq!(numbers(&failed), [0, 1, 2, 3], "the covered segments stay listed"); + assert!( + failed.segments.iter().all(|s| !s.gap), + "nothing past the failure is a gap" ); + test.drain("video1").await; + + let reference = test.listed("video0", 6).await; + assert!(!reference.finished, "the other renditions go on"); } - // A transient error on the reference timeline subscribes again, and every playlist keeps - // advancing. The timeline is durable and longer than a group header restates - // (`CHECKPOINT_RECORDS`), and the source pops records during the outage: those rows go, - // while the retained rows the header omits stay listed. + // A failed reference timeline is not retried. Every playlist ends (`EXT-X-ENDLIST`) instead + // of freezing live, and recording cursors end. #[tokio::test(start_paused = true)] - async fn a_reference_timeline_error_resubscribes_and_every_playlist_advances() { - let mut archive = hang::catalog::Archive::new(); - archive.store = Some("https://objects.example/rec/".parse().unwrap()); - let test = Resubscribe::new(archive); - let (track, mut video0) = test.publish("video0", None); - let (_track, mut video1) = test.publish("video1", None); - // Let the watchers read each record as it lands, so none falls a whole header behind. - for sequence in 0..300 { + async fn a_reference_timeline_error_ends_every_playlist() { + let test = Failing::new(); + let (track, mut video0) = test.publish("video0"); + let (_track, mut video1) = test.publish("video1"); + for sequence in 0..4 { video0.push(&gop(sequence)).unwrap(); video1.push(&gop(sequence)).unwrap(); - settle().await; } - let before = test.listed("video0", 299).await; - assert_eq!(numbers(&before), (0..300).collect::>()); + test.listed("video0", 3).await; cut(track, video0); + assert_eq!(numbers(&test.finished("video0").await), [0, 1, 2, 3]); + test.finished("video1").await; + test.drain("video0").await; + } + + // A malformed reference timeline fails the same way. + #[tokio::test(start_paused = true)] + async fn a_malformed_reference_timeline_ends_every_playlist() { + let test = Failing::new(); + let (track, mut video0) = test.publish("video0"); + video0.push(&gop(0)).unwrap(); + test.listed("video0", 0).await; + + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::now(), &b"garbage"[..]).unwrap(); + test.finished("video0").await; + test.finished("video1").await; + } + + // A timeline subscription still waiting on its track ends with the broadcast, so nothing + // parks. + #[tokio::test(start_paused = true)] + async fn a_pending_timeline_subscription_ends_with_the_broadcast() { + let test = Failing::new(); + // A handler keeps the unpublished timelines' requests waiting rather than refused. + let _handler = test.broadcast.dynamic(); tokio::time::sleep(Duration::from_secs(60)).await; - assert_eq!( - numbers(&test.snapshot("video0")), - numbers(&before), - "the playlist holds" - ); - for sequence in 300..303 { - video1.push(&gop(sequence)).unwrap(); - } - // The source popped ten records while the watcher was away. - let (_track, mut video0) = test.publish("video0", Some((10..300).map(gop).collect())); - video0.push(&gop(300)).unwrap(); - - let reference = test.listed("video0", 300).await; - assert_eq!(numbers(&reference), (10..=300).collect::>()); - assert_eq!(reference.media_sequence, 10); - let rendition = test.listed("video1", 300).await; - assert_eq!(numbers(&rendition), (10..=300).collect::>()); - assert_eq!(gaps(&rendition), [] as [u64; 0]); + assert!(!test.rendition("video0").snapshot().finished); + + test.broadcast.close(); + test.finished("video0").await; + test.drain("video0").await; } } diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 8c97ca9f90..ce4c364fb9 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -467,7 +467,7 @@ impl Rendition { self.run().generation } - /// Mark this rendition's window ended (the timeline finished cleanly). + /// Mark this rendition's window ended (the timeline finished or failed). pub(crate) fn end(&self) { self.live.end(); } @@ -575,23 +575,31 @@ impl Rendition { } }) .collect(); + // A failed timeline ends this playlist at its last resolvable segment. + let failed = window.segments.last().is_some_and(|row| self.is_failed(row)); Snapshot { target_duration, media_sequence: window.sequence, segments, - finished: window.ended, + finished: window.ended || failed, program_date_time, generation, } } - /// `rows` with what each holds, up to the first one this rendition cannot resolve yet: a - /// segment is listed only once its content is final. + /// `rows` with what each holds, up to the first one this rendition cannot resolve yet (or + /// ever, once its timeline failed): a segment is listed only once its content is final. fn resolved<'a>(&'a self, rows: &'a [segments::Row]) -> impl Iterator { rows.iter() .map(|row| (row, self.resolve(row))) - .take_while(|(_, content)| *content != Content::Pending) + .take_while(|(_, content)| !matches!(content, Content::Pending | Content::Failed)) + } + + /// Whether this rendition's timeline failed before reaching `row`'s end, so neither it nor + /// any later row will resolve. + pub(crate) fn is_failed(&self, row: &segments::Row) -> bool { + self.resolve(row) == Content::Failed } /// Whether the playlist has anything to serve yet (at least one segment, or the broadcast @@ -884,7 +892,7 @@ async fn watch_spans( window: Option, ) { let result: Result<()> = async { - let mut timeline = super::timeline::Follower::subscribe(&broadcast, §ion, &track).await?; + let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&broadcast, §ion, &track).await?; while let Some(event) = timeline.next().await? { match event { moq_mux::timeline::Event::Push { entry, .. } => spans.push(entry, window), @@ -896,10 +904,15 @@ async fn watch_spans( Ok(()) } .await; - if let Err(err) = result { - tracing::error!(%track, %err, "rendition timeline failed; its remaining segments are gaps"); + match result { + Ok(()) => spans.end(), + // Never retried (the origin already rides out transient source failures), and not + // papered over with gaps: the rendition's playlist ends at its last record. + Err(err) => { + tracing::error!(%track, %err, "rendition timeline failed; ending its playlist"); + spans.fail(); + } } - spans.end(); } impl Drop for Rendition { diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index 8c5900b809..48c983942d 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -296,7 +296,7 @@ impl Fanout { }); } - /// Mark every living rendition's window ended (the timeline finished cleanly). + /// Mark every living rendition's window ended (the timeline finished or failed). pub fn end_windows(&self) { let mut feed = self.feed.lock().unwrap(); feed.ended = true; diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 1437090c0e..1158bda6f3 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -230,7 +230,7 @@ impl Producer { } } - /// Mark the timeline ended (the broadcast finished cleanly): the playlist gets + /// Mark the timeline ended (it finished, or failed): the playlist gets /// `EXT-X-ENDLIST` and cursors end once drained. pub fn end(&self) { if let Ok(mut state) = self.state.write() { @@ -240,7 +240,7 @@ impl Producer { /// Close the channel: no more rows will arrive. A [`Consumer`] drains the segments it /// can still see and then ends; the serve path keeps reading the frozen window. Call after - /// [`end`](Self::end) on a clean finish, or on its own when the source is lost mid-stream. + /// [`end`](Self::end) when the timeline is over, or on its own when a rendition is retired. pub fn close(&self) { let _ = self.state.close(); } @@ -381,7 +381,8 @@ impl Consumer { self.rendition.init().await } - /// The next segment, with its media; `None` once the rendition ends. + /// The next segment, with its media; `None` once the rendition ends, including when its + /// timeline failed. /// /// Waits for the next segment to resolve on this rendition, then FETCHes and transmuxes its /// frames. A segment whose groups already left the relay cache (or that is a gap for this @@ -393,6 +394,10 @@ impl Consumer { return Ok(None); }; kio::wait(|waiter| self.rendition.poll_resolved(waiter, &row)).await; + // The rendition's timeline failed: it ends here, like its playlist. + if self.rendition.is_failed(&row) { + return Ok(None); + } match self.rendition.fetch(&row).await? { Some(media) => { return Ok(Some(Segment { diff --git a/rs/moq-hls/src/export/spans.rs b/rs/moq-hls/src/export/spans.rs index 769128a5e8..04f79e28c2 100644 --- a/rs/moq-hls/src/export/spans.rs +++ b/rs/moq-hls/src/export/spans.rs @@ -10,7 +10,8 @@ //! from more than one record. //! //! A segment resolves only once the rendition's timeline has reached past its end (or ended), so -//! every edge and every reload agree on its content. +//! every edge and every reload agree on its content. A failed timeline ends the rendition's +//! playlist at its last record instead of listing the rest as gaps. use std::collections::VecDeque; use std::ops::Range; @@ -30,6 +31,9 @@ pub(crate) enum Content { Pending, /// The rendition has no content for the segment. Gap, + /// The rendition's timeline failed before reaching the segment's end, so neither it nor any + /// later segment will resolve. + Failed, /// The frames to fetch, as position ranges, keeping only those whose timestamps fall in /// `filter` when set. Frames { @@ -43,14 +47,21 @@ struct State { records: VecDeque, /// No more records will arrive, so nothing is pending. ended: bool, + /// The timeline failed: no more records will arrive, and a segment past the last one fails + /// rather than resolving as a gap. + failed: bool, } impl State { fn resolve(&self, is_video: bool, span: Range) -> Content { + if self.failed && self.records.back().is_none_or(|last| last.end_time() < span.end) { + return Content::Failed; + } let records = self.records.iter().collect(); + let ended = self.ended || self.failed; match is_video { - true => video(records, self.ended, span), - false => audio(records, self.ended, span), + true => video(records, ended, span), + false => audio(records, ended, span), } } } @@ -66,6 +77,7 @@ impl Spans { state: kio::Producer::new(State { records: VecDeque::new(), ended: false, + failed: false, }), } } @@ -115,6 +127,14 @@ impl Spans { } } + /// The timeline failed: segments its records cover still resolve, and later ones are + /// [`Content::Failed`]. + pub fn fail(&self) { + if let Ok(mut state) = self.state.write() { + state.failed = true; + } + } + /// Resolve the segment spanning `span`, snapping to keyframes when `video`. pub fn resolve(&self, video: bool, span: Range) -> Content { self.state.read().resolve(video, span) @@ -356,4 +376,24 @@ mod tests { let state = spans.state.read(); assert_eq!(state.records.front().unwrap().sequence, 2); } + + #[test] + fn a_failed_timeline_resolves_only_what_its_records_cover() { + let spans = Spans::new(); + for entry in gops(&[0, 2_000, 4_000], 6_000) { + spans.push(entry, None); + } + // Waiting on a keyframe that could still land near the boundary. + assert_eq!(spans.resolve(true, secs(4.0)..secs(6.0)), Content::Pending); + + spans.fail(); + assert_eq!( + spans.resolve(true, secs(4.0)..secs(6.0)), + frames(&[(Position::group(2), Position::group(3))], None) + ); + assert_eq!(spans.resolve(true, secs(6.0)..secs(8.0)), Content::Failed); + assert_eq!(spans.resolve(false, secs(6.0)..secs(8.0)), Content::Failed); + let resolved = spans.poll_resolved(&kio::Waiter::noop(), true, secs(6.0)..secs(8.0)); + assert!(resolved.is_ready(), "nothing waits on a failed timeline"); + } } diff --git a/rs/moq-hls/src/export/timeline.rs b/rs/moq-hls/src/export/timeline.rs deleted file mode 100644 index ddc4610e3b..0000000000 --- a/rs/moq-hls/src/export/timeline.rs +++ /dev/null @@ -1,366 +0,0 @@ -//! One track's timeline, followed across transient subscription failures. -//! -//! A transport error on a timeline subscription (a reset stream, a lost session, the publisher's -//! track aborting) subscribes again rather than ending the timeline. The fresh subscription opens -//! on a group header that restates the retained window, so its events are reconciled against the -//! ones already delivered: rows the source popped during the outage are popped, retained rows the -//! header's checkpoint omitted are kept, and restated records are not delivered twice. -//! -//! Nothing waits on a timer. A re-subscribe is either served, refused (the track is no longer -//! published, or the broadcast went away), or fails before delivering anything; the last two end -//! the timeline with an error. A malformed timeline fails the same way on the first try, since a -//! fresh subscription would only decode the same retained record again. - -use std::collections::VecDeque; -use std::ops::Range; - -use hang::catalog::Archive; -use moq_mux::timeline::{Consumer, Entry, Event}; - -/// Reads one track's timeline, re-subscribing after a transport error. -pub(crate) struct Follower { - broadcast: moq_net::broadcast::Consumer, - section: Archive, - track: String, - consumer: Consumer, - /// Whether `consumer` has not yielded an event yet. - fresh: bool, - /// The window indices delivered and not since popped or skipped, once anything was. - held: Option>, - /// The newest delivered record, to tell a restated timeline from a restarted one. - last: Option, - /// Events reconciliation produced ahead of the one being handled. - queued: VecDeque, -} - -impl Follower { - /// Subscribe to `track`'s timeline as advertised by `section`. - pub async fn subscribe( - broadcast: &moq_net::broadcast::Consumer, - section: &Archive, - track: &str, - ) -> moq_mux::Result { - Ok(Self { - consumer: Consumer::subscribe(broadcast, section, track).await?, - broadcast: broadcast.clone(), - section: section.clone(), - track: track.to_string(), - fresh: true, - held: None, - last: None, - queued: VecDeque::new(), - }) - } - - /// The next window event, or `None` once the timeline ends cleanly. - pub async fn next(&mut self) -> moq_mux::Result> { - loop { - if let Some(event) = self.queued.pop_front() { - return Ok(Some(event)); - } - match self.consumer.next().await { - Ok(Some(event)) => self.reconcile(event), - Ok(None) => return Ok(None), - // A subscription that failed before delivering anything is not retried: it would - // fail the same way again, as fast as the source can refuse it. - Err(err) if is_transport(&err) && !self.fresh => { - tracing::warn!(track = %self.track, %err, "timeline subscription failed; subscribing again"); - self.consumer = Consumer::subscribe(&self.broadcast, &self.section, &self.track).await?; - self.fresh = true; - } - Err(err) => return Err(err), - } - } - } - - /// Queue what `event` means to a reader holding [`held`](Self::held). - fn reconcile(&mut self, event: Event) { - let fresh = std::mem::replace(&mut self.fresh, false); - let Some(held) = self.held.clone() else { - return self.deliver(event); - }; - - if fresh { - // A fresh subscription's first event names the window's front: its header's offset. - let front = match &event { - Event::Push { index, .. } => *index, - Event::Pop(range) | Event::Skip(range) => range.start, - _ => unreachable!("unknown timeline event"), - }; - // The source popped these while this reader was away. - let popped = held.start..front.min(held.end); - if !popped.is_empty() { - self.deliver(Event::Pop(popped)); - } - } - - let held = self.held.clone().expect("held survives a pop"); - match event { - // Records the header restated: this reader holds them unless there's a hole before the - // next one it needs. - Event::Skip(range) if range.end <= held.end => {} - Event::Skip(range) => self.deliver(Event::Skip(held.end..range.end)), - Event::Push { index, entry } if index < held.end => { - // A restated record. The newest one this reader holds must match, or the source - // restarted its timeline under the same track: start over from there. - if index + 1 == held.end && self.last.as_ref().is_some_and(|last| *last != entry) { - self.deliver(Event::Skip(held)); - self.deliver(Event::Push { index, entry }); - } - } - Event::Push { index, entry } if index > held.end => { - // Records pushed and popped while this reader was away. - self.deliver(Event::Skip(held.end..index)); - self.deliver(Event::Push { index, entry }); - } - event => self.deliver(event), - } - } - - /// Queue `event` for the reader and track what it now holds. - fn deliver(&mut self, event: Event) { - let held = self.held.get_or_insert(0..0); - match &event { - Event::Push { index, entry } => { - if held.is_empty() { - held.start = *index; - } - held.end = index + 1; - self.last = Some(entry.clone()); - } - Event::Pop(range) => held.start = held.start.max(range.end).min(held.end), - // A reader forgets everything on a skip, since the next record can't follow its last. - Event::Skip(range) => { - *held = range.end..range.end; - self.last = None; - } - _ => unreachable!("unknown timeline event"), - } - self.queued.push_back(event); - } -} - -/// Whether `err` came from the transport rather than the timeline's content. -fn is_transport(err: &moq_mux::Error) -> bool { - matches!( - err, - moq_mux::Error::Moq(_) | moq_mux::Error::Json(moq_json::Error::Net(_)) - ) -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use hang::timeline::{Position, Record}; - use moq_json::window::Checkpoint; - - use super::*; - - const TRACK: &str = "video0"; - - fn section() -> Archive { - let mut section = Archive::new(); - section - .timelines - .insert(TRACK.to_string(), hang::timeline::default_name(TRACK)); - section - } - - fn record(sequence: u64, pts: u64) -> Record { - Record::new( - sequence, - pts, - 2_000, - Position::group(sequence), - Position::group(sequence + 1), - ) - } - - /// Publish the timeline track, continuing `records` when set, and keep a handle to abort it. - fn publish( - broadcast: &moq_net::broadcast::Producer, - records: Option>, - ) -> (moq_net::track::Producer, moq_mux::timeline::Producer) { - let track = broadcast - .create_track(hang::timeline::default_name(TRACK), moq_mux::timeline::Producer::info()) - .unwrap(); - let timeline = match records { - Some(records) => { - let start = records.first().map_or(0, |record| record.sequence); - let end = records.last().map_or(0, |record| record.sequence + 1); - let checkpoint = Checkpoint { - range: start..end, - records, - }; - moq_mux::timeline::Producer::resume(track.clone(), &checkpoint).unwrap() - } - None => moq_mux::timeline::Producer::new(track.clone()), - }; - (track, timeline) - } - - /// The publisher's session is lost: its track and open group end with an error. - fn abort(track: moq_net::track::Producer, timeline: moq_mux::timeline::Producer) { - drop(timeline); - track.abort(moq_net::Error::SessionClosed).unwrap(); - } - - #[derive(Debug, PartialEq)] - enum Seen { - Push(u64), - Pop(Range), - Skip(Range), - } - - fn seen(event: moq_mux::Result>) -> Seen { - match event.expect("the timeline is readable").expect("the timeline is live") { - Event::Push { index, .. } => Seen::Push(index), - Event::Pop(range) => Seen::Pop(range), - Event::Skip(range) => Seen::Skip(range), - _ => unreachable!("unknown timeline event"), - } - } - - async fn next(follower: &mut Follower) -> Seen { - seen( - tokio::time::timeout(Duration::from_secs(5), follower.next()) - .await - .expect("an event arrives"), - ) - } - - /// Subscribe, then read `count` records as they're pushed, so the reader keeps up. - async fn follow( - broadcast: &moq_net::broadcast::Producer, - timeline: &mut moq_mux::timeline::Producer, - count: u64, - ) -> Follower { - let mut follower = Follower::subscribe(&broadcast.consume(), §ion(), TRACK) - .await - .unwrap(); - for sequence in 0..count { - timeline.push(&record(sequence, sequence * 2_000)).unwrap(); - assert_eq!(next(&mut follower).await, Seen::Push(sequence)); - } - follower - } - - /// Read across an outage: the re-subscribe waits on the track, not a timer, until `restore`. - async fn across(follower: &mut Follower, restore: impl FnOnce() -> T) -> (moq_mux::Result>, T) { - let mut next = std::pin::pin!(follower.next()); - let waited = tokio::time::timeout(Duration::from_secs(60), &mut next).await; - assert!(waited.is_err(), "the re-subscribe did not wait: {waited:?}"); - let restored = restore(); - let next = tokio::time::timeout(Duration::from_secs(5), next) - .await - .expect("nothing parks once the source answers"); - (next, restored) - } - - #[tokio::test(start_paused = true)] - async fn a_reconnect_past_the_checkpoint_keeps_the_retained_records() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let _handler = broadcast.dynamic(); - let (track, mut timeline) = publish(&broadcast, None); - // More records than a group header restates (`CHECKPOINT_RECORDS`, 256). - let mut follower = follow(&broadcast, &mut timeline, 300).await; - abort(track, timeline); - - // The source popped ten records during the outage. Its first header restates only the - // newest 256 of the rest, so the fresh subscription skips 10..44, which the reader holds. - let (first, _source) = across(&mut follower, || { - let retained = (10..300).map(|sequence| record(sequence, sequence * 2_000)).collect(); - let (track, mut timeline) = publish(&broadcast, Some(retained)); - timeline.push(&record(300, 600_000)).unwrap(); - (track, timeline) - }) - .await; - assert_eq!(seen(first), Seen::Pop(0..10)); - assert_eq!(next(&mut follower).await, Seen::Push(300)); - } - - #[tokio::test(start_paused = true)] - async fn a_restarted_timeline_starts_over() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let _handler = broadcast.dynamic(); - let (track, mut timeline) = publish(&broadcast, None); - let mut follower = follow(&broadcast, &mut timeline, 3).await; - abort(track, timeline); - - // A new run numbers from zero again, describing other media. - let (first, _source) = across(&mut follower, || { - let (track, mut timeline) = publish(&broadcast, None); - for sequence in 0..4 { - timeline.push(&record(sequence, 100_000 + sequence * 2_000)).unwrap(); - } - (track, timeline) - }) - .await; - assert_eq!(seen(first), Seen::Skip(0..3)); - assert_eq!(next(&mut follower).await, Seen::Push(2)); - assert_eq!(next(&mut follower).await, Seen::Push(3)); - } - - #[tokio::test(start_paused = true)] - async fn a_refused_resubscribe_ends_the_timeline() { - // No handler, so a track that isn't published is refused rather than awaited. - let broadcast = moq_net::broadcast::Info::new().produce(); - let (track, mut timeline) = publish(&broadcast, None); - let mut follower = follow(&broadcast, &mut timeline, 1).await; - abort(track, timeline); - - let err = follower.next().await.expect_err("the re-subscribe is refused"); - assert!(matches!(err, moq_mux::Error::Moq(moq_net::Error::NotFound)), "{err}"); - } - - #[tokio::test(start_paused = true)] - async fn a_pending_resubscribe_ends_with_the_broadcast() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let _handler = broadcast.dynamic(); - let (track, mut timeline) = publish(&broadcast, None); - let mut follower = follow(&broadcast, &mut timeline, 1).await; - abort(track, timeline); - - let (next, ()) = across(&mut follower, || broadcast.close()).await; - let err = next.expect_err("the broadcast is gone"); - assert!(matches!(err, moq_mux::Error::Moq(moq_net::Error::Unroutable)), "{err}"); - } - - #[tokio::test(start_paused = true)] - async fn a_resubscribe_that_fails_before_any_event_is_not_retried() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let _handler = broadcast.dynamic(); - let (track, mut timeline) = publish(&broadcast, None); - let mut follower = follow(&broadcast, &mut timeline, 1).await; - abort(track, timeline); - - let mut next = std::pin::pin!(follower.next()); - assert!(tokio::time::timeout(Duration::from_secs(60), &mut next).await.is_err()); - // Serve the re-subscribe, then fail it before it delivers a single event. - let (track, timeline) = publish(&broadcast, None); - assert!(tokio::time::timeout(Duration::from_secs(1), &mut next).await.is_err()); - abort(track, timeline); - tokio::time::timeout(Duration::from_secs(5), next) - .await - .expect("a failing source is not subscribed again") - .expect_err("the timeline ends"); - } - - #[tokio::test(start_paused = true)] - async fn a_malformed_timeline_is_not_retried() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let _handler = broadcast.dynamic(); - let (track, mut timeline) = publish(&broadcast, None); - let mut follower = follow(&broadcast, &mut timeline, 1).await; - - // A group that doesn't decode: a fresh subscription would read the same one again. - let mut group = track.append_group().unwrap(); - group.write_frame(moq_net::Timestamp::now(), &b"garbage"[..]).unwrap(); - let err = tokio::time::timeout(Duration::from_secs(5), follower.next()) - .await - .expect("a malformed timeline fails at once") - .expect_err("the timeline is malformed"); - assert!(!is_transport(&err), "{err}"); - } -} From 463583597dfe07c9be262fb9d36a23a7fec589ae Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 16:19:41 -0700 Subject: [PATCH 35/49] perf(archive): binary-search the checkpoint that recovery starts from Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/writer.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index abc3a05e71..34345a7a0a 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -772,11 +772,12 @@ impl Committer { } let (offset, start) = checkpoint(object)?; self.checkpoints.push_back((segment, start)); + // Checkpoint starts never decrease, so a binary search finds the last one at or before + // `offset` without walking a long DVR window on every commit. let keep = self .checkpoints - .iter() - .rposition(|&(_, start)| start <= offset) - .unwrap_or(0); + .partition_point(|&(_, start)| start <= offset) + .saturating_sub(1); let first = self.checkpoints[0].0; self.checkpoints.drain(..keep); Ok(first..self.checkpoints[0].0) From 4005e0bdb2281e66abc1a6cbab3be48930e95d2e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 16:22:28 -0700 Subject: [PATCH 36/49] quest(archive): DVR handover requires wildcard routing Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/dvr.md | 1 + 1 file changed, 1 insertion(+) diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index eacb06a8ff..373feb14f6 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -45,6 +45,7 @@ numbers when the name moves from the live publisher to the archive. - [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - seeks through per-track timelines - [Replay catalog](/quest/m1/archive/replay-catalog.md) - the recording publishes its catalog live, so `moq-hls` finds it after the handover +- [Wildcard](/quest/m0/wildcard/README.md) - the archive's root claim serves the source path once the live announcement ends ## Closes From dc935499261f9b84b16d0ad15cd6f996a36905c9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 16:54:13 -0700 Subject: [PATCH 37/49] quest(archive): note who republishes the catalog behind the root claim Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/dvr.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 373feb14f6..c3bccaef8b 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -35,7 +35,10 @@ path through the root claim, and a live announcement shadows it. So when live ends, `moq-hls` resolves the same name and falls through to the recording: playlists keep serving for rewind and for players finishing the last segments. The fall-through needs the recording to publish its catalog live, -since `moq-hls` subscribes to it rather than FETCHing it. `moq-hls` gains +since `moq-hls` subscribes to it rather than FETCHing it. Open: [Replay +catalog](/quest/m1/archive/replay-catalog.md) keeps that republishing in +`moq-cli`, so decide whether the archive service behind the root claim is a +`moq-cli` owner or the logic moves into `moq-archive`. `moq-hls` gains no linger (the `hls-linger` quest was dropped, because an unannounced broadcast can't be FETCHed and a linger would only serve the cache). Test the handover: a live HLS session keeps its playlist URIs and media sequence From 4aa7062bd9f71efc4a3ae93c163b5232fded5bed Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 19:24:15 -0700 Subject: [PATCH 38/49] quest(archive): DVR rewinds through the durable live timeline; moq-archive republishes the catalog Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/dvr.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index c3bccaef8b..7d498d84ba 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -35,10 +35,15 @@ path through the root claim, and a live announcement shadows it. So when live ends, `moq-hls` resolves the same name and falls through to the recording: playlists keep serving for rewind and for players finishing the last segments. The fall-through needs the recording to publish its catalog live, -since `moq-hls` subscribes to it rather than FETCHing it. Open: [Replay -catalog](/quest/m1/archive/replay-catalog.md) keeps that republishing in -`moq-cli`, so decide whether the archive service behind the root claim is a -`moq-cli` owner or the logic moves into `moq-archive`. `moq-hls` gains +since `moq-hls` subscribes to it rather than FETCHing it. Decided (09-29): +that republishing moves into `moq-archive`, so any host of the archive +behind the root claim does it, not only `moq-cli`; update [Replay +catalog](/quest/m1/archive/replay-catalog.md) to match. + +During live, rewind needs no handover: a recorded broadcast's live timeline +is durable, and every group it lists is promised available, so a seek past +the live window FETCHes old groups through the normal miss chain down to the +archive. `moq-hls` needs no special path. `moq-hls` gains no linger (the `hls-linger` quest was dropped, because an unannounced broadcast can't be FETCHed and a linger would only serve the cache). Test the handover: a live HLS session keeps its playlist URIs and media sequence From f981fd9ec88beeeae7bb14422b9b00235ae8d57d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 4 Oct 2026 21:41:54 -0700 Subject: [PATCH 39/49] fix(archive): build against main's causeless broadcast close and 127 default priority Main removed the deprecated broadcast abort, so `closed()` carries no error, and made 127 the default track priority, which the recorded `.info` reflects. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/proof.rs | 2 +- rs/moq-archive/src/writer.rs | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs index 0177a0c4bd..4a2c316cf6 100644 --- a/rs/moq-archive/src/proof.rs +++ b/rs/moq-archive/src/proof.rs @@ -194,7 +194,7 @@ async fn recordings_are_byte_identical_on_every_backend() { assert_eq!(expected.keys().cloned().collect::>(), layout()); assert_eq!( &expected["rec/video%2F360p/.info"][..], - br#"{"version":1,"priority":0,"timescale":1000}"# + br#"{"version":1,"priority":127,"timescale":1000}"# ); let dir = tempfile::tempdir().unwrap(); diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 34345a7a0a..1fc011f6ac 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -347,11 +347,9 @@ impl Writer { delete(&shared.store, &shared.timeline, &mut pruned, expired).await; } - // A broadcast end carries no cause; only the deprecated `abort` still reports one. - match source.closed().await { - moq_net::Error::Dropped => Ok(()), - err => Err(source_error(err)), - } + // A broadcast end carries no cause. + source.closed().await; + Ok(()) } } @@ -1034,7 +1032,7 @@ mod tests { let object = store.get_groups("video", 2..=2).await.unwrap(); assert_eq!(object.groups[0].frames[1].timestamp, 2500); assert_eq!(object.groups[0].frames[1].payload, "2@2500"); - assert_eq!(store.get_info("video").await.unwrap(), Info::new(0, 1000).unwrap()); + assert_eq!(store.get_info("video").await.unwrap(), Info::new(127, 1000).unwrap()); store.get_info(TIMELINE).await.unwrap(); assert!(store.get_info("ignored").await.is_err()); } @@ -1310,7 +1308,7 @@ mod tests { control.pacing_track("video").await, Err(Error::Priority { existing: 7, - intended: 0 + intended: 127 }) ); } From 59659b7b136fa8de007f41518a5e84b19cfb4563 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:26:44 -0700 Subject: [PATCH 40/49] fix(archive): serve a refresh that lands while a growing group loads An archive FETCH that grows with refreshes marked a refresh seen after loading objects against an older span snapshot, then waited for the next change, so a tail committed during the GET was never served. Reload the index whenever a change is unseen before finishing or waiting. Also align the merge with main: fMP4's nothing-selected test observes the released reservation instead of the old shared-timeline catalog edit, and the archive quests record the review's silent restarted-source drop and the planned 4s minimum. Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/README.md | 2 +- quest/m1/archive/backward-timestamps.md | 23 ++++++++----- quest/m1/archive/track-timeline.md | 4 +-- rs/moq-archive/src/mock.rs | 19 +++++++++++ rs/moq-archive/src/reader/mod.rs | 19 ++++++++--- rs/moq-archive/src/reader/tests.rs | 35 ++++++++++++++++++++ rs/moq-hls/src/import.rs | 5 +-- rs/moq-mux/src/container/fmp4/import_test.rs | 4 ++- 8 files changed, 90 insertions(+), 21 deletions(-) diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 1fad2983f0..1ccbabb27a 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -121,7 +121,7 @@ owned by that prerequisite, not duplicated in archive storage. - [Per-track timelines](/quest/m1/archive/track-timeline.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge - [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay - [Idle flush](/quest/m1/archive/flush.md) - idle tracks are recorded within a bounded wall-clock delay, and `flush()` forces a track's pending record out -- [Backward timestamps](/quest/m1/archive/backward-timestamps.md) - a resumed recording refuses a track whose timestamps go backward past the recovered timeline +- [Backward timestamps](/quest/m1/archive/backward-timestamps.md) - a resumed recording refuses a restarted source instead of dropping its groups or overlapping media time - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback ## Related diff --git a/quest/m1/archive/backward-timestamps.md b/quest/m1/archive/backward-timestamps.md index 1808e73560..348a4fa9a4 100644 --- a/quest/m1/archive/backward-timestamps.md +++ b/quest/m1/archive/backward-timestamps.md @@ -2,17 +2,24 @@ ## Goal -A resumed recording refuses a track whose first group's timestamp is before -the recovered track's last recorded timestamp, failing loud like the existing -group-ID check, instead of writing overlapping media time. The caller starts a -new prefix. +A resumed recording refuses a track whose source restarted, failing loud +instead of recording nothing and then overlapping media time. The caller +starts a new prefix. + +Both signs of a restart count: a first group at or below the recovered +track's committed tail, which the writer drops today with only a `debug` log, +and a first timestamp before the recovered track's last recorded timestamp. ## Plan -Check at enrollment against the recovered timeline, and test both a backward -and a forward restart. The 2026-09-30 audit folded this into the recording -writer, which landed without it. The archive format may break in place since -it is unreleased. +Check at enrollment against the recovered timeline, and test a restart from +group 0, a backward timestamp, and a forward restart that keeps recording. +Mind a writer restarted against a source that kept running: its first group +may be the partially recorded tail group, which resumes rather than refuses. +The 2026-09-30 audit folded this into the recording writer, which landed +without it; the [#4034 review](https://github.com/moq-dev/moq/pull/4034#issuecomment-6005694031) +found the silent group drop. The archive format may break in place since it +is unreleased. ## Related diff --git a/quest/m1/archive/track-timeline.md b/quest/m1/archive/track-timeline.md index 9a4dde0407..3217c266d9 100644 --- a/quest/m1/archive/track-timeline.md +++ b/quest/m1/archive/track-timeline.md @@ -31,8 +31,8 @@ Decisions: - The catalog's root `archive` entry maps each track to its timeline, including the catalog track itself. `replay`, `store`, and `version` stay beside it. - Each track cuts on its own by one rule: at a group boundary between a - minimum and maximum duration (a 4s minimum and a multiple of the declared - duration, a zero minimum only for sparse data such as the catalog), + minimum and maximum duration (a 2s minimum today; 4s and a multiple of the declared + duration once its declared-duration child lands; a zero minimum only for sparse data such as the catalog), splitting a long-lived group by frame at the maximum. Manual cuts stay as an optimization, such as a video keyframe cutting audio so derived segments need fewer objects. - A stored object may hold a frame range of a group, not only whole groups. diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs index c35c9e12c8..8488733a22 100644 --- a/rs/moq-archive/src/mock.rs +++ b/rs/moq-archive/src/mock.rs @@ -36,6 +36,8 @@ struct State { fail_puts: Vec, /// GETs whose path contains any of these return Not Found. hide_gets: Vec, + /// GETs whose path contains the pattern wait for a permit before reading. + stall_gets: Vec<(String, Arc)>, /// Deletes whose path contains any of these fail. fail_deletes: Vec, /// Streaming listings end with an error after every entry. @@ -96,6 +98,13 @@ impl Mock { self } + /// Hold every GET whose path contains `pattern` until the returned semaphore gets a permit. + pub fn stall_gets(&self, pattern: &str) -> Arc { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + self.state().stall_gets.push((pattern.to_string(), gate.clone())); + gate + } + pub fn fail_deletes(&self, pattern: &str) -> &Self { self.state().fail_deletes.push(pattern.to_string()); self @@ -116,6 +125,7 @@ impl Mock { let mut state = self.state(); state.fail_puts.clear(); state.hide_gets.clear(); + state.stall_gets.clear(); state.fail_deletes.clear(); state.fail_lists = false; } @@ -205,6 +215,15 @@ impl ObjectStore for Mock { source: "hidden".into(), }); } + let stall = self + .state() + .stall_gets + .iter() + .find(|(pattern, _)| location.as_ref().contains(pattern)) + .map(|(_, gate)| gate.clone()); + if let Some(gate) = stall { + gate.acquire().await.expect("stall gate closed").forget(); + } self.inner.get_opts(location, options).await } diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs index 67fedca2c6..3da36ac467 100644 --- a/rs/moq-archive/src/reader/mod.rs +++ b/rs/moq-archive/src/reader/mod.rs @@ -414,15 +414,24 @@ async fn serve_group( } let complete = spans.last().is_some_and(|span| span.end.group > sequence); - if complete || *changed.borrow_and_update() { + if complete { let _ = producer.finish(); return; } - // The reader dropping ends the recording as surely as finishing it. - if changed.changed().await.is_err() { - let _ = producer.finish(); - return; + // A change that landed while objects loaded is unseen, so the spans above predate it: + // reload before finishing or waiting, or its records would never be served. + if !changed.has_changed().unwrap_or(false) { + // The caller finished the recording, and these spans are its final index. + if *changed.borrow() { + let _ = producer.finish(); + return; + } + if changed.changed().await.is_err() { + let _ = producer.finish(); + return; + } } + changed.mark_unchanged(); spans = shared.index.lock().unwrap().group(&track, sequence); } } diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index df90591724..2dacaa4dd7 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -311,6 +311,41 @@ async fn an_open_group_grows_as_records_commit() { assert!(group.read_frame().await.unwrap().is_none()); } +/// A refresh that lands while a growing group loads an object is still served, not swallowed +/// by the wait for the next one. +#[tokio::test(start_paused = true)] +async fn a_refresh_during_an_object_load_is_served() { + let mut archive = Archive::new().await; + let head = archive.record("log", Position::group(0), Position::new(0, 2), 0).await; + archive.commit("log", &head, 0).await; + + let (broadcast, mut reader) = open(&archive).await; + let mock = archive.store.inner().clone(); + mock.take(); + let gate = mock.stall_gets("log/segments/"); + let track = broadcast.consume().track("log").unwrap(); + let fetch = tokio::spawn(async move { track.fetch_group(0, group::Fetch::default()).await }); + + // Wait until the head object's GET is in flight, then commit the tail behind it. + while !mock.gets().iter().any(|path| path.contains("log/segments/")) { + tokio::task::yield_now().await; + } + let more = archive.record("log", Position::new(0, 2), Position::new(0, 3), 0).await; + archive.commit("log", &more, 0).await; + reader.refresh().await.unwrap(); + gate.add_permits(16); + + let mut group = fetch.await.unwrap().unwrap(); + for index in 0..3 { + let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) + .await + .expect("the refreshed tail is served") + .unwrap() + .unwrap(); + assert_eq!(frame.payload, frame_payload("log", 0, index)); + } +} + fn frame_payload(track: &str, sequence: u64, index: u64) -> Bytes { frame(track, sequence, index).payload } diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index e9f6ee85a7..c8ce1a29e7 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -1429,10 +1429,7 @@ mod tests { } import.step(OnError::Warn).await.unwrap(); - assert!( - import.video[0].importer.is_none(), - "the dead rendition is retired" - ); + assert!(import.video[0].importer.is_none(), "the dead rendition is retired"); } /// A live window longer than `ANCHOR_SEGMENTS` is joined mid-playlist, which means the diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 1fad00dd18..56e25c6207 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1397,7 +1397,8 @@ fn a_second_moov_is_refused() { ); } -/// A moov with every track deselected publishes at once, since no fragment will ever anchor. +/// A moov with every track deselected releases its reservation at once, since no fragment will +/// ever anchor: the catalog's next change publishes without one. #[tokio::test] async fn a_moov_with_nothing_selected_publishes() { let (init, _, _) = bbb_init(); @@ -1409,6 +1410,7 @@ async fn a_moov_with_nothing_selected_publishes() { .with_select(crate::select::Broadcast::default()); fmp4.decode(&init).unwrap(); + catalog.clone().anchor(moq_net::Timestamp::ZERO).unwrap(); assert_eq!(clocks.drain().len(), 1, "the catalog publishes without a fragment"); } From 4e17a6cd3ca207a2bbcafe2c1c5055fd6d9a6711 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:43:52 -0700 Subject: [PATCH 41/49] feat(hang)!: JS reads the per-track archive timelines @moq/hang's catalog schema still required the shared archive.track, so @moq/watch refused every catalog a Rust, Python, or Go publisher sends now that moq-mux advertises a timelines map. Parse the per-track shape, read one track's records in Timeline.Consumer, and delete the aligned producer the per-track port replaces. Co-Authored-By: Claude Opus 5.5 --- .review-body-4034.md | 28 ++ .status-4034.md | 28 ++ doc/lib/js/hang.md | 4 +- js/hang/src/catalog/archive.ts | 23 +- js/hang/src/catalog/clock.test.ts | 2 +- js/hang/src/catalog/root.test.ts | 24 +- js/hang/src/container/legacy.ts | 24 +- js/hang/src/timeline.test.ts | 578 ++------------------------ js/hang/src/timeline.ts | 532 ++---------------------- quest/m1/archive/README.md | 2 +- quest/m1/archive/declared-duration.md | 4 +- quest/m1/archive/js-timelines.md | 16 +- quest/m1/archive/track-timeline.md | 2 +- 13 files changed, 183 insertions(+), 1084 deletions(-) create mode 100644 .review-body-4034.md create mode 100644 .status-4034.md diff --git a/.review-body-4034.md b/.review-body-4034.md new file mode 100644 index 0000000000..4ea15507ff --- /dev/null +++ b/.review-body-4034.md @@ -0,0 +1,28 @@ +# Codex Adversarial Review + +Target: branch diff against origin/main +Verdict: needs-attention + +Do not ship yet: reference switching reuses segment identities, follow mode misses newly enrolled tracks, and rendition eviction can invalidate advertised segments. Static review against the existing origin/main; sandbox restrictions prevented fetching or running tests. No concrete pruning/recovery defect established. + +Findings: +- [high] Reference switching reuses existing segment URLs (rs/moq-hls/src/export/rendition.rs:401-405) + Segment numbers come directly from the reference track's independent sequence. watch_catalog switches references and clears rows without changing the URL generation. Reproduce: serve video B through sequence 100, then add alphabetically earlier video A whose timeline starts at 0. Surviving rendition playlists rewind and reuse seg/0.m4s for different content; existing cursors skip entries until their previous sequence is exceeded. The prior switching finding remains real. + Recommendation: Assign broadcast-wide monotonic segment identities across reference changes, or explicitly replace the URL namespace and rendition cursors when switching. Test switching between independently numbered timelines. +- [high] Follow mode permanently excludes tracks enrolled after startup (rs/moq-cli/src/archive.rs:101-108) + Import lists timelines once, then only refreshes the Reader's fixed timeline vector. serve() also captures a fixed list of accepted media tracks. Reproduce: start import --follow with audio recorded, then enroll video in the running export. Repeated refreshes never publish video's timeline, and requests for video are rejected as NotFound despite committed objects. The prior enrollment finding remains real. + Recommendation: Discover new timelines during follow refreshes and enroll them into both timeline replay and the shared served-track registry. Add a late-track enrollment integration test. +- [high] Independent span eviction changes still-advertised segment content (rs/moq-hls/src/export/spans.rs:103-105) + Spans evicts against its own newest timestamp, while playlist rows expire against the reference timeline. Reproduce with a 30-second window: pause the reference at 40 seconds while another rendition advances to 80 seconds. That rendition removes records covering the reference's still-listed 10–40 second segments, so previously resolvable segment URLs become gaps. Even ordinary watcher skew can expose this disagreement. The prior eviction finding remains real. + Recommendation: Coordinate span retention with the oldest advertised reference boundary, including snapping tolerance. Handle source pops explicitly so unavailable content is not silently substituted for an already advertised segment. Test a stalled reference with an advancing sibling. +- [medium] Duplicate enrollment resets state still owned by another recorder (rs/moq-mux/src/timeline.rs:507-516) + Timelines::track accepts an already-active name, replaces its segmenter, and returns another Recorder sharing the same Live. Reproduce: retain recorder A, enroll B under the same name, report group 10 through A and group 0 through B. B's report closes the shared timeline on a position error. Dropping A can also flush B's open record. Move-only Recorder does not enforce exclusive ownership here. + Recommendation: Reject enrollment while a recorder owns the name. Require continuing media positions for subsequent enrollment, or a fresh track name when group numbering restarts. Test overlapping recorder lifetimes. + +Next steps: +- Add regression tests for the four reproduction sketches before shipping. +- Separate design concern: writer.rs:261 propagates malformed-track errors out of the entire writer, dropping other tracks and in-flight commits. The prior observation is real and explicitly documented; decide whether recording-wide failure is intended or track isolation is required. + +Adversarial Codex review of `59659b7b1` (branch diff against `main`), focused on the track-timeline fold. Run through the `peer-review` skill; these are suggestions, not a maintainer decision. + +(Written by Codex) diff --git a/.status-4034.md b/.status-4034.md new file mode 100644 index 0000000000..e60d167f91 --- /dev/null +++ b/.status-4034.md @@ -0,0 +1,28 @@ +**Merge held at `59659b7b1`.** `main` is merged in, but this head should not land yet. + +### What changed + +- Merged `origin/main` (21 commits). There were conflicts in `Cargo.lock`, `rs/moq-cli` (`announced` plus `archive`), and fMP4 import. For fMP4 import I kept #4885's reservation-until-first-fragment and dropped main's shared-timeline catalog edit, which the fold replaces. +- #4873's selection refusal now covers `export archive` too (`ignores_selection`). That replaces the archive's own check, and `doc/bin/cli.md` lists it. +- Fixed a stale comment in `moq-hls` about one dead rendition "gating the broadcast's timeline". With per-track timelines a dead rendition only affects its own playlist. `hls-first-catalog.md` no longer cites the removed pass-wide timeline reservation. +- `a_moov_with_nothing_selected_publishes` passed on `main` only because of the shared-timeline catalog edit. It now checks the released reservation directly. +- **Fix (OpenAI review P2 #1):** `serve_group` marked a refresh as seen after loading objects from an older span snapshot. A tail committed during the GET was then never served. The reader now reloads before it finishes or waits. Regression test: `a_refresh_during_an_object_load_is_served`, using a GET stall added to `Mock`. It fails without the fix. +- Quests: `backward-timestamps.md` now covers the silent drop of a restarted source's groups (Grok review #1). `track-timeline.md` marks the 4s minimum as planned. + +`just check` passes. + +### Blocking + +1. **`just test interop --all` fails every Rust, Python, and Go to JS pair.** The JS catalog schema (`js/hang/src/catalog/archive.ts`) still requires `archive.track: string`. `moq-mux` now publishes an `archive.timelines` map for every broadcast, so `@moq/watch` rejects the catalog (`ZodError` at `archive.track`). PR CI does not run the full interop matrix, so the checks stay green. Landing this would break web playback of every native publisher on `main`. Options: + - (a) Pull the schema half of `quest/m1/archive/js-timelines.md` into this PR. **Recommended:** it is the smallest correct fix. + - (b) Land `js-timelines` first. + - (c) Make the JS root schema tolerate an unknown `archive` shape until then. +2. **Review findings needing a design call**, from the OpenAI and Grok reviews and the Codex review above (Codex re-confirmed the first, second, and fourth): + - HLS reference switching reuses segment URLs. + - Span eviction is not tied to the reference window, so a still-listed segment's content can change. + - `ENDLIST` can be sent before a slower rendition's pending tail. + - Follow mode never discovers a track enrolled after startup. + - Re-enrolling an active timeline name shares one segmenter. + - One malformed track ends the whole recording without flushing. This is documented, but isolate-vs-fail-loud is a policy decision. + +(Written by Claude Opus 5.5) diff --git a/doc/lib/js/hang.md b/doc/lib/js/hang.md index 5700f73856..57b9e8e79a 100644 --- a/doc/lib/js/hang.md +++ b/doc/lib/js/hang.md @@ -24,8 +24,8 @@ import * as Container from "@moq/hang/container"; `Catalog.EscapingBroadcast` for a `broadcast` reference that walks above the handle's `path`. Run the same checks on a catalog from another source with `Catalog.checkRenditions(root)` and `Catalog.checkResolvable(root, base)`. -`Hang.Timeline.Consumer.subscribe(broadcast, root.archive)` reads segment -`push`, `pop`, and `skip` events when a root advertises an archive. +`Hang.Timeline.Consumer.subscribe(broadcast, root.archive, track)` reads one +track's timeline as record `push`, `pop`, and `skip` events when a root advertises it. Most apps never import it directly; the elements and `Broadcast` classes in the watch and publish packages do. Reach for it when hand-rolling a catalog diff --git a/js/hang/src/catalog/archive.ts b/js/hang/src/catalog/archive.ts index c73e810ba3..a99b4ee4f0 100644 --- a/js/hang/src/catalog/archive.ts +++ b/js/hang/src/catalog/archive.ts @@ -11,25 +11,24 @@ const timescaleSchema = nonzeroU53Schema.check(z.lte(4_294_967_295)); /** * The recording object format advertised in {@link Archive.version} when a store is present. */ -export const ARCHIVE_VERSION = 1; +export const ARCHIVE_VERSION = 2; /** - * Discovers the broadcast's segment index and any durable archive. + * Discovers each track's timeline and any durable archive. * - * This is the catalog's one name for the segment index: a live publisher sets the - * timeline fields (`track`, `timescale`, `durationMax`) alone, and a recording - * also names the replay broadcast and object store those ranges live under. Every - * advertised range is FETCHable; with a store they are durable. There is no sibling - * `timeline` entry. Wall-clock mapping lives at the catalog root (`clock`), not here. + * A live publisher sets the timeline fields (`timelines`, `timescale`, `durationMax`) + * alone, and a recording also names the replay broadcast and object store those spans + * live under. Every advertised span is FETCHable; with a store they are durable. + * Wall-clock mapping lives at the catalog root (`clock`), not here. */ export const ArchiveSchema = z.object({ - // The name of the MoQ track carrying the broadcast's segment records. - track: z.string(), + // Each indexed track's timeline track, keyed by the indexed track's name. + timelines: z.record(z.string(), z.string()), - // Units per second for the records' timestamps. Defaults to milliseconds. + // Units per second for every timeline's timestamps. Defaults to milliseconds. timescale: z._default(timescaleSchema, u53(1000)), - // The declared upper bound on a segment's duration in timescale units. + // The declared upper bound on a record's duration in timescale units. durationMax: z.optional(u53Schema), // The MoQ broadcast the archive is served back from, relative to this catalog, if any. @@ -45,5 +44,5 @@ export const ArchiveSchema = z.object({ version: z.optional(u53Schema), }); -/** The catalog's root archive entry: the timeline track plus optional replay, store, and version. */ +/** The catalog's root archive entry: each track's timeline plus optional replay, store, and version. */ export type Archive = z.infer; diff --git a/js/hang/src/catalog/clock.test.ts b/js/hang/src/catalog/clock.test.ts index d010f1a4f2..b22465ecd5 100644 --- a/js/hang/src/catalog/clock.test.ts +++ b/js/hang/src/catalog/clock.test.ts @@ -17,7 +17,7 @@ const FIXTURE = { }, }, clock: { wall: 1_751_846_400_000_000, timescale: 1_000_000 }, - archive: { track: "timeline.z", timescale: 1000, durationMax: 2000 }, + archive: { timelines: { video: "video.timeline.z" }, timescale: 1000, durationMax: 2000 }, }; test("timescale defaults to microseconds", () => { diff --git a/js/hang/src/catalog/root.test.ts b/js/hang/src/catalog/root.test.ts index e6e7c64de2..e201b688ac 100644 --- a/js/hang/src/catalog/root.test.ts +++ b/js/hang/src/catalog/root.test.ts @@ -137,7 +137,7 @@ test("clock stays off the wire when absent", () => { test("archive round-trips at the root", () => { const parsed = RootSchema.parse({ archive: { - track: "timeline.z", + timelines: { video: "video.timeline.z" }, durationMax: 2000, replay: "./recordings/clip", store: "https://objects.example/rec/", @@ -145,12 +145,12 @@ test("archive round-trips at the root", () => { }, }); expect(parsed.archive).toMatchObject({ - track: "timeline.z", + timelines: { video: "video.timeline.z" }, timescale: 1000, durationMax: 2000, replay: "recordings/clip", store: "https://objects.example/rec/", - version: 1, + version: 2, }); expect(JSON.stringify(parsed)).not.toContain('"timeline":'); }); @@ -161,7 +161,23 @@ test("a legacy root timeline is not an archive", () => { }); test("an invalid store URL is refused", () => { - expect(() => RootSchema.parse({ archive: { track: "timeline.z", store: "not a url" } })).toThrow(); + expect(() => RootSchema.parse({ archive: { timelines: {}, store: "not a url" } })).toThrow(); +}); + +test("a single shared timeline is refused", () => { + expect(() => RootSchema.parse({ archive: { track: "timeline.z" } })).toThrow(); +}); + +// The exact shape Rust's moq-mux publishes for every broadcast, so a native publisher's catalog +// stays playable. +test("a Rust per-track archive parses", () => { + const parsed = RootSchema.parse({ + archive: { + timelines: { "catalog.json": "catalog.json.timeline.z", video0: "video0.timeline.z" }, + timescale: 1000, + }, + }); + expect(parsed.archive?.timelines.video0).toBe("video0.timeline.z"); }); test("an absent or foreign text section parses to empty", () => { diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index 0904e96754..176a271d1f 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -4,7 +4,6 @@ import { Time } from "@moq/net"; export type { BufferedRange, BufferedRanges, Frame } from "./types"; import type { AudioConfig, VideoConfig } from "../catalog"; -import type { Recorder as TimelineRecorder } from "../timeline"; import type { Format as ContainerFormat } from "./format"; import type { Frame } from "./types"; @@ -59,16 +58,6 @@ export function encodeFrame(source: Uint8Array | Source, timestamp: Time.Micro): return data; } -/** Options for a legacy-container {@link Producer}. */ -export interface ProducerProps { - /** - * Report each group open (sequence + start timestamp) into the broadcast's timeline, so - * consumers can index the media without downloading it. Mint one via - * {@link Timeline.Producer.track}. - */ - timeline?: TimelineRecorder; -} - /** Writes legacy-container frames into a MoQ track, starting a new group on each keyframe. */ export class Producer { #track: Moq.Track.Producer; @@ -76,9 +65,7 @@ export class Producer { #previous?: Time.Micro; #reordered = false; #group?: Moq.Group.Producer; - #timeline?: TimelineRecorder; - // The newest timestamp written, reported to the timeline when the track closes: the last - // group has no successor to bound it, so its segment would be published a group short. + // The newest timestamp written in the current group, which estimates its end. #end?: Time.Micro; // Furthest timestamp in finished groups, used only for discontinuity markers. #liveEdge?: Time.Micro; @@ -91,10 +78,9 @@ export class Producer { #marked = false; /** Wrap a track to publish legacy-container frames into it. */ - constructor(track: Moq.Track.Producer, format: Format, props: ProducerProps = {}) { + constructor(track: Moq.Track.Producer, format: Format) { this.#format = format; this.#track = track; - this.#timeline = props.timeline; } /** Encode and append a frame; a keyframe starts a new group. Throws if the first frame is not a keyframe, a group start goes backwards, or a frame is below the previous group start. */ @@ -111,8 +97,6 @@ export class Producer { this.#group = this.#track.appendGroup(); this.#floor = this.#start; this.#start = timestamp; - // Report the group the moment it opens: its start is this keyframe's timestamp. - this.#timeline?.record(this.#group.sequence, timestamp, true); } else if (!this.#group) { throw new Error("must start with a keyframe"); } @@ -156,8 +140,6 @@ export class Producer { if (this.#format.kind === "data" || this.#marked || timestamp === undefined) return; const group = this.#track.appendGroup(); - this.#timeline?.record(group.sequence, timestamp, false); - this.#timeline?.end(timestamp); group.writeFrame({ payload: encodeFrame(new Uint8Array(), timestamp), timestamp: Time.Timestamp.fromMicros(timestamp), @@ -185,8 +167,6 @@ export class Producer { : undefined; // A presentation endpoint cannot bound the decode-order tail after reordering. this.#format.finishGroup(this.#group, this.#reordered ? undefined : end); - const bound = end ?? this.#end; - if (bound !== undefined) this.#timeline?.end(bound); this.#group.close(); this.#group = undefined; if (this.#end !== undefined) { diff --git a/js/hang/src/timeline.test.ts b/js/hang/src/timeline.test.ts index 200cf2a77b..6c78ba91d0 100644 --- a/js/hang/src/timeline.test.ts +++ b/js/hang/src/timeline.test.ts @@ -1,552 +1,23 @@ import { expect, test } from "bun:test"; import * as Json from "@moq/json"; -import { Broadcast, Time, Track } from "@moq/net"; +import { Broadcast, type Time } from "@moq/net"; import { u53 } from "./catalog"; -import { Consumer, Producer, type Record } from "./timeline.ts"; +import { Consumer, type Record } from "./timeline.ts"; const us = (ms: number): Time.Micro => (ms * 1000) as Time.Micro; -/** A timeline whose published records are decoded back out of its MoQ track. */ -function capture(props?: ConstructorParameters[1]): { - timeline: Producer; - records: () => Promise; -} { - const track = new Track.Producer("timeline.z"); - const consumer = new Json.Window.Consumer({ track: track.subscribe(), compression: true }); - const timeline = new Producer(track, props); - - const records = async () => { - const out: Record[] = []; - for (;;) { - const event = await consumer.next(); - if (!event) return out; - if ("push" in event) { - out.push(event.push.value); - } else { - throw new Error("timeline producer only appends records"); - } - } - }; - return { timeline, records }; -} - -// The coarsest track paces: video GOPs are longer than durationMin, so segments are GOPs. -test("the coarsest track paces the segments", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - // Video keyframes every 2s, audio groups every 500ms, minimum 1s. - video.record(0, us(0)); - for (const [seq, ms] of [ - [0, 0], - [1, 500], - [2, 1000], - [3, 1500], - ] as const) { - audio.record(seq, us(ms)); - } - video.record(1, us(2000)); - for (const [seq, ms] of [ - [4, 2000], - [5, 2500], - [6, 3000], - [7, 3500], - ] as const) { - audio.record(seq, us(ms)); - } - video.record(2, us(4000)); - audio.record(8, us(4000)); - video.close(); - audio.close(); - timeline.finish(); - - // Audio's own candidate (1s) loses to video's (2s), so no segment splits a GOP. - expect(await records()).toEqual([ - { - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], audio0: [{ start: 0, end: 3 }] }, - }, - { - segment: 1, - pts: 2000, - duration: 2000, - tracks: { video0: [{ start: 1, end: 1 }], audio0: [{ start: 4, end: 7 }] }, - }, - { - segment: 2, - pts: 4000, - duration: 0, - tracks: { video0: [{ start: 2, end: 2 }], audio0: [{ start: 8, end: 8 }] }, - }, - ]); -}); - -// A real-time encoder's GOP can dwarf the minimum. Nothing is violated: there is nowhere else -// a segment could start and stay decodable, so the segment is simply long. -test("a GOP longer than the minimum is one segment", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - video.record(1, us(30_000)); - video.end(us(60_000)); - video.close(); - timeline.finish(); - - expect(await records()).toEqual([ - { segment: 0, pts: 0, duration: 30_000, tracks: { video0: [{ start: 0, end: 0 }] } }, - { segment: 1, pts: 30_000, duration: 30_000, tracks: { video0: [{ start: 1, end: 1 }] } }, - ]); -}); - -// Groups shorter than the minimum pack into one segment rather than each becoming one. -test("short groups pack up to the minimum", async () => { - const { timeline, records } = capture({ durationMin: Time.Milli(1500) }); - const audio = timeline.pacingTrack("audio0"); - - for (let seq = 0; seq < 8; seq++) { - audio.record(seq, us(seq * 500)); - } - audio.close(); - timeline.finish(); - - const out = await records(); - // The first group at or past 1500ms ends segment 0, so it holds groups 0..=2. - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 1500, tracks: { audio0: [{ start: 0, end: 2 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 1500, duration: 1500, tracks: { audio0: [{ start: 3, end: 5 }] } }); -}); - -// A track that hasn't reached the minimum yet holds the segment open, so a record never omits -// a rendition that was about to contribute to it. -test("a segment waits for every track", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - video.record(0, us(0)); - audio.record(0, us(0)); - // Video alone would close segment 0 here, but audio hasn't crossed 2s. - video.record(1, us(2000)); - audio.record(1, us(2000)); - video.close(); - audio.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], audio0: [{ start: 0, end: 0 }] }, - }); -}); - -// An application that knows its own boundaries overrides the pacing. -test("explicit cuts override the pacing", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - // Keyframes every second, cut every three: the segments follow the cuts, not the GOPs. - timeline.cut(us(3000)); - timeline.cut(us(6000)); - for (let seq = 0; seq < 7; seq++) { - video.record(seq, us(seq * 1000)); - } - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 3000, tracks: { video0: [{ start: 0, end: 2 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 3000, duration: 3000, tracks: { video0: [{ start: 3, end: 5 }] } }); -}); - -// Every rendition of one import declares the same boundaries, so a cut that would make a -// segment shorter than the minimum is dropped rather than producing a stray segment. -test("a cut below the minimum is ignored", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - timeline.cut(us(2000)); - // A sibling rendition's duplicate, and a boundary too close to be a segment. - timeline.cut(us(2000)); - timeline.cut(us(2500)); - timeline.cut(us(4000)); - video.record(1, us(2000)); - video.record(2, us(4000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out).toHaveLength(3); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }); -}); - -// The declared maximum is a contract with consumers (an HLS EXT-X-TARGETDURATION), so a -// segment that breaks it fails the timeline rather than publishing a record that contradicts -// the catalog. -test("exceeding the declared maximum fails the timeline", async () => { - const { timeline, records } = capture({ durationMin: Time.Milli(1000), durationMax: Time.Milli(3000) }); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - video.record(1, us(2000)); - // A 4s GOP against a declared 3s maximum. - video.record(2, us(6000)); - video.close(); - - expect(() => timeline.finish()).toThrow(/over the declared maximum/); - - // The record that was still true published; the one that would have contradicted the - // catalog did not. - expect(await records()).toEqual([ - { segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }, - ]); -}); - -test("an undeclared maximum is omitted from the catalog", () => { - expect(capture().timeline.section().durationMax).toBeUndefined(); - expect(capture({ durationMax: Time.Milli(2500) }).timeline.section().durationMax).toBe(u53(2500)); -}); - -// The last group of a broadcast has no successor to bound it, so without a reported end the -// final segment's duration collapses to zero (an HLS EXTINF:0). -test("the final segment runs to the reported end", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - video.record(1, us(2000)); - // A closing container producer reports where its content stops. - video.end(us(4000)); - video.close(); - timeline.finish(); - - expect(await records()).toEqual([ - { segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }, - { segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }, - ]); -}); - -// A record is immutable and judged against the tracks enrolled when it flushes, so a producer -// that enrolls its tracks one batch at a time holds flushing back until they are all in. -test("a reservation defers flushing until every track enrolls", async () => { - const { timeline, records } = capture(); - const release = timeline.reserve(); - - // The primary rendition runs a whole batch of segments through before its sibling exists. - const first = timeline.pacingTrack("video0"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - first.record(seq, us(ms)); - } - - const second = timeline.pacingTrack("video1"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - second.record(seq, us(ms)); - } - - release(); - first.close(); - second.close(); - timeline.finish(); - - // Both renditions are indexed from segment 0. Without the hold, segments 0 and 1 would have - // flushed knowing only video0, and video1's first two groups would have folded into - // segment 2. - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], video1: [{ start: 0, end: 0 }] }, - }); - expect(out[1]).toEqual({ - segment: 1, - pts: 2000, - duration: 2000, - tracks: { video0: [{ start: 1, end: 1 }], video1: [{ start: 1, end: 1 }] }, - }); -}); - -// A track that races ahead of the others still lands in the segment its content falls in. -test("groups before the first boundary join the first segment", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - audio.record(0, us(0)); - video.record(0, us(30)); - for (const [seq, ms] of [ - [1, 500], - [2, 1000], - [3, 1500], - [4, 2000], - ] as const) { - audio.record(seq, us(ms)); - } - video.record(1, us(2030)); - audio.record(5, us(2500)); - video.close(); - audio.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - // The segment starts where the earliest content does, not at video's first keyframe. - segment: 0, - pts: 0, - duration: 2030, - tracks: { video0: [{ start: 0, end: 0 }], audio0: [{ start: 0, end: 4 }] }, - }); -}); - -test("sequence gaps split ranges", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - // Elemental-style gap: audio groups 2..=4 never existed inside segment 0. - video.record(0, us(0)); - audio.record(0, us(0)); - audio.record(1, us(300)); - audio.record(5, us(1500)); - video.record(1, us(2000)); - audio.record(6, us(2100)); - video.close(); - audio.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { - video0: [{ start: 0, end: 0 }], - audio0: [ - { start: 0, end: 1 }, - { start: 5, end: 5 }, - ], - }, - }); -}); - -// A track with nothing to contribute is a whole-segment gap: the record simply omits it (an -// HLS exporter renders EXT-X-GAP). Only a closed track produces one, since a live track that -// has merely gone quiet still gets to say where the boundary is. -test("a closed track leaves a whole segment gap", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - video.record(0, us(0)); - audio.record(0, us(0)); - audio.close(); - video.record(1, us(2000)); - video.record(2, us(4000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[1]).toEqual({ segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }); -}); - -test("a non-keyframe range start is flagged", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - // A mid-stream join: the group doesn't open on an IDR. - video.record(1, us(2000), false); - video.record(2, us(4000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[1]?.tracks?.video0[0].keyframe).toBe(false); -}); - -test("the timeline section carries no wall anchor", () => { - // Wall mapping moved to the catalog root clock: the timeline section carries the track, - // timescale, and duration bound, and nothing else time-anchoring. - const section = capture().timeline.section(); - expect(section).toEqual({ track: "timeline.z", timescale: u53(1000), durationMax: undefined }); - expect("wall" in section).toBe(false); -}); - -// Reservations nest like the catalog's, so the first batch to finish doesn't publish records -// the others are still filling in. -test("reservations nest", async () => { - const { timeline, records } = capture(); - - const outer = timeline.reserve(); - const inner = timeline.reserve(); - - const first = timeline.pacingTrack("video0"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - first.record(seq, us(ms)); - } - - // If reservations didn't nest, releasing this one would publish segment 0 without video1. - inner(); - - const second = timeline.pacingTrack("video1"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - second.record(seq, us(ms)); - } - - outer(); - first.close(); - second.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], video1: [{ start: 0, end: 0 }] }, - }); - expect(out[1]).toEqual({ - segment: 1, - pts: 2000, - duration: 2000, - tracks: { video0: [{ start: 1, end: 1 }], video1: [{ start: 1, end: 1 }] }, - }); -}); - -// A cut registered before the media, landing exactly on the first reported group (what the fMP4 -// importer does: it cuts at the keyframe fragment's timestamp, then records that group). The -// spent cut must not block the ones behind it, and durationMin pacing must not race ahead of them. -test("a cut on the first group does not poison later cuts", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - // Source segments every 3s, keyframes every 1s: 3s segments, not the 1s the durationMin - // pacing would produce on its own. - timeline.cut(us(0)); - video.record(0, us(0)); - for (let seq = 1; seq < 10; seq++) { - if (seq % 3 === 0) timeline.cut(us(seq * 1000)); - video.record(seq, us(seq * 1000)); - } - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 3000, tracks: { video0: [{ start: 0, end: 2 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 3000, duration: 3000, tracks: { video0: [{ start: 3, end: 5 }] } }); -}); - -// A catalog publishes a group only when the renditions change, so it can go quiet for the rest -// of the broadcast. Enrolled as a pacing track it would stall the timeline for good; non-pacing, it -// rides along in whichever segment is open. -test("a non-pacing track neither paces nor gates", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const catalog = timeline.track("catalog.json"); - - catalog.record(0, us(0)); - video.record(0, us(0)); - video.record(1, us(2000)); - video.record(2, us(4000)); - video.end(us(6000)); - video.close(); - timeline.finish(); - - expect(await records()).toEqual([ - { - segment: 0, - pts: 0, - duration: 2000, - tracks: { "catalog.json": [{ start: 0, end: 0 }], video0: [{ start: 0, end: 0 }] }, - }, - { segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }, - { segment: 2, pts: 4000, duration: 2000, tracks: { video0: [{ start: 2, end: 2 }] } }, - ]); -}); - -// Nothing waits for a non-pacing track, so a group arriving after its segment already flushed is -// recorded in the next one. Placement is by arrival; the frames still carry their own timestamps. -test("a late non-pacing group lands in the next segment", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const catalog = timeline.track("catalog.json"); - - video.record(0, us(0)); - // Flushes segment 0, which the catalog has contributed nothing to. - video.record(1, us(2000)); - - // The update happened a second in, but only reaches the timeline now. - catalog.record(0, us(1000)); - - video.record(2, us(4000)); - video.end(us(6000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }); - expect(out[1]).toEqual({ - segment: 1, - pts: 2000, - duration: 2000, - tracks: { "catalog.json": [{ start: 0, end: 0 }], video0: [{ start: 1, end: 1 }] }, - }); -}); - -test("a non-pacing track uses arrival and does not extend the tail", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const catalog = timeline.track("catalog.json"); - - video.record(0, us(0)); - catalog.record(0, us(10_000)); - video.record(1, us(2000)); - video.end(us(4000)); - timeline.finish(); - - expect(await records()).toEqual([ - { - segment: 0, - pts: 0, - duration: 2000, - tracks: { "catalog.json": [{ start: 0, end: 0 }], video0: [{ start: 0, end: 0 }] }, - }, - { segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }, - ]); -}); +const archive = (timescale: number) => ({ timelines: { video: "video.timeline.z" }, timescale: u53(timescale) }); test("timeline consumer yields converted push and pop events", async () => { const broadcast = new Broadcast.Producer(); - const track = broadcast.createTrack("timeline.z"); + const track = broadcast.createTrack("video.timeline.z"); const producer = new Json.Window.Producer({ track, compression: true }); - const consumer = Consumer.subscribe(broadcast.consume(), { track: "timeline.z", timescale: u53(1000) }); - producer.push({ segment: 7, pts: 250, duration: 1250, tracks: { video: [{ start: 1, end: 2 }] } }); + const consumer = Consumer.subscribe(broadcast.consume(), archive(1000), "video"); + producer.push({ sequence: 0, pts: 250, duration: 1250, start: { group: 1 }, end: { group: 2, frame: 3 } }); expect(await consumer.next()).toEqual({ push: { index: 0, - entry: { segment: 7, pts: us(250), duration: us(1250), tracks: { video: [{ start: 1, end: 2 }] } }, + entry: { sequence: 0, pts: us(250), duration: us(1250), start: { group: 1 }, end: { group: 2, frame: 3 } }, }, }); producer.pop(1); @@ -558,14 +29,41 @@ test("timeline consumer yields converted push and pop events", async () => { test("timeline consumer floors fractional microseconds without floating point drift", async () => { const broadcast = new Broadcast.Producer(); - const track = broadcast.createTrack("timeline.z"); + const track = broadcast.createTrack("video.timeline.z"); const producer = new Json.Window.Producer({ track, compression: true }); - const consumer = Consumer.subscribe(broadcast.consume(), { track: "timeline.z", timescale: u53(3) }); - producer.push({ segment: 0, pts: 1, duration: 2 }); + const consumer = Consumer.subscribe(broadcast.consume(), archive(3), "video"); + producer.push({ sequence: 0, pts: 1, duration: 2, start: { group: 0 }, end: { group: 1 } }); expect(await consumer.next()).toEqual({ - push: { index: 0, entry: { segment: 0, pts: 333_333 as Time.Micro, duration: 666_666 as Time.Micro } }, + push: { + index: 0, + entry: { + sequence: 0, + pts: 333_333 as Time.Micro, + duration: 666_666 as Time.Micro, + start: { group: 0 }, + end: { group: 1 }, + }, + }, }); producer.finish(); consumer.close(); broadcast.close(); }); + +test("a record numbered unlike its window index is refused", async () => { + const broadcast = new Broadcast.Producer(); + const track = broadcast.createTrack("video.timeline.z"); + const producer = new Json.Window.Producer({ track, compression: true }); + const consumer = Consumer.subscribe(broadcast.consume(), archive(1000), "video"); + producer.push({ sequence: 5, pts: 0, duration: 1000, start: { group: 0 }, end: { group: 1 } }); + await expect(consumer.next()).rejects.toThrow(); + producer.finish(); + consumer.close(); + broadcast.close(); +}); + +test("a track without a timeline is refused", () => { + const broadcast = new Broadcast.Producer(); + expect(() => Consumer.subscribe(broadcast.consume(), archive(1000), "audio")).toThrow(); + broadcast.close(); +}); diff --git a/js/hang/src/timeline.ts b/js/hang/src/timeline.ts index 090e07393f..5b86915701 100644 --- a/js/hang/src/timeline.ts +++ b/js/hang/src/timeline.ts @@ -1,15 +1,11 @@ /** - * Publishing the broadcast's timeline: a single track carrying one record per aligned - * segment, mapping a span of content time to the group ranges that carry it on each media - * track. A consumer can seek (or build an HLS/DASH playlist) without downloading the media. - * See the catalog's root {@link Catalog.Archive} entry that advertises it. + * Reading a track's timeline: a track carrying one record per span of another track, mapping + * its content time to the group and frame positions that carry it. A consumer can seek (or + * build an HLS/DASH playlist) without downloading the media. * - * Facts flow up and policy flows down, meeting in the shared {@link Producer}: each continuous - * media track enrolls with {@link Producer.pacingTrack} and reports every group open through its - * {@link Recorder}. A segment ends at the first group boundary that gives it at least - * {@link ProducerProps.durationMin} on every pacing track, unless the application declares its - * own with {@link Producer.cut}. A segment's record is published only once every pacing track - * has reported past its end (or closed), so records are self-contained and immediately servable. + * Every track has its own timeline, named in the catalog's root {@link Catalog.Archive} entry. + * A span usually holds whole groups; a group that outlives the publisher's maximum span is + * split by frame. Spans are contiguous in position unless the track skipped group sequences. * * @module */ @@ -19,80 +15,57 @@ import type * as Moq from "@moq/net"; import type { Time } from "@moq/net"; import * as z from "@zod/mini"; import type * as Catalog from "./catalog"; -import { u53, u53Schema } from "./catalog"; +import { u53Schema } from "./catalog"; -/** - * A contiguous run of groups a track contributes to a segment, `start` through `end` - * inclusive. More than one range in a segment means the group sequence is discontinuous (a - * gap: groups that never existed). - */ -export interface Range { - start: number; - end: number; - /** - * Whether the run's first group starts with a keyframe, i.e. whether a player can join or - * switch renditions at this range. Omitted when true (the default). - */ - keyframe?: boolean; +/** A frame position within a track: frame `frame` of group `group`. */ +export interface Position { + group: number; + /** Omitted when zero, so a position at a group start reads as just the group. */ + frame?: number; } /** - * One timeline record: a complete aligned segment. `pts` and `duration` bound its span of - * content time (in the timeline's timescale), and `tracks` maps each participating media - * track to the group ranges that carry it. A track absent from `tracks` has no content for - * the span (a gap). + * One timeline record: a span of one track holding every frame from `start` (inclusive) to + * `end` (exclusive). `pts` and `duration` are in the archive's timescale. An `end` at frame zero + * of group `g` holds every frame of group `g - 1` and nothing of `g`. */ export interface Record { - segment: number; + /** The record's number, consecutive within its track's timeline. */ + sequence: number; pts: number; duration: number; - tracks?: { [track: string]: Range[] }; + start: Position; + end: Position; + /** Whether the span starts with a keyframe. Omitted when true (the default). */ + keyframe?: boolean; } +const PositionSchema = z.object({ group: u53Schema, frame: z.optional(u53Schema) }); + const RecordSchema = z.looseObject({ - segment: u53Schema, + sequence: u53Schema, pts: u53Schema, duration: u53Schema, - tracks: z.optional( - z.record( - z.string(), - z.array(z.object({ start: u53Schema, end: u53Schema, keyframe: z.optional(z.boolean()) })), - ), - ), + start: PositionSchema, + end: PositionSchema, + keyframe: z.optional(z.boolean()), }); -/** The default timeline timescale: 1000 units per second (milliseconds). */ -export const DEFAULT_TIMESCALE = 1000; - -/** - * The conventional {@link ProducerProps.durationMin} (1 second), in milliseconds, for callers - * with no opinion of their own. - */ -export const DEFAULT_DURATION_MIN_MS = 1000 as Time.Milli; - -/** Recent segment records repeated when the Window track rolls to a new group. */ -const CHECKPOINT_RECORDS = 256; - -/** - * The conventional name for a broadcast's timeline track (the `.z` marks the - * DEFLATE-compressed window, like the catalog's `.json.z` sibling). The actual name is read - * from the catalog's root `archive` entry, so this is only a default. - */ -export const DEFAULT_NAME = "timeline.z"; +/** The conventional suffix appended to a track name to name its timeline track. */ +export const SUFFIX = ".timeline.z"; -/** A complete segment with timestamps converted to microseconds. */ +/** A record with timestamps converted to microseconds. */ export type Entry = Omit & { pts: Time.Micro; duration: Time.Micro }; /** One change to the visible timeline window. */ export type Event = { push: { index: number; entry: Entry } } | { pop: Json.Window.Span } | { skip: Json.Window.Span }; -/** Reads the timeline advertised by a catalog archive entry. */ +/** Reads one track's timeline, as advertised by a catalog archive entry. */ export class Consumer implements AsyncIterable { readonly #track: Moq.Track.Subscriber; readonly #window: Json.Window.Consumer; readonly #timescale: number; - /** Wrap a subscription to the archive's timeline track. */ private constructor(track: Moq.Track.Subscriber, archive: Catalog.Archive) { this.#track = track; if (!Number.isSafeInteger(archive.timescale) || archive.timescale <= 0) { @@ -102,9 +75,11 @@ export class Consumer implements AsyncIterable { this.#window = new Json.Window.Consumer({ track, compression: true }); } - /** Subscribe to the timeline named by the catalog's archive entry. */ - static subscribe(broadcast: Moq.Broadcast.Consumer, archive: Catalog.Archive): Consumer { - return new Consumer(broadcast.track(archive.track).subscribe(), archive); + /** Subscribe to `track`'s timeline as named by the catalog's archive entry. Throws if it has none. */ + static subscribe(broadcast: Moq.Broadcast.Consumer, archive: Catalog.Archive, track: string): Consumer { + const name = archive.timelines[track]; + if (name === undefined) throw new Error(`no timeline for track ${track}`); + return new Consumer(broadcast.track(name).subscribe(), archive); } /** Get the next timeline event, or `undefined` when the track ends. */ @@ -114,6 +89,7 @@ export class Consumer implements AsyncIterable { if ("push" in event) { const { index } = event.push; const value = RecordSchema.parse(event.push.value); + if (value.sequence !== index) throw new Error("timeline record sequence differs from its index"); const pts = this.#micros(value.pts); const duration = this.#micros(value.duration); return { push: { index, entry: { ...value, pts, duration } } }; @@ -141,437 +117,3 @@ export class Consumer implements AsyncIterable { this.#track.close(); } } - -/** How a {@link Producer} paces its segments. */ -export interface ProducerProps { - /** - * The shortest a segment may be, in milliseconds of media time. - * - * A segment ends at the first point that is a group boundary on every enrolled track and at - * least this far past the segment's start, so the track with the coarsest groups paces the - * broadcast: a 2s GOP against a 1s minimum yields 2s segments, and a real-time encoder with - * a 10 minute GOP yields one 10 minute segment, because there is nowhere else a segment - * could start and stay decodable. A floor rather than a target on purpose: a floor is - * always satisfiable (wait longer), while a ceiling is not. Defaults to - * {@link DEFAULT_DURATION_MIN_MS}. - */ - durationMin?: Time.Milli; - - /** - * The longest a segment may be, in milliseconds of media time, advertised in the catalog - * when set. - * - * Set it only when the publisher can actually promise one, i.e. it controls the encoder's - * keyframe cadence. Consumers that need a bound up front use it (an HLS exporter's - * EXT-X-TARGETDURATION), so it is a contract rather than a hint: a segment that would exceed - * it fails the timeline instead of publishing a record that contradicts the catalog. Leave - * it unset when the media decides, which is the common case for real-time and for anything - * importing a source the publisher doesn't control. - */ - durationMax?: Time.Milli; -} - -/** One enrolled track's report state. */ -interface TrackState { - // Group opens reported and not yet flushed into a record. - pending: { sequence: number; pts: Time.Micro; keyframe: boolean }[]; - // The newest reported timestamp: everything earlier is known. Advanced by a group open (the - // group starts there) and by Recorder.end (the content stops there). - frontier?: Time.Micro; - closed: boolean; - // Whether this track paces boundaries and gates completeness while open. - pacing: boolean; -} - -/** - * Publishes the broadcast's timeline track: one JSON record per complete segment, - * DEFLATE-compressed (a `@moq/json` Window), plus the shared boundary list every media track's - * groups map onto. - * - * One per broadcast. Tracks enroll with {@link track}; continuous media explicitly opts into - * segmentation through {@link pacingTrack}. Both report group opens through the returned - * {@link Recorder}. An application with its own boundaries overrides pacing with {@link cut}. - * Advertise it in the catalog's root `archive` entry via {@link section}. - */ -export class Producer { - #window: Json.Window.Producer; - #trackName: string; - #durationMinUs: number; - #durationMaxUs?: number; - - // Where the open (unflushed) segment starts; undefined until the first report. - #start?: Time.Micro; - // Explicit cut() boundaries not yet reached, in order. - #cuts: Time.Micro[] = []; - // A cut() arrived, so the application owns the boundaries from here on and the durationMin - // pacing stops. Without this the pacing races ahead of a source whose segments are longer - // than the minimum, closing one before its real boundary is declared. - #manual = false; - #nextSegment = 0; - #tracks = new Map(); - // Live reservations: while any exists, no record flushes (more tracks are still enrolling). - #reservers = 0; - // A segment overran durationMax, so the timeline stopped publishing. - #overrun?: Error; - - /** Wrap an already-created MoQ track (named {@link DEFAULT_NAME} by convention). */ - constructor(track: Moq.Track.Producer, props: ProducerProps = {}) { - this.#trackName = track.name; - this.#durationMinUs = (props.durationMin ?? DEFAULT_DURATION_MIN_MS) * 1000; - this.#durationMaxUs = props.durationMax === undefined ? undefined : props.durationMax * 1000; - this.#window = new Json.Window.Producer({ - track, - compression: true, - checkpointRecords: CHECKPOINT_RECORDS, - }); - } - - /** - * Enroll `name` without letting it influence segmentation, returning its {@link Recorder}. - * - * Its groups are recorded into whichever segment is open when they arrive, but the track never - * votes on where a boundary falls and never holds a record back. This safe default is for a - * catalog that emits only when renditions change, or an application's metadata track. - * - * Placement is by arrival rather than by content time: nothing waits for the track, so a group - * that arrives after its segment flushed is recorded in the next one. Frames still carry their - * own timestamps. One recorder per track: enrolling the same name again resets its state. - */ - track(name: string): Recorder { - this.#tracks.set(name, { pending: [], frontier: undefined, closed: false, pacing: false }); - return new Recorder(this, name); - } - - /** - * Enroll the media track `name` as a pacing track, returning its {@link Recorder}. - * - * The track votes on boundaries and gates completeness until its recorder closes. Enroll it - * when it is about to produce: an enrolled but silent pacing track holds every record back, - * since a segment is not complete until every pacing track's content is known. - * - * This opt-in is for continuously publishing media. A sparse track belongs in {@link track}, - * otherwise it can stall the timeline when it goes quiet. - */ - pacingTrack(name: string): Recorder { - this.#tracks.set(name, { pending: [], frontier: undefined, closed: false, pacing: true }); - return new Recorder(this, name); - } - - /** - * Declare a segment boundary at `pts` (microseconds), overriding the - * {@link ProducerProps.durationMin} pacing. - * - * For applications that know their own boundaries (an imported playlist, on-disk CMAF - * segments, an encoder placing keyframes). Cutting ahead of the media is fine: the record - * still waits for every track's groups. A cut that would make a segment shorter than the - * minimum is ignored, so several producers declaring the same boundaries cost nothing. - * - * The first call takes over for good: {@link ProducerProps.durationMin} pacing stops, since it - * would otherwise close a segment just before the caller declares where it really ends. - * - * Throws if a segment already exceeded {@link ProducerProps.durationMax}. - */ - cut(pts: Time.Micro): void { - if (this.#overrun) throw this.#overrun; - - // Even a cut this rejects says the caller owns the boundaries. - this.#manual = true; - - const since = this.#cuts.at(-1) ?? this.#start; - if (since === undefined || pts >= since + this.#durationMinUs) { - this.#cuts.push(pts); - this.#pump(false); - } - } - - /** - * Begin reserving the track set, returning a function that releases the reservation. - * - * The counterpart to the Rust catalog's `reserve`, for the same reason: while any - * reservation is outstanding the track set may still grow, so records are withheld from the - * broadcast. A record is immutable once published and its completeness is judged against the - * tracks enrolled *at that moment*, so a segment that flushes while a sibling rendition is - * still enrolling omits it for good, and that rendition's earlier groups then land in - * whatever segment flushes next. - * - * Take one around whatever brings the tracks up, so a producer that enrolls its renditions - * one at a time publishes nothing until they are all in. Reservations nest, and releasing - * the same one twice is a no-op; the last one released flushes. - */ - reserve(): () => void { - this.#reservers += 1; - let released = false; - return () => { - if (released) return; - released = true; - this.#reservers -= 1; - this.#pump(false); - }; - } - - /** The catalog's root `archive` entry advertising this timeline. */ - section(): Catalog.Archive { - const durationMax = - this.#durationMaxUs === undefined - ? undefined - : u53(Math.ceil((this.#durationMaxUs * DEFAULT_TIMESCALE) / 1_000_000)); - return { - track: this.#trackName, - timescale: u53(DEFAULT_TIMESCALE), - durationMax, - }; - } - - /** - * Flush the final (still open) segment and finish the track. - * - * Throws if a segment exceeded {@link ProducerProps.durationMax}. - */ - finish(): void { - // Nothing more will enroll, so an outstanding reservation has nothing left to wait for. - this.#reservers = 0; - this.#pump(true); - - if (this.#overrun) throw this.#overrun; - - // Skip an empty tail: a boundary with no content after it describes nothing. - const start = this.#start; - this.#start = undefined; - if (start !== undefined) { - for (const track of this.#tracks.values()) { - if (track.pending.length > 0) { - this.#flushSegment(start, undefined); - break; - } - } - } - - this.#window.finish(); - } - - /** @internal A group open reported by a {@link Recorder}. */ - report(name: string, sequence: number, pts: Time.Micro, keyframe: boolean): void { - const track = this.#tracks.get(name); - if (!track) return; - track.pending.push({ sequence, pts, keyframe }); - this.#advance(track, pts); - } - - /** - * @internal A track's content ends at `pts`, without a group opening there. This is what - * gives the final segment an honest duration, since its end is not a boundary anybody cut. - */ - reportEnd(name: string, pts: Time.Micro): void { - const track = this.#tracks.get(name); - if (track) this.#advance(track, pts); - } - - /** @internal A track's recorder closed: stop gating completeness on it. */ - close(name: string): void { - const track = this.#tracks.get(name); - if (track) track.closed = true; - this.#pump(false); - } - - // Raise a track's frontier, then publish whatever that completed. - #advance(track: TrackState, pts: Time.Micro): void { - if (track.frontier === undefined || pts > track.frontier) track.frontier = pts; - - // The first thing a pacing track reports anchors the first segment, so content produced - // before any boundary exists belongs to the oldest segment rather than to nowhere. A - // non-pacing track must not anchor it: a catalog published while the encoder is still warming - // up would otherwise stretch segment 0 across the whole startup gap. - if (this.#start === undefined && track.pacing) this.#start = pts; - - this.#pump(false); - } - - // Publish every segment the media has finalized. `finished` is the terminal pass: no track - // will report again, so one that never reached a boundary stops voting instead of holding - // the timeline open forever. - #pump(finished: boolean): void { - if (this.#tracks.size === 0 || this.#reservers > 0 || this.#overrun) return; - while (this.#closeSegment(finished)) { - if (this.#overrun) break; - } - } - - // Publish the open segment if the media has finalized it, returning whether it did. - #closeSegment(finished: boolean): boolean { - const start = this.#start; - if (start === undefined) return false; - - // Discard boundaries the timeline has already reached. Only the first is ever consulted, - // so leaving a spent one there would block every later cut behind it and silently drop - // the caller back to durationMin pacing. - while (this.#cuts.length > 0 && this.#cuts[0] <= start) this.#cuts.shift(); - - const boundary = this.#boundary(start, finished); - if (!boundary) return false; - const [end, cut] = boundary; - - // Every open pacing track has to have reported at or past the boundary, proving its ranges - // for this segment are final. The track that voted for `end` has by construction; a track - // with shorter groups can still be behind it. A non-pacing track is excluded: it may publish - // rarely or never again, so waiting on it would stall the timeline for good. - if (!finished) { - for (const track of this.#tracks.values()) { - if (track.closed || !track.pacing) continue; - if (track.frontier === undefined || track.frontier < end) return false; - } - } - - this.#flushSegment(start, end); - if (cut) this.#cuts.shift(); - this.#start = end; - return true; - } - - // Where the segment starting at `start` ends: an explicit cut when one is registered, - // otherwise the first group boundary shared by every track that gives the segment its - // minimum duration. The boolean reports which. - #boundary(start: Time.Micro, finished: boolean): [Time.Micro, boolean] | undefined { - const cut = this.#cuts[0]; - if (cut !== undefined && cut > start) return [cut, true]; - - // The application declared a boundary at some point, so it owns them all: pacing here - // would close a segment the caller is about to cut somewhere else. - if (this.#manual) return undefined; - - const threshold = start + this.#durationMinUs; - - let end: Time.Micro | undefined; - for (const track of this.#tracks.values()) { - if (!track.pacing) continue; - const candidate = track.pending.find((group) => group.pts >= threshold)?.pts; - if (candidate === undefined) { - // This track has produced nothing past the minimum yet, so ending the segment - // would strand it. A closed track never will, and neither does anything on the - // terminal pass, so neither one blocks. - if (finished || track.closed) continue; - return undefined; - } - // The latest vote wins: it is a group boundary on the coarsest track, and every finer - // track assigns its groups by start, so no group is split. A closed track still votes: - // it can't report more, but the groups it did report are boundaries like any other. - if (end === undefined || candidate > end) end = candidate; - } - - return end === undefined ? undefined : [end, false]; - } - - // Emit the record for the segment starting at `start`: drain every track's groups before - // `end` (all of them for the final, unbounded segment) into ranges. - #flushSegment(start: Time.Micro, end?: Time.Micro): void { - const pts = Math.floor((start * DEFAULT_TIMESCALE) / 1_000_000); - let endUnits: number; - if (end !== undefined) { - endUnits = Math.floor((end * DEFAULT_TIMESCALE) / 1_000_000); - } else { - // The final segment has no end boundary, so it runs to the newest thing any track - // reported: its end of content when the track reported one, otherwise the last group - // it opened, which undercounts that group's tail. - let max = start; - for (const track of this.#tracks.values()) { - if (!track.pacing) continue; - if (track.frontier !== undefined && track.frontier > max) max = track.frontier; - } - endUnits = Math.floor((max * DEFAULT_TIMESCALE) / 1_000_000); - } - const duration = Math.max(0, endUnits - pts); - - // The catalog promised a bound and this segment breaks it, so the record would contradict - // what consumers were told. Fail the timeline rather than publish it: a declared maximum - // the media can't honor is a bug in the publisher. - if (this.#durationMaxUs !== undefined) { - const maxUnits = Math.ceil((this.#durationMaxUs * DEFAULT_TIMESCALE) / 1_000_000); - if (duration > maxUnits) { - this.#overrun = new Error( - `timeline segment ${this.#nextSegment} lasted ${duration}ms, over the declared maximum ${maxUnits}ms`, - ); - // End the track rather than drop it: the records published before the promise - // broke are still true, and a consumer that has them should keep them. - this.#window.finish(); - return; - } - } - - const record: Record = { segment: this.#nextSegment, pts, duration }; - this.#nextSegment += 1; - - const tracks: { [track: string]: Range[] } = {}; - let any = false; - for (const [name, track] of this.#tracks) { - const ranges: Range[] = []; - while (track.pending.length > 0) { - const group = track.pending[0]; - // A pacing track is assigned by content time. A non-pacing track is assigned by - // arrival, so every group pending when the segment closes belongs to it regardless - // of the timestamp basis carried by that track. - if (track.pacing && end !== undefined && group.pts >= end) break; - track.pending.shift(); - const last = ranges.at(-1); - // Contiguous sequences extend the run; a skip starts a new range (a gap: groups - // that never existed). - if (last && last.end + 1 === group.sequence) { - last.end = group.sequence; - } else { - const range: Range = { start: group.sequence, end: group.sequence }; - if (!group.keyframe) range.keyframe = false; - ranges.push(range); - } - } - if (ranges.length > 0) { - tracks[name] = ranges; - any = true; - } - } - if (any) record.tracks = tracks; - - this.#window.push(record); - } -} - -/** - * Reports one media track's group opens into the shared {@link Producer}. It is the track's - * single reporting handle; {@link close} ends the track's enrollment (segments stop waiting - * on it). Minted by {@link Producer.track} and held by a {@link Legacy.Producer} (its - * `timeline` prop). - */ -export class Recorder { - #timeline: Producer; - #name: string; - - /** @internal Minted by {@link Producer.track}. */ - constructor(timeline: Producer, name: string) { - this.#timeline = timeline; - this.#name = name; - } - - /** - * Report that group `sequence` opened at presentation time `pts` (microseconds), - * `keyframe` stating whether its first frame is one. Reports must be in group order with - * monotonic timestamps. - */ - record(sequence: number, pts: Time.Micro, keyframe = true): void { - this.#timeline.report(this.#name, sequence, pts, keyframe); - } - - /** - * Report that this track's content extends to `pts` (microseconds), without a group opening - * there. - * - * A group open says where content *starts*; the last group of a broadcast has no successor - * to bound it, so its segment would otherwise be published a group short (zero for a segment - * that is a single group). Report the end whenever you know it: closing a group, finishing a - * track. - */ - end(pts: Time.Micro): void { - this.#timeline.reportEnd(this.#name, pts); - } - - /** Close the track's enrollment: segments no longer wait on it. */ - close(): void { - this.#timeline.close(this.#name); - } -} diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 1ccbabb27a..5b4f9cd5c5 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -116,7 +116,7 @@ owned by that prerequisite, not duplicated in archive storage. ## Required - [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md) - each timeline entry declares its segment duration (reported or estimated by the publisher), replacing the root `durationMax` -- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust +- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - `@moq/hang` publishes the same per-track timelines as Rust (it already reads them) - [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, from the reference timeline's declared duration; an overrun is listed with a warning - [Per-track timelines](/quest/m1/archive/track-timeline.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge - [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay diff --git a/quest/m1/archive/declared-duration.md b/quest/m1/archive/declared-duration.md index d7dd7ccb13..c39af1bfa3 100644 --- a/quest/m1/archive/declared-duration.md +++ b/quest/m1/archive/declared-duration.md @@ -18,7 +18,9 @@ the catalog declares a usable target: the archive `timescale`. The root `durationMax` goes away. This is a break in place on this line, with no compatibility path. Update `rs/hang`, `drafts/draft-lcurley-moq-hang.md`, and every `durationMax` reference - (grep it, including `doc/`). The JS port happens in + (grep it, including `doc/`), and the JS catalog schema + (`js/hang/src/catalog/archive.ts`) in the same PR, since `@moq/watch` refuses + a catalog it cannot parse. The rest of the JS port happens in [JS per-track timelines](/quest/m1/archive/js-timelines.md). - **A 4s minimum.** `duration_min` defaults to 4s (from 2s), so an archive doesn't write a stream of tiny objects to S3. A record still ends at the diff --git a/quest/m1/archive/js-timelines.md b/quest/m1/archive/js-timelines.md index da229707d9..9e9bbbc7f5 100644 --- a/quest/m1/archive/js-timelines.md +++ b/quest/m1/archive/js-timelines.md @@ -3,14 +3,20 @@ ## Goal `@moq/hang` publishes one timeline per track with the same records, cuts, and -catalog `archive` map as Rust, and parses them identically. +catalog `archive` map as Rust. ## Plan -Port the landed Rust shape to `js/hang/src/timeline.ts` and its catalog schema, -replacing the aligned timeline and cross-track pacing rather than keeping both. -Cover the same cut rules and a static catalog outliving other tracks' records, -and check the records against Rust output in the interop suite. +The read side landed with the archive line (#4034): the catalog schema parses +the `timelines` map and `Timeline.Consumer` reads one track's records. The +aligned multi-track producer was deleted rather than kept beside it, so JS +publishes no timeline today. + +Port the Rust segmenter, producer, and enrollment +(`rs/moq-mux/src/timeline.rs`) to `js/hang/src/timeline.ts`, and have the +legacy container report its groups again. Cover the same cut rules and a +static catalog outliving other tracks' records, and check the records against +Rust output in the interop suite. ## Required diff --git a/quest/m1/archive/track-timeline.md b/quest/m1/archive/track-timeline.md index 3217c266d9..937cad2cce 100644 --- a/quest/m1/archive/track-timeline.md +++ b/quest/m1/archive/track-timeline.md @@ -63,7 +63,7 @@ leaves two handles on one segmenter ## Required - [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md) - each timeline entry declares its segment duration (reported or estimated by the publisher), replacing the root `durationMax` -- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust +- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - `@moq/hang` publishes the same per-track timelines as Rust (it already reads them) - [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, from the reference timeline's declared duration; an overrun is listed with a warning ## Related From cb8876aa6ad9852bca5ed2b1eb6100a445afe595 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:45:21 -0700 Subject: [PATCH 42/49] fix(moq-mux): refuse enrolling a timeline whose recorder is still alive A second enrollment replaced the segmenter a live Recorder still reported into, so the two producers reset or closed each other's timeline. Enroll a name once at a time; a re-enrollment after the recorder drops still continues the numbering. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/timeline.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs index dc13b55b63..7c7430e46c 100644 --- a/rs/moq-mux/src/timeline.rs +++ b/rs/moq-mux/src/timeline.rs @@ -436,6 +436,8 @@ struct Live { segmenter: Segmenter, /// `None` once a publish failed: the timeline is an optional sidecar, so it stops instead. output: Option, + /// Whether a [`Recorder`] owns the segmenter, so a second enrollment can't share it. + recording: bool, } impl Live { @@ -497,8 +499,9 @@ impl Timelines { /// Enroll the track `name`, its records cut by `config`. /// /// Creates its timeline track, named by [`hang::timeline::default_name`], on first enrollment. - /// Enrolling a name again continues its timeline's numbering for a new producer, cut by the new - /// `config`. Errors when the timeline track name is taken or the timelines finished. + /// Enrolling a name again after its recorder dropped continues its timeline's numbering for a new + /// producer, cut by the new `config`. Errors when the name's recorder is still alive, the timeline + /// track name is taken, or the timelines finished. pub fn track(&self, name: &str, config: Config) -> crate::Result { let mut registry = self.registry.lock().unwrap(); if registry.finished { @@ -508,6 +511,10 @@ impl Timelines { // A re-enrolled track is a new producer whose group sequences may restart, so it gets a // fresh segmenter continuing the record numbering. let mut live = existing.lock().unwrap(); + if live.recording { + return Err(moq_net::Error::Duplicate.into()); + } + live.recording = true; live.segmenter.flush(); live.publish(); let sequence = live.segmenter.sequence(); @@ -522,6 +529,7 @@ impl Timelines { let live = Arc::new(Mutex::new(Live { segmenter: Segmenter::new(config), output: Some(Producer::new(track)), + recording: true, })); registry.tracks.insert(name.to_string(), (timeline, live.clone())); Ok(Recorder { live }) @@ -610,7 +618,10 @@ impl Recorder { impl Drop for Recorder { fn drop(&mut self) { - self.report(Segmenter::flush); + let mut live = self.live.lock().unwrap(); + live.segmenter.flush(); + live.publish(); + live.recording = false; } } @@ -1097,6 +1108,22 @@ mod test { ); } + /// Two live recorders would share one segmenter, each resetting or closing the other's + /// timeline, so a name is enrolled once at a time. + #[tokio::test] + async fn enrolling_an_active_track_is_refused() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut first = timelines.track("video0", Config::default()).unwrap(); + assert!(timelines.track("video0", Config::default()).is_err()); + + // The live recorder is untouched by the refusal. + first.frame(at(0), ms(0), true); + first.frame(at(1), ms(2_000), true); + drop(first); + assert!(timelines.track("video0", Config::default()).is_ok()); + } + #[test] fn a_taken_timeline_name_fails_enrollment() { let broadcast = moq_net::broadcast::Info::new().produce(); From a535c619778212956b4812abde2df057b206f4ae Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:50:39 -0700 Subject: [PATCH 43/49] fix(archive): a malformed track fails the recording only after flushing One track's malformed input returned from Writer::run at once, dropping every track's open record and in-flight commit, and an export that refused a catalog change dropped the writer mid-flight the same way. Stop every track, store what each already reported, then return the error. Control::stop ends a recording the same way without an error, which export archive uses before failing on a catalog it cannot record. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/writer.rs | 131 ++++++++++++++++++++++++++++++----- rs/moq-cli/src/archive.rs | 13 +++- 2 files changed, 125 insertions(+), 19 deletions(-) diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 1cff43db2a..7a03100e65 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -129,6 +129,7 @@ enum Command { Enroll(Box>), Cut(Timestamp), Remove(String), + Stop, } impl Writer { @@ -183,12 +184,14 @@ impl Writer { self.control.clone() } - /// Record until the source broadcast closes and every enrolled track ends. + /// Record until the source broadcast closes and every enrolled track ends, or [`Control::stop`]. /// /// Then flush each track's final record and finish its timeline. Fails when a timeline cannot - /// be committed or stored, or an enrolled track delivers a frame the recording cannot - /// represent: the recording stops at each track's last durable timeline object. Returns the - /// source's error, after finishing, when the broadcast aborted. + /// be committed or stored: the recording stops at each track's last durable timeline object. + /// Also fails when an enrolled track delivers a frame the recording cannot represent, but only + /// after stopping every track and committing what each already reported, so the recording + /// stays readable up to the bad frame. Returns the source's error, after finishing, when the + /// broadcast aborted. pub async fn run(self) -> Result<()> { let Self { control, @@ -209,14 +212,19 @@ impl Writer { let mut closed = false; // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. let mut accepting = true; - // The track the last event touched. Each event touches at most one, so the loop never scans - // the whole table. - let mut touched: Option = None; + // Every track was stopped, by `Control::stop` or malformed input, so `run` ends once flushed. + let mut stopped = false; + // Malformed source input, returned once every track flushed. + let mut failed: Option = None; + // The tracks the last event touched. An event touches at most one, except a failure, so the + // loop never scans the whole table in steady state. + let mut touched: Vec = Vec::new(); loop { - if let Some(name) = touched.take() - && let Some(track) = tracks.get_mut(&name) - { + for name in std::mem::take(&mut touched) { + let Some(track) = tracks.get_mut(&name) else { + continue; + }; if let Some(commit) = track.commit()? { commits.push(commit); } @@ -225,7 +233,7 @@ impl Writer { } } - let idle = closed && tracks.is_empty() && commits.is_empty(); + let idle = (closed || stopped) && tracks.is_empty() && commits.is_empty(); if idle && *enrolling.borrow_and_update() == 0 { // Refuse late commands, so an enrollment racing the end fails instead of vanishing. commands.close(); @@ -241,8 +249,12 @@ impl Writer { None => accepting = false, Some(Command::Enroll(mut track)) => { let subscriber = track.subscriber.take().expect("an enrolling track carries its subscriber"); + // A stopped recording takes no new content; the track just finishes its timeline. + if stopped { + track.remove(); + } reads.push(guard(track.cancelled.clone(), recv(track.name.clone(), subscriber)).boxed()); - touched = Some(track.name.clone()); + touched.push(track.name.clone()); tracks.insert(track.name.clone(), track); } Some(Command::Cut(pts)) => { @@ -253,17 +265,29 @@ impl Writer { Some(Command::Remove(name)) => { if let Some(track) = tracks.get_mut(&name) { track.remove(); - touched = Some(name); + touched.push(name); } } + Some(Command::Stop) => { + stopped = true; + touched.extend(stop(&mut tracks)); + commands.close(); + } + }, + Some(read) = reads.next(), if !reads.is_empty() => match handle(read, &mut tracks, &mut reads) { + Ok(name) => touched.extend(name), + Err(err) => { + tracing::error!(%err, "malformed source input; flushing every track and stopping"); + failed.get_or_insert(err); + stopped = true; + touched.extend(stop(&mut tracks)); + commands.close(); + } }, - Some(read) = reads.next(), if !reads.is_empty() => { - touched = handle(read, &mut tracks, &mut reads)?; - } Some((name, committer, result)) = commits.next(), if !commits.is_empty() => { if let Some(track) = tracks.get_mut(&name) { track.committer = Some(committer); - touched = Some(name); + touched.push(name); } let expired = result?; if let Some(grace) = grace && !expired.is_empty() { @@ -286,6 +310,12 @@ impl Writer { delete(&shared.store, &mut pruned, expired).await; } + if let Some(err) = failed { + return Err(err); + } + if stopped { + return Ok(()); + } // A broadcast end carries no cause. source.closed().await; Ok(()) @@ -312,6 +342,12 @@ impl Control { result } + /// Stop recording every track: store what already arrived, finish each timeline, and end + /// [`run`](Writer::run) without waiting for the source. Later commands are refused. + pub fn stop(&self) -> Result<()> { + self.send(Command::Stop) + } + /// Declare a boundary at `pts` on every enrolled track; see [`Segmenter::cut`]. pub fn cut(&self, pts: Timestamp) -> Result<()> { self.send(Command::Cut(pts)) @@ -579,6 +615,17 @@ async fn frame(name: String, mut group: Box) -> Read { Read::Frame { name, group, result } } +/// Stop reading every track, returning their names so each flushes and finishes. +fn stop(tracks: &mut HashMap>>) -> Vec { + tracks + .iter_mut() + .map(|(name, track)| { + track.remove(); + name.clone() + }) + .collect() +} + /// Resolve to [`Read::Cancelled`] once the track stops reading. async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { tokio::select! { @@ -1368,6 +1415,56 @@ mod tests { ); } + /// A malformed group fails the recording only after committing everything reported before + /// it, including the open record, so the archive stays readable up to the bad frame. + #[tokio::test] + async fn a_failed_recording_flushes_what_was_reported() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().track("video", media()).await.unwrap(); + + group(&video, 0, &[0]); + group(&video, 1, &[1000]); + // Under the minimum, so groups 1 and 2 share the record still open at the failure. + group(&video, 2, &[1500]); + group(&video, 3, &[1200]); + + assert_eq!( + writer.run().await, + Err(Error::Source("track video group 3: timestamp 1200 precedes 1500".into())) + ); + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (1, 2)]); + check_objects(&store, "video", &records).await; + } + + /// Stopping a recording whose source is still live finishes every timeline and ends `run`. + #[tokio::test] + async fn stopping_ends_without_the_source() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + group(&video, 0, &[0]); + control.stop().unwrap(); + + writer.run().await.unwrap(); + let records = window(&store, "video").await; + check_objects(&store, "video", &records).await; + assert!(control.track("audio", media()).await.is_err(), "a stopped recording refuses enrollment"); + drop(source); + } + #[tokio::test] async fn a_stalled_track_holds_only_itself() { let source = broadcast::Info::new().produce(); diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 138c40c45c..459d379d44 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -85,9 +85,18 @@ pub async fn export( notify_ready(); // The writer decides when the recording is over; the catalog only feeds it tracks. + let run = writer.run(); + tokio::pin!(run); tokio::select! { - result = writer.run() => result.context("recording failed"), - Err(err) = enroll(control, catalog) => Err(err), + result = &mut run => result.context("recording failed"), + Err(err) = enroll(control.clone(), catalog) => { + // Store what already arrived before failing, so the recording stays readable. + control.stop()?; + if let Err(stopped) = run.await { + tracing::warn!(%stopped, "recording failed while stopping"); + } + Err(err) + } } } From 1ce3e2c99e8fa6a61c5bef1e06a7ca13210074fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:54:07 -0700 Subject: [PATCH 44/49] fix(archive): follow mode discovers a track recorded after the replay started import archive --follow listed the recording's timelines once and the reader served a fixed track list, so a rendition the recording enrolled later never replayed and its groups answered Not Found. Reader::track serves and follows a new timeline, Store::timelines lists track directories instead of every object, and the follow loop enrolls whatever appeared before each refresh. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 3 +- rs/moq-archive/src/reader/mod.rs | 54 ++++++++++++++++++++---------- rs/moq-archive/src/reader/tests.rs | 25 ++++++++++++++ rs/moq-archive/src/store.rs | 28 ++++++++-------- rs/moq-cli/src/archive.rs | 13 +++++-- 5 files changed, 89 insertions(+), 34 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index c20c3d6d67..9772e184e3 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -366,7 +366,8 @@ the video it was published with. `import archive` republishes a recording: each track's timeline replays as a live track and every track's groups are served on request, one object GET per stored span. By default it replays what is stored and ends the timelines there; -`--follow 2s` keeps checking for new spans of a recording still being made. +`--follow 2s` keeps checking for new spans, and newly recorded tracks, of a +recording still being made. Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. Cloud credentials come from diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs index 3da36ac467..5dc4cc4a6b 100644 --- a/rs/moq-archive/src/reader/mod.rs +++ b/rs/moq-archive/src/reader/mod.rs @@ -25,7 +25,7 @@ mod index; -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::future::Future; use std::sync::{Arc, Mutex}; use std::task::Poll; @@ -71,6 +71,7 @@ impl Config { /// Serves an archive through a [`broadcast::Producer`] and follows its timelines. pub struct Reader { shared: Arc>, + broadcast: broadcast::Producer, dynamic: broadcast::Dynamic, timelines: Vec, } @@ -91,6 +92,8 @@ struct Shared { cache: quick_cache::sync::Cache, Weight>, /// Each track's `.info`, which is immutable, so a re-requested track costs no GET. infos: Mutex>, + /// The tracks a timeline indexes, which are the only ones served. + served: Mutex>, /// Notified whenever the index changes; holds whether the caller declared the recording ended. changed: watch::Sender, } @@ -101,17 +104,6 @@ impl Reader { /// Fails if a timeline's `.info` is missing or invalid, a timeline track already exists, or the /// store cannot list a timeline. pub async fn open(store: Store, broadcast: &broadcast::Producer, config: Config) -> Result { - let mut timelines = Vec::new(); - for (track, timeline) in config.timelines { - let info = track_info(&store.get_info(&timeline).await?)?; - timelines.push(Timeline { - track, - producer: broadcast.create_track(timeline, info)?, - decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), - cursor: None, - }); - } - let items = usize::try_from(config.cache / (1024 * 1024)) .unwrap_or(usize::MAX) .max(16); @@ -120,18 +112,47 @@ impl Reader { index: Mutex::default(), cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), infos: Mutex::default(), + served: Mutex::default(), changed: watch::Sender::new(false), }); let mut reader = Self { shared, + broadcast: broadcast.clone(), dynamic: broadcast.dynamic(), - timelines, + timelines: Vec::new(), }; + for (track, timeline) in config.timelines { + reader.add(track, timeline).await?; + } reader.refresh().await?; Ok(reader) } + /// Serve and follow `track`, whose timeline is the track `timeline`, such as one a recording + /// enrolled after this reader opened. Its timeline replays on the next [`refresh`](Self::refresh). + /// + /// Fails if the timeline's `.info` is missing or invalid, or `track` is already served. + pub async fn track(&mut self, track: &str, timeline: &str) -> Result<()> { + self.add(track.to_string(), timeline.to_string()).await + } + + async fn add(&mut self, track: String, timeline: String) -> Result<()> { + if self.shared.served.lock().unwrap().contains(&track) { + return Err(moq_net::Error::Duplicate.into()); + } + let info = track_info(&self.shared.store.get_info(&timeline).await?)?; + let producer = self.broadcast.create_track(timeline, info)?; + self.shared.served.lock().unwrap().insert(track.clone()); + self.timelines.push(Timeline { + track, + producer, + decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), + cursor: None, + }); + Ok(()) + } + /// Replay timeline objects committed since the last refresh. /// /// Lists each timeline's keys after its last replayed segment, then GETs them in segment order. @@ -165,8 +186,7 @@ impl Reader { where T: 'static, { - let tracks = self.timelines.iter().map(|timeline| timeline.track.clone()).collect(); - serve(self.shared.clone(), self.dynamic.clone(), tracks) + serve(self.shared.clone(), self.dynamic.clone()) } } @@ -259,13 +279,13 @@ impl Timeline { } /// Serve requested tracks until the broadcast closes. -async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic, tracks: Vec) { +async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic) { let mut serving = kio::Tasks::new(); kio::wait(|waiter| { loop { match dynamic.poll_requested_track(waiter) { Poll::Ready(Ok(request)) => { - let known = tracks.iter().any(|track| track == request.name()); + let known = shared.served.lock().unwrap().contains(request.name()); serving.push(task(Box::pin(serve_track(shared.clone(), request, known)))) } Poll::Ready(Err(_)) => return Poll::Ready(()), diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index 2dacaa4dd7..533edba507 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -461,6 +461,31 @@ async fn refresh_follows_new_records_and_pops() { assert_eq!(archive.store.inner().gets(), Vec::::new()); } +/// A track the recording enrolled after the reader opened is found by listing the store again, +/// then served and followed like the rest. +#[tokio::test] +async fn a_track_enrolled_after_opening_is_followed() { + let mut archive = Archive::new().await; + archive.add("audio", 0..2, 1).await; + + let (broadcast, mut reader) = open(&archive).await; + not_found(fetch(&broadcast, "video", 0, 0).await); + + archive.add("video", 0..2, 2).await; + let timelines = archive.store.timelines().await.unwrap(); + assert_eq!(timelines.keys().collect::>(), ["audio", "video"]); + reader.track("video", &timelines["video"]).await.unwrap(); + assert!(reader.track("video", &timelines["video"]).await.is_err(), "a served track is refused"); + reader.refresh().await.unwrap(); + + assert_eq!( + fetch(&broadcast, "video", 1, 0).await.unwrap(), + expected("video", 1, 0..2) + ); + let timeline = broadcast.consume().track(&timeline("video")).unwrap(); + assert!(timeline.subscribe(None).await.is_ok(), "the late timeline replays"); +} + #[tokio::test] async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { let mut archive = Archive::new().await; diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 97cfe04590..83400fb9bb 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -2,7 +2,7 @@ use std::num::NonZeroUsize; use bytes::Bytes; use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; +use futures::StreamExt; use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; use object_store::path::Path; use object_store::{ListResult, ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutPayload}; @@ -191,24 +191,24 @@ impl Store { Object::decode(self.get_bytes(&path).await?) } - /// Each recorded track's timeline track, found by listing the `.info` objects a [`Writer`] - /// creates: a timeline is named by [`hang::timeline::default_name`]. + /// Each recorded track's timeline track, found by listing the track directories a [`Writer`] + /// creates: a timeline is named by [`hang::timeline::default_name`]. One listing of the + /// prefix's directories, so following a long recording doesn't list its objects. /// /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. /// /// [`Writer`]: crate::Writer pub async fn timelines(&self) -> Result> { - let entries: Vec = self.list(&Query::new()).try_collect().await?; - Ok(entries - .into_iter() - .filter_map(|entry| match entry.key { - Key::Info { track } => { - let indexed = track.strip_suffix(hang::timeline::SUFFIX)?.to_string(); - Some((indexed, track)) - } - _ => None, - }) - .collect()) + let listed = self.inner.list_with_delimiter(self.list_path(None).as_ref()).await?; + let mut timelines = std::collections::BTreeMap::new(); + for directory in listed.common_prefixes { + let encoded = directory.filename().ok_or_else(|| Error::Directory(directory.to_string()))?; + let track = crate::path::decode_track(encoded)?; + if let Some(indexed) = track.strip_suffix(hang::timeline::SUFFIX) { + timelines.insert(indexed.to_string(), track); + } + } + Ok(timelines) } /// Delete the object at `key`. diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 459d379d44..32e52354a6 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -106,12 +106,14 @@ pub async fn export( /// request. pub async fn import(origin: moq_net::origin::Producer, name: String, args: ImportArgs) -> anyhow::Result<()> { let store = open(&args.store)?; + // A second handle lists the recording's timelines while the reader owns the first. + let listing = open(&args.store)?; let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; - let timelines = store + let mut timelines = store .timelines() .await .with_context(|| format!("failed to list the recording at {}", args.store))?; - let config = moq_archive::reader::Config::new(timelines); + let config = moq_archive::reader::Config::new(timelines.clone()); let mut reader = moq_archive::Reader::open(store, &broadcast, config) .await .with_context(|| format!("no readable recording at {}", args.store))?; @@ -132,6 +134,13 @@ pub async fn import(origin: moq_net::origin::Producer, name: String, args: Impor let refresh = async { loop { tokio::time::sleep(interval).await; + // A rendition the recording enrolled after the replay started. + for (track, timeline) in listing.timelines().await? { + if !timelines.contains_key(&track) { + reader.track(&track, &timeline).await?; + timelines.insert(track, timeline); + } + } reader.refresh().await?; } }; From a80f9129fe0130ff41919332fce458f3219b83d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 00:25:01 -0700 Subject: [PATCH 45/49] fix(moq-hls)!: segment URLs name their reference, and listed segments stay put - A segment URL carries a tag of the reference rendition whose records number it (seg/[{generation}.]{reference}.{segment}.m4s, and .t{pts} for DASH), so a reference switch starts a new numbering under new URLs instead of reusing old ones for other content. Recording cursors follow the new numbering. - A rendition's own records are trimmed to the oldest listed segment rather than its own newest content, so one running ahead of a stalled reference keeps resolving every listed segment. - A playlist ends (EXT-X-ENDLIST, DASH static) only once the rendition resolved every row, so a slower timeline's tail is listed first. Co-Authored-By: Claude Opus 5.5 --- doc/bin/hls.md | 12 +- rs/moq-cli/src/archive.rs | 6 +- rs/moq-hls/src/export/archive_tests.rs | 38 +++---- rs/moq-hls/src/export/mod.rs | 150 ++++++++++++++++++------- rs/moq-hls/src/export/mpd.rs | 15 ++- rs/moq-hls/src/export/playlist.rs | 22 ++-- rs/moq-hls/src/export/rendition.rs | 70 +++++++++--- rs/moq-hls/src/export/renditions.rs | 2 +- rs/moq-hls/src/export/segments.rs | 94 ++++++++++++---- rs/moq-hls/src/export/spans.rs | 24 ++-- rs/moq-hls/src/server/mod.rs | 6 +- rs/moq-hls/src/server/routes.rs | 94 ++++++++++------ 12 files changed, 365 insertions(+), 168 deletions(-) diff --git a/doc/bin/hls.md b/doc/bin/hls.md index a3af52f2ea..c1b86864fa 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -43,10 +43,16 @@ broadcast by path: /{broadcast}/manifest.mpd /{broadcast}/{video|audio}/{rendition}/media.m3u8 /{broadcast}/{video|audio}/{rendition}/init.{hash}.mp4 -/{broadcast}/{video|audio}/{rendition}/seg/{segment}.m4s -/{broadcast}/{video|audio}/{rendition}/seg/t{pts}.m4s +/{broadcast}/{video|audio}/{rendition}/seg/{reference}.{segment}.m4s +/{broadcast}/{video|audio}/{rendition}/seg/{reference}.t{pts}.m4s ``` +Segment boundaries come from one reference rendition's timeline, and its records +number the segments. `{reference}` is a short hash of that rendition's kind and +name, so every edge derives the same URL, and a reference that changes (a new +first video rendition) starts a new numbering under new URLs rather than reusing +the old ones for other content. + A [`moq-archive`](https://docs.rs/moq-archive) recording replayed through its `Reader` is served the same way, with no second stored copy. Playlists come from the replayed timelines alone, and a segment GETs only its rendition's @@ -63,7 +69,7 @@ declares the recording finished; the store holds no completion marker. The init URL carries a hash of its bytes, so a reconfigured rendition gets a new one. An embedder of the library can also label the publisher's run with `Broadcaster::set_generation`. Every segment URL then carries it -(`seg/{generation}.{segment}.m4s`), since a restarted publisher reuses segment +(`seg/{generation}.{reference}.{segment}.m4s`), since a restarted publisher reuses segment numbers for different media. `--window` sets the live playlist duration (default 16 s) and caps segment diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 32e52354a6..56d462d6e1 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -136,9 +136,9 @@ pub async fn import(origin: moq_net::origin::Producer, name: String, args: Impor tokio::time::sleep(interval).await; // A rendition the recording enrolled after the replay started. for (track, timeline) in listing.timelines().await? { - if !timelines.contains_key(&track) { - reader.track(&track, &timeline).await?; - timelines.insert(track, timeline); + if let std::collections::btree_map::Entry::Vacant(entry) = timelines.entry(track) { + reader.track(entry.key(), &timeline).await?; + entry.insert(timeline); } } reader.refresh().await?; diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 2981880576..8e69bf81dc 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -405,7 +405,7 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { ] { let playlist = replay.playlist(kind, name).await; for segment in 0..3 { - let line = format!("seg/{segment}.m4s\n"); + let line = format!("seg/29b1cd67.{segment}.m4s\n"); assert_eq!(playlist.contains(&line), segment < listed, "{name}: {playlist}"); } assert!(!playlist.contains("#EXT-X-ENDLIST"), "no finality was supplied"); @@ -422,7 +422,7 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { let lists = recording.store.inner().lists(); // Switching renditions downloads only the selected rendition's object. - let low = replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); + let low = replay.rendition(Kind::Video, "360p").listed_segment(1).await.unwrap().unwrap(); assert_eq!(&low[4..8], b"moof"); assert_eq!( recording.gets(), @@ -430,7 +430,7 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { ); let high = replay .rendition(Kind::Video, "1080p") - .segment(2) + .listed_segment(2) .await .unwrap() .unwrap(); @@ -443,7 +443,7 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { // An audio segment spans four groups but still costs one object GET. let audio = replay .rendition(Kind::Audio, "audio") - .segment(1) + .listed_segment(1) .await .unwrap() .unwrap(); @@ -454,7 +454,7 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { ); // A repeated request hits the reader's cache, including the immutable `.info`. - replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); + replay.rendition(Kind::Video, "360p").listed_segment(1).await.unwrap().unwrap(); assert_eq!(recording.gets(), Vec::::new()); assert_eq!(recording.store.inner().lists(), lists, "segments never list"); } @@ -470,7 +470,7 @@ async fn a_bounded_cache_rereads_evicted_objects() { // Each of the segment's four groups misses the cache and GETs the same object again. let audio = replay .rendition(Kind::Audio, "audio") - .segment(1) + .listed_segment(1) .await .unwrap() .unwrap(); @@ -497,24 +497,24 @@ async fn missing_rendition_segments_are_gaps_and_time_jumps_are_discontinuities( .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) .await; let expected = concat!( - "#EXTINF:2.00000,\nseg/0.m4s\n", - "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/1.m4s\n", - "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n", + "#EXTINF:2.00000,\nseg/29b1cd67.0.m4s\n", + "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/29b1cd67.1.m4s\n", + "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/29b1cd67.2.m4s\n", ); assert!(low.contains(expected), "{low}"); let high = replay.playlist(Kind::Video, "1080p").await; assert!(!high.contains("#EXT-X-GAP"), "{high}"); assert!( - high.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n"), + high.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/29b1cd67.2.m4s\n"), "{high}" ); // A gap is never fetched. recording.gets(); let rendition = replay.rendition(Kind::Video, "360p"); - assert!(rendition.segment(1).await.unwrap().is_none()); + assert!(rendition.listed_segment(1).await.unwrap().is_none()); assert!(!recording.gets().iter().any(|path| is_media(path))); - let after = rendition.segment(2).await.unwrap().unwrap(); + let after = rendition.listed_segment(2).await.unwrap().unwrap(); assert_eq!(&after[4..8], b"moof"); } @@ -530,10 +530,10 @@ async fn a_growing_recording_ends_only_on_caller_finality() { let mut reader = replay.reader.take().unwrap(); reader.refresh().await.unwrap(); let playlist = replay - .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/3.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/29b1cd67.3.m4s\n")) .await; assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); - assert!(!playlist.contains("seg/0.m4s"), "{playlist}"); + assert!(!playlist.contains("seg/29b1cd67.0.m4s"), "{playlist}"); assert!( !playlist.contains("#EXT-X-ENDLIST"), "a recording without finality stays live" @@ -549,7 +549,7 @@ async fn a_growing_recording_ends_only_on_caller_finality() { let playlist = replay .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("#EXT-X-ENDLIST")) .await; - assert!(playlist.contains("seg/3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); + assert!(playlist.contains("seg/29b1cd67.3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); } /// A durable timeline lists the whole recording past the default 16s window, and DASH offers @@ -560,10 +560,10 @@ async fn a_durable_timeline_lists_past_the_window() { let recording = segments(12).await; let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; let playlist = replay - .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/11.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/29b1cd67.11.m4s\n")) .await; assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0\n"), "{playlist}"); - assert!(playlist.contains("seg/0.m4s\n"), "{playlist}"); + assert!(playlist.contains("seg/29b1cd67.0.m4s\n"), "{playlist}"); for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { replay.rendition(kind, name).init().await.unwrap(); @@ -576,8 +576,8 @@ async fn a_durable_timeline_lists_past_the_window() { for archive in [live(), elsewhere] { let live = Replay::open(&recording, 64 * 1024 * 1024, archive).await; let playlist = live - .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/11.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/29b1cd67.11.m4s\n")) .await; - assert!(!playlist.contains("seg/0.m4s\n"), "{playlist}"); + assert!(!playlist.contains("seg/29b1cd67.0.m4s\n"), "{playlist}"); } } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 2dcaf26f0f..7b603aa583 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -161,7 +161,7 @@ impl Broadcaster { /// broadcast carries now, or stop versioning with `None` (the default). /// /// A restarted publisher is spliced into the same broadcast and restarts its segment - /// numbers, so `seg/0.m4s` would name different bytes on each run. Supply a new generation + /// numbers, so `seg/{reference}.0.m4s` would name different bytes on each run. Supply a new generation /// before the new run's media flows. Replacing one drops every listed segment and every /// init built from the old run's media, while the first only labels the run already flowing. A segment URL carrying any other generation /// is refused. Init URLs need none: they carry a hash of their bytes. @@ -1079,7 +1079,7 @@ mod tests { let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let unversioned = rendition.media_playlist(None).expect("playable"); - assert!(unversioned.contains("\nseg/0.m4s\n")); + assert!(unversioned.contains("\nseg/656e3d1b.0.m4s\n")); assert!(matches!( broadcaster.set_generation(Some("run.1")), @@ -1091,15 +1091,15 @@ mod tests { let labeled = rendition .media_playlist(None) .expect("the first generation keeps the rows"); - assert!(labeled.contains("\nseg/run-1.0.m4s\n")); - assert!(labeled.contains("\nseg/run-1.1.m4s\n")); + assert!(labeled.contains("\nseg/run-1.656e3d1b.0.m4s\n")); + assert!(labeled.contains("\nseg/run-1.656e3d1b.1.m4s\n")); assert_eq!( init_hash(&labeled), init_hash(&unversioned), "the init needs no generation" ); let manifest = broadcaster.manifest(None).expect("manifest"); - assert!(manifest.contains("media=\"video/video0/seg/run-1.t$Time$.m4s\"")); + assert!(manifest.contains("media=\"video/video0/seg/run-1.656e3d1b.t$Time$.m4s\"")); broadcaster.set_generation(Some("run-2")).unwrap(); assert!( @@ -1110,7 +1110,7 @@ mod tests { media.write(vp8_frame(6_000_000, true)).unwrap(); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; let restarted = rendition.media_playlist(None).expect("the new run lists"); - assert!(restarted.contains("\nseg/run-2.2.m4s\n")); + assert!(restarted.contains("\nseg/run-2.656e3d1b.2.m4s\n")); assert!(!restarted.contains("run-1")); broadcaster.set_generation(None).unwrap(); @@ -1234,22 +1234,22 @@ mod tests { let rendered = rendition.media_playlist(None).expect("playable"); let hash = init_hash(&rendered); - assert!(rendered.contains("seg/0.m4s\n")); - assert!(rendered.contains("seg/1.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.0.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.1.m4s\n")); assert_eq!(rendition.init_versioned(hash).await.unwrap(), Some(init)); assert!(rendition.init_versioned("0000000000000000").await.unwrap().is_none()); // The same render, but carrying a credential into every child URL. let signed = rendition.media_playlist(Some("jwt=abc.def")).expect("playable"); assert!(signed.contains(&format!("#EXT-X-MAP:URI=\"init.{hash}.mp4?jwt=abc.def\"\n"))); - assert!(signed.contains("seg/0.m4s?jwt=abc.def\n")); + assert!(signed.contains("seg/656e3d1b.0.m4s?jwt=abc.def\n")); - let segment = rendition.segment(0).await.unwrap().expect("segment fetched on demand"); + let segment = rendition.listed_segment(0).await.unwrap().expect("segment fetched on demand"); assert_eq!(&segment[4..8], b"moof", "a fetched group transmuxes to moof+mdat"); // The live-edge group isn't a segment yet, and unknown groups miss. - assert!(rendition.segment(2).await.unwrap().is_none()); - assert!(rendition.segment(99).await.unwrap().is_none()); + assert!(rendition.listed_segment(2).await.unwrap().is_none()); + assert!(rendition.listed_segment(99).await.unwrap().is_none()); // Keep the publisher alive for the whole test. drop((media, registration, broadcast)); @@ -1289,7 +1289,7 @@ mod tests { .expect("rendition discovered from the catalog"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; - let segment = rendition.segment(0).await.unwrap().expect("segment fetched on demand"); + let segment = rendition.listed_segment(0).await.unwrap().expect("segment fetched on demand"); assert_eq!(&segment[4..8], b"moof"); drop((media, registration, broadcast)); } @@ -1358,7 +1358,7 @@ mod tests { ); assert!(manifest.contains("id=\"video/video0\"")); assert!(manifest.contains("id=\"audio/audio0\"")); - assert!(manifest.contains("media=\"video/video0/seg/t$Time$.m4s\"")); + assert!(manifest.contains("media=\"video/video0/seg/656e3d1b.t$Time$.m4s\"")); // Both renditions list the same aligned segment spans (the live edge is not listed). assert_eq!(manifest.matches("").count(), 2); assert_eq!(manifest.matches("").count(), 2); @@ -1369,17 +1369,17 @@ mod tests { let hash = between(&signed, "initialization=\"video/video0/init.", ".mp4?jwt=abc.def\""); let init = video_rendition.init().await.unwrap().expect("init segment"); assert_eq!(video_rendition.init_versioned(hash).await.unwrap(), Some(init)); - assert!(signed.contains("media=\"video/video0/seg/t$Time$.m4s?jwt=abc.def\"")); + assert!(signed.contains("media=\"video/video0/seg/656e3d1b.t$Time$.m4s?jwt=abc.def\"")); // $Time$ resolves to the same bytes the aligned number does; unknown times miss. let by_time = video_rendition - .segment_at(2_000) + .listed_segment_at(2_000) .await .unwrap() .expect("segment fetched by pts"); - let by_number = video_rendition.segment(1).await.unwrap().expect("segment by number"); + let by_number = video_rendition.listed_segment(1).await.unwrap().expect("segment by number"); assert_eq!(by_time, by_number); - assert!(video_rendition.segment_at(999).await.unwrap().is_none()); + assert!(video_rendition.listed_segment_at(999).await.unwrap().is_none()); drop((video, audio, video_registration, audio_registration, broadcast)); } @@ -1681,7 +1681,7 @@ mod tests { let snapped = broadcaster .rendition(Kind::Video, "video1") .unwrap() - .segment(1) + .listed_segment(1) .await .unwrap() .expect("the snapped segment is served"); @@ -1694,7 +1694,7 @@ mod tests { "nothing to serve before the rendition starts" ); let video2 = broadcaster.rendition(Kind::Video, "video2").unwrap(); - assert!(video2.segment(0).await.unwrap().is_none(), "a gap is never fetched"); + assert!(video2.listed_segment(0).await.unwrap().is_none(), "a gap is never fetched"); drop((producers, registrations, broadcast)); } @@ -1745,7 +1745,7 @@ mod tests { vec![0, 1, 2] ); assert_eq!(playlist.segments[0].duration, Duration::from_secs(2)); - let segment = rendition.segment(1).await.unwrap().expect("audio segment"); + let segment = rendition.listed_segment(1).await.unwrap().expect("audio segment"); assert_eq!(&segment[4..8], b"moof"); drop((audio, registration, broadcast)); @@ -1868,13 +1868,13 @@ mod tests { // The same URI names the same span of content time on either rendition. let rendered = audio_rendition.media_playlist(None).expect("playable"); - assert!(rendered.contains("seg/0.m4s\n")); - assert!(rendered.contains("seg/1.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.0.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.1.m4s\n")); // A video segment is one group; the matching audio segment transmuxes its four groups. - let video_segment = video_rendition.segment(1).await.unwrap().expect("video segment"); + let video_segment = video_rendition.listed_segment(1).await.unwrap().expect("video segment"); assert_eq!(&video_segment[4..8], b"moof"); - let audio_segment = audio_rendition.segment(1).await.unwrap().expect("audio segment"); + let audio_segment = audio_rendition.listed_segment(1).await.unwrap().expect("audio segment"); assert_eq!(&audio_segment[4..8], b"moof"); assert!( audio_segment.len() > video_segment.len(), @@ -2040,7 +2040,7 @@ mod tests { let (rendition, watcher) = export(&upstream, &config); let _ = tokio::time::timeout(Duration::from_secs(1), rendition.playable()).await; - let served = rendition.segment(0).await.unwrap(); + let served = rendition.listed_segment(0).await.unwrap(); if let Some(served) = &served { assert!( !contains(served, NEW), @@ -2066,7 +2066,7 @@ mod tests { .await .expect("the new publisher's timeline arrives"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the new publisher is servable on its own catalog"); @@ -2151,7 +2151,7 @@ mod tests { settle().await; let served = rendition - .segment(0) + .listed_segment(0) .await .expect("a replaced sibling is an unavailable group, not a server error"); if let Some(served) = &served { @@ -2179,7 +2179,7 @@ mod tests { .await .expect("the timeline reaches the fresh rendition"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the new publisher is servable on a freshly bound rendition"); @@ -2299,7 +2299,7 @@ mod tests { .await .expect("the timeline reaches the rendition"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the original sibling is servable"); @@ -2311,7 +2311,7 @@ mod tests { drop((old_server, old_media, _old_track)); until_empty(&rendition).await; assert!( - rendition.segment(0).await.unwrap().is_none(), + rendition.listed_segment(0).await.unwrap().is_none(), "a row listed for the old publisher answers 404 after replacement" ); @@ -2353,7 +2353,7 @@ mod tests { .expect("a non-gap row after the new bind") .segment; let served = rendition - .segment(listed) + .listed_segment(listed) .await .unwrap() .expect("the rebound sibling is servable"); @@ -2372,7 +2372,7 @@ mod tests { "cursor creation cannot reset the baseline" ); assert!( - rendition.segment(0).await.unwrap().is_none(), + rendition.listed_segment(0).await.unwrap().is_none(), "a row listed for the old publisher is not served from the replacement" ); watcher.abort(); @@ -2410,7 +2410,7 @@ mod tests { .await .expect("the timeline reaches the rendition"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the sibling present at export start is servable"); @@ -2451,13 +2451,13 @@ mod tests { tokio::time::timeout(Duration::from_secs(5), rendition.playable()) .await .expect("the timeline reaches the rendition"); - assert!(rendition.segment(0).await.unwrap().is_some()); + assert!(rendition.listed_segment(0).await.unwrap().is_some()); drop((old_server, old_media, _old_track)); until_empty(&rendition).await; for _ in 0..4 { assert!(rendition.snapshot().segments.is_empty()); - assert!(rendition.segment(0).await.unwrap().is_none()); + assert!(rendition.listed_segment(0).await.unwrap().is_none()); } let new_server = origin.dynamic("media", sibling_route(11)).unwrap(); @@ -2489,12 +2489,12 @@ mod tests { .expect("a non-gap row after the new bind") .segment; let served = rendition - .segment(listed) + .listed_segment(listed) .await .unwrap() .expect("the rebound sibling is servable after an unroutable gap"); assert!(contains(&served, NEW)); - assert!(rendition.segment(0).await.unwrap().is_none()); + assert!(rendition.listed_segment(0).await.unwrap().is_none()); watcher.abort(); drop((new_media, _new_track, new_server, catalog, live)); } @@ -2840,10 +2840,82 @@ mod tests { cut(track, video0); assert_eq!(numbers(&test.finished("video0").await), [0, 1, 2, 3]); - test.finished("video1").await; + // video1's timeline hasn't passed the last segment yet, so ending its playlist now would + // stop players before that segment is listed. + let pending = test.rendition("video1").snapshot(); + assert_eq!(numbers(&pending), [0, 1, 2]); + assert!(!pending.finished, "a pending tail holds back EXT-X-ENDLIST"); + video1.push(&gop(4)).unwrap(); + assert_eq!(numbers(&test.finished("video1").await), [0, 1, 2, 3]); test.drain("video0").await; } + // A rendition running far ahead of a stalled reference keeps the records its listed segments + // resolve to, so a listed segment never turns into a gap. + #[tokio::test(start_paused = true)] + async fn a_rendition_ahead_of_the_reference_keeps_listed_segments() { + let test = Failing::new(); + let (_track0, mut video0) = test.publish("video0"); + let (_track1, mut video1) = test.publish("video1"); + // The reference stalls at 20s while video1 runs on to 80s, far past the 16s window. + for sequence in 0..10 { + video0.push(&gop(sequence)).unwrap(); + } + for sequence in 0..40 { + video1.push(&gop(sequence)).unwrap(); + } + let listed = test.listed("video1", 9).await; + assert!(!listed.segments.is_empty()); + assert!( + listed.segments.iter().all(|s| !s.gap), + "every listed segment still resolves: {:?}", + numbers(&listed) + ); + } + + // A new reference numbers segments from its own records, so its URLs carry its own tag and + // never reuse one the old reference listed for other content. + #[tokio::test(start_paused = true)] + async fn a_reference_switch_never_reuses_a_segment_url() { + let mut test = Failing::new(); + let (_track0, mut video0) = test.publish("video0"); + let (_track1, mut video1) = test.publish("video1"); + for sequence in 0..4 { + video0.push(&gop(sequence)).unwrap(); + } + // video1's records are numbered from 0 but start 3s later, so its segment 0 is other content. + for sequence in 0..4 { + video1 + .push(&hang::timeline::Record::new( + sequence, + sequence * 2_000 + 3_000, + 2_000, + hang::timeline::Position::group(sequence), + hang::timeline::Position::group(sequence + 1), + )) + .unwrap(); + } + let before = test.listed("video0", 2).await; + let old = segments::tag(&(Kind::Video, "video0".to_string())); + assert!(before.segments.iter().all(|s| s.tag == old)); + + // Switch the reference the way `watch_catalog` does. + test.watcher.abort(); + test.renditions.fanout().skip(); + test.watcher = tokio::spawn(watch_timeline( + test.broadcast.consume(), + archive(&["video0", "video1"]), + Arc::new((Kind::Video, "video1".to_string())), + test.renditions.fanout(), + )); + let after = test.listed("video1", 2).await; + let new = segments::tag(&(Kind::Video, "video1".to_string())); + assert_eq!(numbers(&after)[0], 0, "the new reference numbers from its own records"); + assert!(after.segments.iter().all(|s| s.tag == new && s.tag != old)); + let video0 = test.rendition("video0"); + assert!(video0.segment(&old, 0).await.unwrap().is_none(), "the old URL is no longer served"); + } + // A malformed reference timeline fails the same way. #[tokio::test(start_paused = true)] async fn a_malformed_reference_timeline_ends_every_playlist() { diff --git a/rs/moq-hls/src/export/mpd.rs b/rs/moq-hls/src/export/mpd.rs index e90bf22ebf..6a076dd2bc 100644 --- a/rs/moq-hls/src/export/mpd.rs +++ b/rs/moq-hls/src/export/mpd.rs @@ -50,7 +50,9 @@ pub(crate) struct Representation { pub ended: bool, /// The init segment's content hash (`init.{init}.mp4`). pub init: String, - /// The publisher run every segment URL carries (`seg/{generation}.t$Time$.m4s`). + /// The [`tag`](super::segments::tag) of the reference numbering the segments. + pub tag: String, + /// The publisher run every segment URL carries (`seg/{generation}.{tag}.t$Time$.m4s`). pub generation: Option>, } @@ -141,6 +143,7 @@ fn render_representation(out: &mut String, rep: &Representation, suffix: &str) { let kind = rep.kind.as_str(); let name = escape(&rep.name); let init = escape(&rep.init); + let tag = escape(&rep.tag); let generation = rep .generation .as_deref() @@ -172,7 +175,7 @@ fn render_representation(out: &mut String, rep: &Representation, suffix: &str) { let _ = writeln!( out, - " ", + " ", rep.timescale.max(1) ); let _ = writeln!(out, " "); @@ -303,6 +306,7 @@ mod tests { segments, ended, init: "0123abcd".into(), + tag: "ab12cd34".into(), generation: None, } } @@ -322,6 +326,7 @@ mod tests { segments, ended, init: "4567cdef".into(), + tag: "ab12cd34".into(), generation: None, } } @@ -359,7 +364,7 @@ mod tests { "" )); assert!(out.contains( - "" + "" )); assert!(out.contains("")); assert!(out.contains("")); @@ -403,7 +408,7 @@ mod tests { let out = render_manifest(&manifest, Some("jwt=abc.def&x=1")); assert!(out.contains("initialization=\"video/video0/init.0123abcd.mp4?jwt=abc.def&x=1\"")); - assert!(out.contains("media=\"video/video0/seg/t$Time$.m4s?jwt=abc.def&x=1\"")); + assert!(out.contains("media=\"video/video0/seg/ab12cd34.t$Time$.m4s?jwt=abc.def&x=1\"")); } #[test] @@ -421,7 +426,7 @@ mod tests { let out = render_manifest(&manifest, None); assert!(out.contains("initialization=\"video/video0/init.0123abcd.mp4\"")); - assert!(out.contains("media=\"video/video0/seg/run-7.t$Time$.m4s\"")); + assert!(out.contains("media=\"video/video0/seg/run-7.ab12cd34.t$Time$.m4s\"")); } #[test] diff --git a/rs/moq-hls/src/export/playlist.rs b/rs/moq-hls/src/export/playlist.rs index 00f833bd20..64eebbc460 100644 --- a/rs/moq-hls/src/export/playlist.rs +++ b/rs/moq-hls/src/export/playlist.rs @@ -30,14 +30,16 @@ pub(crate) struct Snapshot { /// Wall-clock time of the first listed segment (`EXT-X-PROGRAM-DATE-TIME`), when the /// catalog advertises its root broadcast clock. pub program_date_time: Option, - /// The publisher run every segment URL carries (`seg/{generation}.{segment}.m4s`). + /// The publisher run every segment URL carries (`seg/{generation}.{tag}.{segment}.m4s`). pub generation: Option>, } /// One listed segment. pub(crate) struct Segment { - /// The aligned segment number; the URI is `seg/{segment}.m4s`. + /// The aligned segment number; the URI is `seg/{tag}.{segment}.m4s`. pub segment: u64, + /// The [`tag`](super::segments::tag) of the reference numbering the segment. + pub tag: String, /// `EXTINF` duration. pub duration: Duration, /// The rendition has no content for this span (`EXT-X-GAP`): the segment keeps its slot in @@ -91,7 +93,7 @@ pub(crate) fn render_media(snapshot: &Snapshot, init: &str, query: Option<&str>) let _ = writeln!(out, "#EXT-X-GAP"); } let _ = writeln!(out, "#EXTINF:{:.5},", segment.duration.as_secs_f64()); - let _ = writeln!(out, "seg/{generation}{}.m4s{suffix}", segment.segment); + let _ = writeln!(out, "seg/{generation}{}.{}.m4s{suffix}", segment.tag, segment.segment); } if snapshot.finished { @@ -115,12 +117,14 @@ mod tests { segments: vec![ Segment { segment: 10, + tag: "ab12cd34".into(), duration: Duration::from_secs(2), gap: false, discontinuity: false, }, Segment { segment: 11, + tag: "ab12cd34".into(), duration: Duration::from_millis(1960), gap: false, discontinuity: false, @@ -137,15 +141,15 @@ mod tests { assert!(out.contains("#EXT-X-MEDIA-SEQUENCE:10\n")); assert!(out.contains("#EXT-X-MAP:URI=\"init.0123abcd.mp4\"\n")); assert!(out.contains("#EXT-X-PROGRAM-DATE-TIME:2025-07-07T00:00:00.123Z\n")); - assert!(out.contains("#EXTINF:2.00000,\nseg/10.m4s\n")); - assert!(out.contains("#EXTINF:1.96000,\nseg/11.m4s\n")); + assert!(out.contains("#EXTINF:2.00000,\nseg/ab12cd34.10.m4s\n")); + assert!(out.contains("#EXTINF:1.96000,\nseg/ab12cd34.11.m4s\n")); assert!(!out.contains("#EXT-X-ENDLIST")); // A credential rides every child URL so a header-less player keeps sending it. let signed = render_media(&snapshot, "0123abcd", Some("jwt=abc.def")); assert!(signed.contains("#EXT-X-MAP:URI=\"init.0123abcd.mp4?jwt=abc.def\"\n")); - assert!(signed.contains("\nseg/10.m4s?jwt=abc.def\n")); - assert!(signed.contains("\nseg/11.m4s?jwt=abc.def\n")); + assert!(signed.contains("\nseg/ab12cd34.10.m4s?jwt=abc.def\n")); + assert!(signed.contains("\nseg/ab12cd34.11.m4s?jwt=abc.def\n")); } #[test] @@ -155,6 +159,7 @@ mod tests { media_sequence: 0, segments: vec![Segment { segment: 0, + tag: "ab12cd34".into(), duration: Duration::from_secs(2), gap: false, discontinuity: false, @@ -166,7 +171,7 @@ mod tests { let out = render_media(&snapshot, "0123abcd", Some("jwt=x")); assert!(out.contains("#EXT-X-MAP:URI=\"init.0123abcd.mp4?jwt=x\"\n")); - assert!(out.contains("\nseg/run-7.0.m4s?jwt=x\n")); + assert!(out.contains("\nseg/run-7.ab12cd34.0.m4s?jwt=x\n")); } #[test] @@ -176,6 +181,7 @@ mod tests { media_sequence: 0, segments: vec![Segment { segment: 0, + tag: "ab12cd34".into(), duration: Duration::from_secs(4), gap: false, discontinuity: false, diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 16518b207f..01b7075996 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -367,10 +367,10 @@ impl Rendition { }) } - /// Start following this rendition's own timeline, trimmed to `window` like the segments. + /// Start following this rendition's own timeline, trimmed to the listed segments. /// /// A rendition the catalog indexes no timeline for resolves nothing but the reference's rows. - pub(crate) fn watch(&self, window: Option) { + pub(crate) fn watch(&self) { let Some(_) = self.section.timelines.get(&self.name) else { self.spans.end(); return; @@ -380,7 +380,6 @@ impl Rendition { self.section.clone(), self.name.clone(), self.spans.clone(), - window, )); *self.watcher.lock().expect("watcher lock poisoned") = Some(watcher); } @@ -410,6 +409,14 @@ impl Rendition { discontinuity, }; self.live.push(row, window); + self.trim(); + } + + /// Drop this rendition's own records that no listed segment can resolve to anymore. + fn trim(&self) { + if let Some(oldest) = self.live.window().segments.first() { + self.spans.trim(oldest.pts.into()); + } } /// What `row` holds on this rendition: the reference serves its own record, and every other @@ -442,6 +449,7 @@ impl Rendition { /// Remove source timeline records in `range` from this rendition's window. pub(crate) fn pop(&self, range: std::ops::Range) { self.live.pop(range); + self.trim(); } /// Clear rows that can no longer be followed by a consecutive source timeline record. @@ -574,12 +582,13 @@ impl Rendition { let program_date_time = window.segments.first().and_then(|first| self.wall_clock(first.pts)); let mut previous: Option = None; - let segments = self + let segments: Vec = self .resolved(&window.segments) .map(|(s, content)| { let discontinuity = previous.replace(s.discontinuity).is_some_and(|p| p != s.discontinuity); Segment { segment: s.segment, + tag: segments::tag(&s.reference), duration: s.duration, gap: content == Content::Gap, discontinuity, @@ -588,12 +597,15 @@ impl Rendition { .collect(); // A failed timeline ends this playlist at its last resolvable segment. let failed = window.segments.last().is_some_and(|row| self.is_failed(row)); + // An ended reference ends the playlist only once this rendition resolved every row, so a + // slower timeline's tail is listed before `EXT-X-ENDLIST` stops players polling. + let complete = segments.len() == window.segments.len(); Snapshot { target_duration, media_sequence: window.sequence, segments, - finished: window.ended || failed, + finished: (window.ended && complete) || failed, program_date_time, generation, } @@ -631,6 +643,8 @@ impl Rendition { self.media.sync(&self.live); let window = self.live.window(); let timescale = self.timescale(); + // Static only once every row resolved here, like the playlist's `EXT-X-ENDLIST`. + let complete = self.resolved(&window.segments).count() == window.segments.len(); let segments = self .resolved(&window.segments) .filter(|(_, content)| *content != Content::Gap) @@ -662,22 +676,23 @@ impl Rendition { channel_count, timescale: self.section.timescale.max(1), segments, - ended: window.ended, + ended: (window.ended && complete) || window.segments.last().is_some_and(|row| self.is_failed(row)), + tag: listed_tag(&window.segments), init: init.hash.clone(), generation: run.generation, }) } - /// Fetch and transmux the segment whose timeline `pts` is `time`, in the timeline's - /// timescale (DASH `$Time$` addressing of the same bytes [`segment`](Self::segment) serves - /// by aligned number). + /// Fetch and transmux the segment of the reference tagged `tag` whose timeline `pts` is + /// `time`, in the timeline's timescale (DASH `$Time$` addressing of the same bytes + /// [`segment`](Self::segment) serves by aligned number). #[cfg_attr(not(feature = "server"), allow(dead_code))] - pub(crate) async fn segment_at(&self, time: u64) -> Result> { + pub(crate) async fn segment_at(&self, tag: &str, time: u64) -> Result> { self.media.sync(&self.live); - let Some(segment) = self.live.segment_number_at(time, self.timescale()) else { + let Some(segment) = self.live.segment_number_at(tag, time, self.timescale()) else { return Ok(None); }; - self.segment(segment).await + self.segment(tag, segment).await } /// The timeline's timescale; a declared 0 is clamped rather than trusted. @@ -804,15 +819,16 @@ impl Rendition { Ok(Some(self.cache_init(run.epoch, bytes))) } - /// Fetch and transmux the segment numbered `segment`. + /// Fetch and transmux the segment numbered `segment` by the reference tagged `tag` (the + /// `seg/{tag}.{segment}.m4s` URI). /// /// Fetches every frame the segment resolves to on this rendition (an audio segment packs many /// short groups) and encodes them as a single CMAF fragment. `None` when the segment isn't in /// the playlist window, is a gap or not yet resolved for this rendition, or its groups already /// left the relay cache. - pub async fn segment(&self, segment: u64) -> Result> { + pub async fn segment(&self, tag: &str, segment: u64) -> Result> { self.media.sync(&self.live); - let Some(row) = self.live.row(segment) else { + let Some(row) = self.live.row(tag, segment) else { return Ok(None); }; self.fetch(&row).await @@ -888,6 +904,27 @@ impl Rendition { } } +/// The tag of the reference numbering `rows`, empty when none are listed. A reference switch +/// clears every row, so the listed rows share one. +fn listed_tag(rows: &[segments::Row]) -> String { + rows.first().map(|row| segments::tag(&row.reference)).unwrap_or_default() +} + +#[cfg(test)] +impl Rendition { + /// Fetch the listed segment numbered `segment`, under the tag its rows carry. + pub(crate) async fn listed_segment(&self, segment: u64) -> Result> { + let tag = listed_tag(&self.live.window().segments); + self.segment(&tag, segment).await + } + + /// Fetch the listed segment starting at `time`, under the tag its rows carry. + pub(crate) async fn listed_segment_at(&self, time: u64) -> Result> { + let tag = listed_tag(&self.live.window().segments); + self.segment_at(&tag, time).await + } +} + /// The last group holding a frame of `range`. fn last_group(range: &std::ops::Range) -> u64 { match range.end.frame { @@ -902,13 +939,12 @@ async fn watch_spans( section: Archive, track: String, spans: Arc, - window: Option, ) { let result: Result<()> = async { let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&broadcast, §ion, &track).await?; while let Some(event) = timeline.next().await? { match event { - moq_mux::timeline::Event::Push { entry, .. } => spans.push(entry, window), + moq_mux::timeline::Event::Push { entry, .. } => spans.push(entry), moq_mux::timeline::Event::Pop(range) => spans.pop(range), moq_mux::timeline::Event::Skip(_) => spans.clear(), _ => unreachable!("unknown timeline event"), diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index d0a19a56e0..b5253b506a 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -349,7 +349,7 @@ impl Producer { fn register(&self, rendition: &Arc) { let mut feed = self.fanout.feed.lock().unwrap(); rendition.label(feed.generation.clone()); - rendition.watch(feed.window); + rendition.watch(); for (index, entry, reference, discontinuity) in &feed.history { rendition.push(*index, entry, reference, *discontinuity, feed.window); } diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 4e9595e8c3..9d3abdce6c 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -12,9 +12,11 @@ //! order, exactly once. `next()` waits for the next resolved row, FETCHes and transmuxes its //! frames (via [`Rendition`]), and yields the CMAF bytes. //! -//! A segment is addressed by its number everywhere (the `seg/{segment}.m4s` URI, -//! `EXT-X-MEDIA-SEQUENCE`, the recorder cursor), and the same number names the same span of -//! content time on every rendition, on every edge, and after every reload. +//! A segment is addressed by its reference's [`tag`] and its number (the `seg/{tag}.{segment}.m4s` +//! URI); the number alone is its `EXT-X-MEDIA-SEQUENCE` and the recorder cursor's position. The +//! same URI names the same span of content time on every rendition, on every edge, and after +//! every reload. Each reference numbers segments by its own records, so a new reference starts a +//! new numbering under its own tag rather than reusing the old one's URIs. use std::collections::VecDeque; use std::sync::Arc; @@ -22,6 +24,7 @@ use std::task::Poll; use std::time::{Duration, SystemTime}; use bytes::Bytes; +use sha2::{Digest, Sha256}; use super::{Kind, Rendition}; use crate::Result; @@ -82,12 +85,20 @@ impl Discontinuities { /// The rendition a broadcast's segment boundaries come from. pub(crate) type Reference = Arc<(Kind, String)>; +/// The URI tag naming `reference`'s segment numbering: a hash of its kind and name, so every edge +/// derives the same one and it never contains a URI separator. +pub(crate) fn tag(reference: &(Kind, String)) -> String { + let digest = Sha256::digest(format!("{}/{}", reference.0.as_str(), reference.1)); + // 32 bits: the tag only has to tell apart the references one broadcast ever switches between. + digest[..4].iter().map(|b| format!("{b:02x}")).collect() +} + /// One playlist segment: its number, timing, and the reference record's frames. #[derive(Clone, Debug, PartialEq)] pub(crate) struct Row { /// The reference timeline's record index, used to mirror exact window trims. pub index: u64, - /// The segment number (its URI: `seg/{segment}.m4s`), shared across renditions. + /// The segment number (its URI: `seg/{tag}.{segment}.m4s`), shared across renditions. pub segment: u64, /// The rendition the boundaries come from. pub reference: Reference, @@ -140,14 +151,15 @@ impl State { } } - /// The first segment numbered past `after`, for a cursor. + /// The first segment past `after`, for a cursor. /// /// Rows are complete the moment they arrive. Segments evicted from the front of the /// window before the cursor reached them are skipped: the cursor resumes at the oldest - /// row still in the window. - fn next_after(&self, after: Option) -> Next { + /// row still in the window. A row from another reference starts a new numbering, so it + /// follows `after` whatever its number. + fn next_after(&self, after: Option<&(Reference, u64)>) -> Next { let next = self.rows.iter().find(|r| match after { - Some(after) => r.segment > after, + Some((reference, after)) => r.reference != *reference || r.segment > *after, None => true, }); match next { @@ -251,10 +263,15 @@ impl Producer { self.state.read().window() } - /// Segment `segment`'s row, or `None` if it isn't in the window. - pub fn row(&self, segment: u64) -> Option { + /// The row of segment `segment` numbered by the reference tagged `tag`, or `None` if it isn't + /// in the window. + pub fn row(&self, tag: &str, segment: u64) -> Option { let state = self.state.read(); - state.rows.iter().find(|r| r.segment == segment).cloned() + state + .rows + .iter() + .find(|r| r.segment == segment && self::tag(&r.reference) == tag) + .cloned() } /// The number of the segment whose `pts` is exactly `time` in the timeline's timescale @@ -262,12 +279,12 @@ impl Producer { /// safe because the rendered `S@t` and this lookup convert the same [`Row::pts`] the same /// way. #[cfg_attr(not(feature = "server"), allow(dead_code))] - pub fn segment_number_at(&self, time: u64, timescale: moq_net::Timescale) -> Option { + pub fn segment_number_at(&self, tag: &str, time: u64, timescale: moq_net::Timescale) -> Option { let state = self.state.read(); state .rows .iter() - .find(|row| row.pts.as_scale(timescale) == time as u128) + .find(|row| row.pts.as_scale(timescale) == time as u128 && self::tag(&row.reference) == tag) .map(|row| row.segment) } @@ -321,7 +338,7 @@ impl Producer { /// A segment with its transmuxed media, yielded by a [`Consumer`]. pub struct Segment { - /// The aligned segment number (also its `seg/{segment}.m4s` URI stem), shared across the + /// The aligned segment number (its URI is `seg/{reference}.{segment}.m4s`), shared across the /// broadcast's renditions. pub segment: u64, /// The transmuxed CMAF fragment (`moof`+`mdat`), fetched on demand by [`Consumer::next`]. @@ -348,8 +365,9 @@ pub struct Segment { pub struct Consumer { state: kio::Consumer, rendition: Arc, - /// Last fetched or skipped segment; errors leave it unchanged so callers can retry. - after: Option, + /// Last fetched or skipped segment, with the reference numbering it; errors leave it unchanged + /// so callers can retry. + after: Option<(Reference, u64)>, } impl Consumer { @@ -377,7 +395,7 @@ impl Consumer { return Ok(None); } let media = self.rendition.fetch(&row).await?; - self.after = Some(row.segment); + self.after = Some((row.reference.clone(), row.segment)); if let Some(media) = media { return Ok(Some(Segment { segment: row.segment, @@ -391,7 +409,7 @@ impl Consumer { } fn poll_next(&self, waiter: &kio::Waiter) -> Poll> { - let poll = self.state.poll(waiter, |state| match state.next_after(self.after) { + let poll = self.state.poll(waiter, |state| match state.next_after(self.after.as_ref()) { Next::Ready(row) => Poll::Ready(Some(row)), Next::Ended => Poll::Ready(None), Next::Pending => Poll::Pending, @@ -544,12 +562,14 @@ mod tests { window, ); + let video = (Kind::Video, "video".to_string()); + let tag = tag(&video); assert_eq!( - live.row(0).unwrap().frames, + live.row(&tag, 0).unwrap().frames, hang::timeline::Position::group(0)..hang::timeline::Position::group(1) ); - assert_eq!(live.row(7), None, "unknown segments miss"); - let video = (Kind::Video, "video".to_string()); + assert_eq!(live.row(&tag, 7), None, "unknown segments miss"); + assert_eq!(live.row("00000000", 0), None, "another reference's numbering misses"); assert_eq!( live.latest_keyframe_group(&video), Some(0), @@ -586,16 +606,42 @@ mod tests { panic!("expected a segment"); }; assert_eq!(first.segment, 0); - let Next::Ready(second) = live.state.read().next_after(Some(0)) else { + let after = |segment| Some((first.reference.clone(), segment)); + let Next::Ready(second) = live.state.read().next_after(after(0).as_ref()) else { panic!("expected a segment"); }; assert_eq!(second.segment, 1); assert!( - matches!(live.state.read().next_after(Some(1)), Next::Pending), + matches!(live.state.read().next_after(after(1).as_ref()), Next::Pending), "nothing further while live" ); live.end(); - assert!(matches!(live.state.read().next_after(Some(1)), Next::Ended)); + assert!(matches!(live.state.read().next_after(after(1).as_ref()), Next::Ended)); + } + + /// A new reference numbers segments from its own records, so a cursor past a higher number of + /// the old reference still yields the new reference's rows. + #[test] + fn a_new_reference_restarts_the_cursor() { + let live = Producer::new(); + let window = Some(Duration::from_secs(30)); + live.push(row(5, 5, 10_000, 2_000), window); + let Next::Ready(old) = live.state.read().next_after(None) else { + panic!("expected a segment"); + }; + + live.clear(); + let switched = Row { + reference: Arc::new((Kind::Video, "other".to_string())), + ..row(0, 0, 0, 2_000) + }; + live.push(switched, window); + let after = Some((old.reference.clone(), old.segment)); + let Next::Ready(next) = live.state.read().next_after(after.as_ref()) else { + panic!("the new reference's first row follows"); + }; + assert_eq!(next.segment, 0); + assert_ne!(tag(&next.reference), tag(&old.reference), "its URLs carry another tag"); } } diff --git a/rs/moq-hls/src/export/spans.rs b/rs/moq-hls/src/export/spans.rs index 04f79e28c2..fcc7d17973 100644 --- a/rs/moq-hls/src/export/spans.rs +++ b/rs/moq-hls/src/export/spans.rs @@ -82,9 +82,8 @@ impl Spans { } } - /// Append a record, evicting records that end before `window` (plus the snapping tolerance) - /// of the newest content. With no `window`, only source pops remove records. - pub fn push(&self, entry: Entry, window: Option) { + /// Append a record. Only [`trim`](Self::trim) and source pops remove records. + pub fn push(&self, entry: Entry) { let Ok(mut state) = self.state.write() else { return; }; @@ -97,11 +96,17 @@ impl Spans { state.records.clear(); } state.records.push_back(entry); - let Some(window) = window else { + } + + /// Evict records that end before `oldest`, the start of the oldest listed segment, less the + /// snapping tolerance. Tied to the listed segments rather than this timeline's own newest + /// content, so a rendition running ahead of a stalled reference never drops a record a listed + /// segment still resolves to. + pub fn trim(&self, oldest: Duration) { + let Ok(mut state) = self.state.write() else { return; }; - let newest = state.records.back().expect("just pushed").end_time(); - while state.records.len() >= 2 && state.records[0].end_time() + window + 2 * TOLERANCE < newest { + while state.records.len() >= 2 && state.records[0].end_time() + 2 * TOLERANCE < oldest { state.records.pop_front(); } } @@ -368,11 +373,12 @@ mod tests { } #[test] - fn the_window_keeps_records_a_snap_may_need() { + fn a_trim_keeps_records_a_snap_may_need() { let spans = Spans::new(); for entry in gops(&[0, 2_000, 4_000, 6_000, 8_000, 10_000], 12_000) { - spans.push(entry, Some(secs(4.0))); + spans.push(entry); } + spans.trim(secs(8.0)); let state = spans.state.read(); assert_eq!(state.records.front().unwrap().sequence, 2); } @@ -381,7 +387,7 @@ mod tests { fn a_failed_timeline_resolves_only_what_its_records_cover() { let spans = Spans::new(); for entry in gops(&[0, 2_000, 4_000], 6_000) { - spans.push(entry, None); + spans.push(entry); } // Waiting on a keyframe that could still land near the boundary. assert_eq!(spans.resolve(true, secs(4.0)..secs(6.0)), Content::Pending); diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index 4e15a17b18..2dee9c18c2 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -7,8 +7,8 @@ //! GET /{broadcast}/manifest.mpd //! GET /{broadcast}/{kind}/{rendition}/media.m3u8 //! GET /{broadcast}/{kind}/{rendition}/init.{hash}.mp4 -//! GET /{broadcast}/{kind}/{rendition}/seg/{segment}.m4s -//! GET /{broadcast}/{kind}/{rendition}/seg/t{pts}.m4s +//! GET /{broadcast}/{kind}/{rendition}/seg/{reference}.{segment}.m4s +//! GET /{broadcast}/{kind}/{rendition}/seg/{reference}.t{pts}.m4s //! ``` //! //! `{kind}` is `video` or `audio`, so a video and an audio rendition that share a @@ -17,7 +17,7 @@ //! timeline `pts` (`$Time$`). `{hash}` is a hash of the init segment's bytes, so a //! reconfigured rendition gets a new init URL. A broadcaster given a generation //! ([`Broadcaster::set_generation`](crate::export::Broadcaster::set_generation)) -//! prefixes every segment file with it (`seg/{generation}.{segment}.m4s`), and +//! prefixes every segment file with it (`seg/{generation}.{reference}.{segment}.m4s`), and //! refuses segment URLs carrying any other. //! //! Every request is served. To gate access, wrap [`Server::router`] in your own diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index 324fdd80f8..fb2a4fa247 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -63,7 +63,8 @@ pub enum Resource { /// The hash of the init bytes the URL names. hash: String, }, - /// `{kind}/{rendition}/seg/[{generation}.]{sequence}.m4s`: a segment by its HLS number. + /// `{kind}/{rendition}/seg/[{generation}.]{reference}.{sequence}.m4s`: a segment by its HLS + /// number. #[non_exhaustive] Segment { /// The rendition's kind. @@ -72,11 +73,13 @@ pub enum Resource { rendition: String, /// The publisher run the URL names, if the broadcaster carries one. generation: Option, + /// The tag of the reference rendition whose records number the segment. + reference: String, /// The segment's aligned number. sequence: u64, }, - /// `{kind}/{rendition}/seg/[{generation}.]t{pts}.m4s`: the same bytes, addressed by the - /// DASH timeline pts (`$Time$`). + /// `{kind}/{rendition}/seg/[{generation}.]{reference}.t{pts}.m4s`: the same bytes, addressed + /// by the DASH timeline pts (`$Time$`). #[non_exhaustive] SegmentAt { /// The rendition's kind. @@ -85,6 +88,8 @@ pub enum Resource { rendition: String, /// The publisher run the URL names, if the broadcaster carries one. generation: Option, + /// The tag of the reference rendition whose records number the segment. + reference: String, /// The segment's timeline pts. pts: u64, }, @@ -134,22 +139,26 @@ impl Route { let kind = Kind::parse(kind)?; let rendition = rendition.clone(); let stem = file.strip_suffix(".m4s")?; - // `{generation}.{segment}` when the export carries a generation, `{segment}` otherwise. - let (generation, stem) = match stem.split_once('.') { - Some((generation, stem)) => (Some(generation.to_string()), stem), - None => (None, stem), + // `{generation}.{reference}.{segment}` when the export carries a generation, + // `{reference}.{segment}` otherwise. + let (generation, reference, stem) = match stem.split('.').collect::>().as_slice() { + [reference, stem] => (None, reference.to_string(), *stem), + [generation, reference, stem] => (Some(generation.to_string()), reference.to_string(), *stem), + _ => return None, }; let resource = match stem.strip_prefix('t') { Some(pts) => Resource::SegmentAt { kind, rendition, generation, + reference, pts: pts.parse().ok()?, }, None => Resource::Segment { kind, rendition, generation, + reference, sequence: stem.parse().ok()?, }, }; @@ -201,18 +210,20 @@ impl Server { kind, rendition, generation, + reference, sequence, } => { - let at = SegmentAt::Sequence(*sequence); + let at = SegmentAt::Sequence(reference, *sequence); segment(self, broadcast, *kind, rendition, generation.as_deref(), at).await } Resource::SegmentAt { kind, rendition, generation, + reference, pts, } => { - let at = SegmentAt::Pts(*pts); + let at = SegmentAt::Pts(reference, *pts); segment(self, broadcast, *kind, rendition, generation.as_deref(), at).await } } @@ -267,11 +278,12 @@ async fn init(server: &Server, broadcast: &str, kind: Kind, rendition: &str, has media_result(rendition.init_versioned(hash).await, server) } -/// How a segment URL addresses its bytes: HLS by aligned number (`seg/0.m4s`), DASH by -/// timeline pts (`seg/t2000.m4s`, the SegmentTemplate's `$Time$`). -enum SegmentAt { - Sequence(u64), - Pts(u64), +/// How a segment URL addresses its bytes under its reference's tag: HLS by aligned number +/// (`seg/{tag}.0.m4s`), DASH by timeline pts (`seg/{tag}.t2000.m4s`, the SegmentTemplate's +/// `$Time$`). +enum SegmentAt<'a> { + Sequence(&'a str, u64), + Pts(&'a str, u64), } async fn segment( @@ -280,7 +292,7 @@ async fn segment( kind: Kind, rendition: &str, generation: Option<&str>, - at: SegmentAt, + at: SegmentAt<'_>, ) -> Response { let Some(rendition) = rendition_for(server, broadcast, kind, rendition).await else { return not_found(); @@ -292,8 +304,8 @@ async fn segment( return not_found(); } let result = match at { - SegmentAt::Sequence(sequence) => rendition.segment(sequence).await, - SegmentAt::Pts(pts) => rendition.segment_at(pts).await, + SegmentAt::Sequence(reference, sequence) => rendition.segment(reference, sequence).await, + SegmentAt::Pts(reference, pts) => rendition.segment_at(reference, pts).await, }; // A generation change while fetching may have swapped the rows under the lookup. if !current() { @@ -391,38 +403,42 @@ mod tests { }) ); assert_eq!( - resource("/project/live/video/main/seg/42.m4s"), + resource("/project/live/video/main/seg/ab12cd34.42.m4s"), Some(Resource::Segment { kind: Kind::Video, rendition: "main".to_string(), generation: None, + reference: "ab12cd34".to_string(), sequence: 42, }) ); assert_eq!( - resource("/project/live/video/main/seg/run-1.42.m4s"), + resource("/project/live/video/main/seg/run-1.ab12cd34.42.m4s"), Some(Resource::Segment { kind: Kind::Video, rendition: "main".to_string(), generation: Some("run-1".to_string()), + reference: "ab12cd34".to_string(), sequence: 42, }) ); assert_eq!( - resource("/project/live/video/main/seg/t2000.m4s"), + resource("/project/live/video/main/seg/ab12cd34.t2000.m4s"), Some(Resource::SegmentAt { kind: Kind::Video, rendition: "main".to_string(), generation: None, + reference: "ab12cd34".to_string(), pts: 2000, }) ); assert_eq!( - resource("/project/live/video/main/seg/init.t2000.m4s"), + resource("/project/live/video/main/seg/init.ab12cd34.t2000.m4s"), Some(Resource::SegmentAt { kind: Kind::Video, rendition: "main".to_string(), generation: Some("init".to_string()), + reference: "ab12cd34".to_string(), pts: 2000, }) ); @@ -431,7 +447,11 @@ mod tests { assert!(Route::parse("/project/live/data/main/media.m3u8").is_none()); assert!(Route::parse("/project/live/video/main/init.mp4").is_none()); assert!(Route::parse("/project/live/video/main/init..mp4").is_none()); - assert!(Route::parse("/project/live/video/main/seg/tx.m4s").is_none()); + assert!(Route::parse("/project/live/video/main/seg/ab12cd34.tx.m4s").is_none()); + assert!( + Route::parse("/project/live/video/main/seg/42.m4s").is_none(), + "a segment URL names its reference" + ); assert!(Route::parse("/project/live/video/main/seg/1.ts").is_none()); } @@ -615,7 +635,7 @@ mod tests { if response.status() == StatusCode::OK { let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); let body = String::from_utf8(body.to_vec()).unwrap(); - if body.contains("seg/0.m4s") { + if body.contains("seg/656e3d1b.0.m4s") { break body; } } @@ -626,7 +646,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; }; assert!(body.contains(".mp4?jwt=abc\""), "{body}"); - assert!(body.contains("seg/0.m4s?jwt=abc"), "{body}"); + assert!(body.contains("seg/656e3d1b.0.m4s?jwt=abc"), "{body}"); pair.accept.abort(); } @@ -647,14 +667,14 @@ mod tests { let server = Server::new(origin.consume(), crate::export::Config::default()); let app = server.router(); - wait_listed(&app, "/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; let broadcaster = server.broadcaster("live").await.expect("broadcaster"); broadcaster.set_generation(Some("run-1")).unwrap(); let response = oneshot(app.clone(), "/live/video/video0/media.m3u8").await; let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); let playlist = String::from_utf8(body.to_vec()).unwrap(); - assert!(playlist.contains("\nseg/run-1.0.m4s\n"), "{playlist}"); + assert!(playlist.contains("\nseg/run-1.656e3d1b.0.m4s\n"), "{playlist}"); let map = playlist.lines().find(|line| line.starts_with("#EXT-X-MAP:")).unwrap(); let init = map .strip_prefix("#EXT-X-MAP:URI=\"") @@ -672,24 +692,24 @@ mod tests { StatusCode::NOT_FOUND ); - assert_eq!(status(&app, "/live/video/video0/seg/run-1.0.m4s").await, StatusCode::OK); + assert_eq!(status(&app, "/live/video/video0/seg/run-1.656e3d1b.0.m4s").await, StatusCode::OK); assert_eq!( - status(&app, "/live/video/video0/seg/run-1.t0.m4s").await, + status(&app, "/live/video/video0/seg/run-1.656e3d1b.t0.m4s").await, StatusCode::OK ); assert_eq!( - status(&app, "/live/video/video0/seg/0.m4s").await, + status(&app, "/live/video/video0/seg/656e3d1b.0.m4s").await, StatusCode::NOT_FOUND ); assert_eq!( - status(&app, "/live/video/video0/seg/run-0.0.m4s").await, + status(&app, "/live/video/video0/seg/run-0.656e3d1b.0.m4s").await, StatusCode::NOT_FOUND ); // A new run: the previous generation's URLs are refused even for numbers it reuses. broadcaster.set_generation(Some("run-2")).unwrap(); assert_eq!( - status(&app, "/live/video/video0/seg/run-1.0.m4s").await, + status(&app, "/live/video/video0/seg/run-1.656e3d1b.0.m4s").await, StatusCode::NOT_FOUND ); } @@ -710,9 +730,9 @@ mod tests { write_three_gops(&mut media); let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); - wait_listed(&app, "/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; - let response = oneshot(app, "/live/video/video0/seg/0.m4s").await; + let response = oneshot(app, "/live/video/video0/seg/656e3d1b.0.m4s").await; assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); @@ -728,7 +748,7 @@ mod tests { write_three_gops(&mut media); let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); - wait_listed(&app, "/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; let remote = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("live")) .await @@ -739,7 +759,7 @@ mod tests { .await .expect("publisher close timed out"); - let response = oneshot(app, "/live/video/video0/seg/0.m4s").await; + let response = oneshot(app, "/live/video/video0/seg/656e3d1b.0.m4s").await; assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); @@ -773,7 +793,7 @@ mod tests { write_three_gops(&mut media); let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); - wait_listed(&app, "/room/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/room/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; let remote_media = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("room/source")) .await @@ -785,7 +805,7 @@ mod tests { .await .expect("sibling close timed out"); - let response = oneshot(app, "/room/live/video/video0/seg/0.m4s").await; + let response = oneshot(app, "/room/live/video/video0/seg/656e3d1b.0.m4s").await; assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); From 74bbaef7fff1f95a8c5fecd085ce748481117124 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 00:25:27 -0700 Subject: [PATCH 46/49] test(archive): cover a track that ends during enrollment (#4863) Co-authored-by: Grok 4.7 --- rs/moq-archive/src/mock.rs | 40 ++++++++++++++++++++++++++++++++++++ rs/moq-archive/src/writer.rs | 33 +++++++++++++++++++++++++++++ rs/moq-cli/src/archive.rs | 34 +++++++++++++++--------------- 3 files changed, 90 insertions(+), 17 deletions(-) diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs index 8488733a22..76d1f43adc 100644 --- a/rs/moq-archive/src/mock.rs +++ b/rs/moq-archive/src/mock.rs @@ -44,12 +44,16 @@ struct State { fail_lists: bool, /// Streaming listings yield entries in descending order, like a backend that promises none. unordered: bool, + /// PUTs whose path contains any of these wait until [`Mock::release`]. + holds: Vec, } #[derive(Debug, Clone)] pub(crate) struct Mock { inner: Arc, state: Arc>, + /// Wakes PUTs blocked in [`Self::hold_puts`]. + hold: Arc, } impl Mock { @@ -57,6 +61,7 @@ impl Mock { Self { inner: Arc::new(inner), state: Default::default(), + hold: Arc::new(tokio::sync::Notify::new()), } } @@ -69,6 +74,7 @@ impl Mock { Self { inner: self.inner.clone(), state: Default::default(), + hold: Arc::new(tokio::sync::Notify::new()), } } @@ -120,6 +126,18 @@ impl Mock { self } + /// Hold PUTs whose path contains `pattern` until [`release`](Self::release). + pub fn hold_puts(&self, pattern: &str) -> &Self { + self.state().holds.push(pattern.to_string()); + self + } + + /// Unblock every PUT [`hold_puts`](Self::hold_puts) is waiting on. + pub fn release(&self) { + self.state().holds.clear(); + self.hold.notify_waiters(); + } + /// Clear every injected failure. pub fn heal(&self) { let mut state = self.state(); @@ -128,6 +146,27 @@ impl Mock { state.stall_gets.clear(); state.fail_deletes.clear(); state.fail_lists = false; + state.holds.clear(); + drop(state); + self.hold.notify_waiters(); + } + + /// Wait out a [`hold_puts`](Self::hold_puts) gate. + /// + /// The waiter is armed before the check, so a `release` in between is not missed. + async fn wait_held(&self, location: &Path) { + loop { + let notified = self.hold.notified(); + let held = self + .state() + .holds + .iter() + .any(|pattern| location.as_ref().contains(pattern)); + if !held { + return; + } + notified.await; + } } fn state(&self) -> std::sync::MutexGuard<'_, State> { @@ -196,6 +235,7 @@ impl ObjectStore for Mock { if self.state().fail_puts.iter().any(|p| location.as_ref().contains(p)) { return Err(unsupported("put")); } + self.wait_held(location).await; self.inner.put_opts(location, payload, opts).await } diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 1cff43db2a..b885a02f04 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -1795,4 +1795,37 @@ mod tests { } assert_eq!(stored(&store).await, before); } + + /// The publisher already has the subscriber, and the broadcast ends while `.info` is written. + /// + /// The enroll command is queued only after that write. The recording stays up across the gap, + /// so the track is kept instead of failing with the writer closed. + #[tokio::test] + async fn a_track_that_ends_during_enrollment_is_kept() { + let source = broadcast::Info::new().produce(); + let mut audio = track(&source, "audio"); + group(&audio, 0, &[0, 500]); + + let mock = Mock::memory(); + mock.hold_puts("audio/.info"); + let store = Store::new(mock.clone(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + let run = tokio::spawn(writer.run()); + let enrolling = tokio::spawn(async move { control.track("audio", media()).await }); + + let subscription = audio.subscription_changed().await.unwrap(); + assert!(subscription.is_some(), "enrollment subscribed"); + audio.finish().unwrap(); + source.close(); + mock.release(); + + enrolling.await.unwrap().unwrap(); + run.await.unwrap().unwrap(); + + let records = window(&store, "audio").await; + assert_eq!(groups(&records), vec![(0, 0)]); + } } diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 138c40c45c..000fd8a7b2 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -313,18 +313,17 @@ mod tests { assert!(err.contains("broadcast `source`"), "{err}"); } - /// Wait until the export has enrolled each of `tracks`. + /// Wait until the export's subscription has reached `track`. /// - /// A track's timeline `.info` is stored once its subscription landed, and the writer does not - /// end while that enrollment is still on its way, so the track may finish right after. - async fn enrolled(url: &Url, tracks: &[&str], recording: &tokio::task::JoinHandle>) { - let store = super::open(url).unwrap(); - for track in tracks { - // The store is real disk I/O, so this polls on the wall clock; nextest kills a hang. - while store.get_info(&hang::timeline::default_name(track)).await.is_err() { - assert!(!recording.is_finished(), "the export ended before it enrolled {track}"); - tokio::time::sleep(Duration::from_millis(10)).await; + /// `.info` is written after that, while the enroll command is still being queued. + /// Finishing on the file races that command and the recording ends with "writer closed". + async fn subscribed(track: &mut track::Producer, recording: &mut tokio::task::JoinHandle>) { + tokio::select! { + subscription = track.subscription_changed() => { + let subscription = subscription.expect("the rendition stays open until the export subscribes"); + assert!(subscription.is_some(), "the export subscribed"); } + result = &mut *recording => panic!("the export ended before it subscribed: {result:?}"), } } @@ -343,7 +342,7 @@ mod tests { let info = track::Info::default() .with_timescale(Timescale::MILLI) .with_max_age(Duration::from_secs(3600)); - let track = broadcast.create_track("audio", info).unwrap(); + let mut track = broadcast.create_track("audio", info).unwrap(); catalog .mutate(|catalog| { catalog.audio.renditions.insert("audio".into(), audio()); @@ -356,7 +355,7 @@ mod tests { retention: None, retention_grace: None, }; - let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); for sequence in 0..3 { let mut group = track.create_group(group::Info { sequence }).unwrap(); @@ -367,7 +366,7 @@ mod tests { group.finish().unwrap(); } - enrolled(&url, &["audio"], &recording).await; + subscribed(&mut track, &mut recording).await; track.finish().unwrap(); catalog.finish().unwrap(); broadcast.close(); @@ -416,8 +415,8 @@ mod tests { let info = track::Info::default() .with_timescale(Timescale::MILLI) .with_max_age(Duration::from_secs(3600)); - let first = broadcast.create_track("audio", info.clone()).unwrap(); - let second = broadcast.create_track("audio2", info).unwrap(); + let mut first = broadcast.create_track("audio", info.clone()).unwrap(); + let mut second = broadcast.create_track("audio2", info).unwrap(); for track in [&first, &second] { for sequence in 0..3 { let mut group = track.create_group(group::Info { sequence }).unwrap(); @@ -441,9 +440,10 @@ mod tests { retention: None, retention_grace: None, }; - let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); - enrolled(&url, &["audio", "audio2"], &recording).await; + subscribed(&mut first, &mut recording).await; + subscribed(&mut second, &mut recording).await; first.finish().unwrap(); second.finish().unwrap(); catalog.finish().unwrap(); From 3272d7a1b944d358544729ae8adeadfc20dae343 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 00:31:57 -0700 Subject: [PATCH 47/49] chore: format and drop stray review notes Co-Authored-By: Claude Opus 5.5 --- .review-body-4034.md | 28 -------------------------- .status-4034.md | 28 -------------------------- rs/moq-archive/src/reader/tests.rs | 5 ++++- rs/moq-archive/src/store.rs | 6 ++++-- rs/moq-archive/src/writer.rs | 9 +++++++-- rs/moq-hls/src/export/archive_tests.rs | 26 +++++++++++++++++++----- rs/moq-hls/src/export/mod.rs | 28 +++++++++++++++++++++----- rs/moq-hls/src/export/rendition.rs | 11 ++++------ rs/moq-hls/src/export/segments.rs | 12 ++++++----- rs/moq-hls/src/server/routes.rs | 5 ++++- 10 files changed, 74 insertions(+), 84 deletions(-) delete mode 100644 .review-body-4034.md delete mode 100644 .status-4034.md diff --git a/.review-body-4034.md b/.review-body-4034.md deleted file mode 100644 index 4ea15507ff..0000000000 --- a/.review-body-4034.md +++ /dev/null @@ -1,28 +0,0 @@ -# Codex Adversarial Review - -Target: branch diff against origin/main -Verdict: needs-attention - -Do not ship yet: reference switching reuses segment identities, follow mode misses newly enrolled tracks, and rendition eviction can invalidate advertised segments. Static review against the existing origin/main; sandbox restrictions prevented fetching or running tests. No concrete pruning/recovery defect established. - -Findings: -- [high] Reference switching reuses existing segment URLs (rs/moq-hls/src/export/rendition.rs:401-405) - Segment numbers come directly from the reference track's independent sequence. watch_catalog switches references and clears rows without changing the URL generation. Reproduce: serve video B through sequence 100, then add alphabetically earlier video A whose timeline starts at 0. Surviving rendition playlists rewind and reuse seg/0.m4s for different content; existing cursors skip entries until their previous sequence is exceeded. The prior switching finding remains real. - Recommendation: Assign broadcast-wide monotonic segment identities across reference changes, or explicitly replace the URL namespace and rendition cursors when switching. Test switching between independently numbered timelines. -- [high] Follow mode permanently excludes tracks enrolled after startup (rs/moq-cli/src/archive.rs:101-108) - Import lists timelines once, then only refreshes the Reader's fixed timeline vector. serve() also captures a fixed list of accepted media tracks. Reproduce: start import --follow with audio recorded, then enroll video in the running export. Repeated refreshes never publish video's timeline, and requests for video are rejected as NotFound despite committed objects. The prior enrollment finding remains real. - Recommendation: Discover new timelines during follow refreshes and enroll them into both timeline replay and the shared served-track registry. Add a late-track enrollment integration test. -- [high] Independent span eviction changes still-advertised segment content (rs/moq-hls/src/export/spans.rs:103-105) - Spans evicts against its own newest timestamp, while playlist rows expire against the reference timeline. Reproduce with a 30-second window: pause the reference at 40 seconds while another rendition advances to 80 seconds. That rendition removes records covering the reference's still-listed 10–40 second segments, so previously resolvable segment URLs become gaps. Even ordinary watcher skew can expose this disagreement. The prior eviction finding remains real. - Recommendation: Coordinate span retention with the oldest advertised reference boundary, including snapping tolerance. Handle source pops explicitly so unavailable content is not silently substituted for an already advertised segment. Test a stalled reference with an advancing sibling. -- [medium] Duplicate enrollment resets state still owned by another recorder (rs/moq-mux/src/timeline.rs:507-516) - Timelines::track accepts an already-active name, replaces its segmenter, and returns another Recorder sharing the same Live. Reproduce: retain recorder A, enroll B under the same name, report group 10 through A and group 0 through B. B's report closes the shared timeline on a position error. Dropping A can also flush B's open record. Move-only Recorder does not enforce exclusive ownership here. - Recommendation: Reject enrollment while a recorder owns the name. Require continuing media positions for subsequent enrollment, or a fresh track name when group numbering restarts. Test overlapping recorder lifetimes. - -Next steps: -- Add regression tests for the four reproduction sketches before shipping. -- Separate design concern: writer.rs:261 propagates malformed-track errors out of the entire writer, dropping other tracks and in-flight commits. The prior observation is real and explicitly documented; decide whether recording-wide failure is intended or track isolation is required. - -Adversarial Codex review of `59659b7b1` (branch diff against `main`), focused on the track-timeline fold. Run through the `peer-review` skill; these are suggestions, not a maintainer decision. - -(Written by Codex) diff --git a/.status-4034.md b/.status-4034.md deleted file mode 100644 index e60d167f91..0000000000 --- a/.status-4034.md +++ /dev/null @@ -1,28 +0,0 @@ -**Merge held at `59659b7b1`.** `main` is merged in, but this head should not land yet. - -### What changed - -- Merged `origin/main` (21 commits). There were conflicts in `Cargo.lock`, `rs/moq-cli` (`announced` plus `archive`), and fMP4 import. For fMP4 import I kept #4885's reservation-until-first-fragment and dropped main's shared-timeline catalog edit, which the fold replaces. -- #4873's selection refusal now covers `export archive` too (`ignores_selection`). That replaces the archive's own check, and `doc/bin/cli.md` lists it. -- Fixed a stale comment in `moq-hls` about one dead rendition "gating the broadcast's timeline". With per-track timelines a dead rendition only affects its own playlist. `hls-first-catalog.md` no longer cites the removed pass-wide timeline reservation. -- `a_moov_with_nothing_selected_publishes` passed on `main` only because of the shared-timeline catalog edit. It now checks the released reservation directly. -- **Fix (OpenAI review P2 #1):** `serve_group` marked a refresh as seen after loading objects from an older span snapshot. A tail committed during the GET was then never served. The reader now reloads before it finishes or waits. Regression test: `a_refresh_during_an_object_load_is_served`, using a GET stall added to `Mock`. It fails without the fix. -- Quests: `backward-timestamps.md` now covers the silent drop of a restarted source's groups (Grok review #1). `track-timeline.md` marks the 4s minimum as planned. - -`just check` passes. - -### Blocking - -1. **`just test interop --all` fails every Rust, Python, and Go to JS pair.** The JS catalog schema (`js/hang/src/catalog/archive.ts`) still requires `archive.track: string`. `moq-mux` now publishes an `archive.timelines` map for every broadcast, so `@moq/watch` rejects the catalog (`ZodError` at `archive.track`). PR CI does not run the full interop matrix, so the checks stay green. Landing this would break web playback of every native publisher on `main`. Options: - - (a) Pull the schema half of `quest/m1/archive/js-timelines.md` into this PR. **Recommended:** it is the smallest correct fix. - - (b) Land `js-timelines` first. - - (c) Make the JS root schema tolerate an unknown `archive` shape until then. -2. **Review findings needing a design call**, from the OpenAI and Grok reviews and the Codex review above (Codex re-confirmed the first, second, and fourth): - - HLS reference switching reuses segment URLs. - - Span eviction is not tied to the reference window, so a still-listed segment's content can change. - - `ENDLIST` can be sent before a slower rendition's pending tail. - - Follow mode never discovers a track enrolled after startup. - - Re-enrolling an active timeline name shares one segmenter. - - One malformed track ends the whole recording without flushing. This is documented, but isolate-vs-fail-loud is a policy decision. - -(Written by Claude Opus 5.5) diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index 533edba507..7f2c9eb3ed 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -475,7 +475,10 @@ async fn a_track_enrolled_after_opening_is_followed() { let timelines = archive.store.timelines().await.unwrap(); assert_eq!(timelines.keys().collect::>(), ["audio", "video"]); reader.track("video", &timelines["video"]).await.unwrap(); - assert!(reader.track("video", &timelines["video"]).await.is_err(), "a served track is refused"); + assert!( + reader.track("video", &timelines["video"]).await.is_err(), + "a served track is refused" + ); reader.refresh().await.unwrap(); assert_eq!( diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 83400fb9bb..77ca989a55 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -1,8 +1,8 @@ use std::num::NonZeroUsize; use bytes::Bytes; -use futures::stream::BoxStream; use futures::StreamExt; +use futures::stream::BoxStream; use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; use object_store::path::Path; use object_store::{ListResult, ObjectMeta, ObjectStore, ObjectStoreExt, PutMode, PutPayload}; @@ -202,7 +202,9 @@ impl Store { let listed = self.inner.list_with_delimiter(self.list_path(None).as_ref()).await?; let mut timelines = std::collections::BTreeMap::new(); for directory in listed.common_prefixes { - let encoded = directory.filename().ok_or_else(|| Error::Directory(directory.to_string()))?; + let encoded = directory + .filename() + .ok_or_else(|| Error::Directory(directory.to_string()))?; let track = crate::path::decode_track(encoded)?; if let Some(indexed) = track.strip_suffix(hang::timeline::SUFFIX) { timelines.insert(indexed.to_string(), track); diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 7a03100e65..482028f373 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -1436,7 +1436,9 @@ mod tests { assert_eq!( writer.run().await, - Err(Error::Source("track video group 3: timestamp 1200 precedes 1500".into())) + Err(Error::Source( + "track video group 3: timestamp 1200 precedes 1500".into() + )) ); let records = window(&store, "video").await; assert_eq!(groups(&records), vec![(0, 0), (1, 2)]); @@ -1461,7 +1463,10 @@ mod tests { writer.run().await.unwrap(); let records = window(&store, "video").await; check_objects(&store, "video", &records).await; - assert!(control.track("audio", media()).await.is_err(), "a stopped recording refuses enrollment"); + assert!( + control.track("audio", media()).await.is_err(), + "a stopped recording refuses enrollment" + ); drop(source); } diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 8e69bf81dc..2afa12bf1c 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -422,7 +422,12 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { let lists = recording.store.inner().lists(); // Switching renditions downloads only the selected rendition's object. - let low = replay.rendition(Kind::Video, "360p").listed_segment(1).await.unwrap().unwrap(); + let low = replay + .rendition(Kind::Video, "360p") + .listed_segment(1) + .await + .unwrap() + .unwrap(); assert_eq!(&low[4..8], b"moof"); assert_eq!( recording.gets(), @@ -454,7 +459,12 @@ async fn playlists_read_only_timelines_and_segments_their_own_objects() { ); // A repeated request hits the reader's cache, including the immutable `.info`. - replay.rendition(Kind::Video, "360p").listed_segment(1).await.unwrap().unwrap(); + replay + .rendition(Kind::Video, "360p") + .listed_segment(1) + .await + .unwrap() + .unwrap(); assert_eq!(recording.gets(), Vec::::new()); assert_eq!(recording.store.inner().lists(), lists, "segments never list"); } @@ -530,7 +540,9 @@ async fn a_growing_recording_ends_only_on_caller_finality() { let mut reader = replay.reader.take().unwrap(); reader.refresh().await.unwrap(); let playlist = replay - .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/29b1cd67.3.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.3.m4s\n") + }) .await; assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); assert!(!playlist.contains("seg/29b1cd67.0.m4s"), "{playlist}"); @@ -560,7 +572,9 @@ async fn a_durable_timeline_lists_past_the_window() { let recording = segments(12).await; let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; let playlist = replay - .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/29b1cd67.11.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.11.m4s\n") + }) .await; assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0\n"), "{playlist}"); assert!(playlist.contains("seg/29b1cd67.0.m4s\n"), "{playlist}"); @@ -576,7 +590,9 @@ async fn a_durable_timeline_lists_past_the_window() { for archive in [live(), elsewhere] { let live = Replay::open(&recording, 64 * 1024 * 1024, archive).await; let playlist = live - .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("seg/29b1cd67.11.m4s\n")) + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.11.m4s\n") + }) .await; assert!(!playlist.contains("seg/29b1cd67.0.m4s\n"), "{playlist}"); } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 7b603aa583..8a63897b56 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -1244,7 +1244,11 @@ mod tests { assert!(signed.contains(&format!("#EXT-X-MAP:URI=\"init.{hash}.mp4?jwt=abc.def\"\n"))); assert!(signed.contains("seg/656e3d1b.0.m4s?jwt=abc.def\n")); - let segment = rendition.listed_segment(0).await.unwrap().expect("segment fetched on demand"); + let segment = rendition + .listed_segment(0) + .await + .unwrap() + .expect("segment fetched on demand"); assert_eq!(&segment[4..8], b"moof", "a fetched group transmuxes to moof+mdat"); // The live-edge group isn't a segment yet, and unknown groups miss. @@ -1289,7 +1293,11 @@ mod tests { .expect("rendition discovered from the catalog"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; - let segment = rendition.listed_segment(0).await.unwrap().expect("segment fetched on demand"); + let segment = rendition + .listed_segment(0) + .await + .unwrap() + .expect("segment fetched on demand"); assert_eq!(&segment[4..8], b"moof"); drop((media, registration, broadcast)); } @@ -1377,7 +1385,11 @@ mod tests { .await .unwrap() .expect("segment fetched by pts"); - let by_number = video_rendition.listed_segment(1).await.unwrap().expect("segment by number"); + let by_number = video_rendition + .listed_segment(1) + .await + .unwrap() + .expect("segment by number"); assert_eq!(by_time, by_number); assert!(video_rendition.listed_segment_at(999).await.unwrap().is_none()); @@ -1694,7 +1706,10 @@ mod tests { "nothing to serve before the rendition starts" ); let video2 = broadcaster.rendition(Kind::Video, "video2").unwrap(); - assert!(video2.listed_segment(0).await.unwrap().is_none(), "a gap is never fetched"); + assert!( + video2.listed_segment(0).await.unwrap().is_none(), + "a gap is never fetched" + ); drop((producers, registrations, broadcast)); } @@ -2913,7 +2928,10 @@ mod tests { assert_eq!(numbers(&after)[0], 0, "the new reference numbers from its own records"); assert!(after.segments.iter().all(|s| s.tag == new && s.tag != old)); let video0 = test.rendition("video0"); - assert!(video0.segment(&old, 0).await.unwrap().is_none(), "the old URL is no longer served"); + assert!( + video0.segment(&old, 0).await.unwrap().is_none(), + "the old URL is no longer served" + ); } // A malformed reference timeline fails the same way. diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 01b7075996..981dfeb0e4 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -907,7 +907,9 @@ impl Rendition { /// The tag of the reference numbering `rows`, empty when none are listed. A reference switch /// clears every row, so the listed rows share one. fn listed_tag(rows: &[segments::Row]) -> String { - rows.first().map(|row| segments::tag(&row.reference)).unwrap_or_default() + rows.first() + .map(|row| segments::tag(&row.reference)) + .unwrap_or_default() } #[cfg(test)] @@ -934,12 +936,7 @@ fn last_group(range: &std::ops::Range) -> u64 { } /// Follow `track`'s own timeline into `spans` until it ends, then mark nothing pending. -async fn watch_spans( - broadcast: moq_net::broadcast::Consumer, - section: Archive, - track: String, - spans: Arc, -) { +async fn watch_spans(broadcast: moq_net::broadcast::Consumer, section: Archive, track: String, spans: Arc) { let result: Result<()> = async { let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&broadcast, §ion, &track).await?; while let Some(event) = timeline.next().await? { diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 9d3abdce6c..97e62605d2 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -409,11 +409,13 @@ impl Consumer { } fn poll_next(&self, waiter: &kio::Waiter) -> Poll> { - let poll = self.state.poll(waiter, |state| match state.next_after(self.after.as_ref()) { - Next::Ready(row) => Poll::Ready(Some(row)), - Next::Ended => Poll::Ready(None), - Next::Pending => Poll::Pending, - }); + let poll = self + .state + .poll(waiter, |state| match state.next_after(self.after.as_ref()) { + Next::Ready(row) => Poll::Ready(Some(row)), + Next::Ended => Poll::Ready(None), + Next::Pending => Poll::Pending, + }); match poll { Poll::Ready(Ok(found)) => Poll::Ready(found), // The producer closed without a clean end (broadcast dropped): no more segments. diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index fb2a4fa247..f2dd64ce2b 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -692,7 +692,10 @@ mod tests { StatusCode::NOT_FOUND ); - assert_eq!(status(&app, "/live/video/video0/seg/run-1.656e3d1b.0.m4s").await, StatusCode::OK); + assert_eq!( + status(&app, "/live/video/video0/seg/run-1.656e3d1b.0.m4s").await, + StatusCode::OK + ); assert_eq!( status(&app, "/live/video/video0/seg/run-1.656e3d1b.t0.m4s").await, StatusCode::OK From b6126e671ea8f0c35ca87c13d39a224c450e8192 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 00:59:30 -0700 Subject: [PATCH 48/49] fix(archive,hls): settle cancelled enrollments, list only committed timelines, static DASH waits for all - A Control::track dropped mid-enrollment left its count raised, so a stopped recording never ended. A guard settles the count and the name however the call ends, and the writer keeps watching enrollments after commands close. - Store::timelines skips a timeline directory whose .info isn't stored, so follow mode doesn't fail on one interrupted mid-enrollment. - The DASH manifest turns static only once every representation ended, now that a slower rendition can end after the reference. - Trimming reads the oldest listed segment without copying the window. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/reader/tests.rs | 14 ++++++ rs/moq-archive/src/store.rs | 15 +++++-- rs/moq-archive/src/writer.rs | 69 +++++++++++++++++++++++++++--- rs/moq-hls/src/export/mod.rs | 6 ++- rs/moq-hls/src/export/rendition.rs | 4 +- rs/moq-hls/src/export/segments.rs | 7 ++- 6 files changed, 100 insertions(+), 15 deletions(-) diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index 7f2c9eb3ed..1d625488a8 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -489,6 +489,20 @@ async fn a_track_enrolled_after_opening_is_followed() { assert!(timeline.subscribe(None).await.is_ok(), "the late timeline replays"); } +/// A timeline directory without its `.info`, such as one interrupted mid-enrollment, isn't +/// advertised, so following the recording doesn't fail on it. +#[tokio::test] +async fn a_timeline_without_info_is_not_listed() { + let mut archive = Archive::new().await; + archive.add("audio", 0..2, 1).await; + archive + .raw(&Key::segments(timeline("video"), 0).unwrap(), b"partial") + .await; + + let timelines = archive.store.timelines().await.unwrap(); + assert_eq!(timelines.keys().collect::>(), ["audio"]); +} + #[tokio::test] async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { let mut archive = Archive::new().await; diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 77ca989a55..dfc1c16bfa 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -193,7 +193,9 @@ impl Store { /// Each recorded track's timeline track, found by listing the track directories a [`Writer`] /// creates: a timeline is named by [`hang::timeline::default_name`]. One listing of the - /// prefix's directories, so following a long recording doesn't list its objects. + /// prefix's directories plus a HEAD per timeline, so following a long recording doesn't list + /// its objects. A timeline whose `.info` isn't stored yet, such as one still enrolling, is left + /// out until it is. /// /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. /// @@ -206,9 +208,14 @@ impl Store { .filename() .ok_or_else(|| Error::Directory(directory.to_string()))?; let track = crate::path::decode_track(encoded)?; - if let Some(indexed) = track.strip_suffix(hang::timeline::SUFFIX) { - timelines.insert(indexed.to_string(), track); - } + let Some(indexed) = track.strip_suffix(hang::timeline::SUFFIX) else { + continue; + }; + match self.inner.head(&self.path(&Key::info(&track)?)?).await { + Ok(_) => timelines.insert(indexed.to_string(), track), + Err(object_store::Error::NotFound { .. }) => continue, + Err(err) => return Err(err.into()), + }; } Ok(timelines) } diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs index 0dd3ef7320..e0e4cdf8e4 100644 --- a/rs/moq-archive/src/writer.rs +++ b/rs/moq-archive/src/writer.rs @@ -125,6 +125,23 @@ struct Shared { enrolling: watch::Sender, } +/// One [`Control::track`] in flight: drops its count, and its name unless it enrolled, however +/// the call ends. +struct Enrolling<'a, S> { + shared: &'a Shared, + name: &'a str, + enrolled: bool, +} + +impl Drop for Enrolling<'_, S> { + fn drop(&mut self) { + self.shared.enrolling.send_modify(|count| *count -= 1); + if !self.enrolled { + self.shared.enrolled.lock().unwrap().remove(self.name); + } + } +} + enum Command { Enroll(Box>), Cut(Timestamp), @@ -295,7 +312,8 @@ impl Writer { } } _ = source.closed(), if !closed => closed = true, - _ = enrolling.changed(), if idle && accepting => {} + // Also once commands closed: an enrollment in flight still settles its count. + _ = enrolling.changed(), if idle => {} _ = tokio::time::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { let (_, expired) = deletions.pop_front().unwrap(); delete(&shared.store, &mut pruned, expired).await; @@ -334,12 +352,15 @@ impl Control { return Err(Error::Enrolled(name.to_string())); } self.shared.enrolling.send_modify(|count| *count += 1); - let result = self.subscribe(name, config).await; - self.shared.enrolling.send_modify(|count| *count -= 1); - if result.is_err() { - self.shared.enrolled.lock().unwrap().remove(name); - } - result + // Settles the count and the name even if this future is dropped mid-enrollment. + let mut enrolling = Enrolling { + shared: &self.shared, + name, + enrolled: false, + }; + self.subscribe(name, config).await?; + enrolling.enrolled = true; + Ok(()) } /// Stop recording every track: store what already arrived, finish each timeline, and end @@ -1898,6 +1919,40 @@ mod tests { assert_eq!(stored(&store).await, before); } + /// An enrollment dropped mid-flight still settles its count, so stopping the recording ends it. + #[tokio::test(start_paused = true)] + async fn a_cancelled_enrollment_does_not_block_the_end() { + let source = broadcast::Info::new().produce(); + let mut audio = track(&source, "audio"); + + let mock = Mock::memory(); + mock.hold_puts("audio/.info"); + let store = Store::new(mock.clone(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + let run = tokio::spawn(writer.run()); + let enrolling = { + let control = control.clone(); + tokio::spawn(async move { control.track("audio", media()).await }) + }; + assert!( + audio.subscription_changed().await.unwrap().is_some(), + "enrollment subscribed" + ); + enrolling.abort(); + let _ = enrolling.await; + + control.stop().unwrap(); + tokio::time::timeout(Duration::from_secs(5), run) + .await + .expect("the recording ends") + .unwrap() + .unwrap(); + // The cancelled name was never enrolled, so it isn't taken. + mock.release(); + drop(source); + } + /// The publisher already has the subscriber, and the broadcast ends while `.info` is written. /// /// The enroll command is queued only after that write. The recording stays up across the gap, diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 8a63897b56..d86609ad18 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -251,7 +251,8 @@ impl Broadcaster { if representations().all(|rep| rep.segments.is_empty()) { return None; } - let finished = representations().any(|rep| rep.ended); + // Static only once every representation is final, so none stops being refreshed early. + let finished = representations().all(|rep| rep.ended); if !finished { // A dynamic presentation needs pts 0 anchored to the wall clock. The fallback is // set whenever a record has been pushed, which listing a segment implies. @@ -2860,6 +2861,9 @@ mod tests { let pending = test.rendition("video1").snapshot(); assert_eq!(numbers(&pending), [0, 1, 2]); assert!(!pending.finished, "a pending tail holds back EXT-X-ENDLIST"); + // DASH alike: the manifest turns static only once every representation ended. + assert!(test.rendition("video0").representation().unwrap().ended); + assert!(!test.rendition("video1").representation().unwrap().ended); video1.push(&gop(4)).unwrap(); assert_eq!(numbers(&test.finished("video1").await), [0, 1, 2, 3]); test.drain("video0").await; diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 981dfeb0e4..779f0d0843 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -414,8 +414,8 @@ impl Rendition { /// Drop this rendition's own records that no listed segment can resolve to anymore. fn trim(&self) { - if let Some(oldest) = self.live.window().segments.first() { - self.spans.trim(oldest.pts.into()); + if let Some(oldest) = self.live.oldest() { + self.spans.trim(oldest); } } diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 97e62605d2..f18aba26c6 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -12,7 +12,7 @@ //! order, exactly once. `next()` waits for the next resolved row, FETCHes and transmuxes its //! frames (via [`Rendition`]), and yields the CMAF bytes. //! -//! A segment is addressed by its reference's [`tag`] and its number (the `seg/{tag}.{segment}.m4s` +//! A segment is addressed by its reference's tag and its number (the `seg/{tag}.{segment}.m4s` //! URI); the number alone is its `EXT-X-MEDIA-SEQUENCE` and the recorder cursor's position. The //! same URI names the same span of content time on every rendition, on every edge, and after //! every reload. Each reference numbers segments by its own records, so a new reference starts a @@ -257,6 +257,11 @@ impl Producer { let _ = self.state.close(); } + /// The start of the oldest listed segment, without copying the window. + pub fn oldest(&self) -> Option { + self.state.read().rows.front().map(|row| row.pts.into()) + } + /// Snapshot the current window (serve path). #[cfg_attr(not(feature = "server"), allow(dead_code))] pub fn window(&self) -> Window { From afab42ff7f408c7a2714f7806e792e3329b2d3e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 01:06:44 -0700 Subject: [PATCH 49/49] quest: track the HLS media sequence rewind on a reference switch Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/README.md | 1 + quest/m1/archive/hls-media-sequence.md | 37 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 quest/m1/archive/hls-media-sequence.md diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 5b4f9cd5c5..5dc959e131 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -122,6 +122,7 @@ owned by that prerequisite, not duplicated in archive storage. - [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay - [Idle flush](/quest/m1/archive/flush.md) - idle tracks are recorded within a bounded wall-clock delay, and `flush()` forces a track's pending record out - [Backward timestamps](/quest/m1/archive/backward-timestamps.md) - a resumed recording refuses a restarted source instead of dropping its groups or overlapping media time +- [HLS media sequence](/quest/m1/archive/hls-media-sequence.md) - `EXT-X-MEDIA-SEQUENCE` never decreases when the reference rendition switches - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback ## Related diff --git a/quest/m1/archive/hls-media-sequence.md b/quest/m1/archive/hls-media-sequence.md new file mode 100644 index 0000000000..836e9be983 --- /dev/null +++ b/quest/m1/archive/hls-media-sequence.md @@ -0,0 +1,37 @@ +# [S] HLS media sequence survives a reference switch + +## Goal + +A media playlist's `EXT-X-MEDIA-SEQUENCE` never decreases, as +[RFC 8216 §6.2.2](https://www.rfc-editor.org/rfc/rfc8216.html#section-6.2.2) +requires, even when the reference rendition changes. Players keep loading the +same `media.m3u8` across the switch without stalling. + +## Plan + +Today `moq-hls` numbers segments by the reference rendition's own records. A +switch (a new first video rendition with a timeline, or the reference leaving +the catalog) starts a new numbering under a new URL tag +(`seg/{reference}.{segment}.m4s`, #4034). Its URLs never collide and a +discontinuity marks the switch, but the media sequence can rewind. + +Open design, pick one: + +- **Sticky reference.** Keep the reference while it stays in the catalog, so a + new, earlier-named rendition no longer switches it. Cheap and edge-consistent, + but a reference that leaves the catalog still rewinds. +- **Offset at the switch.** Number the new reference's segments from past the + last listed one. Monotonic, but an edge that starts after the switch numbers + differently, so edges disagree on the media sequence. + +Recommendation: the sticky reference first. It removes the common trigger +without giving up the property that every edge agrees on numbering. Measure +whether a reference leaving the catalog happens in practice before taking on +the offset. + +Test a switch through repeated playlist reloads: the media sequence never +decreases, and no URL names other content. + +## Related + +- [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - also changes how the reference shapes the playlist