diff --git a/Cargo.lock b/Cargo.lock index 28ed1921dc6..f0509ed1d7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1498,6 +1498,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -4094,6 +4104,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "mdns-sd" version = "0.21.4" @@ -4203,15 +4223,22 @@ version = "0.0.11" dependencies = [ "async-trait", "bytes", + "criterion", "futures", + "hang", + "kio 0.6.2", + "moq-json", + "moq-mux", "moq-net", "object_store", "percent-encoding", + "quick_cache", "serde", "serde_json", "tempfile", "thiserror 2.0.21", "tokio", + "tracing", ] [[package]] @@ -4351,9 +4378,12 @@ dependencies = [ "bytes", "hang", "humantime", + "moq-archive", "moq-audio", "moq-auth", "moq-hls", + "moq-json", + "moq-msf", "moq-mux", "moq-relay", "moq-rtc", @@ -4363,6 +4393,7 @@ dependencies = [ "moq-transcode", "moq-video", "mpeg2ts", + "object_store", "pollster", "reqwest", "rustls", @@ -4458,12 +4489,16 @@ dependencies = [ name = "moq-hls" version = "0.5.11" dependencies = [ + "async-trait", "axum", "bytes", + "futures", "hang", "humantime", "kio 0.6.2", "m3u8-rs", + "moq-archive", + "moq-json", "moq-mux", "moq-net", "moq-tokio", @@ -6320,17 +6355,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", + "aws-lc-rs", + "base64 0.23.1", "bytes", "chrono", + "crc-fast", + "form_urlencoded", "futures-channel", "futures-core", "futures-util", "http", + "http-body-util", + "httparse", "humantime", + "hyper", "itertools 0.15.0", + "md-5", "nix", "parking_lot", "percent-encoding", + "quick-xml 0.41.0", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", "thiserror 2.0.21", "tokio", "tracing", @@ -7022,6 +7072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", + "serde", ] [[package]] @@ -7033,6 +7084,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick_cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" +dependencies = [ + "equivalent", + "foldhash", + "hashbrown 0.17.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.12" @@ -7353,6 +7416,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index c76c34e68e1..721180704b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,7 @@ loom = { version = "0.7.2", features = ["futures"] } # DNS-SD advertisement and browsing for LAN peer discovery (moq-tokio's `mdns` feature). # `async` awaits the event channel instead of blocking a thread on it. mdns-sd = { version = "0.21", features = ["async"] } +moq-archive = { version = "0.0.11", path = "rs/moq-archive" } moq-audio = { version = "0.1.10", path = "rs/moq-audio", default-features = false } moq-auth = { version = "0.2.1", path = "rs/moq-auth" } moq-flate = { version = "0.2.0", path = "rs/moq-flate" } diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 31db4cdeb9d..5949ca85ca9 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -19,9 +19,11 @@ Docker; see [Install](/setup/install). | `import` | `capture` | Capture a camera, display, window, or app plus a microphone, and encode natively. | | `import` | `hls ` | Pull a remote HLS playlist. | | `import` | `rtmp`, `srt`, `rtc` | Accept pushes (`--listen`) or pull from a remote (`--connect`). | +| `import` | `archive ` | Replay a recording from an object store. | | `export` | `fmp4`, `mkv`, `ts`, `flv`, `h264`, `h265` | Write a container to stdout. | | `export` | `hls --listen` | Serve the broadcast as HLS over HTTP. | | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | +| `export` | `archive ` | Record the broadcast into an object store. | | `play` | | Decode and play in a native window with sound. | | `transcode` | | Publish a just-in-time rendition ladder next to a broadcast. | | `announced` | `[prefix]` | Follow the broadcasts announced on a relay. | @@ -148,7 +150,7 @@ The `fmp4`, `mkv`, `flv`, `h264`, and `h265` exports select renditions with flags before the sink: `--video-name` and `--audio-name` pick a rendition, `--video-codec` and `--audio-codec` keep a codec family, and `--no-video` or `--no-audio` leaves a role out. `h264` and `h265` refuse `--no-video` and the -audio selection flags. `ts` and the gateways don't apply selection, so they +audio selection flags. `ts`, `archive`, and the gateways don't apply selection, so they refuse these flags. ```bash @@ -338,6 +340,41 @@ refuses any listener, cluster, auth, or `--hop` flag. It gives up after 30 seconds, as `/fetch` does, and exits non-zero when the broadcast or group is not found (before writing anything), the relay refuses, or the deadline passes. +## Archive + +```bash +# Record a broadcast until it ends +moq --connect https://relay.example.com/anon --broadcast event.hang export archive s3://recordings/event + +# Replay it under another name +moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event +``` + +`export archive` records one broadcast with +[moq-archive](https://docs.rs/moq-archive), reading its catalog as it changes. +Every track gets its own timeline, stored in spans cut at group boundaries +between 2s and 10s. The catalog and every text, JSON, and binary track are +sparse data, so each of their groups is stored as soon as it finishes, and a +group that never closes is stored in pieces as it grows. It refuses a rendition served +from another broadcast, and one that returns after the catalog dropped it. The +stage ends once the broadcast does. A store URL that already holds a +recording is continued: each track resumes after its newest stored span. `--retention 1h` keeps only the last hour (a DVR), +deleting expired objects, and timeline objects no longer needed to recover it, +`--retention-grace` (default 30s) after the timeline stops needing them. Every +track keeps at least its newest span, so a catalog that never changes outlives +the video it was published with. + +`import archive` republishes a recording: each track's timeline replays as a +live track and every track's groups are served on request, one object GET per +stored span. By default it replays what is stored and ends the timelines there; +`--follow 2s` keeps checking for new spans, and newly recorded tracks, of a +recording still being made. + +Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, +`gs://bucket/prefix`, or `az://container/prefix`. Cloud credentials come from +the usual `AWS_*`, `GOOGLE_*`, and `AZURE_*` environment variables. The `s3`, +`gcs`, and `azure` cargo features are on by default. + ## Multiple stages Separate stages with `--` to bridge several broadcasts, or both directions, @@ -346,7 +383,8 @@ over one connection: ```bash moq --connect https://relay.example.com/anon \ import --broadcast event.hang srt --listen 0.0.0.0:9000 \ - -- export --broadcast event.hang hls --listen 0.0.0.0:8080 + -- export --broadcast event.hang hls --listen 0.0.0.0:8080 \ + -- export --broadcast event.hang archive file:///recordings/event ``` ## Redundant publishers diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 0b4b2b66587..c1b86864fac 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -17,16 +17,25 @@ moq --connect https://relay.example.com/anon --broadcast my-stream.hang export h moq --connect https://relay.example.com/anon --broadcast my-stream.hang import hls https://example.com/live/master.m3u8 ``` -Export never subscribes to media. It reads the broadcast's -[timeline track](/concept/hang#catalog), a log of complete segments aligned -across every rendition, to build playlists, then fetches exactly the groups a -requested segment covers from the relay's cache and transmuxes them to CMAF on -demand. So a segment is servable for as long as the relay's +Export never subscribes to media. It reads each rendition's +[timeline track](/concept/hang#catalog), a small index of that rendition's +groups, to build playlists, then fetches exactly the frames a requested segment +covers from the relay's cache and transmuxes them to CMAF on demand. So a +segment is servable for as long as the relay's [cache](/bin/relay/config#cache) retains it, and idle renditions cost nothing. -Because segments are aligned, the same number names the same span of content in -every media playlist; a record with nothing for a rendition renders as -`EXT-X-GAP` and a jump in content time as `EXT-X-DISCONTINUITY`. A broadcast -whose catalog advertises no timeline is skipped. One server exposes every +Segment boundaries come from one reference rendition (the first video +rendition by name, or the first audio one without video) and are numbered by +its records, so every edge and every reload agree. Every other video rendition +snaps each boundary to its nearest keyframe within about a second, and a +segment with none in range renders as `EXT-X-GAP`; audio takes every frame +inside the segment's span. A jump in content time renders as +`EXT-X-DISCONTINUITY`. Gaps are a fallback: a publisher wanting clean HLS +export should align video GOPs across renditions. A timeline that fails +(malformed, refused, or lost) is not retried, since the relay already rides out +transient source failures. The reference timeline failing ends every playlist +with `EXT-X-ENDLIST`, and any other rendition's timeline failing ends that +rendition's playlist at the last segment it covers. A broadcast +whose catalog advertises no timelines is skipped. One server exposes every broadcast by path: ```text @@ -34,17 +43,37 @@ broadcast by path: /{broadcast}/manifest.mpd /{broadcast}/{video|audio}/{rendition}/media.m3u8 /{broadcast}/{video|audio}/{rendition}/init.{hash}.mp4 -/{broadcast}/{video|audio}/{rendition}/seg/{segment}.m4s -/{broadcast}/{video|audio}/{rendition}/seg/t{pts}.m4s +/{broadcast}/{video|audio}/{rendition}/seg/{reference}.{segment}.m4s +/{broadcast}/{video|audio}/{rendition}/seg/{reference}.t{pts}.m4s ``` +Segment boundaries come from one reference rendition's timeline, and its records +number the segments. `{reference}` is a short hash of that rendition's kind and +name, so every edge derives the same URL, and a reference that changes (a new +first video rendition) starts a new numbering under new URLs rather than reusing +the old ones for other content. + +A [`moq-archive`](https://docs.rs/moq-archive) recording replayed through its +`Reader` is served the same way, with no second stored copy. Playlists come +from the replayed timelines alone, and a segment GETs only its rendition's +stored objects, so switching renditions never downloads both. An +inline-parameter-set codec with no catalog `description` is the exception: +the first playlist render GETs one keyframe group to build the init segment, +then caches it. Out-of-band configs need no media GET. When the catalog's +`archive` entry names a `store` and no `replay` path, its spans are durable on +this broadcast, so the playlists list the whole retained timeline and only the +recording's own retention trims them; DASH `timeShiftBufferDepth` is the listed +span. The playlist ends with `EXT-X-ENDLIST` only once the reader's caller +declares the recording finished; the store holds no completion marker. + The init URL carries a hash of its bytes, so a reconfigured rendition gets a new one. An embedder of the library can also label the publisher's run with `Broadcaster::set_generation`. Every segment URL then carries it -(`seg/{generation}.{segment}.m4s`), since a restarted publisher reuses segment +(`seg/{generation}.{reference}.{segment}.m4s`), since a restarted publisher reuses segment numbers for different media. -`--window` sets the playlist duration (default 16 s), +`--window` sets the live playlist duration (default 16 s) and caps segment +`Cache-Control: max-age` for every broadcast, `--listen-tls-cert`/`--listen-tls-key` or `--listen-tls-generate` serve HTTPS, and `--cors-origin` opens it to browsers. H.264/H.265 and AAC/Opus renditions are served. Import handles classic HLS; diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 1a7bfb40f6a..156957142c4 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -57,8 +57,8 @@ A few things the catalog can express beyond decoder config: - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Stalled renditions.** A publisher can flag a rendition as temporarily bad so players prefer another one without the track disappearing. First-party video publishers set this flag after more than three frame intervals of source silence or encoding lag while subscribed, and clear it after three on-time completed frames or when idle. Browser and native capture poll while waiting; FLV and MPEG-TS importers observe video silence as container data arrives. The shared detector is `hang::catalog::stalled::Detector` in Rust and `Catalog.Stalled.Detector` in JavaScript. It is a playback diagnostic, not an authorization or routing signal. -- **Archive.** A broadcast may advertise an `archive` entry naming its timeline track (a small index of each complete aligned segment) and, if recorded, the replay MoQ path, object-store URL, and format version. The timeline is what lets the [HLS gateway](/bin/hls) build playlists without subscribing to media. -- **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a segment index. +- **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and, if recorded, the replay MoQ path, object-store URL, and format version. Every track is cut by one rule, at group boundaries between 2s and 10s (a zero minimum for sparse data such as the catalog), and tracks commit and expire independently. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. +- **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a timeline. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. ## Text diff --git a/doc/lib/js/hang.md b/doc/lib/js/hang.md index 5700f73856c..57b9e8e79af 100644 --- a/doc/lib/js/hang.md +++ b/doc/lib/js/hang.md @@ -24,8 +24,8 @@ import * as Container from "@moq/hang/container"; `Catalog.EscapingBroadcast` for a `broadcast` reference that walks above the handle's `path`. Run the same checks on a catalog from another source with `Catalog.checkRenditions(root)` and `Catalog.checkResolvable(root, base)`. -`Hang.Timeline.Consumer.subscribe(broadcast, root.archive)` reads segment -`push`, `pop`, and `skip` events when a root advertises an archive. +`Hang.Timeline.Consumer.subscribe(broadcast, root.archive, track)` reads one +track's timeline as record `push`, `pop`, and `skip` events when a root advertises it. Most apps never import it directly; the elements and `Broadcast` classes in the watch and publish packages do. Reach for it when hand-rolling a catalog diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index b25a36a24aa..dfde9ad1953 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -12,7 +12,7 @@ The [hang media format](/concept/hang) on top of `moq-net`: a live catalog describing renditions with WebCodecs-style decoder configs, and containers that carry a timestamp with every frame. -- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), stalled hints, cross-broadcast rendition references, and the `archive` entry (timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. +- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), stalled hints, cross-broadcast rendition references, and the `archive` entry (each track's timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. - **Containers.** `legacy`, `cmaf`, and `loc`, decoded for you; unknown kinds pass through untouched. - **`Ordered`** reads a media track as timestamped frames, reorders groups, and skips ones that fall past your max age. - **Codecs described**: H.264, H.265, VP8, VP9, AV1, AAC, Opus, PCM. diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index f643a2e52e2..72ccdf15aa7 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -19,7 +19,7 @@ The reference implementation. Every crate is on | [moq-uring](https://docs.rs/moq-uring) | Experimental Linux io\_uring worker: one pinned thread per ring serving moq-lite over its own QUIC stack. | | [hang](/lib/rs/hang) | The media layer: catalog, containers, ordered frame delivery. | | [moq-mux](/lib/rs/moq-mux) | Import and export fMP4/CMAF, MPEG-TS, Matroska, FLV, and Annex-B. | -| [moq-archive](https://docs.rs/moq-archive) | Versioned hang recordings on any `object_store` backend: track layout, `.info` JSON, and segment objects. | +| [moq-archive](https://docs.rs/moq-archive) | Versioned hang recordings on any `object_store` backend: track layout, `.info` JSON, per-track timelines, and span objects. | | [moq-video](/lib/rs/moq-video) | Native capture, hardware encode/decode (Apple, Windows, NVIDIA, VAAPI, V4L2, Android), and GPU rendering. | | [moq-v4l](https://docs.rs/moq-v4l) | Safe Video4Linux 2 bindings with the kernel headers checked in, so a build needs no libclang. | | [moq-audio](/lib/rs/moq-audio) | Microphone and speaker, Opus/PCM/AAC codecs, echo cancellation. | diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 52d5060e68b..38dc8567153 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -141,7 +141,7 @@ For example, a chat entry should name a chat track, not carry individual chat me This way catalog updates are rare and a client MAY choose to not subscribe. The `json` and `binary` sections ({{data}}) are how a track like that is listed. -This specification defines audio, video, and text media tracks, plus an optional `archive` entry ({{archive-catalog}}) naming the timeline track ({{timeline}}) that indexes their segments, an optional root `clock` entry ({{clock}}) mapping content time to wall time, and the application data tracks in {{data}}. +This specification defines audio, video, and text media tracks, plus an optional `archive` entry ({{archive-catalog}}) naming the timeline track ({{timeline}}) that indexes each track, an optional root `clock` entry ({{clock}}) mapping content time to wall time, and the application data tracks in {{data}}. ## Clock {#clock} The catalog's root `clock` field is the broadcast's one continuous clock: @@ -155,12 +155,12 @@ type ClockSchema = { The `wall` field is the wall-clock time of PTS zero, in `timescale` units since the moq epoch, 2020-01-01T00:00:00Z. A consumer derives the wall-clock time of any media timestamp as `wall + pts` after converting that timestamp into this timescale, and Unix time by adding the epoch back (for HLS `EXT-X-PROGRAM-DATE-TIME` or DASH `availabilityStartTime`). -Every media track and the archive index refer to this one mapping after timescale conversion; there are no competing wall epochs. +Every media track and every timeline refer to this one mapping after timescale conversion; there are no competing wall epochs. The `timescale` field is the units per second for `wall`, defaulting to 1000000 (microseconds) when absent. A zero `timescale` is invalid, as is a `wall` outside the JSON-safe integer range; both are refused rather than truncated. -The mapping is fixed for the broadcast and independent of `archive`: a live-only publisher exposes its clock without creating a segment index. +The mapping is fixed for the broadcast and independent of `archive`: a live-only publisher exposes its clock without creating a timeline. A discontinuity marker is a delivery event, not a new epoch, and a system-clock adjustment never retimes the mapping. ## Video @@ -626,7 +626,7 @@ Some metadata tracks are compressed. Compression is signalled per track, never inferred from the track's name. A data track ({{data}}) declares it with the `compression` field ({{field-compression}}). -The two tracks this specification defines as always compressed, `catalog.json.z` ({{catalog}}) and the timeline track ({{timeline}}), are identified by their role instead. +The tracks this specification defines as always compressed, `catalog.json.z` ({{catalog}}) and the timeline tracks ({{timeline}}), are identified by their role instead. The `.z` suffix on those names is a naming convention, and a consumer MUST NOT treat it as a signal. Such a track is compressed per {{moqflate}}: each group is one raw DEFLATE stream, sync flushed at each frame boundary. @@ -634,22 +634,21 @@ The suffix is the declaration. # Timeline {#timeline} -The timeline track is the broadcast's segment index. +A timeline track indexes one track's content. MoQ groups carry only an opaque sequence number; the timestamps live inside the media frames. -The timeline republishes the broadcast's segmentation as metadata: one record per segment, mapping a span of content time to the group ranges that carry it on each media track. -A consumer can answer "which groups cover time T on track X" and "where is the live edge" from a few bytes per segment, without downloading media. -This is sufficient to render an HLS or DASH playlist, seek a VOD recording, or index an archive. +A timeline republishes a track's spans as metadata: one record per span, mapping a span of content time to the frame positions that carry it. +A consumer can answer "which groups cover time T on track X" and "where is the live edge" from a few bytes per span, without downloading media. +This is sufficient to derive an HLS or DASH playlist, seek a VOD recording, or index an archive. -The timeline is optional. -There is one timeline per broadcast, because its purpose is that segments are aligned across the broadcast's tracks: segment N covers the same span of content time on every track, which is what HLS requires of switchable renditions. -A broadcast that does not need aligned segments simply omits it. +Timelines are optional, and a track has at most one. +Each track's timeline is cut, published, and expired independently; nothing in one timeline waits for another track. ## Catalog Section {#archive-catalog} -The catalog's root `archive` field is the one name for the segment index, and for any durable recording of those ranges: +The catalog's root `archive` field names every timeline, and any durable recording of their spans: ~~~ type ArchiveSchema = { - "track": string, + "timelines": Map, "timescale": number | undefined, "durationMax": number | undefined, "replay": string | undefined, @@ -658,41 +657,38 @@ type ArchiveSchema = { } ~~~ -The `track` field names the MoQ track carrying the segment records. -The name `timeline.z` is RECOMMENDED; a consumer MUST use the advertised name rather than assuming it. -A live publisher without a store advertises `archive` with the timeline fields (`track`, `timescale`, `durationMax`) alone. -Every range the timeline advertises is FETCHable; with a store they are also durable. -There is no sibling `timeline` entry and no generation: a client that must tell recordings apart compares `replay` and `store`. +The `timelines` field maps each indexed track to the MoQ track carrying its records. +Any track may be indexed, including the catalog track itself and data tracks ({{data}}). +Naming a timeline after its track with a `.timeline.z` suffix is RECOMMENDED; a consumer MUST use the advertised names rather than assuming them. +A live publisher without a store advertises `archive` with the timeline fields (`timelines`, `timescale`, `durationMax`) alone. +Every span a timeline advertises is FETCHable; with a store it is also durable. +There is no generation: a client that must tell recordings apart compares `replay` and `store`. The `timescale` field is the units per second for the records' `pts` and `duration` values, and for `durationMax`. If absent, it defaults to 1000 (milliseconds). A zero `timescale` is invalid and MUST be refused. -The `durationMax` field, if present, is the declared upper bound on a segment's `duration`, in `timescale` units. -A publisher that controls its encoder knows its keyframe cadence up front, so a consumer can size buffers or write an HLS `EXT-X-TARGETDURATION` from the catalog alone, before observing a single segment. -The value MUST NOT change for the life of the broadcast, and a publisher MUST NOT emit a record whose `duration` exceeds it. -A publisher that cannot honor that MUST omit the field rather than emit a record contradicting it. - -The field is absent when the media decides the segmentation instead, which is the common case: a real-time encoder places keyframes on demand and a single GOP may be minutes long, and a publisher importing a source it does not control cannot promise anything about that source. -A consumer needing a bound then derives one from the records it has seen, raising it as longer segments arrive. +The `durationMax` field, if present, is the declared upper bound on a record's `duration`, in `timescale` units. +A publisher MUST NOT emit a record whose `duration` exceeds it; splitting a group between frames ({{timeline-cutting}}) makes the bound achievable for any content, and a record whose frames are farther apart than the bound ends at the bound, leaving the rest of the gap uncovered. +The value MUST NOT change for the life of the broadcast. -Wall-clock mapping is the catalog root `clock` ({{clock}}), not this section: a consumer derives the wall-clock time of any segment as `clock.wall + pts` after converting `pts` into the clock's timescale. +Wall-clock mapping is the catalog root `clock` ({{clock}}), not this section: a consumer derives the wall-clock time of any record as `clock.wall + pts` after converting `pts` into the clock's timescale. The `replay` field, if present, is a relative MoQ broadcast path ({{field-broadcast}}) the archive is served back from. -Absent, the timeline lives on the catalog's own broadcast. +Absent, the timelines live on the catalog's own broadcast. A wildcard replay path names no generation. The `store` field, if present, is the object-store URL the recording objects ({{recording}}) live under. Authorization for `replay` and `store` is external. The `version` field is the recording object format version ({{recording-track}}). -It is 1 for this specification. +It is 2 for this specification. A publisher that exposes a store MUST set it; a publisher that does not MUST omit it. A catalog that composes another broadcast's renditions MUST preserve that child's `archive` entry instead of synthesizing one. ## Track Framing {#timeline-framing} -The timeline track is a sliding window of records. +A timeline track is a sliding window of records. An unbounded publisher only appends records, while a DVR publisher also removes records from the front as they expire. The first frame of each group is a UTF-8 JSON object containing a checkpoint of the retained window: @@ -717,74 +713,67 @@ A consumer that missed records reports their absolute index range as skipped bef The frames are DEFLATE-compressed ({{!RFC1951}}) within each group. The publisher ends each frame's compressed data with an empty sync-flush block (the `0x00 0x00 0xff 0xff` trailer is removed, as in {{?RFC7692}}), so a consumer decompresses frames incrementally from the group's first frame. -The `.z` suffix on the RECOMMENDED track name marks this compression, mirroring the catalog's `catalog.json.z` sibling. +The `.z` suffix on the RECOMMENDED track names marks this compression, mirroring the catalog's `catalog.json.z` sibling. ## Records {#timeline-records} -Each record describes one complete segment: +Each record describes one span of its track: ~~~ type TimelineRecord = { - "segment": number, + "sequence": number, "pts": number, "duration": number, - "tracks": Map | undefined, + "start": TimelinePosition, + "end": TimelinePosition, + "keyframe": boolean | undefined, } -type TimelineRange = { - "start": number, - "end": number, - "keyframe": boolean | undefined, +type TimelinePosition = { + "group": number, + "frame": number | undefined, } ~~~ -The `segment` field is the segment's number. -Numbers are consecutive within a broadcast, anchoring HLS `EXT-X-MEDIA-SEQUENCE`; they are explicit rather than implied by record order so a reader joining mid-stream, or reading a windowed recording, keeps stable numbering. +The `sequence` field is the record's number, consecutive within its timeline and equal to the record's window index. -The `pts` field is the segment's start and `duration` its length, both in the timeline's timescale. +The `pts` field is the timestamp of the span's first frame and `duration` its length, both in the timescale of the catalog section. The next record's `pts` equals `pts + duration` unless content time itself jumped; a consumer SHOULD treat such a jump as a discontinuity. -The `tracks` field maps each participating track name to the group ranges it contributes. -Each range covers groups `start` through `end` inclusive, as used by moq-lite FETCH and SUBSCRIBE. -More than one range means the group sequence is discontinuous inside the segment: the skipped groups never existed. -A track absent from the map has no content for the span (a gap; HLS `EXT-X-GAP`). +A position names frame `frame` of group `group`, as used by moq-lite FETCH and SUBSCRIBE; `frame` defaults to 0. +Positions order by group, then frame. +The span holds every frame from `start` (inclusive) to `end` (exclusive). +An `end` at frame 0 of group `g` holds every frame of group `g - 1` and none of `g`, whether or not `g` exists; an `end` at a later frame of `g` holds the frames of `g` before it. +Every group from `start.group` to the last group holding a frame exists and holds at least one frame of the span. +A record starts where the previous one ended, unless the track skipped group sequences between them. -Participating tracks need not be audio or video. -A catalog, or an application's own metadata track such as a chat log, is listed exactly like a media track, which is what lets a recording ({{recording}}) address all of them the same way. -A consumer that only wants renditions therefore MUST select tracks by consulting the catalog rather than by assuming every name in the map is media. -A record MUST tolerate and SHOULD preserve unknown fields, like the catalog. +The `keyframe` field states whether the span's first frame is a keyframe, i.e. whether a player can join or switch renditions there. +If absent, it defaults to true. -The `keyframe` field states whether the range's first group starts with a keyframe, i.e. whether a player can join or switch renditions there. -If absent, it defaults to true; a publisher sets `false` when a source resumes without one, so an exporter knows not to advertise the segment as independently decodable. +A record MUST tolerate and SHOULD preserve unknown fields, like the catalog. -## Segmentation {#timeline-segmentation} -A segment is a span of content time shared by every media track. -A track contributes every group whose start falls inside the span, so a segment boundary SHOULD land on a group start: every group already begins with a keyframe ({{container}}), so a boundary at a group start lets each track contribute whole groups and remain independently decodable. -A segment MAY span multiple groups of a track (short groups packed into a longer segment). +## Cutting {#timeline-cutting} +Where a span ends is publisher policy. +A publisher MUST NOT publish a record until its span is final: every frame between `start` and `end` has been published. +The newest record is therefore the live edge of its track. -How boundaries are chosen is publisher policy: following a source's existing segmentation (an imported HLS playlist, CMAF segments on disk), or pacing by a minimum duration. -A publisher pacing itself SHOULD end a segment at the earliest point that is a group start on every enrolled track and at least the minimum past the segment's start, which makes the track with the coarsest groups pace the broadcast and leaves no track's group split across a boundary. -A minimum is always satisfiable, whereas a maximum is not: a single group longer than it cannot be divided. -Where no such point exists because two tracks have different coarse cadences, a publisher MUST choose one of them rather than a point interior to any track's group. +The RECOMMENDED policy is one rule for every track: -Whatever the policy, a publisher MUST NOT emit a record until the segment is complete: every *pacing* track's groups for the span are known. -Records are therefore self-contained and immediately servable, and the newest record is the live edge. -A pacing track that has produced nothing for the span holds the record back; a publisher that knows a track has stopped for good closes it, and the record then simply omits it (a gap). +- A record ends at the first group boundary at least a minimum duration past its own start, so short groups (audio) pack into one record and long ones (video) get one each. +- A group still open a maximum duration past the record's start is split between frames, so a group that never closes (an append-log such as a `moq-json` stream) is indexed as it grows; `durationMax` advertises this maximum. +- A skipped group sequence ends the record at its newest known content, not where the next group starts, so a pause is not counted as content. -A pacing track is one whose groups arrive continuously, which is what makes them usable as boundaries. -A track that publishes on its own schedule cannot pace: a catalog emits a group only when the renditions change, so a timeline waiting for it would stall the moment it went quiet. -Such a track is *non-pacing*: its groups are listed in whichever segment is open when they arrive, but it never determines a boundary and never holds a record back. -A publisher SHOULD record its catalog this way, so a recording can resolve the renditions in effect at any segment. +A minimum of 2 seconds and a maximum of 10 seconds are RECOMMENDED. +Sparse data that publishes on its own schedule, such as a catalog, SHOULD use a minimum of zero: each group is then its own record, ended as soon as the group finishes, so its newest group is indexed without waiting for another that may never come. -Placement of a non-pacing track's groups is therefore by arrival rather than by content time: nothing waits for them, so a group that arrives after its segment has already been published is listed in the next one. -When a segment closes, every non-pacing group that arrived while it was open belongs to that segment regardless of the timestamp basis carried by the non-pacing track. -The frames still carry their own timestamps, so no timing information is lost. -A non-pacing track's timestamps do not extend the final segment's duration. -A non-pacing track whose group never closes (an append-log such as a `moq-json` stream) is listed once, in the segment its group opened in. -A publisher that needs such content addressable per segment SHOULD roll the group at segment boundaries, which costs the shared compression window but makes each segment self-contained. +A publisher MAY add boundaries of its own, such as following a source's segmentation or cutting audio where video starts a group, so a derived segment needs fewer records. +The final record of an ended track has no successor; its `duration` runs to the newest known content, which a publisher SHOULD carry past the start of the last frame when it knows where that frame ends. -A group that starts before the first boundary belongs to the first segment. -The final segment of an ended broadcast has no closing boundary; its `duration` runs to the newest known content. -A publisher SHOULD carry the end of the last group's content into that value, since a publisher that knows only where each group *started* would report a duration one group short, and zero for a final segment that is a single group. +## Derived Formats {#timeline-derived} +Segmented formats such as HLS and DASH need one segment numbering across renditions, which per-track timelines do not provide on their own. +An edge deriving one takes the segment boundaries from a reference rendition's records, the first video rendition by name with a timeline, or the first such audio rendition when there is none, and numbers the segments by that rendition's record `sequence`, so every edge and every reload agree. +Another video rendition snaps each boundary to its nearest record that starts a group on a keyframe; a segment with no such record nearby has no content on that rendition (a gap; HLS `EXT-X-GAP`). +Other renditions take the frames whose timestamps fall inside each segment's span. +A publisher wanting such an export SHOULD start video groups at the same timestamps across renditions. # MPEG-TS Service Information {#mpegts-si} @@ -844,21 +833,18 @@ Carrying the source's time rather than synthesizing one keeps the clock consiste A live broadcast is bounded history: moq-lite {{moql}} serves the present with SUBSCRIBE and the recent past with FETCH, both ending at the publisher's cache. A *recording* is the persistent tier, writing a broadcast to a filesystem or object store so it can be served back long after the live session ended. -A recording is addressed by segment. -The timeline ({{timeline}}) names every segment and the groups that carry it. -Each track's object is addressed by its smallest and largest stored group IDs; the timeline supplies those bounds without reading media. -A reader that wants segment N of a track issues one whole-object GET, with no range header and no second request, which is what both an HLS or DASH origin and a player seeking a VOD recording need. +A recording is addressed by record. +Each recorded track has its own timeline ({{timeline}}), and each of its records is stored as one object named by the record's `sequence`. +A reader that wants a record of a track issues one whole-object GET, with no range header and no second request, which is what both an HLS or DASH origin and a player seeking a VOD recording need. -A recording covers the tracks the timeline describes, plus the catalog and the timeline itself. -A broadcast with no timeline has no segments and cannot be recorded this way. +A recording covers the tracks its writer selects, including the catalog, plus each one's timeline. ## Layout {#recording-layout} A recording is a set of objects under a common prefix: ~~~ //.info -//groups/. -//segments/ +//segments/ ~~~ The common prefix is application-defined and is not interpreted by this format. @@ -866,22 +852,17 @@ The common prefix is application-defined and is not interpreted by this format. `` is the track's UTF-8 name with every byte outside `A-Z a-z 0-9 _ -` percent-encoded as `%` followed by two uppercase hexadecimal digits. For example, `catalog.json` becomes `catalog%2Ejson`. An encoded name never contains `/` or begins with `.`, so a track cannot address anything outside the prefix. -`.info`, `groups`, and `segments` are reserved within a track directory. +`.info` and `segments` are reserved within a track directory. + +A recording names each track's timeline by appending `.timeline.z` to the track's name, so a recording is recoverable from its objects alone; a recorded track's own name MUST NOT end in `.timeline.z`. +For a recorded track, `segments/` holds the frames of record `sequence` of its timeline. +For a timeline track, `segments/` holds the complete Window groups ({{timeline-framing}}) closed while committing record `sequence` of the track it indexes, at least one per object. -`` and `` are the inclusive first and last group sequence numbers in the object. -Recorded group and segment IDs MUST be integers in the range 0 through 9007199254740991 (2^53 - 1), so timeline JSON preserves them exactly. -Each filename field is written as exactly 19 decimal digits, padded with leading zeros, so lexical and numeric order agree. +`` is written as exactly 19 decimal digits, padded with leading zeros, so lexical and numeric order agree. +Recorded sequences, group IDs, and frame indices MUST be integers in the range 0 through 9007199254740991 (2^53 - 1), so timeline JSON preserves them exactly. Writers and readers MUST reject IDs outside this range, including reconstructed group IDs in the binary table. -A range-named object MUST contain at least one group, and its table's first and last sequences MUST match the filename. -For each track, object ranges MUST be nonoverlapping and strictly increasing in timeline segment order; gaps are allowed both within and between objects. -A track with no stored groups for a segment has no object or range in that record. -There are no empty index objects or duplicate copies addressed by segment number. - -This layout applies to every recorded track, including the catalog, except the recording-owned timeline track identified by the catalog's `archive` field ({{archive-catalog}}). -The timeline uses `segments/` with the same binary envelope and at least one complete Window group per object. -`` is the committed timeline segment ID, encoded as 19 zero-padded decimal digits within the recording ID range. -After its first segment, the recording MUST commit consecutive segment IDs, including all-gap segments. -A writer MUST stop before allocating an ID beyond the limit; IDs never wrap. +After its first record, a timeline MUST commit consecutive sequences. +A writer MUST stop before allocating a sequence beyond the limit; sequences never wrap. Each track has its own objects so a reader can fetch one rendition without downloading the others. ## Track Objects {#recording-track} @@ -889,30 +870,31 @@ Each track has its own objects so a reader can fetch one rendition without downl ~~~ { - "version": 1, + "version": 2, "priority": 0, "timescale": 1000000 } ~~~ All three fields are required integers. -`version` identifies this recording format and MUST be 1. +`version` identifies this recording format and MUST be 2. `priority` and `timescale` have the meanings of moq-lite `TRACK_INFO` {{moql}}: `priority` is in the range 0 through 255, and `timescale` is in the range 1 through 9007199254740991, so JSON consumers can preserve it exactly. A reader MUST preserve integer values exactly. `Publisher Max Age` is not stored; a reader supplies its own serving policy. -The track object MUST be durable before its first segment object is stored, including for the timeline track. +The track object MUST be durable before its first segment object is stored, including for a timeline track. It is immutable for the lifetime of the recording. A reader MUST refuse an unknown version or invalid track properties. On an existing `.info`, a writer MUST validate and compare the parsed `version`, `priority`, and `timescale` values; JSON whitespace and member order do not affect equality. Different property values MUST fail enrollment, and the existing object MUST NOT be rewritten. ## Segment Objects {#recording-segments} -A segment object holds one track's complete groups for one segment: +A segment object holds one record's frames: ~~~ Segment Object { - Version (i) = 1 + Version (i) = 2 + Frame Start (i) Group Count (i) Group Table { Sequence Delta (i) @@ -928,11 +910,11 @@ Segment Object { ~~~ Fields annotated `(i)` are variable-length integers using the QUIC encoding ({{!RFC9000}}, Section 16). +`Frame Start` is the index within its group of the first group's first stored frame; every later group starts at frame 0. `Group Count` gives the number of group entries; each `Frame Count` gives the number of frame entries in that group. The first `Sequence Delta` is the absolute group sequence number. Each subsequent value is the group sequence number minus the previous sequence number minus one; zero therefore means the next consecutive group. Groups MUST have strictly ascending sequence numbers; reconstruction MUST reject values exceeding the recording ID limit ({{recording-layout}}). -For range-named objects, the sequences MUST match the ranges in the timeline. Frame entries appear in their original order within each group. `Timestamp` is the frame's absolute timestamp in the track's `timescale` units, not a delta. It MUST be in the range 0 through 9007199254740991 (2^53 - 1), so browser replay preserves it exactly; writers and readers MUST reject larger values. @@ -944,103 +926,93 @@ A reader can locate a group or a frame index directly from the table without par A decoder MUST validate the complete table before accessing any payload. Counts and entries MUST fit in the retrieved object, and every offset and length MUST describe a range within its payload bytes, with overflow checked before arithmetic. -An unknown version or malformed table is a missing segment, not an invalid recording. +An unknown version or malformed table is a missing record, not an invalid recording. -A segment object MUST contain whole groups. -Segmentation forbids a boundary inside a track's group ({{timeline-segmentation}}), so a reader never has to stitch a group across objects. +A recorded track's object MUST hold exactly the frames of its record's span ({{timeline-records}}): its first group and `Frame Start` match `start`, its groups are consecutive and none is empty, and its last group ends at `end`. +A group may span several objects when a record was split between frames; a reader stitches it from consecutive records. Segment objects are immutable once written and are parseable without the timeline. -The timeline uses the same envelope and stores only complete Window groups ({{timeline-framing}}) closed while committing that segment. -Its own groups are discovered by listing, not included in the record's `tracks`, avoiding a record that must index itself. +The timeline objects use the same envelope with a `Frame Start` of zero. +Their own groups are discovered by listing, not indexed by a record, avoiding a record that must index itself. A reader replays their checkpoints and operations in group order, preserving group boundaries and their independent DEFLATE windows. Neither timeline nor media objects are appended to or rewritten. One writer owns a recording prefix. An existing segment object key MAY be reused only for identical object bytes; a conflicting create MUST fail. ## Writer Behavior {#recording-writer} -A writer subscribes to the broadcast and buffers the in-progress segment independently of the publisher or relay cache. -It writes a track's segment object once that track's groups for the span are known. -Per-track objects are written independently: a track whose content is complete does not wait for a slower one. +A writer subscribes to the broadcast and buffers each track's open span independently of the publisher or relay cache. +It cuts each track's timeline itself ({{timeline-cutting}}) and commits each track independently: a track whose span is complete does not wait for any other track. The recording contract MUST NOT depend on a relay retaining those groups while storage catches up. -A writer MUST make each track's `.info` and segment object durable before publishing the timeline record that references them. -The recording owns its timeline encoder so it can publish the ranges actually stored. - -If a track's object cannot be made durable, the writer MUST omit only that track from the record's `tracks`; successfully stored tracks retain their ranges. -The segment number and timing are unchanged, including when every track is omitted. -The writer pushes the resulting record and every subsequent record through its own Window encoder. -It MUST NOT copy source frames whose Window position or group-local DEFLATE dictionary depends on a record it changed. - A writer MUST accept new groups for each track in strictly increasing sequence order and MUST refuse a duplicate or decreasing sequence; it does not reorder arrivals. -Groups already accepted MAY complete in any order; the writer buffers them and writes their table in sequence order. -An incomplete group omitted by an application-forced segment cut MUST NOT be inserted into a later object if that would overlap or precede an earlier object's range. -The application MAY force a cut or remove a stalled track; storage does not impose a timeout. -A writer MUST NOT insert objects for earlier segments after committing a later segment. +Groups already accepted MAY complete in any order; a group's frames are indexed once every earlier accepted group has finished, and a group still open is indexed frame by frame as it grows. +A writer MUST NOT insert frames before the end of a record it already committed. -For segment N, the writer publishes the stored ranges, applies any retention pops, closes the timeline's current Window group, and stores the complete groups under the timeline track's `segments/N` key, with N encoded as specified in {{recording-layout}}. -It MUST make that object durable before committing the next segment or deleting expired objects. +To commit record N of a track, the writer makes the track's `segments/N` object durable, publishes the record through the track's timeline encoder, applies any retention pops, closes the timeline's current Window group, and stores the complete groups under the timeline track's `segments/N` key. +It MUST make that timeline object durable before committing the track's next record or deleting expired objects. +If a track's object cannot be made durable, the writer drops the record: its frames are lost, and the next record takes sequence N. If the timeline object cannot be made durable, the recording stops at the preceding durable timeline object. +The recording owns its timeline encoders so they publish only what was stored. -On a clean end, the writer MUST flush the final partial segment, finish the timeline track, and make its final complete groups durable. +On a clean end, the writer MUST close each track's final record, finish its timeline track, and make its final complete groups durable. There is no completion marker; object listing alone does not distinguish a clean end from an interruption. ## Retention {#recording-retention} A recording has one of two retention modes: -- An *archive* is unbounded and retains every complete segment until explicitly deleted. -- A *DVR* retains a configured duration of complete segments and expires the oldest whole segments as newer ones become durable. +- An *archive* is unbounded and retains every record until explicitly deleted. +- A *DVR* retains a configured duration of each track's records and expires each track's oldest records as newer ones become durable. An application offering DVR without an explicit retention value SHOULD default to at least 30 seconds. -The writer removes the oldest segment only when the remaining complete segments still cover the configured duration. +The writer removes a track's oldest record only while the track's remaining records still span the configured duration, measured from the second-oldest record's `pts` to the newest record's end. +A track's newest record is never removed, so a catalog that never changes outlives the media recorded with it. Retention is measured from timeline records, not from wall-clock arrival or relay cache state. -Trimming occurs only as part of committing a new segment, before that segment's timeline object is finalized ({{recording-writer}}). -A recording MUST NOT trim after its final segment is committed. -Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired segments' objects. -The writer MUST retain the latest timeline object, even for an all-gap segment, and enough earlier timeline groups to recover the retained window from a checkpoint. +Trimming occurs only as part of committing a new record of the same track, before that record's timeline object is finalized ({{recording-writer}}). +A recording MUST NOT trim after a track's final record is committed. +Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired records' objects. +The writer MUST retain each timeline's latest object and enough earlier timeline groups to recover the retained window from a checkpoint. +It MAY delete a timeline's earlier objects, oldest first, so its remaining record IDs stay consecutive. The index therefore never advertises media the retention process has already deleted, although media objects MAY temporarily outlive the index. Relay cache eviction does not change the recording timeline. -Before accepting new groups, a restarting DVR writer MUST recover the complete retained timeline and list every recorded track's `groups/` prefix under exclusive ownership of the recording prefix. -After waiting the configured deletion grace period from successful recovery, it MUST delete group objects whose keys are absent from the recovered retained records, including expired objects and uncommitted uploads left by a crash. -It MUST NOT perform this cleanup if timeline recovery or listing fails or is incomplete, or while another writer can create or commit objects. -This completes previously committed expiration; it does not pop additional records after a final segment. -Timeline objects needed for checkpoint recovery and `.info` objects are not candidates for this cleanup. - ## Bootstrap and Recovery {#recording-recovery} -A reader or restarting writer lists the timeline track's `segments/` prefix and replays its objects in numeric segment order, including retention operations, from a retained checkpoint. -The catalog supplies the timeline track's name ({{archive-catalog}}). -The recovered records determine the committed track object keys; a missing or malformed referenced object MUST NOT be served. -Objects not referenced by the recovered timeline do not advertise content on their own. +A reader or restarting writer lists each timeline track's `segments/` prefix and replays its objects in numeric sequence order, including retention operations, from a retained checkpoint. +The catalog supplies the timeline names ({{archive-catalog}}); a restarting writer finds them by their suffix ({{recording-layout}}). +The recovered records determine the committed object keys; a missing or malformed referenced object MUST NOT be served. +Objects not referenced by a recovered timeline do not advertise content on their own. Listing is not an atomic snapshot across tracks; absence from an earlier listing MUST NOT override a successful GET of an object referenced by a later durable timeline record. -After replaying segment N below the recording ID limit, a reader follows an active recording by GET of `segments/N+1`, without refreshing media listings. +Before accepting new groups, a restarting writer MUST recover each retained timeline under exclusive ownership of the recording prefix. +A track's objects at or past its next sequence were never referenced by a timeline, so the writer deletes them before reusing their keys. +It resumes each track after the end of its newest committed record, including partway through a group that record split. +A restarting DVR writer also lists every recorded track's `segments/` prefix and, after waiting the configured deletion grace period from successful recovery, deletes objects whose keys are absent from the recovered retained records, including expired objects left by interrupted expiration. +It MUST NOT perform this cleanup if timeline recovery or listing fails or is incomplete, or while another writer can create or commit objects. +This completes previously committed expiration; it does not pop additional records after a final record. +Timeline objects needed for checkpoint recovery and `.info` objects are not candidates for this cleanup. + +After replaying a timeline's record N below the recording ID limit, a reader follows an active recording by GET of that timeline's `segments/N+1`, without refreshing media listings. At the limit there is no next key; this does not establish a clean recording end. -Not Found does not distinguish a pending commit, an expired DVR segment, or an interrupted recording. +Not Found does not distinguish a pending commit, an expired DVR record, or an interrupted recording. A reader that has fallen behind retention MUST bootstrap again from a retained checkpoint. Alternatively, a reader MAY list timeline keys after its last replayed key, consume all pages, sort the results, and replay them in order. A continuation token is used only within that enumeration; the next refresh starts from the last replayed key. -A reader MUST NOT advance its replay cursor past a missing segment without recovering from a retained checkpoint. -Retention operations evict expired ranges from the reader's cached index; incremental listing alone does not report deletions. +A reader MUST NOT advance its replay cursor past a missing object without recovering from a retained checkpoint. +Retention operations evict expired records from the reader's cached index; incremental listing alone does not report deletions. ## Reader Behavior {#recording-reader} -Reading segment N of track T is: take the minimum and maximum group IDs from T's ranges in record N, GET `groups/.`, and parse the groups. -The table MUST contain exactly the group sequences advertised for that track in the record, including its gaps. +Reading record N of track T is a GET of T's `segments/N`; the table MUST hold exactly the frames the record names. Nothing on that path reads media the consumer did not ask for, and nothing requires a second request. A reader MAY serve moq-lite FETCH from a recording. -Given a group, a reader locates the candidate object from a cached filename range index or the timeline's track ranges, then uses the table for the group and any requested `frame_start` {{moql}}. -A filename listing requires no media GETs. -On a backend guaranteeing lexical listing order and exclusive offsets, listing after `groups/` (19 padded digits, without the dot) selects the first candidate with an upper bound at least the requested group. -A reader MUST check the lower bound and committed timeline membership before serving it; an internal gap is resolved by the candidate's table. -An unordered listing MUST be collected and sorted before selecting a candidate; taking its first result is insufficient. +Given a group, a reader finds the retained records whose spans hold it, in order, and serves their frames from the requested `frame_start` {{moql}}. +A group whose earliest retained record starts after the requested frame is not served. +A group whose records do not reach its end yet, such as an append-log still being recorded, grows as later records commit, and ends where the recording does once the reader learns it ended. The reader reconstructs moq-lite FRAME headers from the stored timestamps and lengths, preserving the original payload bytes. A group absent from the recording is a normal FETCH failure. Recording catalog groups does not establish which catalog update applies to a media group; this format does not define that correlation. -A reader deriving a presentation-ordered format renders it from the timeline and transmuxes segment objects on demand. -Nothing derived needs to be stored: the playlist or manifest is a function of the timeline, and a media segment is a function of one recorded object. -An HLS segment URI can carry the track and both group bounds, allowing the handler to resolve the exact object without listing or a segment-ID index. -HLS sequence numbers remain timeline metadata and need not occur in object names. +A reader deriving a presentation-ordered format renders it from the timelines ({{timeline-derived}}) and transmuxes record objects on demand. +Nothing derived needs to be stored: the playlist or manifest is a function of the timelines, and a media segment is a function of the records its span covers. # Rooms @@ -1072,7 +1044,7 @@ Clamping a reference that escapes above that root would silently redirect the su TODO Security A consumer parsing a recording ({{recording}}) is parsing data at rest that it did not necessarily write. -It MUST validate the group/frame table against the bytes actually retrieved before allocating from its counts or accessing payloads ({{recording-segments}}), and treat a malformed object as a missing segment rather than letting it invalidate the recording. +It MUST validate the group/frame table against the bytes actually retrieved before allocating from its counts or accessing payloads ({{recording-segments}}), and treat a malformed object as a missing record rather than letting it invalidate the recording. It MUST reject a track object with a zero `timescale`. Varint fields are subject to the same limits as moq-lite {{moql}}. A recording inherits the confidentiality and integrity properties of the storage holding it; encryption at rest is transparent to the format and out of scope. @@ -1088,6 +1060,15 @@ This document has no IANA actions. # Appendix A: Changelog {:numbered="false"} +## moq-hang-04 +{:numbered="false"} + +- Replaced the broadcast's one aligned timeline with one timeline per track: the catalog `archive` entry's `track` became a `timelines` map from each indexed track, the catalog included, to its timeline track. +- Replaced the segment record with a per-track record: `sequence`, `pts`, `duration`, and a `start`/`end` range of group and frame positions, dropping cross-track pacing and completeness. +- One cutting rule for every track: a record ends at the first group boundary past a minimum (2 seconds RECOMMENDED, zero for sparse data such as a catalog) and splits a group between frames at a maximum (10 seconds RECOMMENDED), so a group that never closes is indexed as it grows and `durationMax` bounds every record. +- Recording format version 2: each track stores record N at `segments/N`, beside its timeline's `segments/N`, with a `Frame Start` field in the segment object. Tracks commit and expire independently, and a DVR keeps each track's newest record. +- Described deriving HLS and DASH at the edge from a reference rendition's records. + ## moq-hang-03 {:numbered="false"} @@ -1100,6 +1081,7 @@ A publisher MAY estimate an unknown final duration from the frame cadence, but M - Specified version 1 recording objects: JSON track properties and binary group/frame tables with ascending, delta-encoded group sequences. - Addressed track objects by inclusive group bounds and timeline objects by consecutive segment IDs, with incremental discovery and per-track omission on storage failure. - Restricted retention updates to segment commits and removed completion markers. +- Allowed a DVR to delete timeline objects no checkpoint recovery needs, oldest first. - Limited recorded group and segment IDs and frame timestamps to JSON-safe integers, including delta reconstruction. - Compared existing track properties by parsed values rather than JSON serialization. - Required exclusive DVR restart recovery to remove unreferenced group objects left by interrupted expiration. diff --git a/js/hang/src/catalog/archive.ts b/js/hang/src/catalog/archive.ts index c73e810ba3e..a99b4ee4f08 100644 --- a/js/hang/src/catalog/archive.ts +++ b/js/hang/src/catalog/archive.ts @@ -11,25 +11,24 @@ const timescaleSchema = nonzeroU53Schema.check(z.lte(4_294_967_295)); /** * The recording object format advertised in {@link Archive.version} when a store is present. */ -export const ARCHIVE_VERSION = 1; +export const ARCHIVE_VERSION = 2; /** - * Discovers the broadcast's segment index and any durable archive. + * Discovers each track's timeline and any durable archive. * - * This is the catalog's one name for the segment index: a live publisher sets the - * timeline fields (`track`, `timescale`, `durationMax`) alone, and a recording - * also names the replay broadcast and object store those ranges live under. Every - * advertised range is FETCHable; with a store they are durable. There is no sibling - * `timeline` entry. Wall-clock mapping lives at the catalog root (`clock`), not here. + * A live publisher sets the timeline fields (`timelines`, `timescale`, `durationMax`) + * alone, and a recording also names the replay broadcast and object store those spans + * live under. Every advertised span is FETCHable; with a store they are durable. + * Wall-clock mapping lives at the catalog root (`clock`), not here. */ export const ArchiveSchema = z.object({ - // The name of the MoQ track carrying the broadcast's segment records. - track: z.string(), + // Each indexed track's timeline track, keyed by the indexed track's name. + timelines: z.record(z.string(), z.string()), - // Units per second for the records' timestamps. Defaults to milliseconds. + // Units per second for every timeline's timestamps. Defaults to milliseconds. timescale: z._default(timescaleSchema, u53(1000)), - // The declared upper bound on a segment's duration in timescale units. + // The declared upper bound on a record's duration in timescale units. durationMax: z.optional(u53Schema), // The MoQ broadcast the archive is served back from, relative to this catalog, if any. @@ -45,5 +44,5 @@ export const ArchiveSchema = z.object({ version: z.optional(u53Schema), }); -/** The catalog's root archive entry: the timeline track plus optional replay, store, and version. */ +/** The catalog's root archive entry: each track's timeline plus optional replay, store, and version. */ export type Archive = z.infer; diff --git a/js/hang/src/catalog/clock.test.ts b/js/hang/src/catalog/clock.test.ts index d010f1a4f26..b22465ecd54 100644 --- a/js/hang/src/catalog/clock.test.ts +++ b/js/hang/src/catalog/clock.test.ts @@ -17,7 +17,7 @@ const FIXTURE = { }, }, clock: { wall: 1_751_846_400_000_000, timescale: 1_000_000 }, - archive: { track: "timeline.z", timescale: 1000, durationMax: 2000 }, + archive: { timelines: { video: "video.timeline.z" }, timescale: 1000, durationMax: 2000 }, }; test("timescale defaults to microseconds", () => { diff --git a/js/hang/src/catalog/root.test.ts b/js/hang/src/catalog/root.test.ts index e6e7c64de2e..e201b688ac8 100644 --- a/js/hang/src/catalog/root.test.ts +++ b/js/hang/src/catalog/root.test.ts @@ -137,7 +137,7 @@ test("clock stays off the wire when absent", () => { test("archive round-trips at the root", () => { const parsed = RootSchema.parse({ archive: { - track: "timeline.z", + timelines: { video: "video.timeline.z" }, durationMax: 2000, replay: "./recordings/clip", store: "https://objects.example/rec/", @@ -145,12 +145,12 @@ test("archive round-trips at the root", () => { }, }); expect(parsed.archive).toMatchObject({ - track: "timeline.z", + timelines: { video: "video.timeline.z" }, timescale: 1000, durationMax: 2000, replay: "recordings/clip", store: "https://objects.example/rec/", - version: 1, + version: 2, }); expect(JSON.stringify(parsed)).not.toContain('"timeline":'); }); @@ -161,7 +161,23 @@ test("a legacy root timeline is not an archive", () => { }); test("an invalid store URL is refused", () => { - expect(() => RootSchema.parse({ archive: { track: "timeline.z", store: "not a url" } })).toThrow(); + expect(() => RootSchema.parse({ archive: { timelines: {}, store: "not a url" } })).toThrow(); +}); + +test("a single shared timeline is refused", () => { + expect(() => RootSchema.parse({ archive: { track: "timeline.z" } })).toThrow(); +}); + +// The exact shape Rust's moq-mux publishes for every broadcast, so a native publisher's catalog +// stays playable. +test("a Rust per-track archive parses", () => { + const parsed = RootSchema.parse({ + archive: { + timelines: { "catalog.json": "catalog.json.timeline.z", video0: "video0.timeline.z" }, + timescale: 1000, + }, + }); + expect(parsed.archive?.timelines.video0).toBe("video0.timeline.z"); }); test("an absent or foreign text section parses to empty", () => { diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index 0904e967541..176a271d1fb 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -4,7 +4,6 @@ import { Time } from "@moq/net"; export type { BufferedRange, BufferedRanges, Frame } from "./types"; import type { AudioConfig, VideoConfig } from "../catalog"; -import type { Recorder as TimelineRecorder } from "../timeline"; import type { Format as ContainerFormat } from "./format"; import type { Frame } from "./types"; @@ -59,16 +58,6 @@ export function encodeFrame(source: Uint8Array | Source, timestamp: Time.Micro): return data; } -/** Options for a legacy-container {@link Producer}. */ -export interface ProducerProps { - /** - * Report each group open (sequence + start timestamp) into the broadcast's timeline, so - * consumers can index the media without downloading it. Mint one via - * {@link Timeline.Producer.track}. - */ - timeline?: TimelineRecorder; -} - /** Writes legacy-container frames into a MoQ track, starting a new group on each keyframe. */ export class Producer { #track: Moq.Track.Producer; @@ -76,9 +65,7 @@ export class Producer { #previous?: Time.Micro; #reordered = false; #group?: Moq.Group.Producer; - #timeline?: TimelineRecorder; - // The newest timestamp written, reported to the timeline when the track closes: the last - // group has no successor to bound it, so its segment would be published a group short. + // The newest timestamp written in the current group, which estimates its end. #end?: Time.Micro; // Furthest timestamp in finished groups, used only for discontinuity markers. #liveEdge?: Time.Micro; @@ -91,10 +78,9 @@ export class Producer { #marked = false; /** Wrap a track to publish legacy-container frames into it. */ - constructor(track: Moq.Track.Producer, format: Format, props: ProducerProps = {}) { + constructor(track: Moq.Track.Producer, format: Format) { this.#format = format; this.#track = track; - this.#timeline = props.timeline; } /** Encode and append a frame; a keyframe starts a new group. Throws if the first frame is not a keyframe, a group start goes backwards, or a frame is below the previous group start. */ @@ -111,8 +97,6 @@ export class Producer { this.#group = this.#track.appendGroup(); this.#floor = this.#start; this.#start = timestamp; - // Report the group the moment it opens: its start is this keyframe's timestamp. - this.#timeline?.record(this.#group.sequence, timestamp, true); } else if (!this.#group) { throw new Error("must start with a keyframe"); } @@ -156,8 +140,6 @@ export class Producer { if (this.#format.kind === "data" || this.#marked || timestamp === undefined) return; const group = this.#track.appendGroup(); - this.#timeline?.record(group.sequence, timestamp, false); - this.#timeline?.end(timestamp); group.writeFrame({ payload: encodeFrame(new Uint8Array(), timestamp), timestamp: Time.Timestamp.fromMicros(timestamp), @@ -185,8 +167,6 @@ export class Producer { : undefined; // A presentation endpoint cannot bound the decode-order tail after reordering. this.#format.finishGroup(this.#group, this.#reordered ? undefined : end); - const bound = end ?? this.#end; - if (bound !== undefined) this.#timeline?.end(bound); this.#group.close(); this.#group = undefined; if (this.#end !== undefined) { diff --git a/js/hang/src/timeline.test.ts b/js/hang/src/timeline.test.ts index 200cf2a77b7..6c78ba91d06 100644 --- a/js/hang/src/timeline.test.ts +++ b/js/hang/src/timeline.test.ts @@ -1,552 +1,23 @@ import { expect, test } from "bun:test"; import * as Json from "@moq/json"; -import { Broadcast, Time, Track } from "@moq/net"; +import { Broadcast, type Time } from "@moq/net"; import { u53 } from "./catalog"; -import { Consumer, Producer, type Record } from "./timeline.ts"; +import { Consumer, type Record } from "./timeline.ts"; const us = (ms: number): Time.Micro => (ms * 1000) as Time.Micro; -/** A timeline whose published records are decoded back out of its MoQ track. */ -function capture(props?: ConstructorParameters[1]): { - timeline: Producer; - records: () => Promise; -} { - const track = new Track.Producer("timeline.z"); - const consumer = new Json.Window.Consumer({ track: track.subscribe(), compression: true }); - const timeline = new Producer(track, props); - - const records = async () => { - const out: Record[] = []; - for (;;) { - const event = await consumer.next(); - if (!event) return out; - if ("push" in event) { - out.push(event.push.value); - } else { - throw new Error("timeline producer only appends records"); - } - } - }; - return { timeline, records }; -} - -// The coarsest track paces: video GOPs are longer than durationMin, so segments are GOPs. -test("the coarsest track paces the segments", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - // Video keyframes every 2s, audio groups every 500ms, minimum 1s. - video.record(0, us(0)); - for (const [seq, ms] of [ - [0, 0], - [1, 500], - [2, 1000], - [3, 1500], - ] as const) { - audio.record(seq, us(ms)); - } - video.record(1, us(2000)); - for (const [seq, ms] of [ - [4, 2000], - [5, 2500], - [6, 3000], - [7, 3500], - ] as const) { - audio.record(seq, us(ms)); - } - video.record(2, us(4000)); - audio.record(8, us(4000)); - video.close(); - audio.close(); - timeline.finish(); - - // Audio's own candidate (1s) loses to video's (2s), so no segment splits a GOP. - expect(await records()).toEqual([ - { - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], audio0: [{ start: 0, end: 3 }] }, - }, - { - segment: 1, - pts: 2000, - duration: 2000, - tracks: { video0: [{ start: 1, end: 1 }], audio0: [{ start: 4, end: 7 }] }, - }, - { - segment: 2, - pts: 4000, - duration: 0, - tracks: { video0: [{ start: 2, end: 2 }], audio0: [{ start: 8, end: 8 }] }, - }, - ]); -}); - -// A real-time encoder's GOP can dwarf the minimum. Nothing is violated: there is nowhere else -// a segment could start and stay decodable, so the segment is simply long. -test("a GOP longer than the minimum is one segment", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - video.record(1, us(30_000)); - video.end(us(60_000)); - video.close(); - timeline.finish(); - - expect(await records()).toEqual([ - { segment: 0, pts: 0, duration: 30_000, tracks: { video0: [{ start: 0, end: 0 }] } }, - { segment: 1, pts: 30_000, duration: 30_000, tracks: { video0: [{ start: 1, end: 1 }] } }, - ]); -}); - -// Groups shorter than the minimum pack into one segment rather than each becoming one. -test("short groups pack up to the minimum", async () => { - const { timeline, records } = capture({ durationMin: Time.Milli(1500) }); - const audio = timeline.pacingTrack("audio0"); - - for (let seq = 0; seq < 8; seq++) { - audio.record(seq, us(seq * 500)); - } - audio.close(); - timeline.finish(); - - const out = await records(); - // The first group at or past 1500ms ends segment 0, so it holds groups 0..=2. - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 1500, tracks: { audio0: [{ start: 0, end: 2 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 1500, duration: 1500, tracks: { audio0: [{ start: 3, end: 5 }] } }); -}); - -// A track that hasn't reached the minimum yet holds the segment open, so a record never omits -// a rendition that was about to contribute to it. -test("a segment waits for every track", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - video.record(0, us(0)); - audio.record(0, us(0)); - // Video alone would close segment 0 here, but audio hasn't crossed 2s. - video.record(1, us(2000)); - audio.record(1, us(2000)); - video.close(); - audio.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], audio0: [{ start: 0, end: 0 }] }, - }); -}); - -// An application that knows its own boundaries overrides the pacing. -test("explicit cuts override the pacing", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - // Keyframes every second, cut every three: the segments follow the cuts, not the GOPs. - timeline.cut(us(3000)); - timeline.cut(us(6000)); - for (let seq = 0; seq < 7; seq++) { - video.record(seq, us(seq * 1000)); - } - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 3000, tracks: { video0: [{ start: 0, end: 2 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 3000, duration: 3000, tracks: { video0: [{ start: 3, end: 5 }] } }); -}); - -// Every rendition of one import declares the same boundaries, so a cut that would make a -// segment shorter than the minimum is dropped rather than producing a stray segment. -test("a cut below the minimum is ignored", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - timeline.cut(us(2000)); - // A sibling rendition's duplicate, and a boundary too close to be a segment. - timeline.cut(us(2000)); - timeline.cut(us(2500)); - timeline.cut(us(4000)); - video.record(1, us(2000)); - video.record(2, us(4000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out).toHaveLength(3); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }); -}); - -// The declared maximum is a contract with consumers (an HLS EXT-X-TARGETDURATION), so a -// segment that breaks it fails the timeline rather than publishing a record that contradicts -// the catalog. -test("exceeding the declared maximum fails the timeline", async () => { - const { timeline, records } = capture({ durationMin: Time.Milli(1000), durationMax: Time.Milli(3000) }); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - video.record(1, us(2000)); - // A 4s GOP against a declared 3s maximum. - video.record(2, us(6000)); - video.close(); - - expect(() => timeline.finish()).toThrow(/over the declared maximum/); - - // The record that was still true published; the one that would have contradicted the - // catalog did not. - expect(await records()).toEqual([ - { segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }, - ]); -}); - -test("an undeclared maximum is omitted from the catalog", () => { - expect(capture().timeline.section().durationMax).toBeUndefined(); - expect(capture({ durationMax: Time.Milli(2500) }).timeline.section().durationMax).toBe(u53(2500)); -}); - -// The last group of a broadcast has no successor to bound it, so without a reported end the -// final segment's duration collapses to zero (an HLS EXTINF:0). -test("the final segment runs to the reported end", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - video.record(1, us(2000)); - // A closing container producer reports where its content stops. - video.end(us(4000)); - video.close(); - timeline.finish(); - - expect(await records()).toEqual([ - { segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }, - { segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }, - ]); -}); - -// A record is immutable and judged against the tracks enrolled when it flushes, so a producer -// that enrolls its tracks one batch at a time holds flushing back until they are all in. -test("a reservation defers flushing until every track enrolls", async () => { - const { timeline, records } = capture(); - const release = timeline.reserve(); - - // The primary rendition runs a whole batch of segments through before its sibling exists. - const first = timeline.pacingTrack("video0"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - first.record(seq, us(ms)); - } - - const second = timeline.pacingTrack("video1"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - second.record(seq, us(ms)); - } - - release(); - first.close(); - second.close(); - timeline.finish(); - - // Both renditions are indexed from segment 0. Without the hold, segments 0 and 1 would have - // flushed knowing only video0, and video1's first two groups would have folded into - // segment 2. - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], video1: [{ start: 0, end: 0 }] }, - }); - expect(out[1]).toEqual({ - segment: 1, - pts: 2000, - duration: 2000, - tracks: { video0: [{ start: 1, end: 1 }], video1: [{ start: 1, end: 1 }] }, - }); -}); - -// A track that races ahead of the others still lands in the segment its content falls in. -test("groups before the first boundary join the first segment", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - audio.record(0, us(0)); - video.record(0, us(30)); - for (const [seq, ms] of [ - [1, 500], - [2, 1000], - [3, 1500], - [4, 2000], - ] as const) { - audio.record(seq, us(ms)); - } - video.record(1, us(2030)); - audio.record(5, us(2500)); - video.close(); - audio.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - // The segment starts where the earliest content does, not at video's first keyframe. - segment: 0, - pts: 0, - duration: 2030, - tracks: { video0: [{ start: 0, end: 0 }], audio0: [{ start: 0, end: 4 }] }, - }); -}); - -test("sequence gaps split ranges", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - // Elemental-style gap: audio groups 2..=4 never existed inside segment 0. - video.record(0, us(0)); - audio.record(0, us(0)); - audio.record(1, us(300)); - audio.record(5, us(1500)); - video.record(1, us(2000)); - audio.record(6, us(2100)); - video.close(); - audio.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { - video0: [{ start: 0, end: 0 }], - audio0: [ - { start: 0, end: 1 }, - { start: 5, end: 5 }, - ], - }, - }); -}); - -// A track with nothing to contribute is a whole-segment gap: the record simply omits it (an -// HLS exporter renders EXT-X-GAP). Only a closed track produces one, since a live track that -// has merely gone quiet still gets to say where the boundary is. -test("a closed track leaves a whole segment gap", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const audio = timeline.pacingTrack("audio0"); - - video.record(0, us(0)); - audio.record(0, us(0)); - audio.close(); - video.record(1, us(2000)); - video.record(2, us(4000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[1]).toEqual({ segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }); -}); - -test("a non-keyframe range start is flagged", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - video.record(0, us(0)); - // A mid-stream join: the group doesn't open on an IDR. - video.record(1, us(2000), false); - video.record(2, us(4000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[1]?.tracks?.video0[0].keyframe).toBe(false); -}); - -test("the timeline section carries no wall anchor", () => { - // Wall mapping moved to the catalog root clock: the timeline section carries the track, - // timescale, and duration bound, and nothing else time-anchoring. - const section = capture().timeline.section(); - expect(section).toEqual({ track: "timeline.z", timescale: u53(1000), durationMax: undefined }); - expect("wall" in section).toBe(false); -}); - -// Reservations nest like the catalog's, so the first batch to finish doesn't publish records -// the others are still filling in. -test("reservations nest", async () => { - const { timeline, records } = capture(); - - const outer = timeline.reserve(); - const inner = timeline.reserve(); - - const first = timeline.pacingTrack("video0"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - first.record(seq, us(ms)); - } - - // If reservations didn't nest, releasing this one would publish segment 0 without video1. - inner(); - - const second = timeline.pacingTrack("video1"); - for (const [seq, ms] of [ - [0, 0], - [1, 2000], - [2, 4000], - ] as const) { - second.record(seq, us(ms)); - } - - outer(); - first.close(); - second.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ - segment: 0, - pts: 0, - duration: 2000, - tracks: { video0: [{ start: 0, end: 0 }], video1: [{ start: 0, end: 0 }] }, - }); - expect(out[1]).toEqual({ - segment: 1, - pts: 2000, - duration: 2000, - tracks: { video0: [{ start: 1, end: 1 }], video1: [{ start: 1, end: 1 }] }, - }); -}); - -// A cut registered before the media, landing exactly on the first reported group (what the fMP4 -// importer does: it cuts at the keyframe fragment's timestamp, then records that group). The -// spent cut must not block the ones behind it, and durationMin pacing must not race ahead of them. -test("a cut on the first group does not poison later cuts", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - - // Source segments every 3s, keyframes every 1s: 3s segments, not the 1s the durationMin - // pacing would produce on its own. - timeline.cut(us(0)); - video.record(0, us(0)); - for (let seq = 1; seq < 10; seq++) { - if (seq % 3 === 0) timeline.cut(us(seq * 1000)); - video.record(seq, us(seq * 1000)); - } - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 3000, tracks: { video0: [{ start: 0, end: 2 }] } }); - expect(out[1]).toEqual({ segment: 1, pts: 3000, duration: 3000, tracks: { video0: [{ start: 3, end: 5 }] } }); -}); - -// A catalog publishes a group only when the renditions change, so it can go quiet for the rest -// of the broadcast. Enrolled as a pacing track it would stall the timeline for good; non-pacing, it -// rides along in whichever segment is open. -test("a non-pacing track neither paces nor gates", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const catalog = timeline.track("catalog.json"); - - catalog.record(0, us(0)); - video.record(0, us(0)); - video.record(1, us(2000)); - video.record(2, us(4000)); - video.end(us(6000)); - video.close(); - timeline.finish(); - - expect(await records()).toEqual([ - { - segment: 0, - pts: 0, - duration: 2000, - tracks: { "catalog.json": [{ start: 0, end: 0 }], video0: [{ start: 0, end: 0 }] }, - }, - { segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }, - { segment: 2, pts: 4000, duration: 2000, tracks: { video0: [{ start: 2, end: 2 }] } }, - ]); -}); - -// Nothing waits for a non-pacing track, so a group arriving after its segment already flushed is -// recorded in the next one. Placement is by arrival; the frames still carry their own timestamps. -test("a late non-pacing group lands in the next segment", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const catalog = timeline.track("catalog.json"); - - video.record(0, us(0)); - // Flushes segment 0, which the catalog has contributed nothing to. - video.record(1, us(2000)); - - // The update happened a second in, but only reaches the timeline now. - catalog.record(0, us(1000)); - - video.record(2, us(4000)); - video.end(us(6000)); - video.close(); - timeline.finish(); - - const out = await records(); - expect(out[0]).toEqual({ segment: 0, pts: 0, duration: 2000, tracks: { video0: [{ start: 0, end: 0 }] } }); - expect(out[1]).toEqual({ - segment: 1, - pts: 2000, - duration: 2000, - tracks: { "catalog.json": [{ start: 0, end: 0 }], video0: [{ start: 1, end: 1 }] }, - }); -}); - -test("a non-pacing track uses arrival and does not extend the tail", async () => { - const { timeline, records } = capture(); - const video = timeline.pacingTrack("video0"); - const catalog = timeline.track("catalog.json"); - - video.record(0, us(0)); - catalog.record(0, us(10_000)); - video.record(1, us(2000)); - video.end(us(4000)); - timeline.finish(); - - expect(await records()).toEqual([ - { - segment: 0, - pts: 0, - duration: 2000, - tracks: { "catalog.json": [{ start: 0, end: 0 }], video0: [{ start: 0, end: 0 }] }, - }, - { segment: 1, pts: 2000, duration: 2000, tracks: { video0: [{ start: 1, end: 1 }] } }, - ]); -}); +const archive = (timescale: number) => ({ timelines: { video: "video.timeline.z" }, timescale: u53(timescale) }); test("timeline consumer yields converted push and pop events", async () => { const broadcast = new Broadcast.Producer(); - const track = broadcast.createTrack("timeline.z"); + const track = broadcast.createTrack("video.timeline.z"); const producer = new Json.Window.Producer({ track, compression: true }); - const consumer = Consumer.subscribe(broadcast.consume(), { track: "timeline.z", timescale: u53(1000) }); - producer.push({ segment: 7, pts: 250, duration: 1250, tracks: { video: [{ start: 1, end: 2 }] } }); + const consumer = Consumer.subscribe(broadcast.consume(), archive(1000), "video"); + producer.push({ sequence: 0, pts: 250, duration: 1250, start: { group: 1 }, end: { group: 2, frame: 3 } }); expect(await consumer.next()).toEqual({ push: { index: 0, - entry: { segment: 7, pts: us(250), duration: us(1250), tracks: { video: [{ start: 1, end: 2 }] } }, + entry: { sequence: 0, pts: us(250), duration: us(1250), start: { group: 1 }, end: { group: 2, frame: 3 } }, }, }); producer.pop(1); @@ -558,14 +29,41 @@ test("timeline consumer yields converted push and pop events", async () => { test("timeline consumer floors fractional microseconds without floating point drift", async () => { const broadcast = new Broadcast.Producer(); - const track = broadcast.createTrack("timeline.z"); + const track = broadcast.createTrack("video.timeline.z"); const producer = new Json.Window.Producer({ track, compression: true }); - const consumer = Consumer.subscribe(broadcast.consume(), { track: "timeline.z", timescale: u53(3) }); - producer.push({ segment: 0, pts: 1, duration: 2 }); + const consumer = Consumer.subscribe(broadcast.consume(), archive(3), "video"); + producer.push({ sequence: 0, pts: 1, duration: 2, start: { group: 0 }, end: { group: 1 } }); expect(await consumer.next()).toEqual({ - push: { index: 0, entry: { segment: 0, pts: 333_333 as Time.Micro, duration: 666_666 as Time.Micro } }, + push: { + index: 0, + entry: { + sequence: 0, + pts: 333_333 as Time.Micro, + duration: 666_666 as Time.Micro, + start: { group: 0 }, + end: { group: 1 }, + }, + }, }); producer.finish(); consumer.close(); broadcast.close(); }); + +test("a record numbered unlike its window index is refused", async () => { + const broadcast = new Broadcast.Producer(); + const track = broadcast.createTrack("video.timeline.z"); + const producer = new Json.Window.Producer({ track, compression: true }); + const consumer = Consumer.subscribe(broadcast.consume(), archive(1000), "video"); + producer.push({ sequence: 5, pts: 0, duration: 1000, start: { group: 0 }, end: { group: 1 } }); + await expect(consumer.next()).rejects.toThrow(); + producer.finish(); + consumer.close(); + broadcast.close(); +}); + +test("a track without a timeline is refused", () => { + const broadcast = new Broadcast.Producer(); + expect(() => Consumer.subscribe(broadcast.consume(), archive(1000), "audio")).toThrow(); + broadcast.close(); +}); diff --git a/js/hang/src/timeline.ts b/js/hang/src/timeline.ts index 090e07393f4..5b869157011 100644 --- a/js/hang/src/timeline.ts +++ b/js/hang/src/timeline.ts @@ -1,15 +1,11 @@ /** - * Publishing the broadcast's timeline: a single track carrying one record per aligned - * segment, mapping a span of content time to the group ranges that carry it on each media - * track. A consumer can seek (or build an HLS/DASH playlist) without downloading the media. - * See the catalog's root {@link Catalog.Archive} entry that advertises it. + * Reading a track's timeline: a track carrying one record per span of another track, mapping + * its content time to the group and frame positions that carry it. A consumer can seek (or + * build an HLS/DASH playlist) without downloading the media. * - * Facts flow up and policy flows down, meeting in the shared {@link Producer}: each continuous - * media track enrolls with {@link Producer.pacingTrack} and reports every group open through its - * {@link Recorder}. A segment ends at the first group boundary that gives it at least - * {@link ProducerProps.durationMin} on every pacing track, unless the application declares its - * own with {@link Producer.cut}. A segment's record is published only once every pacing track - * has reported past its end (or closed), so records are self-contained and immediately servable. + * Every track has its own timeline, named in the catalog's root {@link Catalog.Archive} entry. + * A span usually holds whole groups; a group that outlives the publisher's maximum span is + * split by frame. Spans are contiguous in position unless the track skipped group sequences. * * @module */ @@ -19,80 +15,57 @@ import type * as Moq from "@moq/net"; import type { Time } from "@moq/net"; import * as z from "@zod/mini"; import type * as Catalog from "./catalog"; -import { u53, u53Schema } from "./catalog"; +import { u53Schema } from "./catalog"; -/** - * A contiguous run of groups a track contributes to a segment, `start` through `end` - * inclusive. More than one range in a segment means the group sequence is discontinuous (a - * gap: groups that never existed). - */ -export interface Range { - start: number; - end: number; - /** - * Whether the run's first group starts with a keyframe, i.e. whether a player can join or - * switch renditions at this range. Omitted when true (the default). - */ - keyframe?: boolean; +/** A frame position within a track: frame `frame` of group `group`. */ +export interface Position { + group: number; + /** Omitted when zero, so a position at a group start reads as just the group. */ + frame?: number; } /** - * One timeline record: a complete aligned segment. `pts` and `duration` bound its span of - * content time (in the timeline's timescale), and `tracks` maps each participating media - * track to the group ranges that carry it. A track absent from `tracks` has no content for - * the span (a gap). + * One timeline record: a span of one track holding every frame from `start` (inclusive) to + * `end` (exclusive). `pts` and `duration` are in the archive's timescale. An `end` at frame zero + * of group `g` holds every frame of group `g - 1` and nothing of `g`. */ export interface Record { - segment: number; + /** The record's number, consecutive within its track's timeline. */ + sequence: number; pts: number; duration: number; - tracks?: { [track: string]: Range[] }; + start: Position; + end: Position; + /** Whether the span starts with a keyframe. Omitted when true (the default). */ + keyframe?: boolean; } +const PositionSchema = z.object({ group: u53Schema, frame: z.optional(u53Schema) }); + const RecordSchema = z.looseObject({ - segment: u53Schema, + sequence: u53Schema, pts: u53Schema, duration: u53Schema, - tracks: z.optional( - z.record( - z.string(), - z.array(z.object({ start: u53Schema, end: u53Schema, keyframe: z.optional(z.boolean()) })), - ), - ), + start: PositionSchema, + end: PositionSchema, + keyframe: z.optional(z.boolean()), }); -/** The default timeline timescale: 1000 units per second (milliseconds). */ -export const DEFAULT_TIMESCALE = 1000; - -/** - * The conventional {@link ProducerProps.durationMin} (1 second), in milliseconds, for callers - * with no opinion of their own. - */ -export const DEFAULT_DURATION_MIN_MS = 1000 as Time.Milli; - -/** Recent segment records repeated when the Window track rolls to a new group. */ -const CHECKPOINT_RECORDS = 256; - -/** - * The conventional name for a broadcast's timeline track (the `.z` marks the - * DEFLATE-compressed window, like the catalog's `.json.z` sibling). The actual name is read - * from the catalog's root `archive` entry, so this is only a default. - */ -export const DEFAULT_NAME = "timeline.z"; +/** The conventional suffix appended to a track name to name its timeline track. */ +export const SUFFIX = ".timeline.z"; -/** A complete segment with timestamps converted to microseconds. */ +/** A record with timestamps converted to microseconds. */ export type Entry = Omit & { pts: Time.Micro; duration: Time.Micro }; /** One change to the visible timeline window. */ export type Event = { push: { index: number; entry: Entry } } | { pop: Json.Window.Span } | { skip: Json.Window.Span }; -/** Reads the timeline advertised by a catalog archive entry. */ +/** Reads one track's timeline, as advertised by a catalog archive entry. */ export class Consumer implements AsyncIterable { readonly #track: Moq.Track.Subscriber; readonly #window: Json.Window.Consumer; readonly #timescale: number; - /** Wrap a subscription to the archive's timeline track. */ private constructor(track: Moq.Track.Subscriber, archive: Catalog.Archive) { this.#track = track; if (!Number.isSafeInteger(archive.timescale) || archive.timescale <= 0) { @@ -102,9 +75,11 @@ export class Consumer implements AsyncIterable { this.#window = new Json.Window.Consumer({ track, compression: true }); } - /** Subscribe to the timeline named by the catalog's archive entry. */ - static subscribe(broadcast: Moq.Broadcast.Consumer, archive: Catalog.Archive): Consumer { - return new Consumer(broadcast.track(archive.track).subscribe(), archive); + /** Subscribe to `track`'s timeline as named by the catalog's archive entry. Throws if it has none. */ + static subscribe(broadcast: Moq.Broadcast.Consumer, archive: Catalog.Archive, track: string): Consumer { + const name = archive.timelines[track]; + if (name === undefined) throw new Error(`no timeline for track ${track}`); + return new Consumer(broadcast.track(name).subscribe(), archive); } /** Get the next timeline event, or `undefined` when the track ends. */ @@ -114,6 +89,7 @@ export class Consumer implements AsyncIterable { if ("push" in event) { const { index } = event.push; const value = RecordSchema.parse(event.push.value); + if (value.sequence !== index) throw new Error("timeline record sequence differs from its index"); const pts = this.#micros(value.pts); const duration = this.#micros(value.duration); return { push: { index, entry: { ...value, pts, duration } } }; @@ -141,437 +117,3 @@ export class Consumer implements AsyncIterable { this.#track.close(); } } - -/** How a {@link Producer} paces its segments. */ -export interface ProducerProps { - /** - * The shortest a segment may be, in milliseconds of media time. - * - * A segment ends at the first point that is a group boundary on every enrolled track and at - * least this far past the segment's start, so the track with the coarsest groups paces the - * broadcast: a 2s GOP against a 1s minimum yields 2s segments, and a real-time encoder with - * a 10 minute GOP yields one 10 minute segment, because there is nowhere else a segment - * could start and stay decodable. A floor rather than a target on purpose: a floor is - * always satisfiable (wait longer), while a ceiling is not. Defaults to - * {@link DEFAULT_DURATION_MIN_MS}. - */ - durationMin?: Time.Milli; - - /** - * The longest a segment may be, in milliseconds of media time, advertised in the catalog - * when set. - * - * Set it only when the publisher can actually promise one, i.e. it controls the encoder's - * keyframe cadence. Consumers that need a bound up front use it (an HLS exporter's - * EXT-X-TARGETDURATION), so it is a contract rather than a hint: a segment that would exceed - * it fails the timeline instead of publishing a record that contradicts the catalog. Leave - * it unset when the media decides, which is the common case for real-time and for anything - * importing a source the publisher doesn't control. - */ - durationMax?: Time.Milli; -} - -/** One enrolled track's report state. */ -interface TrackState { - // Group opens reported and not yet flushed into a record. - pending: { sequence: number; pts: Time.Micro; keyframe: boolean }[]; - // The newest reported timestamp: everything earlier is known. Advanced by a group open (the - // group starts there) and by Recorder.end (the content stops there). - frontier?: Time.Micro; - closed: boolean; - // Whether this track paces boundaries and gates completeness while open. - pacing: boolean; -} - -/** - * Publishes the broadcast's timeline track: one JSON record per complete segment, - * DEFLATE-compressed (a `@moq/json` Window), plus the shared boundary list every media track's - * groups map onto. - * - * One per broadcast. Tracks enroll with {@link track}; continuous media explicitly opts into - * segmentation through {@link pacingTrack}. Both report group opens through the returned - * {@link Recorder}. An application with its own boundaries overrides pacing with {@link cut}. - * Advertise it in the catalog's root `archive` entry via {@link section}. - */ -export class Producer { - #window: Json.Window.Producer; - #trackName: string; - #durationMinUs: number; - #durationMaxUs?: number; - - // Where the open (unflushed) segment starts; undefined until the first report. - #start?: Time.Micro; - // Explicit cut() boundaries not yet reached, in order. - #cuts: Time.Micro[] = []; - // A cut() arrived, so the application owns the boundaries from here on and the durationMin - // pacing stops. Without this the pacing races ahead of a source whose segments are longer - // than the minimum, closing one before its real boundary is declared. - #manual = false; - #nextSegment = 0; - #tracks = new Map(); - // Live reservations: while any exists, no record flushes (more tracks are still enrolling). - #reservers = 0; - // A segment overran durationMax, so the timeline stopped publishing. - #overrun?: Error; - - /** Wrap an already-created MoQ track (named {@link DEFAULT_NAME} by convention). */ - constructor(track: Moq.Track.Producer, props: ProducerProps = {}) { - this.#trackName = track.name; - this.#durationMinUs = (props.durationMin ?? DEFAULT_DURATION_MIN_MS) * 1000; - this.#durationMaxUs = props.durationMax === undefined ? undefined : props.durationMax * 1000; - this.#window = new Json.Window.Producer({ - track, - compression: true, - checkpointRecords: CHECKPOINT_RECORDS, - }); - } - - /** - * Enroll `name` without letting it influence segmentation, returning its {@link Recorder}. - * - * Its groups are recorded into whichever segment is open when they arrive, but the track never - * votes on where a boundary falls and never holds a record back. This safe default is for a - * catalog that emits only when renditions change, or an application's metadata track. - * - * Placement is by arrival rather than by content time: nothing waits for the track, so a group - * that arrives after its segment flushed is recorded in the next one. Frames still carry their - * own timestamps. One recorder per track: enrolling the same name again resets its state. - */ - track(name: string): Recorder { - this.#tracks.set(name, { pending: [], frontier: undefined, closed: false, pacing: false }); - return new Recorder(this, name); - } - - /** - * Enroll the media track `name` as a pacing track, returning its {@link Recorder}. - * - * The track votes on boundaries and gates completeness until its recorder closes. Enroll it - * when it is about to produce: an enrolled but silent pacing track holds every record back, - * since a segment is not complete until every pacing track's content is known. - * - * This opt-in is for continuously publishing media. A sparse track belongs in {@link track}, - * otherwise it can stall the timeline when it goes quiet. - */ - pacingTrack(name: string): Recorder { - this.#tracks.set(name, { pending: [], frontier: undefined, closed: false, pacing: true }); - return new Recorder(this, name); - } - - /** - * Declare a segment boundary at `pts` (microseconds), overriding the - * {@link ProducerProps.durationMin} pacing. - * - * For applications that know their own boundaries (an imported playlist, on-disk CMAF - * segments, an encoder placing keyframes). Cutting ahead of the media is fine: the record - * still waits for every track's groups. A cut that would make a segment shorter than the - * minimum is ignored, so several producers declaring the same boundaries cost nothing. - * - * The first call takes over for good: {@link ProducerProps.durationMin} pacing stops, since it - * would otherwise close a segment just before the caller declares where it really ends. - * - * Throws if a segment already exceeded {@link ProducerProps.durationMax}. - */ - cut(pts: Time.Micro): void { - if (this.#overrun) throw this.#overrun; - - // Even a cut this rejects says the caller owns the boundaries. - this.#manual = true; - - const since = this.#cuts.at(-1) ?? this.#start; - if (since === undefined || pts >= since + this.#durationMinUs) { - this.#cuts.push(pts); - this.#pump(false); - } - } - - /** - * Begin reserving the track set, returning a function that releases the reservation. - * - * The counterpart to the Rust catalog's `reserve`, for the same reason: while any - * reservation is outstanding the track set may still grow, so records are withheld from the - * broadcast. A record is immutable once published and its completeness is judged against the - * tracks enrolled *at that moment*, so a segment that flushes while a sibling rendition is - * still enrolling omits it for good, and that rendition's earlier groups then land in - * whatever segment flushes next. - * - * Take one around whatever brings the tracks up, so a producer that enrolls its renditions - * one at a time publishes nothing until they are all in. Reservations nest, and releasing - * the same one twice is a no-op; the last one released flushes. - */ - reserve(): () => void { - this.#reservers += 1; - let released = false; - return () => { - if (released) return; - released = true; - this.#reservers -= 1; - this.#pump(false); - }; - } - - /** The catalog's root `archive` entry advertising this timeline. */ - section(): Catalog.Archive { - const durationMax = - this.#durationMaxUs === undefined - ? undefined - : u53(Math.ceil((this.#durationMaxUs * DEFAULT_TIMESCALE) / 1_000_000)); - return { - track: this.#trackName, - timescale: u53(DEFAULT_TIMESCALE), - durationMax, - }; - } - - /** - * Flush the final (still open) segment and finish the track. - * - * Throws if a segment exceeded {@link ProducerProps.durationMax}. - */ - finish(): void { - // Nothing more will enroll, so an outstanding reservation has nothing left to wait for. - this.#reservers = 0; - this.#pump(true); - - if (this.#overrun) throw this.#overrun; - - // Skip an empty tail: a boundary with no content after it describes nothing. - const start = this.#start; - this.#start = undefined; - if (start !== undefined) { - for (const track of this.#tracks.values()) { - if (track.pending.length > 0) { - this.#flushSegment(start, undefined); - break; - } - } - } - - this.#window.finish(); - } - - /** @internal A group open reported by a {@link Recorder}. */ - report(name: string, sequence: number, pts: Time.Micro, keyframe: boolean): void { - const track = this.#tracks.get(name); - if (!track) return; - track.pending.push({ sequence, pts, keyframe }); - this.#advance(track, pts); - } - - /** - * @internal A track's content ends at `pts`, without a group opening there. This is what - * gives the final segment an honest duration, since its end is not a boundary anybody cut. - */ - reportEnd(name: string, pts: Time.Micro): void { - const track = this.#tracks.get(name); - if (track) this.#advance(track, pts); - } - - /** @internal A track's recorder closed: stop gating completeness on it. */ - close(name: string): void { - const track = this.#tracks.get(name); - if (track) track.closed = true; - this.#pump(false); - } - - // Raise a track's frontier, then publish whatever that completed. - #advance(track: TrackState, pts: Time.Micro): void { - if (track.frontier === undefined || pts > track.frontier) track.frontier = pts; - - // The first thing a pacing track reports anchors the first segment, so content produced - // before any boundary exists belongs to the oldest segment rather than to nowhere. A - // non-pacing track must not anchor it: a catalog published while the encoder is still warming - // up would otherwise stretch segment 0 across the whole startup gap. - if (this.#start === undefined && track.pacing) this.#start = pts; - - this.#pump(false); - } - - // Publish every segment the media has finalized. `finished` is the terminal pass: no track - // will report again, so one that never reached a boundary stops voting instead of holding - // the timeline open forever. - #pump(finished: boolean): void { - if (this.#tracks.size === 0 || this.#reservers > 0 || this.#overrun) return; - while (this.#closeSegment(finished)) { - if (this.#overrun) break; - } - } - - // Publish the open segment if the media has finalized it, returning whether it did. - #closeSegment(finished: boolean): boolean { - const start = this.#start; - if (start === undefined) return false; - - // Discard boundaries the timeline has already reached. Only the first is ever consulted, - // so leaving a spent one there would block every later cut behind it and silently drop - // the caller back to durationMin pacing. - while (this.#cuts.length > 0 && this.#cuts[0] <= start) this.#cuts.shift(); - - const boundary = this.#boundary(start, finished); - if (!boundary) return false; - const [end, cut] = boundary; - - // Every open pacing track has to have reported at or past the boundary, proving its ranges - // for this segment are final. The track that voted for `end` has by construction; a track - // with shorter groups can still be behind it. A non-pacing track is excluded: it may publish - // rarely or never again, so waiting on it would stall the timeline for good. - if (!finished) { - for (const track of this.#tracks.values()) { - if (track.closed || !track.pacing) continue; - if (track.frontier === undefined || track.frontier < end) return false; - } - } - - this.#flushSegment(start, end); - if (cut) this.#cuts.shift(); - this.#start = end; - return true; - } - - // Where the segment starting at `start` ends: an explicit cut when one is registered, - // otherwise the first group boundary shared by every track that gives the segment its - // minimum duration. The boolean reports which. - #boundary(start: Time.Micro, finished: boolean): [Time.Micro, boolean] | undefined { - const cut = this.#cuts[0]; - if (cut !== undefined && cut > start) return [cut, true]; - - // The application declared a boundary at some point, so it owns them all: pacing here - // would close a segment the caller is about to cut somewhere else. - if (this.#manual) return undefined; - - const threshold = start + this.#durationMinUs; - - let end: Time.Micro | undefined; - for (const track of this.#tracks.values()) { - if (!track.pacing) continue; - const candidate = track.pending.find((group) => group.pts >= threshold)?.pts; - if (candidate === undefined) { - // This track has produced nothing past the minimum yet, so ending the segment - // would strand it. A closed track never will, and neither does anything on the - // terminal pass, so neither one blocks. - if (finished || track.closed) continue; - return undefined; - } - // The latest vote wins: it is a group boundary on the coarsest track, and every finer - // track assigns its groups by start, so no group is split. A closed track still votes: - // it can't report more, but the groups it did report are boundaries like any other. - if (end === undefined || candidate > end) end = candidate; - } - - return end === undefined ? undefined : [end, false]; - } - - // Emit the record for the segment starting at `start`: drain every track's groups before - // `end` (all of them for the final, unbounded segment) into ranges. - #flushSegment(start: Time.Micro, end?: Time.Micro): void { - const pts = Math.floor((start * DEFAULT_TIMESCALE) / 1_000_000); - let endUnits: number; - if (end !== undefined) { - endUnits = Math.floor((end * DEFAULT_TIMESCALE) / 1_000_000); - } else { - // The final segment has no end boundary, so it runs to the newest thing any track - // reported: its end of content when the track reported one, otherwise the last group - // it opened, which undercounts that group's tail. - let max = start; - for (const track of this.#tracks.values()) { - if (!track.pacing) continue; - if (track.frontier !== undefined && track.frontier > max) max = track.frontier; - } - endUnits = Math.floor((max * DEFAULT_TIMESCALE) / 1_000_000); - } - const duration = Math.max(0, endUnits - pts); - - // The catalog promised a bound and this segment breaks it, so the record would contradict - // what consumers were told. Fail the timeline rather than publish it: a declared maximum - // the media can't honor is a bug in the publisher. - if (this.#durationMaxUs !== undefined) { - const maxUnits = Math.ceil((this.#durationMaxUs * DEFAULT_TIMESCALE) / 1_000_000); - if (duration > maxUnits) { - this.#overrun = new Error( - `timeline segment ${this.#nextSegment} lasted ${duration}ms, over the declared maximum ${maxUnits}ms`, - ); - // End the track rather than drop it: the records published before the promise - // broke are still true, and a consumer that has them should keep them. - this.#window.finish(); - return; - } - } - - const record: Record = { segment: this.#nextSegment, pts, duration }; - this.#nextSegment += 1; - - const tracks: { [track: string]: Range[] } = {}; - let any = false; - for (const [name, track] of this.#tracks) { - const ranges: Range[] = []; - while (track.pending.length > 0) { - const group = track.pending[0]; - // A pacing track is assigned by content time. A non-pacing track is assigned by - // arrival, so every group pending when the segment closes belongs to it regardless - // of the timestamp basis carried by that track. - if (track.pacing && end !== undefined && group.pts >= end) break; - track.pending.shift(); - const last = ranges.at(-1); - // Contiguous sequences extend the run; a skip starts a new range (a gap: groups - // that never existed). - if (last && last.end + 1 === group.sequence) { - last.end = group.sequence; - } else { - const range: Range = { start: group.sequence, end: group.sequence }; - if (!group.keyframe) range.keyframe = false; - ranges.push(range); - } - } - if (ranges.length > 0) { - tracks[name] = ranges; - any = true; - } - } - if (any) record.tracks = tracks; - - this.#window.push(record); - } -} - -/** - * Reports one media track's group opens into the shared {@link Producer}. It is the track's - * single reporting handle; {@link close} ends the track's enrollment (segments stop waiting - * on it). Minted by {@link Producer.track} and held by a {@link Legacy.Producer} (its - * `timeline` prop). - */ -export class Recorder { - #timeline: Producer; - #name: string; - - /** @internal Minted by {@link Producer.track}. */ - constructor(timeline: Producer, name: string) { - this.#timeline = timeline; - this.#name = name; - } - - /** - * Report that group `sequence` opened at presentation time `pts` (microseconds), - * `keyframe` stating whether its first frame is one. Reports must be in group order with - * monotonic timestamps. - */ - record(sequence: number, pts: Time.Micro, keyframe = true): void { - this.#timeline.report(this.#name, sequence, pts, keyframe); - } - - /** - * Report that this track's content extends to `pts` (microseconds), without a group opening - * there. - * - * A group open says where content *starts*; the last group of a broadcast has no successor - * to bound it, so its segment would otherwise be published a group short (zero for a segment - * that is a single group). Report the end whenever you know it: closing a group, finishing a - * track. - */ - end(pts: Time.Micro): void { - this.#timeline.reportEnd(this.#name, pts); - } - - /** Close the track's enrollment: segments no longer wait on it. */ - close(): void { - this.#timeline.close(this.#name); - } -} diff --git a/js/json/src/window/encoder.ts b/js/json/src/window/encoder.ts index e403ef919c3..a5836a25078 100644 --- a/js/json/src/window/encoder.ts +++ b/js/json/src/window/encoder.ts @@ -129,6 +129,15 @@ export class Encoder { return this.#start + this.#window.length; } + /** + * Discard group-local state, so the next edit opens a new group with a header. + * + * Call this whenever the caller closes the current group behind the encoder's back. + */ + reset(): void { + this.#resyncGroup(); + } + /** Discard group-local state after an encoded frame did not reach the wire. */ #resyncGroup(): void { this.#flate = undefined; diff --git a/js/json/src/window/producer.ts b/js/json/src/window/producer.ts index b7051f60f7b..20457f5e134 100644 --- a/js/json/src/window/producer.ts +++ b/js/json/src/window/producer.ts @@ -90,6 +90,24 @@ export class Producer { encoded.commit(); } + /** + * Finish the open group, leaving the next edit to open a replacement with a header. + * + * Idempotent: cutting when no group is open does nothing. A caller that stores complete groups + * cuts after its edits, so every edit so far sits in a group no later frame can extend. + */ + cut(): void { + if (!this.#group) return; + + // Reset first: the group closes either way below, and a throw must not leave the encoder + // appending ops to a group that is gone. + this.#encoder.reset(); + + const group = this.#group; + this.#group = undefined; + group.close(); + } + /** Finish the track, closing any open group. */ finish(): void { if (this.#finished) return; diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index 95d2f0bedd9..f5bdb63cc57 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -75,6 +75,28 @@ test("push and pop round-trip", async () => { ]); }); +test("a cut is invisible to the consumer", async () => { + const live = new Live(); + await live.push(0); + live.producer.cut(); + live.producer.cut(); + await live.push(1); + + expect(await live.finish()).toEqual([ + { push: { index: 0, value: { n: 0 } } }, + { push: { index: 1, value: { n: 1 } } }, + ]); +}); + +test("reset makes the next edit a header", () => { + const encoder = new Encoder({}); + encoder.push({ n: 0 }).commit(); + expect(encoder.push({ n: 1 }).keyframe).toBe(false); + + encoder.reset(); + expect(encoder.push({ n: 2 }).keyframe).toBe(true); +}); + test("concurrent consumer reads are rejected", async () => { const track = new Track.Producer("test"); const consumer = new Consumer({ track: track.subscribe() }); diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index fbc5f955689..5dc959e1319 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -28,76 +28,79 @@ recordings, later format changes go through the entry's format version. ### Landed -The segment engine is in `rs/moq-mux/src/timeline.rs`: - -- `Segmenter` (:477) builds aligned `{ segment, pts, duration, tracks }` - records (`rs/hang/src/timeline.rs:105-121`) from pacing and non-pacing - tracks, with explicit `cut(pts)` boundaries. -- `Producer::deferred` (:917) returns a `Deferred` (:619) whose `next` yields - a cancellation-safe `Pending` record (:702). Dropping it requeues the record, - `Pending::gap()` (:712) turns it into a gap, and `Producer::push` (:948) - commits it into the visible timeline in segment order. -- The timeline track is a `moq_json::window::Producer` (:742). - `Producer::pop` (:983) trims the oldest records and `finish` (:1015) closes - the track. -- `moq-hls` renders live playlists from the timeline alone and FETCHes media - per HTTP request (`rs/moq-hls/src/export/mod.rs:3-8`). A clean timeline - finish ends every window with `EXT-X-ENDLIST` (:325-327). - -`rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`: -percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list/delete. +Every track has its own timeline (`rs/moq-mux/src/timeline.rs`): + +- A `Segmenter` builds one track's `{ sequence, pts, duration, start, end }` + records (`rs/hang/src/timeline.rs`) from its frame reports: a record ends at + the first group start past a minimum, a sparse track (zero minimum) records + each group as it finishes, a group open past `durationMax` splits between + frames, and `cut(pts)` adds application boundaries. +- A `Producer` publishes records onto a timeline track, a + `moq_json::window::Producer` whose `pop` trims the oldest and `flush` closes + the open group. `Timelines` pairs them for live publishing and names each + track's timeline in the catalog's root `archive` entry. +- `moq-hls` renders playlists from the timelines alone and FETCHes media per + HTTP request. Segment boundaries come from a reference rendition's records; + other renditions resolve each segment against their own timelines + (`rs/moq-hls/src/export/spans.rs`). A clean reference timeline finish ends + every window with `EXT-X-ENDLIST`. +- The same exporter serves a recording replayed through `moq_archive::Reader` + with no archive-specific code (`rs/moq-hls/src/export/archive_tests.rs`): + playlists read only the timelines, and a segment GETs only its rendition's + objects. The caller supplies the catalog. +- A catalog `archive` entry with a `store` and no `replay` path declares its + spans durable on that broadcast, so the exporter lists the whole retained + timeline and only its pops trim it (`durable` in + `rs/moq-hls/src/export/mod.rs`). + +`rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`. +`moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) cuts each enrolled track +with its own `Segmenter`, streams frames as they arrive, and commits each track +independently: the record's object, the record, retention pops, then the +timeline's `segments/`. A failed object PUT drops that record. A DVR prunes +each timeline's oldest objects no checkpoint recovery reads. On a prefix +that already holds a recording, it replays each retained timeline through +`timeline::Producer::resume`, deletes objects past each track's committed tail, +resumes after the newest record's end (partway through a split group), and a +DVR deletes unreferenced objects and unneeded timeline objects one grace period +after recovery. +`moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays every timeline +onto a supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` +with a byte-bounded object LRU, stitching a group split across records and +growing a group whose records do not reach its end yet. `Reader::refresh` +follows by listing each timeline's keys after its cursor; `Reader::finish` +applies out-of-band finality. `rs/moq-archive/src/proof.rs` records one +multi-rendition broadcast end to end. ### Format The format is the draft's -[Recording section](/drafts/draft-lcurley-moq-hang.md#recording). +[Recording section](/drafts/draft-lcurley-moq-hang.md#recording), version 2. The application chooses the object prefix, selected tracks, retention, and credentials; `moq-archive` owns the portable layout and codecs: ```text //.info -//groups/. -//segments/ +//segments/ ``` -`.info` is versioned JSON with the immutable priority and timescale. A segment -object is a versioned binary envelope: a group/frame table with timestamps and -payload offsets, then the original payloads. One GET populates an LRU with the -adjacent groups, while audio-only or low-rendition playback never downloads -unrelated tracks. - -There is no `.head`, manifest, `.complete`, or `.timeline` object. The archive -timeline uses the same envelope under `segments/`, with consecutive -IDs for incremental GETs. Other tracks use `groups/.` with -19-digit zero-padded inclusive bounds, strictly increasing nonoverlapping -ranges, and no segment number or secondary index file. The catalog's `archive` entry names the -timeline track, plus any replay path, store URL, and format version. Listing -bootstraps recovery; following the next timeline key updates the cached ranges. - -### Writer and reader - -The application passes a `broadcast::Consumer` and opts arbitrary track names -in as pacing or non-pacing, including catalog and JSON tracks; `moq-archive` -never parses a media catalog. For each closed segment the writer PUTs one -object per participating track, drops the ranges of any track whose PUT -failed, then pushes the record. A clean source end flushes the final partial -segment and finishes the timeline; there is no completion marker. - -The reader takes a `broadcast::Producer` and uses `track::Dynamic` -(`rs/moq-net/src/model/track.rs:1652`) to answer FETCH for the tracks and -groups the timeline advertises: map the request to the track's group-range key, GET -once, validate, cache, and replay the original timestamps. Any ingest that -produces a broadcast (RTMP, SRT, WHIP) is archivable without its own -implementation. +A recorded track's `segments/` holds record `n` of its timeline; its +timeline, named with a `.timeline.z` suffix, stores the window groups committed +with record `n` under its own `segments/`. `.info` is versioned JSON with +the immutable priority and timescale. A segment object is a versioned binary +envelope: a frame start, a group/frame table with timestamps and payload +offsets, then the original payloads. One GET populates an LRU with the adjacent +groups, while audio-only or low-rendition playback never downloads unrelated +tracks. There is no `.head`, manifest, `.complete`, or index object. ### Retention -An unbounded archive only pushes records. During each new segment commit, a -DVR pops expired records before closing and storing that segment's timeline -groups, then waits the configured grace period before deleting expired objects. Timeline -objects use `segments/` for the segment being committed. Retention stops after -the final segment; no existing object is rewritten. HLS is a derived view of -the archive, never a second stored copy. +An unbounded archive only pushes records. A DVR pops each track's expired +records while committing that track's next record, before storing its timeline +groups, then waits the configured grace period before deleting expired objects. +Each track keeps its newest record, so a static catalog outlives the video +recorded with it. HLS is a derived view of the archive, never a second stored +copy. ### Managed boundary @@ -112,12 +115,15 @@ owned by that prerequisite, not duplicated in archive storage. ## Required -- [Recording writer](/quest/m1/archive/writer.md) - feed the segmenter from a `broadcast::Consumer`, store each segment, then commit its record -- [Recording reader](/quest/m1/archive/reader.md) - serve archived FETCH through a supplied `broadcast::Producer` -- [Replay provenance](/quest/m1/archive/provenance.md) - a replay's catalog names its timeline, replay path, store URL, and format version +- [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md) - each timeline entry declares its segment duration (reported or estimated by the publisher), replacing the root `durationMax` +- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - `@moq/hang` publishes the same per-track timelines as Rust (it already reads them) +- [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, from the reference timeline's declared duration; an overrun is listed with a warning +- [Per-track timelines](/quest/m1/archive/track-timeline.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge +- [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay +- [Idle flush](/quest/m1/archive/flush.md) - idle tracks are recorded within a bounded wall-clock delay, and `flush()` forces a track's pending record out +- [Backward timestamps](/quest/m1/archive/backward-timestamps.md) - a resumed recording refuses a restarted source instead of dropping its groups or overlapping media time +- [HLS media sequence](/quest/m1/archive/hls-media-sequence.md) - `EXT-X-MEDIA-SEQUENCE` never decreases when the reference rendition switches - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback -- [Enrollment flake](/quest/m1/archive/enrollment-flake.md) - the opening-snapshot test waits for real enrollment, not the `.info` file -- [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, and exact FETCH replay; its HLS step is done on the line branch ## Related diff --git a/quest/m1/archive/backward-timestamps.md b/quest/m1/archive/backward-timestamps.md new file mode 100644 index 00000000000..348a4fa9a45 --- /dev/null +++ b/quest/m1/archive/backward-timestamps.md @@ -0,0 +1,26 @@ +# [XS] Archive refuses backward timestamps + +## Goal + +A resumed recording refuses a track whose source restarted, failing loud +instead of recording nothing and then overlapping media time. The caller +starts a new prefix. + +Both signs of a restart count: a first group at or below the recovered +track's committed tail, which the writer drops today with only a `debug` log, +and a first timestamp before the recovered track's last recorded timestamp. + +## Plan + +Check at enrollment against the recovered timeline, and test a restart from +group 0, a backward timestamp, and a forward restart that keeps recording. +Mind a writer restarted against a source that kept running: its first group +may be the partially recorded tail group, which resumes rather than refuses. +The 2026-09-30 audit folded this into the recording writer, which landed +without it; the [#4034 review](https://github.com/moq-dev/moq/pull/4034#issuecomment-6005694031) +found the silent group drop. The archive format may break in place since it +is unreleased. + +## Related + +- [Per-track timelines](/quest/m1/archive/track-timeline.md) - reshapes the recovery this check reads diff --git a/quest/m1/archive/declared-duration.md b/quest/m1/archive/declared-duration.md new file mode 100644 index 00000000000..c39af1bfa31 --- /dev/null +++ b/quest/m1/archive/declared-duration.md @@ -0,0 +1,64 @@ +# [M] Timelines declare their segment duration + +## Goal + +Each timeline in the catalog `archive` map declares its own nominal segment +duration, derived from what the application reports (like bitrate or +framerate), or estimated by the publisher when the application doesn't know +it. The broadcast-wide `durationMax`, filled from moq-mux's 10s default, is deleted. Every edge reads +the same declared value, so an HLS edge can fix its target duration from it. + +## Plan + +Decided in planning (09-29), after the hls-target agent found that nothing in +the catalog declares a usable target: + +- **Shape.** Each `timelines` entry becomes an object: + `timelines: { video: { track: "video.timeline.z", duration: 4000 } }`, in + the archive `timescale`. The root `durationMax` goes away. This is a break in + place on this line, with no compatibility path. Update `rs/hang`, + `drafts/draft-lcurley-moq-hang.md`, and every `durationMax` reference + (grep it, including `doc/`), and the JS catalog schema + (`js/hang/src/catalog/archive.ts`) in the same PR, since `@moq/watch` refuses + a catalog it cannot parse. The rest of the JS port happens in + [JS per-track timelines](/quest/m1/archive/js-timelines.md). +- **A 4s minimum.** `duration_min` defaults to 4s (from 2s), so an archive + doesn't write a stream of tiny objects to S3. A record still ends at the + first group boundary at or past the minimum. This also packs audio, where + moq-audio publishes each ~20 ms codec packet as its own group (Codex on + this plan's PR). +- **Meaning: a nominal target, not a hard max.** Records start on keyframes. A + GOP longer than the declared duration overruns, and moq-hls lists the overrun + with a warning. A mid-group split remains only as a safety ceiling at a + multiple of the declared duration (pick one, for example 3x), which is + internal and not declared. Before an unhinted track declares, the ceiling + is a fixed default (today's 10s). A hard max that splits at the target was + rejected: jitter makes slivers, and segments would start without a keyframe. +- **Reported by the application.** moq-mux takes an optional per-track GOP + hint. moq-video's encoder sets it from its `encode::Gop`. Importers pass + nothing and estimate. With a hint, the declared duration is the smallest + multiple of the hint that is at least `duration_min`, the record length a + regular GOP produces. +- **Estimated by the publisher when not reported.** moq-mux holds that + timeline's catalog entry until its first record closes, then declares that + record's duration, rounded up to the timescale. If the first record hits the + safety ceiling, it declares the ceiling. Estimating at the edge was rejected + because edges and restarts would disagree. A rolling estimate was rejected + because it conflicts with a fixed target. The user chose a single sample + over the longer of two, accepting that an irregular first GOP sets the value. + The sample is the first record, not the first group, because a group can be + a single audio packet. +- **Fixed per timeline.** The declared value never changes for a timeline + track's life. A reconfigure that makes a new timeline track declares its own. +- Sparse timelines (the catalog's own, zero minimum) keep their current + cutting. Decide what, if anything, they declare, and keep it out of HLS. + +Tests: a hinted track declares the smallest multiple of its hint at or past +the minimum. An unhinted track holds its entry until its first record closes, +then declares that record's duration. A first record that hits the ceiling +declares the ceiling. Audio made of one packet per group declares about 4s, +not 20 ms. The declared value survives later, longer records. + +## Related + +- [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - consumes the declared duration diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 930571650ff..ef9003c0456 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -7,9 +7,9 @@ same timeline and group-range objects as an unbounded archive. ## Plan -The recording writer prerequisite owns retention, checkpoint recovery, deletion -grace, and restart cleanup. This quest consumes that contract and owns viewer -seek and return-to-live behavior, not a second writer implementation. +The recording writer owns retention, deletion grace, checkpoint recovery, and +restart cleanup. This quest consumes that contract and owns viewer seek and +return-to-live behavior, not a second writer implementation. The player reads the archive timeline, FETCHes old groups through the normal miss chain, and splices back to SUBSCRIBE at the live edge without opening a @@ -22,10 +22,38 @@ Test seeks within the retained window, expiry during a seek, missing groups, restart recovery, and return to live without duplicated or rewound playback. Use the writer/reader fixtures; a retention defect is fixed in its owning layer. +The reader evicts popped spans from its object cache, but a group it already +served stays in `moq_net`'s track cache until the pool reclaims it. Decide +whether expiry during a seek needs a group eviction API in `moq-net`. + +`moq-hls` reads a timeline from the catalog's own broadcast, and an +`archive.replay` path only marks it non-durable. Decided (09-29): the exporter +does not follow `replay`, and viewers don't address a separate replay +broadcast. A recording is the broadcast. Under the +[wildcard](/quest/m0/wildcard/README.md) plan the archive serves the source +path through the root claim, and a live announcement shadows it. So when live +ends, `moq-hls` resolves the same name and falls through to the recording: +playlists keep serving for rewind and for players finishing the last +segments. The fall-through needs the recording to publish its catalog live, +since `moq-hls` subscribes to it rather than FETCHing it. Decided (09-29): +that republishing moves into `moq-archive`, so any host of the archive +behind the root claim does it, not only `moq-cli`; update [Replay +catalog](/quest/m1/archive/replay-catalog.md) to match. + +During live, rewind needs no handover: a recorded broadcast's live timeline +is durable, and every group it lists is promised available, so a seek past +the live window FETCHes old groups through the normal miss chain down to the +archive. `moq-hls` needs no special path. `moq-hls` gains +no linger (the `hls-linger` quest was dropped, because an unannounced +broadcast can't be FETCHed and a linger would only serve the cache). Test +the handover: a live HLS session keeps its playlist URIs and media sequence +numbers when the name moves from the live publisher to the archive. + ## Required -- [Recording writer](/quest/m1/archive/writer.md) -- [Recording reader](/quest/m1/archive/reader.md) +- [Per-track timelines](/quest/m1/archive/track-timeline.md) - seeks through per-track timelines +- [Replay catalog](/quest/m1/archive/replay-catalog.md) - the recording publishes its catalog live, so `moq-hls` finds it after the handover +- [Wildcard](/quest/m0/wildcard/README.md) - the archive's root claim serves the source path once the live announcement ends ## Closes diff --git a/quest/m1/archive/enrollment-flake.md b/quest/m1/archive/enrollment-flake.md deleted file mode 100644 index f8a02346e09..00000000000 --- a/quest/m1/archive/enrollment-flake.md +++ /dev/null @@ -1,26 +0,0 @@ -# [XS] The opening-snapshot test waits for real enrollment - -## Goal - -`archive::tests::an_opening_snapshot_records_every_rendition` in -`rs/moq-cli/src/archive.rs` passes under load. It fails with "writer closed" -because it treats a rendition's `.info` file as proof the export enrolled -that track, then finishes the tracks and the catalog; under load the writer -can still be subscribing when the tracks end. Seen while landing -[#4169](https://github.com/moq-dev/moq/pull/4169). - -## Plan - -- Wait on the signal enrollment actually produces (the subscription reaching - the publisher, or an explicit event from the exporter), not a file that - appears earlier. No longer timeout and no retry. -- If the exporter can genuinely lose a rendition that ends right after it - appears in the catalog, that is a bug in the exporter, not the test; fix it - there. - -Public API: none. Wire: none. - -## Related - -- [More tests hold up under load](/quest/m1/test-flakes-2/README.md) - the same - round of load-only failures on `main` diff --git a/quest/m1/archive/flush.md b/quest/m1/archive/flush.md new file mode 100644 index 00000000000..895bf4283a2 --- /dev/null +++ b/quest/m1/archive/flush.md @@ -0,0 +1,42 @@ +# [M] Idle tracks still get recorded + +## Goal + +A recording closes the record holding every frame within a bounded wall-clock +delay of its arrival, and hands it to the committer, even when its track goes +idle, and a caller can force one track's pending +record out with `flush()`. An append-only group that stops growing, or a track +whose publisher stops sending, no longer waits for the next frame or the end of +the recording. + +## Plan + +Decided: + +- Time decisions use elapsed = max(wall clock, pts) everywhere, so the existing + cutting rule does the work with no new setting: an open group idle past + `duration_max` (10 s by default) splits between frames, and a finished sparse + group (minimum 0) stores at once. Peer timestamps alone can never stall a + cut, since the local wall clock bounds them. A forward pts leap can close a + record early; a new record then starts at the leaping frame, so each leap + costs at most one extra record. Say in the PR how repeated leaps are bounded. +- The segmenter stays pure: it takes `now` as an input and exposes the next + wall-clock deadline at which a record must close. Callers arm the timer. The + moq-archive writer uses tokio time; moq-mux's `Recorder` must stay wasm-safe + (moq-mux builds for wasm32), so it uses moq-net's runtime `Timer` or + `web_async` time, never `tokio::time`. +- Manual flush is per track: `Control::flush(name)` on the writer, beside the + broadcast-wide `cut`, and `Recorder::flush()` on a timeline. +- JS mirrors it: `@moq/hang`'s `Timeline.Recorder` gets the same idle + deadline and a `flush()`, so Rust and JS timelines stay one API. Without it, + an idle browser track keeps today's unbounded behavior (Codex on #4301). +- The bound covers closing a record, not upload latency: a slow object-store + PUT still delays the durable commit, which the writer already serializes. +- Tests inject `now` into the segmenter and run the writer on paused tokio + time, per the repository's mock-time rule. Cover an idle open group, an idle + track after a finished group, a manual flush mid-group, and a publisher whose + timestamps run ahead of or behind the wall clock. + +## Required + +- [Per-track timelines](/quest/m1/archive/track-timeline.md) - the per-track segmenter, writer, and JS recorder this extends, so the flush lands on the per-track shape rather than the aligned one it replaces diff --git a/quest/m1/archive/hls-media-sequence.md b/quest/m1/archive/hls-media-sequence.md new file mode 100644 index 00000000000..836e9be983a --- /dev/null +++ b/quest/m1/archive/hls-media-sequence.md @@ -0,0 +1,37 @@ +# [S] HLS media sequence survives a reference switch + +## Goal + +A media playlist's `EXT-X-MEDIA-SEQUENCE` never decreases, as +[RFC 8216 §6.2.2](https://www.rfc-editor.org/rfc/rfc8216.html#section-6.2.2) +requires, even when the reference rendition changes. Players keep loading the +same `media.m3u8` across the switch without stalling. + +## Plan + +Today `moq-hls` numbers segments by the reference rendition's own records. A +switch (a new first video rendition with a timeline, or the reference leaving +the catalog) starts a new numbering under a new URL tag +(`seg/{reference}.{segment}.m4s`, #4034). Its URLs never collide and a +discontinuity marks the switch, but the media sequence can rewind. + +Open design, pick one: + +- **Sticky reference.** Keep the reference while it stays in the catalog, so a + new, earlier-named rendition no longer switches it. Cheap and edge-consistent, + but a reference that leaves the catalog still rewinds. +- **Offset at the switch.** Number the new reference's segments from past the + last listed one. Monotonic, but an edge that starts after the switch numbers + differently, so edges disagree on the media sequence. + +Recommendation: the sticky reference first. It removes the common trigger +without giving up the property that every edge agrees on numbering. Measure +whether a reference leaving the catalog happens in practice before taking on +the offset. + +Test a switch through repeated playlist reloads: the media sequence never +decreases, and no URL names other content. + +## Related + +- [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - also changes how the reference shapes the playlist diff --git a/quest/m1/archive/hls-target.md b/quest/m1/archive/hls-target.md new file mode 100644 index 00000000000..5c76ce24672 --- /dev/null +++ b/quest/m1/archive/hls-target.md @@ -0,0 +1,52 @@ +# [S] Fixed HLS target duration + +## Goal + +A `moq-hls` media playlist advertises one `EXT-X-TARGETDURATION` for the +whole run, taken from the reference timeline's declared duration, so every +edge and every reload agree on it. Segments stay one-to-one with timeline +records: the edge never skips, splits, or renumbers one. + +## Plan + +[#4280](https://github.com/moq-dev/moq/pull/4280) derives the target from the +current window's longest segment (`snapshot_as` in +`rs/moq-hls/src/export/rendition.rs`), so it rises when a long record arrives +and falls when that record is evicted. Codex flagged it +([r4113921580](https://github.com/moq-dev/moq/pull/4280#discussion_r4113921580)). +The 09-28 audit reversed #4280's decision 4 (the observed maximum), and the +09-29 planning settled the rest: + +- The target is `ceil` of the reference timeline's declared duration from + [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md). + Until that timeline's entry is declared, the rendition has no playlist yet. +- A segment whose rounded `EXTINF` exceeds the target (a GOP that overran the + publisher's declared value) is listed anyway, with a rate-limited warning. + This deliberately deviates from RFC 8216 4.3.3.1's MUST, because refusing + the segment loses content, and splitting or skipping it at the edge breaks + the stable media sequence numbers of RFC 8216 6.2.2. Document the + deviation. hls.js tolerates it; Apple's mediastreamvalidator flags it. +- A record the publisher split mid-group at its safety ceiling is listed as + its own segment. Stop merging split halves back into one range + (`a_frame_split_record_is_not_a_video_boundary` in `spans.rs`). This is legal + because moq-hls never emits `EXT-X-INDEPENDENT-SEGMENTS`, so a segment may + start without a keyframe. +- A non-reference rendition keeps the reference record's `EXTINF`, even though + its keyframe-snapped content can run longer. The user decided this needs no + change. + +Replace `target_duration_follows_the_observed_segments` with tests that pin +the declared target across a window whose segment durations vary, and that +cover an overrun at the rounding boundary (listed, warned). Fix its stale +comment ("default 1s minimum"; the minimum is 2s). Hold the target per +playlist URI, not per `Rendition`: `renditions::Producer::sync` replaces the +`Rendition` on a catalog reconfigure, so test that a reconfigure mid-run +keeps the original target. + +## Required + +- [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md) - the declared value the target is taken from + +## Related + +- [Per-track timelines](/quest/m1/archive/track-timeline.md) - the quest this blocks diff --git a/quest/m1/archive/js-timelines.md b/quest/m1/archive/js-timelines.md new file mode 100644 index 00000000000..9e9bbbc7f55 --- /dev/null +++ b/quest/m1/archive/js-timelines.md @@ -0,0 +1,23 @@ +# [M] JS per-track timelines + +## Goal + +`@moq/hang` publishes one timeline per track with the same records, cuts, and +catalog `archive` map as Rust. + +## Plan + +The read side landed with the archive line (#4034): the catalog schema parses +the `timelines` map and `Timeline.Consumer` reads one track's records. The +aligned multi-track producer was deleted rather than kept beside it, so JS +publishes no timeline today. + +Port the Rust segmenter, producer, and enrollment +(`rs/moq-mux/src/timeline.rs`) to `js/hang/src/timeline.ts`, and have the +legacy container report its groups again. Cover the same cut rules and a +static catalog outliving other tracks' records, and check the records against +Rust output in the interop suite. + +## Required + +- [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md) - the final `timelines` entry shape to port, so JS ports it once diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md deleted file mode 100644 index 0ffdb48f1ce..00000000000 --- a/quest/m1/archive/proof.md +++ /dev/null @@ -1,67 +0,0 @@ -# [M] Archive proof - -## Goal - -Prove deterministic segment storage, exact FETCH replay, selective rendition -reads, and timeline-only HLS generation from one multi-rendition broadcast. - -## Plan - -Record explicitly enrolled video, audio, catalog, and non-media tracks. Cut at -aligned keyframe boundaries, including multiple audio groups per segment, then -replay their original sequences, timestamps, and payloads through -`track::Dynamic`. - -Verify the exact object keys and bytes on memory, local, and S3-compatible -`object_store` implementations, including the percent-encoded track names. A -360p or audio-only FETCH must not GET the 1080p object, while adjacent group -requests should hit the segment LRU. - -Cover the persistence boundary: a crash after segment PUT but before timeline -commit leaves invisible orphan data that a listing bootstrap ignores; a failed -or mismatched `.info` exposes no ranges; a failed independent track PUT omits -only that track while the record's other tracks stay advertised; later -segments remain usable. Catalog-to-group applicability is outside this proof. Also -cover a stalled pacing track forced to a gap, sparse group ranges, malformed -table offsets, an unknown envelope or `.info` version treated as a missing -segment, segment create collisions under the single-writer rule, a missing -tail, and a clean end without a completion marker. Accept equivalent `.info` -JSON with reordered members or different whitespace without rewriting it; -reject differing properties and malformed or unsupported metadata. - -Exercise group and segment IDs 0 and 2^53 - 1; reject 2^53 and the largest -QUIC varint in keys and reconstructed group IDs. Cover consecutive zero -deltas, sparse deltas, overflow, stopping at ID exhaustion without wrapping -or inferring clean finality, empty objects, mismatched filename bounds, and overlapping -ranges across segments. Reject decreasing or duplicate group arrivals while -allowing accepted groups to complete out of order. Verify JSON-safe timescales -and timestamps, accepting timestamp 2^53 - 1 and rejecting 2^53. -Test ordered S3 lookup at both endpoints and between ranges, unordered paginated -results, incremental cursor recovery, DVR expiration while a reader is offline, -and stale media listings preceding a new timeline commit. Following N+1 must -not refresh all media listings. Wire these cases into CI for the store, writer, -and reader implementations; do not add an unconnected standalone proof script. - -The store's own tests also cover recording-prefix isolation (`rec` beside -`rec-other`), empty prefixes, track-prefix listings, continuation pages, and -every supported pagination option. A page must not lose directory entries -silently or fail because the backend matched a neighbouring recording. Every -publicly constructible key either serializes to a path its parser accepts or -fails before storage; direct `Key::Groups` construction must not bypass range -validation. These cases belong beside the store and codec code and need no -new public API or format change. - -Crash a DVR writer after its pop becomes durable but before media deletion. -On exclusive restart, prove that expired and uncommitted group objects are -removed after the grace period while retained media, `.info`, and checkpoint -objects survive. Failed or incomplete recovery/listing must delete nothing; -restart must finish this cleanup before accepting new groups. - -Finally render and reload HLS playlists while rejecting every media-object GET -until a segment URI is requested. The range-bearing URI must resolve one -object directly without any listing or separate index object. This HLS step -is done on the archive line branch (#4115, #4169). - -## Required - -- [Recording writer](/quest/m1/archive/writer.md) diff --git a/quest/m1/archive/provenance.md b/quest/m1/archive/provenance.md deleted file mode 100644 index c766a0dc702..00000000000 --- a/quest/m1/archive/provenance.md +++ /dev/null @@ -1,18 +0,0 @@ -# [S] Replay provenance - -## Goal - -The catalog a replay publishes carries the root `archive` entry the line -promises: the timeline track, the replay path, the store URL, and the format -version. A player or tool can tell a replay from the live source and find the -recording. - -## Plan - -- Check which of these fields `rs/hang` and `js/hang` already define. Add any - missing ones as optional fields in both, and in the Recording section of - `drafts/draft-lcurley-moq-hang.md`. -- Never advertise credentials: strip userinfo and query from the store URL, and - let the importer opt out of advertising the URL at all. -- `moq import archive` fills the entry. Test that a round trip through - `export archive` and `import archive` advertises it. diff --git a/quest/m1/archive/reader.md b/quest/m1/archive/reader.md deleted file mode 100644 index f317970680e..00000000000 --- a/quest/m1/archive/reader.md +++ /dev/null @@ -1,49 +0,0 @@ -# [L] Recording reader - -## Goal - -A reader takes an archive and a caller-provided `broadcast::Producer`, then -serves FETCH requests for every track and group the archive timeline -advertises. - -## Plan - -Bootstrap by listing the timeline track's `segments/` keys, sorting them, and -replaying from a retained Window checkpoint. Records derive each track's -`groups/.` key from its minimum and maximum advertised IDs. -Follow a growing archive with GET of timeline segment N+1; media listings do -not need refreshing. Not Found is not finality. A reader behind DVR retention -re-bootstraps from a retained checkpoint, and timeline pops evict cached ranges. -Alternatively enumerate timeline keys after the last replayed key with -`list_with_offset`, consuming and sorting the full result before replay. -Pagination tokens continue one enumeration, not future refreshes. - -Filename listings provide a cached group-range index without media GETs. A -backend with verified ordered offset listing may find a cold FETCH candidate -using `PaginatedListStore` with offset `groups/` (19 padded -digits, no dot) and `max_keys = 1`. Check the lower bound and committed timeline -membership; the table resolves internal gaps. Generic listings are unordered, -so never take their first entry as the nearest match. A stale listing cannot -veto a successful GET referenced by a newer durable timeline record. - -Use `track::Dynamic` (`rs/moq-net/src/model/track.rs:1652`, minted by -`Producer::dynamic` :1587) to accept requested tracks and groups on the -supplied producer; a cache-miss `Consumer::fetch_group` (:2352) parks on it. -Map `(track, group)` to its range-named object through the cached index, GET -that one object, validate the envelope version and every group/frame table -entry against the retrieved length, filename bounds, and exact committed -ranges, and place it in a byte-bounded LRU. -Adjacent group FETCHes reuse the same object. A request for one audio track or -rendition never downloads another track's object. - -Reproduce the original group sequence, frame timestamps, and payload bytes, -including a requested `frame_start`. A group absent from the timeline, a -missing object, an unknown envelope version, or a malformed table behaves -exactly like a group the source never delivered; siblings and later segments -remain usable. - -The reader does not interpret media, own routing, or expose storage paths. It -does not infer a terminal broadcast state: a caller such as a managed -recordings API may supply finality out of band, and the reader then finishes -the replayed timeline track. Otherwise an archive may be growing, crashed, or -missing its tail, and remains readable. diff --git a/quest/m1/archive/replay-catalog.md b/quest/m1/archive/replay-catalog.md new file mode 100644 index 00000000000..3435387cbf2 --- /dev/null +++ b/quest/m1/archive/replay-catalog.md @@ -0,0 +1,45 @@ +# [S] Replay catalog + +## Goal + +A replayed recording plays as VOD HLS from the stock server: `moq export +archive`, then `moq import archive`, then `moq export hls` lists the whole +recording with no embedder supplying a catalog. The replay broadcast publishes +its recorded catalog live, stamped with the recording's `store` and `version`, +so the exporter treats its timeline as durable and lists past `--window`. A +`--follow` replay grows like an event playlist. + +Choosing which catalog applies to which media group stays with +[Catalog track identity](/quest/m2/catalog-tracks.md). + +## Plan + +Today the reader writes live groups only for timelines; the recorded catalog +is FETCH-only, so a SUBSCRIBE to it on the replay broadcast never sees a group +and `moq-hls` never finds the `archive` entry. The HLS archive tests work around +this by hand-building a catalog. + +- Republish each recorded catalog group live in its timeline's order, so the newest + one is at the live edge and a `--follow` replay picks up catalogs recorded + after it opened. +- Stamp `store` with the URL passed to `import archive`, and `version` with the + recording format. Refuse a URL carrying userinfo and strip its query so + credentials never land in a catalog, and let the importer opt out of + advertising the URL at all. `replay` stays unset: the timelines live on this broadcast. +- Keep the logic in `moq-cli`; `moq-archive` stays catalog-agnostic. Select the + catalog track with the CLI's catalog format as `export archive` does: hang + and hang.z are stamped, MSF is refused. +- The recorded catalog's `archive` entry describes the source's live + timelines, not the recording's, so replace it with the timelines the reader + replays rather than trusting the recorded ones. + +Add a CI test that records a broadcast longer than the default window, replays +it, and asserts the stock exporter lists segment 0 with a durable +`timeShiftBufferDepth`. Consider moving the HLS archive tests onto the +republished catalog instead of their hand-built one. + +Update `doc/bin/cli.md` and `doc/bin/hls.md` inline with an export, import, and +serve example. + +Cover a DVR recording whose catalog outlived its first video segment, and fail +loudly on a recording with no catalog. diff --git a/quest/m1/archive/track-timeline.md b/quest/m1/archive/track-timeline.md new file mode 100644 index 00000000000..937cad2cced --- /dev/null +++ b/quest/m1/archive/track-timeline.md @@ -0,0 +1,71 @@ +# [S] Per-track timelines + +## Goal + +Every track carries its own timeline, so tracks segment, commit, and expire +independently. A DVR keeps each track's newest group, such as a catalog that +never changes, and an append-only group stays addressable as frames arrive. An +edge like `moq-hls` derives HLS and DASH from group timestamps, so a publisher +never needs to know about HLS. + +Nothing on `main` has users yet: break the timeline, catalog `archive` entry, +and recording format in place, with no compatibility path, even though hang +0.21 released the current shape. The Rust side lands on `main` with the +[archive line](/quest/m1/archive/README.md) (#4034); the children follow as +their own PRs to `main`. + +## Plan + +The Rust side has landed: a `hang::timeline::Record` is one span of its own +track (`sequence`, `pts`, `duration`, `start`/`end` group and frame positions), +`moq_mux::timeline::Timelines` publishes one timeline per enrolled track, +`moq-archive` writes recording version 2 (`/segments/` beside its +timeline's `segments/`), and `moq-hls` derives segments from a reference +rendition's records (`rs/moq-hls/src/export/spans.rs`). The draft is updated +in moq-hang-04. + +Decisions: + +- One timeline track per track, live and recorded. `moq-mux` publishes them for + every broadcast; an unsubscribed track costs nothing. +- The catalog's root `archive` entry maps each track to its timeline, including + the catalog track itself. `replay`, `store`, and `version` stay beside it. +- Each track cuts on its own by one rule: at a group boundary between a + minimum and maximum duration (a 2s minimum today; 4s and a multiple of the declared + duration once its declared-duration child lands; a zero minimum only for sparse data such as the catalog), + splitting a long-lived group by frame at the maximum. Manual cuts stay as an optimization, such as a + video keyframe cutting audio so derived segments need fewer objects. +- A stored object may hold a frame range of a group, not only whole groups. +- HLS and DASH segments are derived at the edge from group timestamps, not + from storage objects. Fetching extra objects is fine when they land in the + reader's cache for the next request. +- `moq-hls` never retries a failed timeline, because the origin already reconnects + transient source failures. A replacement publisher that restarts group + numbering is a publisher bug: a broadcast, track, or group name always means + the same content, and MoQ has no ETag-style invalidation + ([#4556](https://github.com/moq-dev/moq/pull/4556)). A failed timeline + fails loud instead: the reference ends every playlist with `EXT-X-ENDLIST`, + and another rendition's playlist ends at its last covered segment rather than + listing gaps. +- HLS `EXT-X-TARGETDURATION` is fixed for the run, not the observed maximum + #4280 shipped. This reverses that PR's decision 4 (09-28 merged-PR audit). + The target comes from each timeline's declared duration, which replaces the + broadcast-wide `durationMax` (09-29 planning). + +For triage, not blocking: two Codex P2s arrived after #4280 merged and are +unanswered. A timeline record whose `sequence` differs from its window index +is passed through unvalidated +([review](https://github.com/moq-dev/moq/pull/4280#pullrequestreview-5332725290)), +and `Timelines::track` re-enrolling a name while its old `Recorder` is alive +leaves two handles on one segmenter +([r4117516035](https://github.com/moq-dev/moq/pull/4280#discussion_r4117516035)). + +## Required + +- [Timelines declare their segment duration](/quest/m1/archive/declared-duration.md) - each timeline entry declares its segment duration (reported or estimated by the publisher), replacing the root `durationMax` +- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - `@moq/hang` publishes the same per-track timelines as Rust (it already reads them) +- [Fixed HLS target duration](/quest/m1/archive/hls-target.md) - one `EXT-X-TARGETDURATION` for the run, from the reference timeline's declared duration; an overrun is listed with a warning + +## Related + +- [Catalog track identity](/quest/m2/catalog-tracks.md) - the catalog's own timeline gives it timestamps, but which catalog applies to a group stays there diff --git a/quest/m1/archive/writer.md b/quest/m1/archive/writer.md deleted file mode 100644 index 3a1329e1f74..00000000000 --- a/quest/m1/archive/writer.md +++ /dev/null @@ -1,88 +0,0 @@ -# [L] Recording writer - -## Goal - -A generic `moq-archive` writer consumes explicitly selected tracks from a -`broadcast::Consumer`, feeds their complete groups to the existing segmenter, -stores one object per track per segment, and commits the timeline record only -after the objects it names are durable. - -## Plan - -The application supplies the broadcast, object prefix, and arbitrary track -registrations, each pacing or non-pacing; the writer does not parse a catalog -or know which tracks are media. - -Enrollment creates the track's `.info` before accepting any group. If that -create fails, enrollment fails and no range for the track can enter the -timeline. On `AlreadyExists`, validate the immutable `.info` and accept only matching -parsed `version`, `priority`, and `timescale` values; a priority or timescale mismatch fails enrollment. - -Feed the existing segmenter (`rs/moq-mux/src/timeline.rs`): take -`Producer::deferred` (:917), enroll each registration through -`Deferred::track` or `pacing_track` (:625-631), and report every complete -group read from the consumer through its `Recorder` (:1096). The application -calls `cut(pts)` (:635) when it knows an aligned keyframe boundary. A segment -may hold many groups per track, especially one-group-per-frame audio. A pacing -track that stops without closing blocks segment completion on purpose; the -application applies its own deadline and calls `cut(pts)` or removes the -track. Storage does not invent a timeout. - -Accept new group IDs in strictly increasing order per track; refuse duplicate -or decreasing arrivals. Already accepted groups may finish in any order, so -buffer completion independently and encode in sequence order. Require -nonoverlapping object ranges across segments, including forced cuts; never -backfill a closed segment or move its stalled groups into an overlapping range. - -For each `Pending` record from `Deferred::next` (:658), encode and PUT one -object per participating track at `groups/.`, buffering -groups independently of relay retention. The bounds come from the exact -recorded ranges; tracks with no complete groups have no object. After all PUTs settle, drop the ranges of every track whose PUT -failed and commit with `Producer::push` (:948). That needs a per-track -omission on `Pending` beside `Pending::gap()` (:712-719), which clears every -track; add it here. Never publish a range first and hope the relay still has -it. A later segment continues normally after any omission. - -Catalog update applicability and cross-track configuration dependencies belong -to [Catalog track identity](/quest/m2/catalog-tracks.md). Recording a catalog -as an ordinary track does not bind its updates to media groups; this writer -does not introduce catalog-specific commit or retention prerequisites. - -The archive timeline uses the same object envelope but is not enrolled in its -own records. After pushing segment N and applying retention pops, close the -recording-owned Window group and store the complete groups under the timeline -track's `segments/N` key (19-digit padded N). Commit IDs consecutively, -including all-gap segments. Make it durable before committing N+1; a timeline PUT -failure stops the recording at its preceding durable timeline object. Never -cut a source group. On a clean source end, drain with -`Deferred::finish` (:650), commit the final partial segment, and call -`Producer::finish` (:1015). Do not write a completion marker. - -Retention is writer policy. During the next segment commit, use -`Producer::pop` (:983) for expired records before closing that segment's -timeline group. Make it durable, wait the configured grace period, then delete -the corresponding segment objects. No trimming occurs after the final segment -is committed, and timeline objects are never rewritten. - -Before a DVR writer resumes input, recover its full retained timeline and -reconcile a complete media-key listing under exclusive prefix ownership. -Wait the deletion grace period from successful recovery, then delete unreferenced `groups/` objects -left by interrupted expiration or uploads. Failed or incomplete recovery must -prevent deletion. Preserve `.info` and timeline checkpoint objects. - -On resume, fail loud on a track that goes backward. The resume step already -refuses a group ID at or below the recovered track's last one; also refuse a -first group whose timestamp is before the recovered track's last recorded -timestamp, instead of writing overlapping media time. The caller starts a new -prefix. Test a backward and a forward restart. Decided in the 2026-09-30 -audit: folded in from a separate quest, and the archive format may break in -place since it is unreleased. - -Keep archive policy out of protocol libraries. As the native application that -owns its storage and track choices, `moq-cli` attaches the writer to every -import path and enrolls the resulting `broadcast::Consumer` tracks. Downstream -(moq.pro) gateways attach the same writer once it ships in a release. - -## Closes - -- [#2281](https://github.com/moq-dev/moq/issues/2281) - close this issue when the quest finishes diff --git a/quest/m1/hls-bounded.md b/quest/m1/hls-bounded.md index 3cfb9de98bf..bf3bbb187b0 100644 --- a/quest/m1/hls-bounded.md +++ b/quest/m1/hls-bounded.md @@ -27,8 +27,8 @@ timelines need it first. The offline archive HLS that `quest/m1/archive/hls.md` used to describe is done on the archive line's branch (#4115, #4155, #4169). The per-track gap -behaviour lives on that line's track-timeline questline -(`quest/m1/archive/track-timeline/README.md`, #4280, `spans.rs`), which +behaviour lives on that line's +[Per-track timelines](/quest/m1/archive/track-timeline.md) (#4280, `spans.rs`), which derives segments at the edge from group timestamps and emits `EXT-X-GAP` for a rendition with no group start in a span. Whichever lands second rebases onto the other, and the rules here hold for both shapes. diff --git a/quest/m1/hls-first-catalog.md b/quest/m1/hls-first-catalog.md index b2034dfac6a..4d7d3b55840 100644 --- a/quest/m1/hls-first-catalog.md +++ b/quest/m1/hls-first-catalog.md @@ -22,11 +22,8 @@ is reserved. Guidance: -- `step` already holds a pass-wide timeline reservation for the same reason - ("a record flushed mid-pass would omit every rendition that hasn't loaded - its init segment yet"). A catalog `Reserved` held the same way, at least - across the first pass, is the likely shape: taken after `ensure_tracks` - and before the first `ingest`, beside the timeline reservation. +- A catalog `Reserved` held across at least the first pass is the likely + shape: taken after `ensure_tracks` and before the first `ingest`. - Don't let a hold outlive its pass. A rendition with no segments yet never reaches `ensure_map`, and one whose init fetch fails under `OnError::Warn` never reserves. Neither may withhold the catalog for the whole import, and a diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 5d9095d498d..16de2758ed0 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -33,8 +33,3 @@ Public API: none. Wire: none. - [Subscription cut by disconnect](/quest/m1/test-flakes-2/subscription-cut.md) - a publisher disconnect never ends a subscription clean - [Media late join](/quest/m1/test-flakes-2/media-late-join.md) - `just test media` late join stays within one GOP, or the regression is fixed - [Rejoin after idle](/quest/m1/test-flakes-2/rejoin-idle-race.md) - an IETF rejoin never gets the stale warm cache first, because the copy goes idle before the cancel - -## Related - -- [Archive enrollment](/quest/m1/archive/enrollment-flake.md) - the same - kind of flake on the archive line, where its test lives diff --git a/quest/m2/README.md b/quest/m2/README.md index d71aba5b456..4bcdf46cf9f 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -63,6 +63,7 @@ waiting on an upstream release waits in [m4](/quest/m4/README.md). - [AV1 metadata separation](/quest/m2/av1-metadata.md) - retain metadata OBUs inline while evaluating separate delivery - [Catalog track identity](/quest/m2/catalog-tracks.md) - a changed track configuration becomes a new track name or epoch, never a mutated definition - [Catalog colour model](/quest/m2/color-catalog.md) - the catalog describes a rendition's colour and HDR properties once a renderer consumes them +- [Archive S3 wire proof](/quest/m2/archive-s3.md) - the archive proof also runs through the S3 client against an in-process S3-compatible server - [Archive recovery listing](/quest/m2/archive-recovery-listing.md) - a resumed DVR lists what changed since its checkpoint, not every stored group - [Relay io_uring packages](/quest/m2/relay-io-uring-package.md) - Linux relay packages ship io_uring once the ring is on par with tokio - [iOS capture](/quest/m2/mobile-capture-ios.md) - Rust captures the camera and screen on iOS diff --git a/quest/m2/archive-s3.md b/quest/m2/archive-s3.md new file mode 100644 index 00000000000..d190490d8c4 --- /dev/null +++ b/quest/m2/archive-s3.md @@ -0,0 +1,22 @@ +# [S] Archive S3 wire proof + +## Goal + +The `moq-archive` proof (`rs/moq-archive/src/proof.rs`) also runs through +`object_store`'s S3 client against an in-process S3-compatible server, in CI +with no external network. + +## Plan + +The proof covers memory, local disk, and an S3-style listing fake +(`rs/moq-archive/src/mock.rs`), but not the S3 client: percent-encoded keys +such as `catalog%2Ejson` inside request URLs, `PutMode::Create` through +`If-None-Match`, `start-after` listing, and continuation tokens. Evaluate a +maintained in-process server (for example `s3s` with `s3s-fs`) on loopback; +refuse one that ignores conditional creates, since collisions would pass +silently. Gate it behind a dev-dependency feature if the build cost is large, +and wire it into at least the nightly workflow. + +## Related + +- [Timeline-indexed MoQ archives](/quest/m1/archive/README.md) diff --git a/quest/m3/archive-browser.md b/quest/m3/archive-browser.md index 97b34cd8773..1f909533d3d 100644 --- a/quest/m3/archive-browser.md +++ b/quest/m3/archive-browser.md @@ -9,16 +9,15 @@ layout, and FETCH behavior as native `moq-archive` users. Port the archive contract to the JS packages with memory and OPFS storage. The application explicitly enrolls video, audio, catalog, or arbitrary data tracks; -the archive does not infer them from Hang. The JS timeline already publishes -through the Window (`js/hang/src/timeline.ts`, its `Consumer` and `Producer`, backed by -`js/json/src/window/`) with pacing tracks and application-driven cuts; add the -deferred commit Rust has (`Producer::deferred`, -`rs/moq-mux/src/timeline.rs:917`). - -Persist one range-named object per track per segment after its groups complete, -then publish the archive timeline record. Match the 19-digit group-bound keys, -ascending delta-encoded IDs, and sequential timeline discovery used by Rust. A typical audio segment contains many -one-group-per-frame audio groups. Match the Rust binary envelope bytes and `.info` property values, per the [Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without inferring catalog-to-group applicability. +the archive does not infer them from Hang. Record against the per-track +timelines from [JS per-track timelines](/quest/m1/archive/js-timelines.md). + +Commit each track independently: persist each stored span, including a frame +range of a still-open group, then publish that track's timeline record. Match +the object keys, envelope bytes, `.info` property values, and timeline +discovery the Rust writer uses, per the +[Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without +inferring catalog-to-group applicability. [Catalog track identity](/quest/m2/catalog-tracks.md) addresses that separately. Use [JS ranges](/quest/m1/subscribe-ranges/js.md)'s on-demand range requests to @@ -38,4 +37,5 @@ lands first. - [JS ranges](/quest/m1/subscribe-ranges/js.md) - generic on-demand range serving in `@moq/net` - [JavaScript FETCH](/quest/m1/js-fetch.md) - IETF FETCH dispatch onto it +- [JS per-track timelines](/quest/m1/archive/js-timelines.md) - the timeline this archive records against - [Archive](/quest/m1/archive/README.md) - the native writer, reader, and DVR contract this ports diff --git a/quest/m3/archive-paced-replay.md b/quest/m3/archive-paced-replay.md index 2729d00580b..5f30e4c8d59 100644 --- a/quest/m3/archive-paced-replay.md +++ b/quest/m3/archive-paced-replay.md @@ -27,7 +27,3 @@ who joins late joins mid-replay. current group, and that two tracks stay aligned. - Deferred in the 2026-09-30 audit and moved to m3 in the 2026-10-05 audit: no named consumer; FETCH replay already serves DVR and HLS. - -## Required - -- [Recording reader](/quest/m1/archive/reader.md) - the FETCH reader this adds live publishing to diff --git a/rs/hang/fixtures/catalog-clock.json b/rs/hang/fixtures/catalog-clock.json index 1db86f67f8f..2a63ca1dbd3 100644 --- a/rs/hang/fixtures/catalog-clock.json +++ b/rs/hang/fixtures/catalog-clock.json @@ -20,5 +20,9 @@ } }, "clock": { "wall": 1751846400000000, "timescale": 1000000 }, - "archive": { "track": "timeline.z", "timescale": 1000, "durationMax": 2000 } + "archive": { + "timelines": { "video": "video.timeline.z", "audio": "audio.timeline.z" }, + "timescale": 1000, + "durationMax": 2000 + } } diff --git a/rs/hang/src/catalog/archive.rs b/rs/hang/src/catalog/archive.rs index 2cf3a5e8e23..9c7199761a5 100644 --- a/rs/hang/src/catalog/archive.rs +++ b/rs/hang/src/catalog/archive.rs @@ -1,29 +1,31 @@ +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; /// The moq epoch (2020-01-01T00:00:00Z) in Unix-epoch milliseconds. pub const MOQ_EPOCH_UNIX_MILLIS: u64 = 1_577_836_800_000; -/// Discovers the broadcast's segment index and any durable archive. +/// Discovers each track's timeline and any durable archive. /// -/// A live publisher sets the timeline fields (`track`, `timescale`, `duration_max`) alone, and a -/// recording also names the replay broadcast and object store those ranges live under. Every -/// advertised range is FETCHable; with a store they are durable. +/// A live publisher sets the timeline fields (`timelines`, `timescale`, `duration_max`) alone, +/// and a recording also names the replay broadcast and object store those spans live under. +/// Every advertised span is FETCHable; with a store they are durable. #[serde_with::skip_serializing_none] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] #[non_exhaustive] pub struct Archive { - /// The MoQ track carrying the broadcast's segment records. - pub track: String, + /// Each indexed track's timeline track, keyed by the indexed track's name. + pub timelines: BTreeMap, - /// Units per second for the timeline's timestamps. Defaults to milliseconds. + /// Units per second for every timeline's timestamps. Defaults to milliseconds. #[serde( default = "Archive::default_timescale", deserialize_with = "super::deserialize_timescale_or_default" )] pub timescale: u32, - /// The declared upper bound on a segment's duration, in [`timescale`](Self::timescale) units. + /// The declared upper bound on a record's duration, in [`timescale`](Self::timescale) units. pub duration_max: Option, /// The MoQ broadcast the archive is served back from, relative to this catalog, if any. @@ -41,17 +43,17 @@ pub struct Archive { impl Archive { /// The recording object format advertised in [`version`](Self::version). - pub const VERSION: u32 = 1; + pub const VERSION: u32 = 2; /// The default timeline timescale, milliseconds. pub const fn default_timescale() -> u32 { 1000 } - /// An archive naming `track` as its timeline, with no duration bound or durable storage. - pub fn new(track: impl Into) -> Self { + /// An archive with no timelines, duration bound, or durable storage. + pub fn new() -> Self { Self { - track: track.into(), + timelines: BTreeMap::new(), timescale: Self::default_timescale(), duration_max: None, replay: None, @@ -61,23 +63,34 @@ impl Archive { } } +impl Default for Archive { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod test { use super::*; + fn live() -> Archive { + let mut archive = Archive::new(); + archive + .timelines + .insert("video".to_string(), "video.timeline.z".to_string()); + archive + } + #[test] fn live_publisher_is_timeline_fields_alone() { - let json = serde_json::to_string(&Archive::new("timeline.z")).unwrap(); - assert_eq!(json, r#"{"track":"timeline.z","timescale":1000}"#); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - Archive::new("timeline.z") - ); + let json = serde_json::to_string(&live()).unwrap(); + assert_eq!(json, r#"{"timelines":{"video":"video.timeline.z"},"timescale":1000}"#); + assert_eq!(serde_json::from_str::(&json).unwrap(), live()); } #[test] fn recording_roundtrips_replay_store_and_version() { - let mut archive = Archive::new("timeline.z"); + let mut archive = live(); archive.replay = Some(moq_net::path::RelativeOwned::new("./recordings/clip")); archive.store = Some("https://objects.example/rec/".parse().unwrap()); archive.version = Some(Archive::VERSION); @@ -85,20 +98,25 @@ mod test { let json = serde_json::to_string(&archive).unwrap(); assert_eq!( json, - r#"{"track":"timeline.z","timescale":1000,"replay":"recordings/clip","store":"https://objects.example/rec/","version":1}"# + r#"{"timelines":{"video":"video.timeline.z"},"timescale":1000,"replay":"recordings/clip","store":"https://objects.example/rec/","version":2}"# ); assert_eq!(serde_json::from_str::(&json).unwrap(), archive); } #[test] fn zero_timescale_is_refused() { - serde_json::from_str::(r#"{"track":"timeline.z","timescale":0}"#) + serde_json::from_str::(r#"{"timelines":{},"timescale":0}"#) .expect_err("a zero timescale must not decode"); } #[test] fn invalid_store_url_is_refused() { - serde_json::from_str::(r#"{"track":"timeline.z","store":"not a url"}"#) + serde_json::from_str::(r#"{"timelines":{},"store":"not a url"}"#) .expect_err("an invalid store URL must not decode"); } + + #[test] + fn a_single_shared_timeline_is_refused() { + serde_json::from_str::(r#"{"track":"timeline.z"}"#).expect_err("the old shape names no timelines"); + } } diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 540b928748d..85ed58b84da 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; /// A catalog track, created by a broadcaster to describe the tracks available in a broadcast. /// /// The base catalog carries the media sections (`video`, `audio`, `text`), the optional -/// `archive` (the segment index and any durable recording), the optional `clock` (the +/// `archive` (each track's timeline and any durable recording), the optional `clock` (the /// broadcast's one wall-clock mapping), and the data sections /// (`json`, `binary`) for application tracks that aren't media. /// Applications extend it with their own root sections (e.g. `scte35`) through `E`. The catalog @@ -42,14 +42,14 @@ pub struct Catalog { #[serde(default)] pub audio: Audio, - /// The broadcast's segment index and any durable archive, if the publisher offers one. + /// Each track's timeline and any durable archive, if the publisher offers them. /// See [`Archive`](crate::catalog::Archive) and the [`timeline`](crate::timeline) module. pub archive: Option, /// The broadcast's one continuous clock, if the publisher exposes one. /// /// `wall` is the wall-clock time of PTS zero in `timescale` units since the moq epoch - /// (2020-01-01); every media track and the archive index refer to this mapping after + /// (2020-01-01); every media track and every timeline refer to this mapping after /// timescale conversion. Independent of [`archive`](Self::archive): a live-only publisher /// exposes its clock without creating a segment index. See /// [`Clock`](crate::catalog::Clock). @@ -685,7 +685,7 @@ mod test { assert_eq!(clock.wall.scale().as_u64(), 1_000_000); let archive = catalog.archive.expect("the fixture carries an archive"); - assert_eq!(archive.track, "timeline.z"); + assert_eq!(archive.timelines["video"], "video.timeline.z"); assert_eq!(archive.timescale, 1000); // Archive PTS 2000 (ms) lands 2s after the wall epoch, whatever the track timescale. @@ -701,7 +701,10 @@ mod test { #[test] fn archive_roundtrips_at_the_root() { - let mut archive = crate::catalog::Archive::new("timeline.z"); + let mut archive = crate::catalog::Archive::new(); + archive + .timelines + .insert("video".to_string(), "video.timeline.z".to_string()); archive.duration_max = Some(2000); archive.replay = Some(moq_net::path::RelativeOwned::new("recordings/clip")); archive.version = Some(crate::catalog::Archive::VERSION); @@ -712,7 +715,10 @@ mod test { }; let json = catalog.to_json().expect("failed to encode"); - assert!(json.contains(r#""archive":{"track":"timeline.z""#), "{json}"); + assert!( + json.contains(r#""archive":{"timelines":{"video":"video.timeline.z"}"#), + "{json}" + ); assert!( !json.contains(r#""timeline":"#), "the old root key must not appear: {json}" diff --git a/rs/hang/src/lib.rs b/rs/hang/src/lib.rs index f4705d0aa46..cfe65b273ed 100644 --- a/rs/hang/src/lib.rs +++ b/rs/hang/src/lib.rs @@ -22,7 +22,7 @@ pub mod catalog; /// The container is the contents of each media track. pub mod container; -/// The timeline indexes the broadcast's aligned segments. +/// Per-track timelines index each track's spans. pub mod timeline; /// Export the moq-net version we use. diff --git a/rs/hang/src/timeline.rs b/rs/hang/src/timeline.rs index 757cfd255d2..71e1cc750b7 100644 --- a/rs/hang/src/timeline.rs +++ b/rs/hang/src/timeline.rs @@ -1,47 +1,47 @@ -//! The timeline track: the broadcast's segment index, one record per aligned segment. +//! Per-track timelines: each track's index of stored spans, one record per span. //! //! MoQ groups carry only an opaque sequence number; the timestamps live inside the media -//! frames. The timeline track republishes the broadcast's segmentation as metadata: each -//! record describes one *segment*, a span of content time shared by every media track, and -//! maps it to the group ranges that carry it on each track. A consumer can answer "which -//! groups cover time T on track X" (and "where is the live edge") from a few bytes per -//! segment without downloading media. That is exactly what an HLS/DASH origin needs to render -//! playlists without touching media bytes, and the index a VOD player seeks with. +//! frames. A timeline track republishes one track's spans as metadata: each record maps a span +//! of content time to the group and frame positions that carry it. A consumer can answer +//! "which groups cover time T on track X" (and "where is the live edge") from a few bytes per +//! span without downloading media. That is what an HLS/DASH origin needs to render playlists +//! without touching media bytes, the index a VOD player seeks with, and the object index of a +//! recording. //! -//! ## Segments +//! ## Spans //! -//! A segment covers the same span of content time on every track of the broadcast, which is -//! what HLS requires of switchable renditions. Segment boundaries land on video keyframes -//! (every group already opens on one), so a video track contributes whole groups to each -//! segment; an audio track packs however many of its shorter groups fall inside the span. -//! A record is published once the segment is *complete*: every participating track's groups -//! for the span are known, so the record is self-contained and immediately servable. +//! Every track has its own timeline, so tracks cut, commit, and expire independently. A span +//! usually holds whole groups; a group that outlives the publisher's maximum span is split by +//! frame, so an append-only group that never closes is still indexed as its frames arrive. +//! Spans are contiguous in position: a record starts where the previous one ended unless the +//! track skipped group sequences in between. //! -//! There is one timeline per broadcast, advertised by the catalog's root -//! [`Archive`](crate::catalog::Archive) entry (its `track` field names this track, -//! [`DEFAULT_NAME`](crate::timeline::DEFAULT_NAME) by convention). A broadcast that doesn't need aligned segments simply -//! doesn't publish one. +//! The catalog's root [`Archive`](crate::catalog::Archive) entry maps each track to its +//! timeline track, conventionally the track name plus [`SUFFIX`](crate::timeline::SUFFIX). //! -//! On the wire the track is a DEFLATE-compressed `moq-json` window (see `moq_json::window`). +//! On the wire each timeline is a DEFLATE-compressed `moq-json` window (see `moq_json::window`). //! Each group starts with a checkpoint and continues with push/pop operations; group rolls are an //! encoding detail that consumers do not surface as duplicate records. Like the catalog, a record //! tolerates and preserves unknown fields: extend it by flattening a //! [`Record`](crate::timeline::Record) into your own struct via //! [`RecordExt`](crate::timeline::RecordExt). -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use crate::Result; -/// The conventional name for a broadcast's timeline track (the `.z` marks the -/// DEFLATE-compressed stream, like the catalog's `.json.z` sibling). +/// The conventional suffix appended to a track name to name its timeline track (the `.z` marks +/// the DEFLATE-compressed stream, like the catalog's `.json.z` sibling). /// -/// A publisher records the actual name in the catalog's root -/// [`Archive`](crate::catalog::Archive) entry (`timeline.track`); a consumer reads it from the -/// catalog rather than assuming, so this is only a default. -pub const DEFAULT_NAME: &str = "timeline.z"; +/// A publisher records the actual names in the catalog's root +/// [`Archive`](crate::catalog::Archive) entry; a consumer reads them from the catalog rather than +/// assuming, so this is only a default. +pub const SUFFIX: &str = ".timeline.z"; + +/// The conventional timeline track name for `track`: the name plus [`SUFFIX`]. +pub fn default_name(track: &str) -> String { + format!("{track}{SUFFIX}") +} /// The application extension carried alongside a record's base fields. /// @@ -51,74 +51,69 @@ pub const DEFAULT_NAME: &str = "timeline.z"; pub trait RecordExt: serde::Serialize + serde::de::DeserializeOwned + Default + Clone + Send + Unpin + 'static {} impl RecordExt for () {} -/// A contiguous run of groups a track contributes to a segment, `start` through `end` -/// inclusive. +/// A frame position within a track: frame `frame` of group `group`. /// -/// A track's segment entry is an array of these: more than one range means the group -/// sequence is discontinuous inside the segment (groups that never existed, e.g. a gappy -/// source). -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct Range { - /// The first group of the run, as used by FETCH/SUBSCRIBE on the media track. - pub start: u64, - - /// The last group of the run, inclusive. - pub end: u64, - - /// Whether the run's first group starts with a keyframe, i.e. whether a player can join or - /// switch renditions at this range. Defaults to `true` (and is omitted from the JSON when - /// true); a publisher sets `false` when a source resumes without one, so an exporter knows - /// not to advertise the segment as independently decodable. - #[serde(default = "Range::default_keyframe", skip_serializing_if = "Clone::clone")] - pub keyframe: bool, +/// Positions order by group, then frame. `frame` is omitted from the JSON when zero, so a +/// position at a group start reads as just the group. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct Position { + /// The group sequence, as used by FETCH/SUBSCRIBE on the track. + pub group: u64, + + /// The frame index within the group. + #[serde(default, skip_serializing_if = "is_zero")] + pub frame: u64, +} + +fn is_zero(v: &u64) -> bool { + *v == 0 } -impl Range { - fn default_keyframe() -> bool { - true +impl Position { + /// Frame `frame` of group `group`. + pub const fn new(group: u64, frame: u64) -> Self { + Self { group, frame } } - /// A range covering groups `start` through `end` inclusive, starting on a keyframe. - pub fn new(start: u64, end: u64) -> Self { - Self { - start, - end, - keyframe: true, - } + /// The start of group `group`. + pub const fn group(group: u64) -> Self { + Self { group, frame: 0 } } } -/// One timeline record: a complete aligned segment, mapping its span of content time to the -/// group ranges that carry it on each track. +/// One timeline record: a span of one track, mapping its content time to frame positions. /// -/// Records are self-contained: `pts` and `duration` bound the span (no peeking at the next -/// record), and `tracks` names every participating media track's group ranges. A track absent -/// from `tracks` has no content for the span (a gap; HLS `EXT-X-GAP`). `pts`/`duration` are in -/// the timescale declared by the catalog's [`Archive`](crate::catalog::Archive) entry -/// (default milliseconds). Extend with a typed [`RecordExt`]. +/// The span holds every frame from [`start`](Self::start) (inclusive) to [`end`](Self::end) +/// (exclusive). An `end` at frame zero of group `g` means the span holds every frame of group +/// `g - 1` and nothing of `g`, whether or not `g` exists. `pts`/`duration` are in the timescale +/// declared by the catalog's [`Archive`](crate::catalog::Archive) entry (default milliseconds). +/// Extend with a typed [`RecordExt`]. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde( rename_all = "camelCase", bound(serialize = "E: serde::Serialize", deserialize = "E: serde::de::DeserializeOwned") )] pub struct Record { - /// The segment's number. Consecutive within a broadcast, so it anchors HLS - /// `EXT-X-MEDIA-SEQUENCE`; explicit (rather than implied by record order) so a reader - /// joining mid-stream or across a windowed recording keeps stable numbering. - pub segment: u64, + /// The record's number, consecutive within its track's timeline and equal to its window index. + pub sequence: u64, - /// The segment's start, in the timeline's timescale. + /// The span's start, the timestamp of its first frame, in the timeline's timescale. pub pts: u64, - /// The segment's duration, in the timeline's timescale. The next record's `pts` is + /// The span's duration, in the timeline's timescale. The next record's `pts` is /// `pts + duration` unless content time itself jumped (a discontinuity). pub duration: u64, - /// The group ranges each participating media track contributes to this segment, keyed by - /// track name. A track absent from the map has no content in this span. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub tracks: BTreeMap>, + /// The first frame of the span, inclusive. + pub start: Position, + + /// The end of the span, exclusive. + pub end: Position, + + /// Whether the span's first frame is a keyframe, i.e. whether a player can join or switch + /// renditions at this record. Defaults to `true` (and is omitted from the JSON when true). + #[serde(default = "default_keyframe", skip_serializing_if = "Clone::clone")] + pub keyframe: bool, /// The application extension, flattened into the record's JSON object (nothing for the /// default `()`). See [`RecordExt`]. @@ -126,19 +121,33 @@ pub struct Record { pub ext: E, } +fn default_keyframe() -> bool { + true +} + impl Record { - /// A record with no tracks and the default (empty) extension; fill in - /// [`tracks`](Self::tracks) afterward. - pub fn new(segment: u64, pts: u64, duration: u64) -> Self { + /// A keyframe record spanning `start..end` with the default (empty) extension. + pub fn new(sequence: u64, pts: u64, duration: u64, start: Position, end: Position) -> Self { Self { - segment, + sequence, pts, duration, - tracks: BTreeMap::new(), + start, + end, + keyframe: true, ext: E::default(), } } + /// The groups holding at least one frame of the span, in order. + pub fn groups(&self) -> std::ops::RangeInclusive { + let last = match self.end.frame { + 0 => self.end.group.saturating_sub(1), + _ => self.end.group, + }; + self.start.group..=last + } + /// Parse a record from a slice of bytes. pub fn from_slice(v: &[u8]) -> Result { Ok(serde_json::from_slice(v)?) @@ -156,54 +165,38 @@ mod test { #[test] fn roundtrip() { - let mut record = Record::<()>::new(3, 84_000, 6_000); - record.tracks.insert("video".to_string(), vec![Range::new(42, 42)]); - record - .tracks - .insert("audio".to_string(), vec![Range::new(512, 540), Range::new(550, 560)]); - + let record = Record::<()>::new(3, 84_000, 6_000, Position::group(42), Position::group(44)); let json = record.to_vec().unwrap(); assert_eq!( std::str::from_utf8(&json).unwrap(), - concat!( - r#"{"segment":3,"pts":84000,"duration":6000,"tracks":{"#, - r#""audio":[{"start":512,"end":540},{"start":550,"end":560}],"#, - r#""video":[{"start":42,"end":42}]}}"# - ) + r#"{"sequence":3,"pts":84000,"duration":6000,"start":{"group":42},"end":{"group":44}}"# ); assert_eq!(Record::<()>::from_slice(&json).unwrap(), record); + assert_eq!(record.groups(), 42..=43); } #[test] - fn keyframe_false_roundtrips_and_true_is_omitted() { - let mut range = Range::new(7, 9); - range.keyframe = false; - let mut record = Record::<()>::new(0, 0, 2_000); - record.tracks.insert("video".to_string(), vec![range]); - + fn a_frame_split_roundtrips() { + let mut record = Record::<()>::new(0, 0, 10_000, Position::new(7, 300), Position::new(7, 600)); + record.keyframe = false; let json = record.to_vec().unwrap(); assert_eq!( std::str::from_utf8(&json).unwrap(), - r#"{"segment":0,"pts":0,"duration":2000,"tracks":{"video":[{"start":7,"end":9,"keyframe":false}]}}"# + r#"{"sequence":0,"pts":0,"duration":10000,"start":{"group":7,"frame":300},"end":{"group":7,"frame":600},"keyframe":false}"# ); + assert_eq!(Record::<()>::from_slice(&json).unwrap(), record); + assert_eq!(record.groups(), 7..=7); - let decoded = Record::<()>::from_slice(&json).unwrap(); - assert_eq!(decoded, record); // An omitted flag decodes as the keyframe default. let plain: Record = - Record::from_slice(br#"{"segment":0,"pts":0,"duration":1,"tracks":{"v":[{"start":1,"end":1}]}}"#).unwrap(); - assert!(plain.tracks["v"][0].keyframe); + Record::from_slice(br#"{"sequence":0,"pts":0,"duration":1,"start":{"group":1},"end":{"group":2}}"#) + .unwrap(); + assert!(plain.keyframe); } #[test] - fn empty_tracks_is_omitted() { - let record = Record::<()>::new(1, 1_000, 500); - let json = record.to_vec().unwrap(); - assert_eq!( - std::str::from_utf8(&json).unwrap(), - r#"{"segment":1,"pts":1000,"duration":500}"# - ); - assert_eq!(Record::<()>::from_slice(&json).unwrap(), record); + fn default_name_appends_the_suffix() { + assert_eq!(default_name("video0"), "video0.timeline.z"); } #[test] @@ -216,17 +209,12 @@ mod test { } impl RecordExt for Ext {} - let record = Record { - segment: 2, - pts: 14_000, - duration: 2_000, - tracks: BTreeMap::new(), - ext: Ext { discontinuity: true }, - }; + let mut record = Record::::new(2, 14_000, 2_000, Position::group(1), Position::group(2)); + record.ext.discontinuity = true; let json = record.to_vec().unwrap(); assert_eq!( std::str::from_utf8(&json).unwrap(), - r#"{"segment":2,"pts":14000,"duration":2000,"discontinuity":true}"# + r#"{"sequence":2,"pts":14000,"duration":2000,"start":{"group":1},"end":{"group":2},"discontinuity":true}"# ); assert_eq!(Record::::from_slice(&json).unwrap(), record); } diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index ae6a0403f7e..c9a1da6940b 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -15,15 +15,27 @@ categories = ["multimedia", "filesystem"] [dependencies] bytes = { workspace = true } futures = { workspace = true } +hang = { workspace = true } +kio = { workspace = true } +moq-json = { workspace = true } +moq-mux = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } +quick_cache = "0.7" serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "sync", "time"] } +tracing = { workspace = true } [dev-dependencies] async-trait = "0.1" +criterion = { workspace = true } object_store = { workspace = true, features = ["fs"] } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt", "test-util"] } + +[[bench]] +name = "writer" +harness = false diff --git a/rs/moq-archive/benches/writer.rs b/rs/moq-archive/benches/writer.rs new file mode 100644 index 00000000000..8df214a4848 --- /dev/null +++ b/rs/moq-archive/benches/writer.rs @@ -0,0 +1,78 @@ +//! `Writer::run` recording every enrolled track into memory, swept over the number of tracks and +//! the groups each one publishes. Throughput is per frame, so a per-event cost that grows with the +//! track table shows up as a slope across the track counts. +//! +//! Run with `cargo bench -p moq-archive --bench writer`. + +use std::time::{Duration, Instant}; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::{Store, Writer, writer}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; + +const FRAMES_PER_GROUP: u64 = 2; + +/// A writer with `tracks` enrolled tracks, each already holding `groups` one-second groups. +async fn setup(tracks: usize, groups: u64) -> Writer { + let source = broadcast::Info::new().produce(); + let store = Store::new(InMemory::new(), "bench"); + let writer = Writer::new(store, source.consume(), writer::Config::default()) + .await + .unwrap(); + let control = writer.control(); + let config = moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)); + + for index in 0..tracks { + let name = format!("track{index}"); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let track = source.create_track(name.as_str(), info).unwrap(); + control.track(&name, config.clone()).await.unwrap(); + for sequence in 0..groups { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for frame in 0..FRAMES_PER_GROUP { + let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); + group.write_frame(pts, vec![0u8; 64]).unwrap(); + } + group.finish().unwrap(); + } + track.finish().unwrap(); + } + source.close(); + writer +} + +fn writer(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + let mut bench = c.benchmark_group("writer"); + bench.sample_size(10); + bench.warm_up_time(Duration::from_secs(1)); + bench.measurement_time(Duration::from_secs(2)); + for tracks in [1, 10, 100] { + for groups in [10, 100] { + bench.throughput(Throughput::Elements(tracks as u64 * groups * FRAMES_PER_GROUP)); + bench.bench_function(BenchmarkId::new(format!("{groups} groups"), tracks), |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let writer = runtime.block_on(setup(tracks, groups)); + let start = Instant::now(); + runtime.block_on(writer.run()).unwrap(); + total += start.elapsed(); + } + total + }); + }); + } + } + bench.finish(); +} + +criterion_group!(benches, writer); +criterion_main!(benches); diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index ad372a32096..0b7b396ba99 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -10,7 +10,7 @@ pub enum Error { #[error("not found: {0}")] NotFound(String), - /// The recording format version is not 1. + /// The recording format version is not the current one. #[error("unknown version {0}")] Version(u64), @@ -38,9 +38,9 @@ pub enum Error { #[error("group sequences are not strictly ascending")] Sequence, - /// A group range is empty, reversed, or does not match an object's table. - #[error("invalid or mismatched group bounds {smallest}..={largest}")] - Bounds { smallest: u64, largest: u64 }, + /// An object's table does not hold exactly the frames its record names. + #[error("object does not match its record")] + Span, /// The binary table is truncated, overlapping, gapped, or out of range. #[error("malformed table")] @@ -73,6 +73,26 @@ pub enum Error { /// `.info` JSON is malformed. #[error("json: {0}")] Json(String), + + /// Publishing the replayed archive failed. + #[error("moq: {0}")] + Moq(String), + + /// The track was already enrolled, or is the recording's own timeline. + #[error("track already enrolled: {0}")] + Enrolled(String), + + /// The source broadcast or one of its tracks failed. + #[error("source: {0}")] + Source(String), + + /// The recording's timeline could not be recovered, segmented, or published. + #[error("timeline: {0}")] + Timeline(String), + + /// The writer stopped accepting commands. + #[error("writer closed")] + Closed, } impl From for Error { @@ -84,6 +104,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: moq_net::Error) -> Self { + Self::Moq(err.to_string()) + } +} + impl From for Error { fn from(err: serde_json::Error) -> Self { Self::Json(err.to_string()) diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index 75e92acc20f..c17d7d7ad96 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -17,7 +17,7 @@ pub struct Info { } impl Info { - /// Format version 1 with `priority` and `timescale`. + /// The current format version with `priority` and `timescale`. pub fn new(priority: u8, timescale: u64) -> Result { let info = Self { version: VERSION, @@ -62,16 +62,16 @@ mod tests { fn roundtrip() { let info = Info::new(0, 1_000_000).unwrap(); let bytes = info.encode().unwrap(); - assert_eq!(bytes.as_ref(), br#"{"version":1,"priority":0,"timescale":1000000}"#); + assert_eq!(bytes.as_ref(), br#"{"version":2,"priority":0,"timescale":1000000}"#); assert_eq!(Info::decode(&bytes).unwrap(), info); } #[test] fn whitespace_and_member_order_do_not_affect_equality() { - let info = Info::decode(br#"{"timescale":1000,"priority":7,"version":1}"#).unwrap(); + let info = Info::decode(br#"{"timescale":1000,"priority":7,"version":2}"#).unwrap(); assert_eq!( info, - Info::decode(br#"{ "version": 1, "priority": 7, "timescale": 1000 }"#).unwrap() + Info::decode(br#"{ "version": 2, "priority": 7, "timescale": 1000 }"#).unwrap() ); assert_eq!(info.priority, 7); assert_eq!(info.timescale, 1000); @@ -79,10 +79,10 @@ mod tests { #[test] fn unknown_version_is_refused() { - assert!(matches!( - Info::decode(br#"{"version":2,"priority":0,"timescale":1000}"#), - Err(Error::Version(2)) - )); + for version in [1, 3] { + let json = format!(r#"{{"version":{version},"priority":0,"timescale":1000}}"#); + assert!(matches!(Info::decode(json.as_bytes()), Err(Error::Version(v)) if v == version)); + } } #[test] @@ -96,11 +96,11 @@ mod tests { #[test] fn unknown_members_and_missing_fields_are_refused() { assert!(matches!( - Info::decode(br#"{"version":1,"priority":0,"timescale":1,"extra":true}"#), + Info::decode(br#"{"version":2,"priority":0,"timescale":1,"extra":true}"#), Err(Error::Json(_)) )); assert!(matches!( - Info::decode(br#"{"version":1,"priority":0}"#), + Info::decode(br#"{"version":2,"priority":0}"#), Err(Error::Json(_)) )); } diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 603284461ca..d1bc87a73c9 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -1,12 +1,14 @@ //! Versioned hang recording objects on any [`object_store::ObjectStore`]. //! //! The crate owns the portable layout and codecs: percent-encoded track names, `.info` JSON, -//! the binary segment envelope, and put/get/list/delete. Callers that need runtime dispatch +//! the binary segment envelope, and put/get/list/delete. A [`Writer`] records selected tracks of +//! a broadcast into those objects, and a [`Reader`] serves a recording back through a `moq_net` +//! broadcast. Callers that need runtime dispatch //! supply `Arc`; the archive API itself stays generic. //! -//! Group bounds are finite inclusive ranges in first-to-last order: +//! Every object is keyed by its track and a record sequence: //! ``` -//! let key = moq_archive::Key::groups("video", 5..=7)?; +//! let key = moq_archive::Key::segments("video", 7)?; //! assert_eq!(key.track(), "video"); //! # Ok::<(), moq_archive::Error>(()) //! ``` @@ -15,21 +17,30 @@ pub use object_store; mod error; pub mod info; +#[cfg(test)] +mod mock; mod path; +#[cfg(test)] +mod proof; +pub mod reader; +mod recover; pub mod segment; pub mod store; +pub mod writer; pub use error::{Error, Result}; pub use info::Info; pub use path::Key; +pub use reader::Reader; pub use segment::{Frame, Group, Object}; pub use store::Store; +pub use writer::Writer; /// Recording format version written into `.info` and the binary envelope. -pub const VERSION: u64 = 1; +pub const VERSION: u64 = 2; /// Largest group, segment, or timestamp value the format allows (`2^53 - 1`). pub const ID_MAX: u64 = (1 << 53) - 1; -/// Decimal width of group and segment ID filename fields. +/// Decimal width of segment ID filename fields. pub(crate) const ID_WIDTH: usize = 19; diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs new file mode 100644 index 00000000000..76d1f43adc7 --- /dev/null +++ b/rs/moq-archive/src/mock.rs @@ -0,0 +1,347 @@ +//! A test object store: any inner store plus an operation log, injected failures, and +//! S3-style paginated listing. + +use std::sync::{Arc, Mutex}; + +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; +use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; +use object_store::memory::InMemory; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, + PutOptions, PutPayload, PutResult, +}; + +/// S3's ListObjectsV2 page limit. +const MAX_KEYS: usize = 1000; + +/// One call made against a [`Mock`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Op { + Get(String), + Put(String), + Delete(String), + /// A streaming or paginated listing, with its prefix and exclusive offset. + List { + prefix: String, + offset: Option, + }, +} + +#[derive(Debug, Default)] +struct State { + ops: Vec, + /// PUTs whose path contains any of these fail. + fail_puts: Vec, + /// GETs whose path contains any of these return Not Found. + hide_gets: Vec, + /// GETs whose path contains the pattern wait for a permit before reading. + stall_gets: Vec<(String, Arc)>, + /// Deletes whose path contains any of these fail. + fail_deletes: Vec, + /// Streaming listings end with an error after every entry. + fail_lists: bool, + /// Streaming listings yield entries in descending order, like a backend that promises none. + unordered: bool, + /// PUTs whose path contains any of these wait until [`Mock::release`]. + holds: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct Mock { + inner: Arc, + state: Arc>, + /// Wakes PUTs blocked in [`Self::hold_puts`]. + hold: Arc, +} + +impl Mock { + pub fn new(inner: impl ObjectStore) -> Self { + Self { + inner: Arc::new(inner), + state: Default::default(), + hold: Arc::new(tokio::sync::Notify::new()), + } + } + + pub fn memory() -> Self { + Self::new(InMemory::new()) + } + + /// Share this store's objects, without its log or failures. + pub fn fork(&self) -> Self { + Self { + inner: self.inner.clone(), + state: Default::default(), + hold: Arc::new(tokio::sync::Notify::new()), + } + } + + /// Every call since the last take. + pub fn take(&self) -> Vec { + std::mem::take(&mut self.state().ops) + } + + /// Paths of every GET since the last take. + pub fn gets(&self) -> Vec { + self.take() + .into_iter() + .filter_map(|op| match op { + Op::Get(path) => Some(path), + _ => None, + }) + .collect() + } + + pub fn fail_puts(&self, pattern: &str) -> &Self { + self.state().fail_puts.push(pattern.to_string()); + self + } + + pub fn hide_gets(&self, pattern: &str) -> &Self { + self.state().hide_gets.push(pattern.to_string()); + self + } + + /// Hold every GET whose path contains `pattern` until the returned semaphore gets a permit. + pub fn stall_gets(&self, pattern: &str) -> Arc { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + self.state().stall_gets.push((pattern.to_string(), gate.clone())); + gate + } + + pub fn fail_deletes(&self, pattern: &str) -> &Self { + self.state().fail_deletes.push(pattern.to_string()); + self + } + + pub fn fail_lists(&self) -> &Self { + self.state().fail_lists = true; + self + } + + pub fn unordered(&self) -> &Self { + self.state().unordered = true; + self + } + + /// Hold PUTs whose path contains `pattern` until [`release`](Self::release). + pub fn hold_puts(&self, pattern: &str) -> &Self { + self.state().holds.push(pattern.to_string()); + self + } + + /// Unblock every PUT [`hold_puts`](Self::hold_puts) is waiting on. + pub fn release(&self) { + self.state().holds.clear(); + self.hold.notify_waiters(); + } + + /// Clear every injected failure. + pub fn heal(&self) { + let mut state = self.state(); + state.fail_puts.clear(); + state.hide_gets.clear(); + state.stall_gets.clear(); + state.fail_deletes.clear(); + state.fail_lists = false; + state.holds.clear(); + drop(state); + self.hold.notify_waiters(); + } + + /// Wait out a [`hold_puts`](Self::hold_puts) gate. + /// + /// The waiter is armed before the check, so a `release` in between is not missed. + async fn wait_held(&self, location: &Path) { + loop { + let notified = self.hold.notified(); + let held = self + .state() + .holds + .iter() + .any(|pattern| location.as_ref().contains(pattern)); + if !held { + return; + } + notified.await; + } + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap() + } + + fn log(&self, op: Op) { + self.state().ops.push(op); + } + + fn listed( + &self, + prefix: Option<&Path>, + offset: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.log(Op::List { + prefix: prefix.map(ToString::to_string).unwrap_or_default(), + offset: offset.map(ToString::to_string), + }); + let listed = match offset { + Some(offset) => self.inner.list_with_offset(prefix, offset), + None => self.inner.list(prefix), + }; + let state = self.state(); + let listed = match state.unordered { + true => futures::stream::once(async move { + let mut metas: Vec = listed.try_collect().await?; + metas.sort_by(|a, b| b.location.cmp(&a.location)); + Ok::<_, object_store::Error>(futures::stream::iter(metas.into_iter().map(Ok))) + }) + .try_flatten() + .boxed(), + false => listed, + }; + match state.fail_lists { + true => listed + .chain(futures::stream::once(async { Err(unsupported("list")) })) + .boxed(), + false => listed, + } + } +} + +fn unsupported(operation: &str) -> object_store::Error { + object_store::Error::NotImplemented { + operation: operation.into(), + implementer: "Mock".into(), + } +} + +impl std::fmt::Display for Mock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Mock({})", self.inner) + } +} + +#[async_trait::async_trait] +impl ObjectStore for Mock { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.log(Op::Put(location.to_string())); + if self.state().fail_puts.iter().any(|p| location.as_ref().contains(p)) { + return Err(unsupported("put")); + } + self.wait_held(location).await; + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.log(Op::Get(location.to_string())); + if self.state().hide_gets.iter().any(|p| location.as_ref().contains(p)) { + return Err(object_store::Error::NotFound { + path: location.to_string(), + source: "hidden".into(), + }); + } + let stall = self + .state() + .stall_gets + .iter() + .find(|(pattern, _)| location.as_ref().contains(pattern)) + .map(|(_, gate)| gate.clone()); + if let Some(gate) = stall { + gate.acquire().await.expect("stall gate closed").forget(); + } + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + let state = self.state.clone(); + let locations = locations + .and_then(move |path| { + let mut state = state.lock().unwrap(); + state.ops.push(Op::Delete(path.to_string())); + let failed = state.fail_deletes.iter().any(|p| path.as_ref().contains(p)); + futures::future::ready(match failed { + true => Err(unsupported("delete")), + false => Ok(path), + }) + }) + .boxed(); + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.listed(prefix, None) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, object_store::Result> { + self.listed(prefix, Some(offset)) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// ListObjectsV2 semantics: a raw string prefix, lexical order, an exclusive `start-after` +/// offset, at most 1000 keys, and an opaque continuation token that resumes after the last key. +#[async_trait::async_trait] +impl PaginatedListStore for Mock { + async fn list_paginated( + &self, + prefix: Option<&str>, + opts: PaginatedListOptions, + ) -> object_store::Result { + if opts.delimiter.is_some() { + return Err(unsupported("delimiter")); + } + self.log(Op::List { + prefix: prefix.unwrap_or_default().to_string(), + offset: opts.offset.clone(), + }); + + let mut metas: Vec = self.inner.list(None).try_collect().await?; + metas.sort_by(|a, b| a.location.as_ref().cmp(b.location.as_ref())); + let after = opts.page_token.or(opts.offset); + metas.retain(|meta| { + let key = meta.location.as_ref(); + prefix.is_none_or(|prefix| key.starts_with(prefix)) && after.as_deref().is_none_or(|after| key > after) + }); + + let take = opts.max_keys.unwrap_or(MAX_KEYS).clamp(1, MAX_KEYS); + let page_token = (metas.len() > take).then(|| metas[take - 1].location.to_string()); + metas.truncate(take); + Ok(PaginatedListResult { + result: ListResult { + objects: metas, + common_prefixes: Vec::new(), + extensions: Default::default(), + }, + page_token, + }) + } +} diff --git a/rs/moq-archive/src/path.rs b/rs/moq-archive/src/path.rs index 86a8294e7b7..576067c3ff3 100644 --- a/rs/moq-archive/src/path.rs +++ b/rs/moq-archive/src/path.rs @@ -1,5 +1,3 @@ -use std::ops::RangeInclusive; - use object_store::path::{Path, PathPart}; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; @@ -28,7 +26,7 @@ pub(crate) fn decode_track(encoded: &str) -> Result { Ok(decoded.into_owned()) } -/// Write a group or segment ID as 19 zero-padded decimal digits. +/// Write a segment ID as 19 zero-padded decimal digits. pub(crate) fn format_id(id: u64) -> Result { check_id(id)?; Ok(format!("{id:0width$}", width = ID_WIDTH)) @@ -56,18 +54,12 @@ pub enum Key { /// The unencoded track name. track: String, }, - /// `/groups/.` - Groups { - /// The unencoded track name. - track: String, - /// Inclusive group sequences in the object, from first to last. - range: RangeInclusive, - }, - /// `/segments/` + /// `/segments/`: record `segment` of a track's timeline, or for a + /// timeline track, the window groups closed while committing that record. Segments { /// The unencoded track name. track: String, - /// The committed timeline segment ID. + /// The record's sequence. segment: u64, }, } @@ -80,15 +72,7 @@ impl Key { Ok(Self::Info { track }) } - /// A groups object named by its inclusive first-to-last sequence range. - pub fn groups(track: impl Into, range: RangeInclusive) -> Result { - let track = track.into(); - encode_track(&track)?; - check_range(&range)?; - Ok(Self::Groups { track, range }) - } - - /// A timeline object at `segments/`. + /// The object at `segments/`. pub fn segments(track: impl Into, segment: u64) -> Result { let track = track.into(); encode_track(&track)?; @@ -99,7 +83,7 @@ impl Key { /// The unencoded track name this key belongs to. pub fn track(&self) -> &str { match self { - Self::Info { track } | Self::Groups { track, .. } | Self::Segments { track, .. } => track, + Self::Info { track } | Self::Segments { track, .. } => track, } } @@ -108,10 +92,6 @@ impl Key { let path = push(prefix, &encode_track(self.track())?)?; match self { Self::Info { .. } => push(&path, ".info"), - Self::Groups { range, .. } => { - let path = push(&path, "groups")?; - push(&path, &range_name(range)?) - } Self::Segments { segment, .. } => { let path = push(&path, "segments")?; push(&path, &format_id(*segment)?) @@ -142,14 +122,6 @@ fn parse_parts<'a>(mut parts: impl Iterator>, location: &Pat } Ok(Key::Info { track }) } - "groups" => { - let name = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; - if parts.next().is_some() { - return Err(Error::Path(location.to_string())); - } - let range = parse_range(name.as_ref())?; - Ok(Key::Groups { track, range }) - } "segments" => { let name = parts.next().ok_or_else(|| Error::Path(location.to_string()))?; if parts.next().is_some() { @@ -168,21 +140,11 @@ pub(crate) fn track_prefix(track: &str) -> Result { push(&Path::ROOT, &encode_track(track)?) } -pub(crate) fn groups_prefix(prefix: &Path, track: &str) -> Result { - let path = append(prefix, &track_prefix(track)?); - push(&path, "groups") -} - pub(crate) fn segments_prefix(prefix: &Path, track: &str) -> Result { let path = append(prefix, &track_prefix(track)?); push(&path, "segments") } -pub(crate) fn groups_offset(prefix: &Path, track: &str, group: u64) -> Result { - let path = groups_prefix(prefix, track)?; - push(&path, &format_id(group)?) -} - fn append(base: &Path, relative: &Path) -> Path { let mut path = base.clone(); path.extend(relative.parts()); @@ -194,35 +156,6 @@ pub(crate) fn push(base: &Path, segment: &str) -> Result { Ok(base.clone().join(part)) } -fn range_name(range: &RangeInclusive) -> Result { - check_range(range)?; - let (smallest, largest) = (*range.start(), *range.end()); - Ok(format!("{}.{}", format_id(largest)?, format_id(smallest)?)) -} - -fn parse_range(name: &str) -> Result> { - let (largest, smallest) = name.split_once('.').ok_or_else(|| Error::Path(name.to_string()))?; - if smallest.contains('.') { - return Err(Error::Path(name.to_string())); - } - let largest = parse_id(largest)?; - let smallest = parse_id(smallest)?; - if largest < smallest { - return Err(Error::Bounds { smallest, largest }); - } - Ok(smallest..=largest) -} - -pub(crate) fn check_range(range: &RangeInclusive) -> Result<()> { - let (smallest, largest) = (*range.start(), *range.end()); - check_id(smallest)?; - check_id(largest)?; - if range.is_empty() { - return Err(Error::Bounds { smallest, largest }); - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -280,6 +213,10 @@ mod tests { parse_id("0009007199254740992"), Err(Error::Id(n)) if n == ID_MAX + 1 )); + // The largest QUIC varint still fits the 19-digit field. + let varint = (1u64 << 62) - 1; + assert!(matches!(format_id(varint), Err(Error::Id(n)) if n == varint)); + assert!(matches!(parse_id("4611686018427387903"), Err(Error::Id(n)) if n == varint)); assert!(matches!(parse_id("5"), Err(Error::Path(_)))); assert!(matches!(parse_id("000000000000000000X"), Err(Error::Path(_)))); } @@ -289,7 +226,7 @@ mod tests { let prefix = Path::from("rec/1"); for key in [ Key::info("catalog.json").unwrap(), - Key::groups("video", 5..=10).unwrap(), + Key::segments("video", 5).unwrap(), Key::segments("timeline.z", 0).unwrap(), Key::segments("timeline.z", ID_MAX).unwrap(), ] { @@ -305,60 +242,46 @@ mod tests { } #[test] - fn inverted_range_is_rejected() { - let (smallest, largest) = (2, 1); - assert!(matches!( - Key::groups("v", smallest..=largest), - Err(Error::Bounds { - smallest: 2, - largest: 1 - }) - )); - } - - #[test] - fn exhausted_range_is_rejected() { - let mut range = 1..=1; - assert_eq!(range.next(), Some(1)); - assert!(matches!(Key::groups("v", range), Err(Error::Bounds { .. }))); - } + fn direct_construction_is_validated_when_serialized() { + let prefix = Path::from("rec"); + let empty = Key::Info { track: String::new() }; + assert!(matches!(empty.path(&prefix), Err(Error::Track))); + let segment = Key::Segments { + track: "t".to_string(), + segment: ID_MAX + 1, + }; + assert!(matches!(segment.path(&prefix), Err(Error::Id(_)))); - #[test] - fn range_id_endpoints_are_valid() { - let key = Key::groups("v", 0..=ID_MAX).unwrap(); - assert_eq!( - key.path(&Path::from("rec")).unwrap().as_ref(), - "rec/v/groups/0009007199254740991.0000000000000000000" - ); - assert!(matches!(Key::groups("v", 0..=ID_MAX + 1), Err(Error::Id(_)))); - assert!(matches!(Key::groups("v", ID_MAX + 1..=ID_MAX + 1), Err(Error::Id(_)))); + // Everything that does serialize parses back to the same key. + for key in [ + Key::Info { + track: "a/b".to_string(), + }, + Key::Segments { + track: ".x".to_string(), + segment: ID_MAX, + }, + Key::Segments { + track: "é".to_string(), + segment: 0, + }, + ] { + let path = key.path(&prefix).unwrap(); + assert_eq!(Key::parse(&prefix, &path).unwrap(), key); + } } #[test] - fn direct_inverted_range_is_rejected_when_serialized() { - let (smallest, largest) = (2, 1); - let key = Key::Groups { - track: "v".to_string(), - range: smallest..=largest, - }; - assert!(matches!( - key.path(&Path::from("rec")), - Err(Error::Bounds { - smallest: 2, - largest: 1 - }) - )); - - let key = Key::Groups { - track: "v".to_string(), - range: 0..=ID_MAX + 1, - }; - assert!(matches!(key.path(&Path::from("rec")), Err(Error::Id(_)))); + fn keys_parse_under_an_empty_prefix() { + let key = Key::segments("catalog.json", 2).unwrap(); + let path = key.path(&Path::ROOT).unwrap(); + assert_eq!(path.as_ref(), "catalog%2Ejson/segments/0000000000000000002"); + assert_eq!(Key::parse(&Path::ROOT, &path).unwrap(), key); } #[test] - fn groups_offset_has_no_dot() { - let path = groups_offset(&Path::from("rec"), "video", 5).unwrap(); - assert_eq!(path.as_ref(), "rec/video/groups/0000000000000000005"); + fn a_retired_directory_is_not_a_key() { + let path = Path::from("video/groups/0000000000000000001.0000000000000000001"); + assert!(matches!(Key::parse(&Path::ROOT, &path), Err(Error::Path(_)))); } } diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs new file mode 100644 index 00000000000..15d396e47f0 --- /dev/null +++ b/rs/moq-archive/src/proof.rs @@ -0,0 +1,396 @@ +//! End to end: one multi-rendition broadcast recorded by a [`Writer`], stored byte-identically on +//! every backend, and replayed exactly through a [`Reader`]. + +use std::collections::BTreeMap; +use std::time::Duration; + +use bytes::Bytes; +use futures::TryStreamExt; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::local::LocalFileSystem; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::mock::{Mock, Op}; +use crate::reader::Config as ReaderConfig; +use crate::writer::{Config, Retention}; +use crate::{Reader, Store, Writer}; + +const RENDITIONS: [&str; 2] = ["video/1080p", "video/360p"]; +const MEDIA: [&str; 3] = ["video/1080p", "video/360p", "audio"]; +const TRACKS: [&str; 5] = ["video/1080p", "video/360p", "audio", "catalog.json", "chat"]; + +/// Frames of one source group: (millisecond timestamp, payload). +type Frames = Vec<(u64, Bytes)>; + +/// Source segments: three 2s segments, then a one-frame tail. +const SEGMENTS: u64 = 4; + +/// The source groups of one segment: (track, sequence, frames). +/// +/// Video renditions share a 2s GOP and audio has four 500ms groups per segment, which its own +/// timeline packs into 2s records by the default minimum. The catalog and a non-media chat track publish sparsely, +/// including a sequence skip and an empty payload. +fn plan(segment: u64) -> Vec<(&'static str, u64, Frames)> { + let pts = segment * 2000; + let (frames, audio) = if segment + 1 < SEGMENTS { (4, 4) } else { (1, 1) }; + let mut groups = Vec::new(); + for track in RENDITIONS { + let frames = (0..frames) + .map(|i| (pts + i * 500, Bytes::from(format!("{track} {segment}.{i}")))) + .collect(); + groups.push((track, segment, frames)); + } + for sequence in segment * 4..segment * 4 + audio { + let frames = (0..2) + .map(|i| (sequence * 500 + i * 250, Bytes::from(format!("audio {sequence}.{i}")))) + .collect(); + groups.push(("audio", sequence, frames)); + } + let catalog = |sequence: u64| Bytes::from(format!(r#"{{"version":{sequence}}}"#)); + let chat = |text: &'static str| Bytes::from_static(text.as_bytes()); + match segment { + 0 => { + groups.push(("catalog.json", 0, vec![(0, catalog(0))])); + groups.push(("chat", 0, vec![(1200, chat("hello")), (1300, Bytes::new())])); + } + 2 => { + groups.push(("catalog.json", 1, vec![(4000, catalog(1))])); + groups.push(("chat", 3, vec![(4100, chat("skip"))])); + groups.push(("chat", 5, vec![(5100, chat("bye"))])); + } + _ => {} + } + groups +} + +fn ms(value: u64) -> Timestamp { + Timestamp::new(value, Timescale::MILLI).unwrap() +} + +fn write(track: &track::Producer, sequence: u64, frames: &Frames) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for (timestamp, payload) in frames { + group.write_frame(ms(*timestamp), payload.clone()).unwrap(); + } + group.finish().unwrap(); +} + +/// Let the writer read and report everything already published. Reads never touch the store. +async fn settle() { + for _ in 0..256 { + tokio::task::yield_now().await; + } +} + +/// Media cut at 1s, so one-second test groups get a record each. +fn media() -> moq_mux::timeline::Config { + moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)) +} + +fn timeline(track: &str) -> String { + hang::timeline::default_name(track) +} + +/// Every recorded track's timeline, as the catalog's `archive` entry names them. +fn config() -> ReaderConfig { + ReaderConfig::new( + TRACKS + .iter() + .map(|track| (track.to_string(), timeline(track))) + .collect(), + ) +} + +/// Record every segment of [`plan`] into `store`. +async fn record(store: &Store) { + let source = broadcast::Info::new().produce(); + let tracks: BTreeMap<&str, track::Producer> = TRACKS + .iter() + .map(|&name| { + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + (name, source.create_track(name, info).unwrap()) + }) + .collect(); + + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + for name in TRACKS { + let config = match MEDIA.contains(&name) { + true => moq_mux::timeline::Config::default(), + false => moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO), + }; + control.track(name, config).await.unwrap(); + } + let run = tokio::spawn(writer.run()); + + // Each track cuts its own timeline, so the interleave across tracks changes nothing stored. + for segment in 0..SEGMENTS { + for (name, sequence, frames) in plan(segment) { + write(&tracks[name], sequence, &frames); + } + settle().await; + } + for track in tracks.values() { + track.finish().unwrap(); + } + source.close(); + run.await.unwrap().unwrap(); +} + +/// Every object under the store's prefix, by path relative to the store root. +async fn objects(store: &Store) -> BTreeMap { + let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); + let mut objects = BTreeMap::new(); + for meta in metas { + let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); + objects.insert(meta.location.to_string(), bytes); + } + objects +} + +fn id(value: u64) -> String { + format!("{value:019}") +} + +/// Each track's record count: a GOP per video record, four audio groups per record, and a group per +/// sparse record. +const RECORDS: [(&str, u64); 5] = [ + ("video%2F1080p", 4), + ("video%2F360p", 4), + ("audio", 4), + ("catalog%2Ejson", 2), + ("chat", 3), +]; + +/// The exact layout [`plan`] produces: no manifest, index, or completion marker. +fn layout() -> Vec { + let mut keys = Vec::new(); + for (track, records) in RECORDS { + for directory in [track.to_string(), format!("{track}%2Etimeline%2Ez")] { + keys.push(format!("rec/{directory}/.info")); + keys.extend((0..records).map(|record| format!("rec/{directory}/segments/{}", id(record)))); + } + } + keys.sort(); + keys +} + +/// FETCH one whole group, returning its frames. +async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> moq_net::Result { + let track = broadcast.consume().track(track)?; + let mut group = track.fetch_group(sequence, group::Fetch::default()).await?; + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await? { + frames.push(( + frame.timestamp.convert(Timescale::MILLI).unwrap().value(), + frame.payload, + )); + } + Ok(frames) +} + +#[tokio::test] +async fn recordings_are_byte_identical_on_every_backend() { + let memory = Store::new(Mock::memory(), "rec"); + record(&memory).await; + let expected = objects(&memory).await; + assert_eq!(expected.keys().cloned().collect::>(), layout()); + assert_eq!( + &expected["rec/video%2F360p/.info"][..], + br#"{"version":2,"priority":127,"timescale":1000}"# + ); + + let dir = tempfile::tempdir().unwrap(); + let local = Store::new(Mock::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap()), "rec"); + record(&local).await; + assert_eq!(objects(&local).await, expected, "local disk"); + + // A backend with no listing order, and a sibling recording sharing the prefix's stem. + let unordered = Mock::memory(); + unordered.unordered(); + let sibling = Store::new(unordered.clone(), "rec-other"); + record(&sibling).await; + let unordered = Store::new(unordered, "rec"); + record(&unordered).await; + assert_eq!(objects(&unordered).await, expected, "unordered listing"); + assert_eq!(objects(&sibling).await.len(), expected.len()); + + // The media objects hold exactly the source groups, in order, byte for byte. + let mut stored: BTreeMap<(String, u64), Frames> = BTreeMap::new(); + for (path, bytes) in &expected { + let key = crate::Key::parse(&Path::from("rec"), &Path::parse(path).unwrap()).unwrap(); + if let crate::Key::Segments { track, .. } = key + && !track.ends_with(hang::timeline::SUFFIX) + { + let object = crate::Object::decode(bytes.clone()).unwrap(); + assert_eq!(object.frame_start, 0, "no source group outlived the maximum"); + for group in object.groups { + let frames = group.frames.into_iter().map(|f| (f.timestamp, f.payload)).collect(); + stored.insert((track.clone(), group.sequence), frames); + } + } + } + let source: BTreeMap<(String, u64), Frames> = (0..SEGMENTS) + .flat_map(plan) + .map(|(track, sequence, frames)| ((track.to_string(), sequence), frames)) + .collect(); + assert_eq!(stored, source); +} + +#[tokio::test] +async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + record(&store).await; + + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(store, &broadcast, config()).await.unwrap(); + tokio::spawn(reader.serve()); + mock.take(); + + // Low-rendition playback never downloads the 1080p object. + for sequence in 0..SEGMENTS { + let expected = plan(sequence) + .into_iter() + .find(|(name, ..)| *name == "video/360p") + .unwrap() + .2; + assert_eq!(fetch(&broadcast, "video/360p", sequence).await.unwrap(), expected); + } + let object = |track: &str, record: u64| format!("rec/{track}/segments/{}", id(record)); + // Each track request also GETs that track's `.info`, but nothing of another track. + let media = |gets: Vec, track: &str| { + let prefix = format!("rec/{track}/"); + assert!(gets.iter().all(|path| path.starts_with(&prefix)), "{gets:?}"); + gets.into_iter() + .filter(|path| path.contains("/segments/")) + .collect::>() + }; + assert_eq!( + media(mock.gets(), "video%2F360p"), + (0..4).map(|record| object("video%2F360p", record)).collect::>() + ); + + // Audio-only playback: four adjacent groups per GET, the rest from the cache. + for segment in 0..SEGMENTS { + for (_, sequence, frames) in plan(segment).into_iter().filter(|(name, ..)| *name == "audio") { + assert_eq!(fetch(&broadcast, "audio", sequence).await.unwrap(), frames); + } + } + assert_eq!( + media(mock.gets(), "audio"), + (0..4).map(|record| object("audio", record)).collect::>() + ); + + // Every other enrolled group replays its original sequence, timestamps, and payloads. + for segment in 0..SEGMENTS { + for (name, sequence, frames) in plan(segment) { + assert_eq!( + fetch(&broadcast, name, sequence).await.unwrap(), + frames, + "{name} {sequence}" + ); + } + } + for (track, sequence) in [("chat", 1), ("chat", 4), ("chat", 6), ("audio", 13), ("video/1080p", 4)] { + assert!( + matches!(fetch(&broadcast, track, sequence).await, Err(moq_net::Error::NotFound)), + "{track} {sequence} was never recorded" + ); + } + assert!( + !mock.take().iter().any(|op| matches!(op, Op::List { .. })), + "a group request resolves its object without listing" + ); +} + +#[tokio::test] +async fn an_offline_reader_follows_dvr_expiry() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let source = broadcast::Info::new().produce(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let video = source.create_track("video", info).unwrap(); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + let run = tokio::spawn(writer.run()); + + let frames = |sequence: u64| vec![(sequence * 1000, Bytes::from(format!("video {sequence}")))]; + let committed = |segment: u64| { + let store = store.clone(); + async move { + while store.get_segments(&timeline("video"), segment).await.is_err() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + }; + + for sequence in 0..3 { + write(&video, sequence, &frames(sequence)); + } + committed(1).await; + + let broadcast = broadcast::Info::new().produce(); + let config = ReaderConfig::new([("video".to_string(), timeline("video"))].into()); + let mut reader = Reader::open(store.clone(), &broadcast, config).await.unwrap(); + tokio::spawn(reader.serve()); + assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), frames(0)); + + // The reader is offline while the DVR commits and expires several records. + for sequence in 3..10 { + write(&video, sequence, &frames(sequence)); + } + committed(8).await; + // Records 7 and 8 hold the 2s window, and the newest timeline object restates both. + while store.get_segments("video", 6).await.is_ok() || store.get_segments(&timeline("video"), 7).await.is_ok() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + + mock.take(); + reader.refresh().await.unwrap(); + let ops = mock.take(); + let (list, gets) = ops.split_first().unwrap(); + assert_eq!( + list, + &Op::List { + prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), + offset: Some(format!("rec/video%2Etimeline%2Ez/segments/{}", id(1))), + } + ); + assert_eq!( + gets, + &[Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{}", id(8)))], + "only the retained timeline key is read" + ); + + // Expired groups are gone from the index, so they cost no media GET. + for sequence in 0..7 { + assert!( + matches!( + fetch(&broadcast, "video", sequence).await, + Err(moq_net::Error::NotFound) + ), + "group {sequence} expired" + ); + } + let gets = mock.gets(); + assert!( + !gets.iter().any(|path| path.starts_with("rec/video/segments/")), + "{gets:?}" + ); + for sequence in 7..9 { + assert_eq!(fetch(&broadcast, "video", sequence).await.unwrap(), frames(sequence)); + } + + video.finish().unwrap(); + source.close(); + run.await.unwrap().unwrap(); +} diff --git a/rs/moq-archive/src/reader/index.rs b/rs/moq-archive/src/reader/index.rs new file mode 100644 index 00000000000..4cafc63d2c9 --- /dev/null +++ b/rs/moq-archive/src/reader/index.rs @@ -0,0 +1,170 @@ +use std::collections::{BTreeMap, HashMap}; +use std::ops::Range; + +use hang::timeline::{Position, Record}; + +use crate::path::check_id; + +/// One committed record: the object at `segments/` and the frames it holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Span { + /// The record's sequence, which names its object. + pub sequence: u64, + /// The first frame, inclusive. + pub start: Position, + /// The end, exclusive. + pub end: Position, +} + +impl Span { + /// Build from a record, refusing an empty span or IDs past the recording range. + fn new(record: &Record) -> Option { + check_id(record.sequence).ok()?; + check_id(record.start.group).ok()?; + check_id(record.end.group).ok()?; + (record.start < record.end).then_some(Self { + sequence: record.sequence, + start: record.start, + end: record.end, + }) + } +} + +/// One track's committed spans. +#[derive(Default)] +struct Spans { + /// Each span keyed by its start. + spans: BTreeMap, + /// Each span's start keyed by its sequence, so a pop can find it. + sequences: BTreeMap, +} + +/// The spans each track's replayed timeline window commits. +#[derive(Default)] +pub(super) struct Index { + tracks: HashMap, +} + +impl Index { + /// Add `track`'s record. + /// + /// A malformed record or one overlapping another is left out, so its frames behave like ones + /// the source never delivered; the track's other records stay usable. + pub fn push(&mut self, track: &str, record: &Record) { + let Some(span) = Span::new(record) else { + tracing::warn!(track, sequence = record.sequence, "ignoring a malformed archive record"); + return; + }; + + let spans = self.tracks.entry(track.to_string()).or_default(); + let before = spans.spans.range(..span.end).next_back(); + if before.is_some_and(|(_, prev)| prev.end > span.start) || spans.sequences.contains_key(&span.sequence) { + tracing::warn!( + track, + sequence = record.sequence, + "ignoring an overlapping archive record" + ); + return; + } + spans.spans.insert(span.start, span); + spans.sequences.insert(span.sequence, span.start); + } + + /// Remove `track`'s records with these sequences, returning the evicted spans. + pub fn pop(&mut self, track: &str, sequences: Range) -> Vec { + let Some(spans) = self.tracks.get_mut(track) else { + return Vec::new(); + }; + let popped: Vec = spans + .sequences + .range(sequences) + .map(|(sequence, _)| *sequence) + .collect(); + popped + .into_iter() + .filter_map(|sequence| { + let start = spans.sequences.remove(&sequence)?; + spans.spans.remove(&start) + }) + .collect() + } + + /// The spans holding frames of `group` on `track`, in order. + pub fn group(&self, track: &str, group: u64) -> Vec { + let Some(spans) = self.tracks.get(track) else { + return Vec::new(); + }; + let head = Position::group(group); + let mut found: Vec = spans + .spans + .range(..Position::group(group.saturating_add(1))) + .rev() + .map(|(_, span)| *span) + .take_while(|span| span.end > head) + .collect(); + found.reverse(); + found + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ID_MAX; + + fn record(sequence: u64, start: Position, end: Position) -> Record { + Record::new(sequence, sequence * 1000, 1000, start, end) + } + + fn at(group: u64) -> Position { + Position::group(group) + } + + #[test] + fn lookup_finds_every_span_of_a_group() { + let mut index = Index::default(); + index.push("audio", &record(0, at(0), at(3))); + index.push("audio", &record(1, at(5), at(6))); + // Group 7 split across two records, then whole groups again. + index.push("video", &record(0, at(7), Position::new(7, 3))); + index.push("video", &record(1, Position::new(7, 3), at(9))); + + let sequences = |spans: Vec| spans.iter().map(|span| span.sequence).collect::>(); + assert_eq!(sequences(index.group("audio", 1)), vec![0]); + assert!(index.group("audio", 3).is_empty(), "the gap between records"); + assert_eq!(sequences(index.group("audio", 5)), vec![1]); + assert!(index.group("audio", 6).is_empty()); + assert_eq!(sequences(index.group("video", 7)), vec![0, 1]); + assert_eq!(sequences(index.group("video", 8)), vec![1]); + assert!(index.group("chat", 0).is_empty()); + } + + #[test] + fn pop_evicts_only_the_popped_records() { + let mut index = Index::default(); + index.push("video", &record(0, at(0), at(2))); + index.push("video", &record(1, at(2), at(4))); + + let evicted = index.pop("video", 0..1); + assert_eq!(evicted.len(), 1); + assert_eq!(evicted[0].start, at(0)); + assert!(index.group("video", 0).is_empty()); + assert_eq!(index.group("video", 2).len(), 1); + assert!(index.pop("video", 0..1).is_empty(), "already popped"); + } + + #[test] + fn malformed_and_overlapping_records_are_ignored() { + let mut index = Index::default(); + index.push("video", &record(0, at(4), at(7))); + index.push("video", &record(1, at(6), at(9))); + assert_eq!(index.group("video", 8), Vec::new(), "overlapping the earlier span"); + index.push("video", &record(2, at(3), at(5))); + assert_eq!(index.group("video", 3), Vec::new(), "overlapping the later span"); + + index.push("audio", &record(0, at(3), at(3))); + assert!(index.group("audio", 3).is_empty(), "an empty span"); + index.push("data", &record(0, at(ID_MAX + 1), at(ID_MAX + 2))); + assert!(index.group("data", ID_MAX + 1).is_empty(), "past the recording range"); + } +} diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs new file mode 100644 index 00000000000..5dc4cc4a6b8 --- /dev/null +++ b/rs/moq-archive/src/reader/mod.rs @@ -0,0 +1,500 @@ +//! Serve an archive's groups through a caller-supplied broadcast. +//! +//! A [`Reader`] replays each track's timeline onto its timeline track and answers FETCH for every +//! frame those timelines commit. A group maps to the records holding it, each one object fetched +//! with a single GET, validated against its record, and kept in a byte-bounded cache so adjacent +//! groups reuse it. A request for one track never downloads another track's object. A group whose +//! records do not reach its end yet, such as an append-only log, grows as later records commit. +//! +//! The reader never infers that a recording ended: it keeps following until the caller, who may +//! know finality out of band, calls [`Reader::finish`]. +//! +//! ```no_run +//! # async fn example(store: moq_archive::Store) -> moq_archive::Result<()> { +//! use moq_archive::reader::{Config, Reader}; +//! +//! let broadcast = moq_net::broadcast::Info::new().produce(); +//! let timelines = [("video".to_string(), "video.timeline.z".to_string())].into(); +//! let mut reader = Reader::open(store, &broadcast, Config::new(timelines)).await?; +//! let serve = reader.serve(); +//! // Spawn `serve` on any executor, then call `reader.refresh()` to follow a growing archive. +//! # drop(serve); +//! # Ok(()) +//! # } +//! ``` + +mod index; + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use hang::timeline::{Position, Record}; +use moq_json::window; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStore; +use tokio::sync::watch; + +use self::index::{Index, Span}; +use crate::store::list::Query; +use crate::{Error, Key, Object, Result, Store}; + +/// Configuration for [`Reader::open`]. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Config { + /// Each served track's timeline track, from the catalog's `archive` entry. + pub timelines: BTreeMap, + /// Upper bound on cached object bytes shared by every track. + pub cache: u64, +} + +impl Config { + /// Serve the tracks `timelines` indexes with a 64 MiB object cache. + pub fn new(timelines: BTreeMap) -> Self { + Self { + timelines, + cache: 64 * 1024 * 1024, + } + } + + /// Set [`cache`](Self::cache). + pub fn with_cache(mut self, bytes: u64) -> Self { + self.cache = bytes; + self + } +} + +/// Serves an archive through a [`broadcast::Producer`] and follows its timelines. +pub struct Reader { + shared: Arc>, + broadcast: broadcast::Producer, + dynamic: broadcast::Dynamic, + timelines: Vec, +} + +/// One track's timeline, replayed onto its own timeline track. +struct Timeline { + /// The track it indexes. + track: String, + producer: track::Producer, + decoder: window::Decoder, + /// The last timeline segment replayed. + cursor: Option, +} + +struct Shared { + store: Store, + index: Mutex, + cache: quick_cache::sync::Cache, Weight>, + /// Each track's `.info`, which is immutable, so a re-requested track costs no GET. + infos: Mutex>, + /// The tracks a timeline indexes, which are the only ones served. + served: Mutex>, + /// Notified whenever the index changes; holds whether the caller declared the recording ended. + changed: watch::Sender, +} + +impl Reader { + /// Create every timeline track on `broadcast`, then replay each track's retained timeline. + /// + /// Fails if a timeline's `.info` is missing or invalid, a timeline track already exists, or the + /// store cannot list a timeline. + pub async fn open(store: Store, broadcast: &broadcast::Producer, config: Config) -> Result { + let items = usize::try_from(config.cache / (1024 * 1024)) + .unwrap_or(usize::MAX) + .max(16); + let shared = Arc::new(Shared { + store, + index: Mutex::default(), + cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), + infos: Mutex::default(), + served: Mutex::default(), + changed: watch::Sender::new(false), + }); + + let mut reader = Self { + shared, + broadcast: broadcast.clone(), + dynamic: broadcast.dynamic(), + timelines: Vec::new(), + }; + for (track, timeline) in config.timelines { + reader.add(track, timeline).await?; + } + reader.refresh().await?; + Ok(reader) + } + + /// Serve and follow `track`, whose timeline is the track `timeline`, such as one a recording + /// enrolled after this reader opened. Its timeline replays on the next [`refresh`](Self::refresh). + /// + /// Fails if the timeline's `.info` is missing or invalid, or `track` is already served. + pub async fn track(&mut self, track: &str, timeline: &str) -> Result<()> { + self.add(track.to_string(), timeline.to_string()).await + } + + async fn add(&mut self, track: String, timeline: String) -> Result<()> { + if self.shared.served.lock().unwrap().contains(&track) { + return Err(moq_net::Error::Duplicate.into()); + } + let info = track_info(&self.shared.store.get_info(&timeline).await?)?; + let producer = self.broadcast.create_track(timeline, info)?; + self.shared.served.lock().unwrap().insert(track.clone()); + self.timelines.push(Timeline { + track, + producer, + decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), + cursor: None, + }); + Ok(()) + } + + /// Replay timeline objects committed since the last refresh. + /// + /// Lists each timeline's keys after its last replayed segment, then GETs them in segment order. + /// Each stored group opens with a window checkpoint, so a segment that expired or is malformed + /// is skipped and the next one recovers the retained window, evicting whatever it popped. A + /// malformed newest object is retried on the next refresh. + pub async fn refresh(&mut self) -> Result<()> { + for timeline in &mut self.timelines { + timeline.refresh(&self.shared).await?; + } + Ok(()) + } + + /// Finish every replayed timeline track once the caller knows the recording ended. + /// + /// Following stops, and a group still growing is served as it stands. Groups already committed + /// stay servable through [`serve`](Self::serve). + pub fn finish(self) -> Result<()> { + self.shared.changed.send_replace(true); + for timeline in self.timelines { + timeline.producer.finish()?; + } + Ok(()) + } + + /// Answer track and group requests on the broadcast until it closes. + /// + /// The future is independent of this reader, so it keeps serving after + /// [`finish`](Self::finish); run it on any executor. + pub fn serve(&self) -> impl Future + Send + 'static + where + T: 'static, + { + serve(self.shared.clone(), self.dynamic.clone()) + } +} + +impl Timeline { + async fn refresh(&mut self, shared: &Shared) -> Result<()> { + let name = self.producer.name().to_string(); + let mut query = Query::segments(&name)?; + if let Some(cursor) = self.cursor { + query = query.after(&Key::segments(&name, cursor)?)?; + } + + let entries: Vec<_> = shared.store.list(&query).try_collect().await?; + let mut segments: Vec = entries + .into_iter() + .filter_map(|entry| match entry.key { + Key::Segments { segment, .. } => Some(segment), + _ => None, + }) + .filter(|segment| self.cursor.is_none_or(|cursor| *segment > cursor)) + .collect(); + segments.sort_unstable(); + segments.dedup(); + + for segment in segments { + match shared.store.get_segments(&name, segment).await { + Ok(object) => self.replay(shared, &object)?, + Err(Error::Store(err)) => return Err(Error::Store(err)), + Err(err) => { + tracing::warn!(timeline = name, segment, %err, "skipping unreadable timeline segment"); + continue; + } + } + self.cursor = Some(segment); + } + + Ok(()) + } + + /// Index one timeline object's groups, then republish each on the timeline track. + fn replay(&mut self, shared: &Shared, object: &Object) -> Result<()> { + for stored in &object.groups { + let mut group = self.decoder.group(); + let decoded = stored.frames.iter().try_for_each(|frame| group.decode(&frame.payload)); + + // Apply whatever decoded, even from a group that failed partway: the next group's + // checkpoint restates the window regardless. + let mut index = shared.index.lock().unwrap(); + while let Some(event) = group.next_event() { + match event { + window::Event::Push { index: at, value } if at == value.sequence => index.push(&self.track, &value), + window::Event::Push { index: at, value } => { + tracing::warn!( + track = self.track, + at, + sequence = value.sequence, + "ignoring a misnumbered record" + ); + } + window::Event::Pop(range) | window::Event::Skip(range) => { + for span in index.pop(&self.track, range) { + if let Ok(key) = Key::segments(self.track.clone(), span.sequence) { + shared.cache.remove(&key); + } + } + } + _ => {} + } + } + drop(index); + shared.changed.send_modify(|_| {}); + + if let Err(err) = decoded { + tracing::warn!(group = stored.sequence, %err, "skipping malformed timeline group"); + continue; + } + + let mut producer = match self.producer.create_group(stored.sequence.into()) { + Ok(producer) => producer, + Err(moq_net::Error::Duplicate) => continue, + Err(err) => return Err(err.into()), + }; + let timescale = producer.timescale(); + for frame in &stored.frames { + producer.write_frame(timestamp(frame.timestamp, timescale)?, frame.payload.clone())?; + } + producer.finish()?; + } + Ok(()) + } +} + +/// Serve requested tracks until the broadcast closes. +async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic) { + let mut serving = kio::Tasks::new(); + kio::wait(|waiter| { + loop { + match dynamic.poll_requested_track(waiter) { + Poll::Ready(Ok(request)) => { + let known = shared.served.lock().unwrap().contains(request.name()); + serving.push(task(Box::pin(serve_track(shared.clone(), request, known)))) + } + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } + } + let _ = serving.poll(waiter); + Poll::Pending + }) + .await +} + +/// Accept a track a timeline indexes, then serve its group requests until nobody uses it. +async fn serve_track(shared: Arc>, request: track::Request, known: bool) { + let name = request.name().to_string(); + if !known { + request.reject(moq_net::Error::NotFound); + return; + } + + let cached = shared.infos.lock().unwrap().get(&name).cloned(); + let info = match cached { + Some(info) => info, + None => match shared.store.get_info(&name).await.and_then(|info| track_info(&info)) { + Ok(info) => { + shared.infos.lock().unwrap().insert(name.clone(), info.clone()); + info + } + Err(err) => { + tracing::warn!(track = name, %err, "archived track has no usable .info"); + request.reject(moq_net::Error::NotFound); + return; + } + }, + }; + + let timescale = info.timescale; + let dynamic = request.dynamic(); + let demand = dynamic.demand(); + let _producer = request.accept(info); + let mut groups = kio::Tasks::new(); + + kio::wait(|waiter| { + loop { + match dynamic.poll_requested_group(waiter) { + Poll::Ready(Ok(request)) => { + let serve = serve_group(shared.clone(), name.clone(), timescale, request); + groups.push(task(Box::pin(serve))); + } + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } + } + match groups.poll(waiter).is_ready() && demand.poll_unused(waiter).is_ready() { + true => Poll::Ready(()), + false => Poll::Pending, + } + }) + .await +} + +/// Answer one group request from the records holding it, or reject it as never delivered. +/// +/// A group whose records stop short of its end keeps its producer open and grows as later records +/// commit, until one reaches the group's end or the caller finishes the reader. +async fn serve_group( + shared: Arc>, + track: String, + timescale: Timescale, + request: group::Request, +) { + let sequence = request.sequence(); + let start = request.frame_start(); + let mut changed = shared.changed.subscribe(); + let mut spans = shared.index.lock().unwrap().group(&track, sequence); + + // The earliest retained record must hold the requested frame; an expired head is gone. + if spans + .first() + .is_none_or(|span| span.start > Position::new(sequence, start)) + { + request.reject(moq_net::Error::NotFound); + return; + } + // Validate the first object before accepting, so an unreadable group is a plain miss. Keep it, + // so a cache too small to hold it costs no second GET. + let mut first = match object(&shared, &track, &spans[0]).await { + Ok(object) => Some((spans[0].sequence, object)), + Err(err) => { + tracing::warn!(track, sequence, %err, "archived group is unavailable"); + request.reject(moq_net::Error::NotFound); + return; + } + }; + let Ok(mut producer) = request.accept(None) else { + return; + }; + if producer.start_at(start).is_err() { + return; + } + + let mut next = start; + loop { + for span in &spans { + if span.end <= Position::new(sequence, next) { + continue; + } + let loaded = match first.take_if(|(loaded, _)| *loaded == span.sequence) { + Some((_, object)) => Ok(object), + None => object(&shared, &track, span).await, + }; + let object = match loaded { + Ok(object) if span.start <= Position::new(sequence, next) => object, + Ok(_) => { + let _ = producer.abort(moq_net::Error::NotFound); + return; + } + Err(err) => { + tracing::warn!(track, sequence, %err, "archived group is unavailable"); + let _ = producer.abort(moq_net::Error::NotFound); + return; + } + }; + // The span was validated against its object, so the group is present. + let Some(stored) = object.groups.iter().find(|group| group.sequence == sequence) else { + continue; + }; + let first = match object.groups[0].sequence == sequence { + true => object.frame_start, + false => 0, + }; + for (index, frame) in (first..).zip(&stored.frames) { + if index < next { + continue; + } + let Ok(timestamp) = timestamp(frame.timestamp, timescale) else { + let _ = producer.abort(moq_net::Error::TimestampMismatch); + return; + }; + if let Err(err) = producer.write_frame(timestamp, frame.payload.clone()) { + let _ = producer.abort(err); + return; + } + next = index + 1; + } + } + + let complete = spans.last().is_some_and(|span| span.end.group > sequence); + if complete { + let _ = producer.finish(); + return; + } + // A change that landed while objects loaded is unseen, so the spans above predate it: + // reload before finishing or waiting, or its records would never be served. + if !changed.has_changed().unwrap_or(false) { + // The caller finished the recording, and these spans are its final index. + if *changed.borrow() { + let _ = producer.finish(); + return; + } + if changed.changed().await.is_err() { + let _ = producer.finish(); + return; + } + } + changed.mark_unchanged(); + spans = shared.index.lock().unwrap().group(&track, sequence); + } +} + +/// A span's object, from the cache or one GET validated against the span. +async fn object(shared: &Shared, track: &str, span: &Span) -> Result> { + let key = Key::segments(track.to_string(), span.sequence)?; + shared + .cache + .get_or_insert_async(&key, async { + let object = shared.store.get_segments(track, span.sequence).await?; + object.check_span(span.start, span.end)?; + Ok(Arc::new(object)) + }) + .await +} + +fn track_info(info: &crate::Info) -> Result { + let timescale = Timescale::new(info.timescale).map_err(|_| Error::Timescale(info.timescale))?; + Ok(track::Info::default() + .with_timescale(timescale) + .with_priority(info.priority)) +} + +fn timestamp(value: u64, timescale: Timescale) -> Result { + Timestamp::new(value, timescale).map_err(|_| Error::Id(value)) +} + +/// Adapt a boxed future into a [`kio::Tasks`] task. +fn task(mut future: BoxFuture<'static, ()>) -> impl FnMut(&kio::Waiter) -> Poll<()> + Send { + move |waiter| waiter.poll_future(future.as_mut()) +} + +/// Weighs a cached object by its payload bytes plus a small per-frame overhead. +#[derive(Clone)] +struct Weight; + +impl quick_cache::Weighter> for Weight { + fn weight(&self, _key: &Key, object: &Arc) -> u64 { + let frames = object.groups.iter().flat_map(|group| &group.frames); + frames.map(|frame| frame.payload.len() as u64 + 32).sum::() + 64 + } +} + +#[cfg(test)] +mod tests; diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs new file mode 100644 index 00000000000..1d625488a89 --- /dev/null +++ b/rs/moq-archive/src/reader/tests.rs @@ -0,0 +1,708 @@ +use std::collections::HashMap; +use std::time::Duration; + +use bytes::Bytes; +use hang::timeline::{Position, Record}; +use moq_json::window; +use moq_net::{broadcast, group}; +use object_store::ObjectStoreExt; + +use super::*; +use crate::Info; +use crate::mock::{Mock, Op}; +use crate::segment::{Frame, Group}; + +fn timeline(track: &str) -> String { + hang::timeline::default_name(track) +} + +/// One track's timeline encoder, as a recording writer drives it. +struct Encoder { + encoder: window::Encoder, + /// Next timeline group sequence. + group: u64, + /// Next record sequence. + record: u64, +} + +/// Writes archive objects the way a recording writer lays them out. +struct Archive { + store: Store, + op_ratio: u32, + encoders: HashMap, +} + +impl Archive { + async fn new() -> Self { + Self::with_op_ratio(0).await + } + + /// `op_ratio` 0 makes every window edit its own checkpoint group. + async fn with_op_ratio(op_ratio: u32) -> Self { + Self { + store: Store::new(Mock::memory(), "rec"), + op_ratio, + encoders: HashMap::new(), + } + } + + async fn encoder(&mut self, track: &str) -> &mut Encoder { + if !self.encoders.contains_key(track) { + self.store.put_info(track, &Info::new(1, 1000).unwrap()).await.unwrap(); + self.store + .put_info(&timeline(track), &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(self.op_ratio); + self.encoders.insert( + track.to_string(), + Encoder { + encoder: window::Encoder::new(config), + group: 0, + record: 0, + }, + ); + } + self.encoders.get_mut(track).unwrap() + } + + /// The next record of `track`, spanning `start..end` where every group holds `frames` frames. + /// Stores its object unless `store` is false. + async fn record(&mut self, track: &str, start: Position, end: Position, frames: u64) -> Record { + let encoder = self.encoder(track).await; + let sequence = encoder.record; + encoder.record += 1; + let record = Record::new(sequence, start.group * 100, 100, start, end); + + let last = match end.frame { + 0 => end.group - 1, + _ => end.group, + }; + let groups = (start.group..=last) + .map(|group| { + let first = if group == start.group { start.frame } else { 0 }; + let stop = if group == end.group { end.frame } else { frames }; + Group { + sequence: group, + frames: (first..stop).map(|i| frame(track, group, i)).collect(), + } + }) + .collect(); + let object = Object { + frame_start: start.frame, + groups, + }; + self.store.put_segments(track, sequence, &object).await.unwrap(); + record + } + + /// Store and commit a record of whole groups `groups`, each holding `frames` frames. + async fn add(&mut self, track: &str, groups: std::ops::Range, frames: u64) -> Record { + let record = self + .record( + track, + Position::group(groups.start), + Position::group(groups.end), + frames, + ) + .await; + self.commit(track, &record, 0).await; + record + } + + /// Push `record`, pop `pop` records, and return the resulting timeline groups. + async fn edit(&mut self, track: &str, record: &Record, pop: u64) -> Vec { + let encoder = self.encoder(track).await; + let mut frames = Vec::new(); + let pending = encoder.encoder.push(record).unwrap(); + frames.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = encoder.encoder.pop(pop).unwrap() { + frames.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + + let mut groups: Vec = Vec::new(); + for (keyframe, payload) in frames { + if keyframe { + encoder.group += 1; + } + if groups.last().is_none_or(|group| group.sequence != encoder.group - 1) { + groups.push(Group { + sequence: encoder.group - 1, + frames: Vec::new(), + }); + } + let timestamp = record.pts; + groups.last_mut().unwrap().frames.push(Frame { timestamp, payload }); + } + groups + } + + async fn timeline(&self, track: &str, segment: u64, groups: Vec) { + self.store + .put_segments(&timeline(track), segment, &Object::new(groups)) + .await + .unwrap(); + } + + /// Commit `record` as its track's timeline segment, popping `pop` older records. + async fn commit(&mut self, track: &str, record: &Record, pop: u64) { + let groups = self.edit(track, record, pop).await; + self.timeline(track, record.sequence, groups).await; + } + + async fn raw(&self, key: &Key, bytes: &'static [u8]) { + let path = self.store.path(key).unwrap(); + self.store + .inner() + .put(&path, Bytes::from_static(bytes).into()) + .await + .unwrap(); + } + + fn config(&self) -> Config { + Config::new( + self.encoders + .keys() + .map(|track| (track.clone(), timeline(track))) + .collect(), + ) + } +} + +fn frame(track: &str, sequence: u64, index: u64) -> Frame { + Frame { + timestamp: sequence * 100 + index, + payload: Bytes::from(format!("{track}/{sequence}/{index}")), + } +} + +async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { + open_with(archive, archive.config()).await +} + +async fn open_with(archive: &Archive, config: Config) -> (broadcast::Producer, Reader) { + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(archive.store.clone(), &broadcast, config).await.unwrap(); + tokio::spawn(reader.serve()); + (broadcast, reader) +} + +/// FETCH one group starting at `frame_start`, returning each frame's timestamp and payload. +async fn fetch( + broadcast: &broadcast::Producer, + track: &str, + sequence: u64, + frame_start: u64, +) -> std::result::Result, moq_net::Error> { + let track = broadcast.consume().track(track)?; + let options = group::Fetch::default().with_frame_start(frame_start); + let mut group = track.fetch_group(sequence, options).await?; + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await? { + frames.push((frame.timestamp.value(), frame.payload)); + } + Ok(frames) +} + +fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<(u64, Bytes)> { + frames + .map(|i| { + let frame = frame(track, sequence, i); + (frame.timestamp, frame.payload) + }) + .collect() +} + +fn not_found(result: std::result::Result, moq_net::Error>) { + assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); +} + +#[tokio::test] +async fn fetch_replays_original_groups() { + let mut archive = Archive::new().await; + archive.add("video", 0..2, 3).await; + archive.add("video", 2..3, 2).await; + archive.add("audio", 0..2, 1).await; + archive.add("audio", 5..6, 2).await; + + let (broadcast, _reader) = open(&archive).await; + + assert_eq!( + fetch(&broadcast, "video", 0, 0).await.unwrap(), + expected("video", 0, 0..3) + ); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..2) + ); + assert_eq!( + fetch(&broadcast, "audio", 5, 0).await.unwrap(), + expected("audio", 5, 0..2) + ); + + // A requested frame_start keeps the original frame indices and timestamps. + let tail = fetch(&broadcast, "video", 1, 1).await.unwrap(); + assert_eq!(tail, expected("video", 1, 1..3)); + + let track = broadcast.consume().track("video").unwrap(); + let info = track.query().await.unwrap(); + assert_eq!(info.priority, 1); + assert_eq!(info.timescale.as_u64(), 1000); +} + +#[tokio::test] +async fn a_group_split_across_records_is_served_whole() { + let mut archive = Archive::new().await; + let head = archive + .record("video", Position::group(7), Position::new(7, 3), 5) + .await; + archive.commit("video", &head, 0).await; + let tail = archive + .record("video", Position::new(7, 3), Position::group(9), 5) + .await; + archive.commit("video", &tail, 0).await; + + let (broadcast, _reader) = open(&archive).await; + assert_eq!( + fetch(&broadcast, "video", 7, 0).await.unwrap(), + expected("video", 7, 0..5) + ); + assert_eq!( + fetch(&broadcast, "video", 7, 4).await.unwrap(), + expected("video", 7, 4..5) + ); + assert_eq!( + fetch(&broadcast, "video", 8, 0).await.unwrap(), + expected("video", 8, 0..5) + ); +} + +#[tokio::test] +async fn an_open_group_grows_as_records_commit() { + let mut archive = Archive::new().await; + let head = archive.record("log", Position::group(0), Position::new(0, 2), 0).await; + archive.commit("log", &head, 0).await; + + let (broadcast, mut reader) = open(&archive).await; + let track = broadcast.consume().track("log").unwrap(); + let mut group = tokio::time::timeout(Duration::from_secs(1), track.fetch_group(0, group::Fetch::default())) + .await + .expect("fetch") + .unwrap(); + for index in 0..2 { + let frame = group.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.payload, frame_payload("log", 0, index)); + } + let pending = tokio::time::timeout(Duration::from_millis(50), group.read_frame()).await; + assert!(pending.is_err(), "the group waits for its next record"); + + let more = archive.record("log", Position::new(0, 2), Position::new(0, 3), 0).await; + archive.commit("log", &more, 0).await; + reader.refresh().await.unwrap(); + let frame = group.read_frame().await.unwrap().unwrap(); + assert_eq!(frame.payload, frame_payload("log", 0, 2)); + + // Finishing the reader ends the group where the recording stopped. + reader.finish().unwrap(); + assert!(group.read_frame().await.unwrap().is_none()); +} + +/// A refresh that lands while a growing group loads an object is still served, not swallowed +/// by the wait for the next one. +#[tokio::test(start_paused = true)] +async fn a_refresh_during_an_object_load_is_served() { + let mut archive = Archive::new().await; + let head = archive.record("log", Position::group(0), Position::new(0, 2), 0).await; + archive.commit("log", &head, 0).await; + + let (broadcast, mut reader) = open(&archive).await; + let mock = archive.store.inner().clone(); + mock.take(); + let gate = mock.stall_gets("log/segments/"); + let track = broadcast.consume().track("log").unwrap(); + let fetch = tokio::spawn(async move { track.fetch_group(0, group::Fetch::default()).await }); + + // Wait until the head object's GET is in flight, then commit the tail behind it. + while !mock.gets().iter().any(|path| path.contains("log/segments/")) { + tokio::task::yield_now().await; + } + let more = archive.record("log", Position::new(0, 2), Position::new(0, 3), 0).await; + archive.commit("log", &more, 0).await; + reader.refresh().await.unwrap(); + gate.add_permits(16); + + let mut group = fetch.await.unwrap().unwrap(); + for index in 0..3 { + let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) + .await + .expect("the refreshed tail is served") + .unwrap() + .unwrap(); + assert_eq!(frame.payload, frame_payload("log", 0, index)); + } +} + +fn frame_payload(track: &str, sequence: u64, index: u64) -> Bytes { + frame(track, sequence, index).payload +} + +#[tokio::test] +async fn an_expired_head_is_not_found() { + let mut archive = Archive::new().await; + let head = archive + .record("video", Position::group(7), Position::new(7, 3), 5) + .await; + archive.commit("video", &head, 0).await; + let tail = archive + .record("video", Position::new(7, 3), Position::group(8), 5) + .await; + archive.commit("video", &tail, 1).await; + + let (broadcast, _reader) = open(&archive).await; + not_found(fetch(&broadcast, "video", 7, 0).await); + assert_eq!( + fetch(&broadcast, "video", 7, 3).await.unwrap(), + expected("video", 7, 3..5) + ); +} + +#[tokio::test] +async fn requests_download_only_their_object() { + let mut archive = Archive::new().await; + archive.add("video", 0..1, 1).await; + archive.add("audio", 0..3, 1).await; + + let (broadcast, _reader) = open(&archive).await; + archive.store.inner().take(); + + for sequence in 0..3 { + assert_eq!( + fetch(&broadcast, "audio", sequence, 0).await.unwrap(), + expected("audio", sequence, 0..1) + ); + } + + let gets = archive.store.inner().gets(); + assert_eq!( + gets, + vec![ + "rec/audio/.info".to_string(), + "rec/audio/segments/0000000000000000000".to_string(), + ], + "adjacent groups reuse one GET and never touch video" + ); +} + +#[tokio::test] +async fn unadvertised_or_unreadable_groups_are_not_found() { + let mut archive = Archive::new().await; + archive.add("audio", 0..2, 1).await; + archive.add("audio", 4..5, 1).await; + // The object holds groups 0 and 1, but its record names group 2 too. + let mismatched = archive.record("video", Position::group(0), Position::group(2), 1).await; + archive + .commit( + "video", + &Record::new(0, 0, 100, Position::group(0), Position::group(3)), + 0, + ) + .await; + let bad = archive.record("bad", Position::group(0), Position::group(1), 1).await; + archive.raw(&Key::segments("bad", 0).unwrap(), b"\x02garbage").await; + archive.commit("bad", &bad, 0).await; + let missing = Record::new(0, 0, 100, Position::group(0), Position::group(1)); + archive.encoder("missing").await; + archive.commit("missing", &missing, 0).await; + assert_eq!(mismatched.end, Position::group(2)); + + let (broadcast, _reader) = open(&archive).await; + + not_found(fetch(&broadcast, "audio", 2, 0).await); // between records + not_found(fetch(&broadcast, "audio", 5, 0).await); // past the last record + not_found(fetch(&broadcast, "video", 0, 0).await); // table disagrees with the record + not_found(fetch(&broadcast, "bad", 0, 0).await); // malformed envelope + not_found(fetch(&broadcast, "missing", 0, 0).await); // no object + not_found(fetch(&broadcast, "chat", 0, 0).await); // a track no timeline indexes + + // Siblings remain usable. + assert_eq!( + fetch(&broadcast, "audio", 4, 0).await.unwrap(), + expected("audio", 4, 0..1) + ); +} + +#[tokio::test] +async fn refresh_follows_new_records_and_pops() { + let mut archive = Archive::new().await; + archive.add("video", 0..2, 1).await; + + let (broadcast, mut reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 0, 0).await.is_ok()); + + let next = archive.record("video", Position::group(2), Position::group(3), 1).await; + // Refreshing before the commit finds nothing new; Not Found is not finality. + reader.refresh().await.unwrap(); + not_found(fetch(&broadcast, "video", 2, 0).await); + + archive.commit("video", &next, 1).await; + reader.refresh().await.unwrap(); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..1) + ); + + // The popped record's groups are gone, although its object is still stored and was cached. + archive.store.inner().take(); + not_found(fetch(&broadcast, "video", 1, 0).await); + assert_eq!(archive.store.inner().gets(), Vec::::new()); +} + +/// A track the recording enrolled after the reader opened is found by listing the store again, +/// then served and followed like the rest. +#[tokio::test] +async fn a_track_enrolled_after_opening_is_followed() { + let mut archive = Archive::new().await; + archive.add("audio", 0..2, 1).await; + + let (broadcast, mut reader) = open(&archive).await; + not_found(fetch(&broadcast, "video", 0, 0).await); + + archive.add("video", 0..2, 2).await; + let timelines = archive.store.timelines().await.unwrap(); + assert_eq!(timelines.keys().collect::>(), ["audio", "video"]); + reader.track("video", &timelines["video"]).await.unwrap(); + assert!( + reader.track("video", &timelines["video"]).await.is_err(), + "a served track is refused" + ); + reader.refresh().await.unwrap(); + + assert_eq!( + fetch(&broadcast, "video", 1, 0).await.unwrap(), + expected("video", 1, 0..2) + ); + let timeline = broadcast.consume().track(&timeline("video")).unwrap(); + assert!(timeline.subscribe(None).await.is_ok(), "the late timeline replays"); +} + +/// A timeline directory without its `.info`, such as one interrupted mid-enrollment, isn't +/// advertised, so following the recording doesn't fail on it. +#[tokio::test] +async fn a_timeline_without_info_is_not_listed() { + let mut archive = Archive::new().await; + archive.add("audio", 0..2, 1).await; + archive + .raw(&Key::segments(timeline("video"), 0).unwrap(), b"partial") + .await; + + let timelines = archive.store.timelines().await.unwrap(); + assert_eq!(timelines.keys().collect::>(), ["audio"]); +} + +#[tokio::test] +async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { + let mut archive = Archive::new().await; + archive.add("video", 0..1, 1).await; + // Record 1 is committed to the window but its timeline object is unreadable. + let lost = archive.record("video", Position::group(1), Position::group(2), 1).await; + archive.edit("video", &lost, 0).await; + archive + .raw(&Key::segments(timeline("video"), 1).unwrap(), b"\x02") + .await; + archive.add("video", 2..3, 1).await; + + let (broadcast, _reader) = open(&archive).await; + + // Record 2's checkpoint restates record 1, so its group is still served. + for sequence in 0..3 { + assert_eq!( + fetch(&broadcast, "video", sequence, 0).await.unwrap(), + expected("video", sequence, 0..1) + ); + } +} + +#[tokio::test] +async fn a_missing_tail_is_retried_on_the_next_refresh() { + let mut archive = Archive::new().await; + for group in 0..3 { + archive.add("video", group..group + 1, 1).await; + } + // Listed, but not yet readable. + archive.store.inner().hide_gets("segments/0000000000000000002"); + + let (broadcast, mut reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); + not_found(fetch(&broadcast, "video", 2, 0).await); + + archive.store.inner().heal(); + archive.store.inner().take(); + reader.refresh().await.unwrap(); + assert_eq!( + archive.store.inner().take(), + [ + Op::List { + prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), + offset: Some("rec/video%2Etimeline%2Ez/segments/0000000000000000001".to_string()), + }, + Op::Get("rec/video%2Etimeline%2Ez/segments/0000000000000000002".to_string()), + ], + "the cursor stays before the missing tail" + ); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..1) + ); +} + +#[tokio::test] +async fn following_lists_only_new_timeline_keys() { + let mut archive = Archive::new().await; + archive.add("video", 0..1, 1).await; + let (broadcast, mut reader) = open(&archive).await; + + for group in 1..4 { + // A media object stored ahead of its commit is invisible until the timeline names it. + let record = archive + .record("video", Position::group(group), Position::group(group + 1), 1) + .await; + reader.refresh().await.unwrap(); + not_found(fetch(&broadcast, "video", group, 0).await); + + archive.commit("video", &record, 0).await; + archive.store.inner().take(); + reader.refresh().await.unwrap(); + let previous = format!("rec/video%2Etimeline%2Ez/segments/{:019}", group - 1); + assert_eq!( + archive.store.inner().take(), + [ + Op::List { + prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), + offset: Some(previous), + }, + Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{group:019}")), + ], + "following record {group} touches no media listing" + ); + assert_eq!( + fetch(&broadcast, "video", group, 0).await.unwrap(), + expected("video", group, 0..1) + ); + } +} + +#[tokio::test] +async fn an_unordered_listing_replays_in_segment_order() { + let mut archive = Archive::new().await; + archive.store.inner().unordered(); + for group in 0..4 { + let record = archive + .record("video", Position::group(group), Position::group(group + 1), 1) + .await; + archive.commit("video", &record, u64::from(group >= 2)).await; + } + + let (broadcast, mut reader) = open(&archive).await; + + // The cursor is the newest segment, not the last one listed. + let record = archive.record("video", Position::group(4), Position::group(5), 1).await; + archive.commit("video", &record, 1).await; + archive.store.inner().take(); + reader.refresh().await.unwrap(); + assert_eq!( + archive.store.inner().gets(), + ["rec/video%2Etimeline%2Ez/segments/0000000000000000004"] + ); + + for group in 0..3 { + not_found(fetch(&broadcast, "video", group, 0).await); + } + for group in 3..5 { + assert_eq!( + fetch(&broadcast, "video", group, 0).await.unwrap(), + expected("video", group, 0..1) + ); + } +} + +#[tokio::test] +async fn a_track_without_usable_info_is_not_found() { + let mut archive = Archive::new().await; + archive.add("audio", 0..1, 1).await; + archive.add("bare", 0..1, 1).await; + archive.add("future", 0..1, 1).await; + let path = archive.store.path(&Key::info("bare").unwrap()).unwrap(); + archive.store.inner().delete(&path).await.unwrap(); + let path = archive.store.path(&Key::info("future").unwrap()).unwrap(); + archive.store.inner().delete(&path).await.unwrap(); + archive + .raw( + &Key::info("future").unwrap(), + br#"{"version":3,"priority":0,"timescale":1000}"#, + ) + .await; + + let (broadcast, _reader) = open(&archive).await; + for track in ["bare", "future"] { + not_found(fetch(&broadcast, track, 0, 0).await); + } + assert_eq!( + fetch(&broadcast, "audio", 0, 0).await.unwrap(), + expected("audio", 0, 0..1) + ); +} + +#[tokio::test] +async fn multi_frame_timeline_groups_decode() { + let mut archive = Archive::with_op_ratio(64).await; + let first = archive.record("video", Position::group(0), Position::group(1), 1).await; + let second = archive.record("video", Position::group(1), Position::group(2), 1).await; + let mut groups = archive.edit("video", &first, 0).await; + let ops = archive.edit("video", &second, 0).await; + assert!( + ops[0].sequence == groups[0].sequence, + "the push stays in the open group" + ); + groups[0].frames.extend(ops.into_iter().flat_map(|group| group.frames)); + archive.timeline("video", 0, groups).await; + + let (broadcast, _reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); +} + +#[tokio::test] +async fn timeline_tracks_are_republished_and_finished_on_request() { + let mut archive = Archive::new().await; + archive.add("video", 0..1, 1).await; + archive.add("video", 1..2, 1).await; + archive.add("audio", 0..1, 1).await; + + let (broadcast, reader) = open(&archive).await; + let track = broadcast.consume().track(&timeline("video")).unwrap(); + let subscriber = track.subscribe(None).await.unwrap(); + let mut timeline = + window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); + + reader.finish().unwrap(); + + let mut records = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let window::Event::Push { value, .. } = event { + records.push(value.sequence); + } + } + assert_eq!(records, vec![0, 1]); +} + +#[tokio::test] +async fn open_requires_every_timeline_info() { + let store = Store::new(Mock::memory(), "rec"); + let broadcast = broadcast::Info::new().produce(); + let config = Config::new([("video".to_string(), timeline("video"))].into()); + let result = Reader::open(store, &broadcast, config).await; + assert!(matches!(result, Err(Error::NotFound(_)))); +} diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs new file mode 100644 index 00000000000..e1eeffdb178 --- /dev/null +++ b/rs/moq-archive/src/recover.rs @@ -0,0 +1,209 @@ +//! Recover a recording so a restarted [`Writer`](crate::Writer) continues it. + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::ops::{Range, RangeInclusive}; + +use futures::TryStreamExt; +use hang::timeline::{Position, Record}; +use moq_json::window::{self, Checkpoint}; +use object_store::ObjectStore; + +use crate::store::list::Query; +use crate::{Error, Key, Object, Result, Store}; + +/// What a restarted writer continues from. +#[derive(Default)] +pub(crate) struct Recovery { + /// Each recorded track's retained timeline, keyed by the recorded track's name. + pub tracks: HashMap, + /// Committed objects no retained record references, such as an interrupted expiration. Only + /// collected for a DVR, which deletes them after its grace. + pub orphans: Vec, + /// Objects at or past a track's next record: uploads a crash left before their commit. No + /// timeline ever referenced them, and the resumed writer reuses their keys. + pub uncommitted: Vec, + /// Each timeline's stored segments older than any recovery reads. Only collected for a DVR. + pub stale: Vec<(String, Range)>, +} + +/// One track's retained timeline. +pub(crate) struct Resume { + /// The retained window. + pub checkpoint: Checkpoint, + /// The next stored timeline group sequence, so the stored numbering keeps increasing. + pub sequence: u64, + /// The timeline objects recovery replayed, oldest first, with the first record index each + /// one's opening checkpoint restates. Only collected for a DVR. + pub checkpoints: VecDeque<(u64, u64)>, +} + +impl Resume { + /// Where the newest committed record ended: the resumed track refuses anything before it. + pub fn floor(&self) -> Option { + self.checkpoint.records.last().map(|record| record.end) + } +} + +/// List the whole recording, then replay each track's timeline from a retained checkpoint. +/// +/// A DVR (`complete`) replays far enough back to recover every retained record and reports the +/// unreferenced objects; an archive only needs the newest checkpoint. Any listing, GET, or decode +/// failure fails recovery, so a caller never acts on a partial view. +pub(crate) async fn recover(store: &Store, complete: bool) -> Result { + let entries: Vec<_> = store.list(&Query::new()).try_collect().await?; + let mut segments: BTreeMap> = BTreeMap::new(); + for entry in entries { + if let Key::Segments { track, segment } = entry.key { + segments.entry(track).or_default().push(segment); + } + } + + let mut recovery = Recovery::default(); + for (timeline, ids) in &mut segments { + let Some(track) = timeline.strip_suffix(hang::timeline::SUFFIX) else { + continue; + }; + ids.sort_unstable(); + let (first, last) = (ids[0], ids[ids.len() - 1]); + if ids.len() as u64 != last - first + 1 { + return Err(Error::Timeline(format!( + "{timeline} segments {first}..={last} are not contiguous" + ))); + } + let resume = replay(store, timeline, first..=last, complete).await?; + // The newest object is segment `last`, so the window must end on the next one. A shorter + // window would resume onto that segment and collide with it. + let next = last.checked_add(1).ok_or(Error::Overflow)?; + if resume.checkpoint.range.end != next { + return Err(Error::Timeline(format!( + "{timeline} window ends at {}, not segment {next}", + resume.checkpoint.range.end + ))); + } + if let Some(&(needed, _)) = resume.checkpoints.front() + && first < needed + { + recovery.stale.push((timeline.clone(), first..needed)); + } + recovery.tracks.insert(track.to_string(), resume); + } + + for (track, ids) in segments { + if track.ends_with(hang::timeline::SUFFIX) { + continue; + } + let resume = recovery.tracks.get(&track); + let next = resume.map_or(0, |resume| resume.checkpoint.range.end); + let retained: HashSet = resume + .iter() + .flat_map(|resume| &resume.checkpoint.records) + .map(|record| record.sequence) + .collect(); + for id in ids { + let key = Key::segments(track.clone(), id)?; + if id >= next { + recovery.uncommitted.push(key); + } else if complete && !retained.contains(&id) { + recovery.orphans.push(key); + } + } + } + + Ok(recovery) +} + +/// Replay `segments` from the newest checkpoint that restates every record `complete` needs. +async fn replay( + store: &Store, + timeline: &str, + segments: RangeInclusive, + complete: bool, +) -> Result { + // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, + // so walk back until a checkpoint restates from there. The writer prunes by the same rule. + let mut objects = VecDeque::new(); + let mut checkpoints = VecDeque::new(); + let mut needed = None; + for segment in segments.rev() { + let object = store.get_segments(timeline, segment).await?; + let (offset, start) = checkpoint(&object)?; + let needed = *needed.get_or_insert(offset); + objects.push_front(object); + if complete { + checkpoints.push_front((segment, start)); + } + if !complete || start <= needed { + break; + } + } + + let mut decoder = decoder(); + let mut records = BTreeMap::new(); + for object in &objects { + for stored in &object.groups { + let mut group = decoder.group(); + for frame in &stored.frames { + group.decode(&frame.payload).map_err(json_error)?; + } + } + while let Some(event) = decoder.next_event() { + match event { + window::Event::Push { index, value } => { + records.insert(index, value); + } + window::Event::Pop(range) | window::Event::Skip(range) => { + let tail = records.split_off(&range.end); + records.retain(|index, _| *index < range.start); + records.extend(tail); + } + _ => {} + } + } + } + + // The records must be a contiguous suffix of the window, and all of it for a DVR. + let range = decoder.range(); + let start = records.keys().next().copied().unwrap_or(range.end); + if records.len() as u64 != range.end - start || (complete && start != range.start) { + return Err(Error::Timeline(format!( + "cannot recover {timeline} window {range:?} from the retained checkpoints" + ))); + } + + let last = objects.back().and_then(|object| object.groups.last()); + let sequence = last.map_or(Ok(0), |group| group.sequence.checked_add(1).ok_or(Error::Overflow))?; + let checkpoint = Checkpoint { + range, + records: records.into_values().collect(), + }; + Ok(Resume { + checkpoint, + sequence, + checkpoints, + }) +} + +/// The retained offset and first restated index of the checkpoint opening `object`. +pub(crate) fn checkpoint(object: &Object) -> Result<(u64, u64)> { + let frame = object + .groups + .first() + .and_then(|group| group.frames.first()) + .ok_or_else(|| Error::Timeline("timeline object has no checkpoint".into()))?; + let mut decoder = decoder(); + decoder.group().decode(&frame.payload).map_err(json_error)?; + let offset = decoder.range().start; + let start = match decoder.next_event() { + Some(window::Event::Skip(skipped)) => skipped.end, + _ => offset, + }; + Ok((offset, start)) +} + +fn decoder() -> window::Decoder { + window::Decoder::new(window::ConsumerConfig::default().with_compression(true)) +} + +fn json_error(err: moq_json::Error) -> Error { + Error::Timeline(err.to_string()) +} diff --git a/rs/moq-archive/src/segment.rs b/rs/moq-archive/src/segment.rs index 9b1cac5a892..a582b77afec 100644 --- a/rs/moq-archive/src/segment.rs +++ b/rs/moq-archive/src/segment.rs @@ -1,12 +1,11 @@ -use std::ops::RangeInclusive; - use bytes::{Buf, BufMut, Bytes, BytesMut}; +use hang::timeline::Position; use moq_net::varint; -use crate::path::{check_id, check_range}; +use crate::path::check_id; use crate::{Error, Result, VERSION}; -/// One complete group in a segment object, in sequence order. +/// One group's stored frames in a segment object, in sequence order. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Group { /// Group sequence number. @@ -27,15 +26,50 @@ pub struct Frame { /// A versioned group/frame table followed by concatenated payload bytes. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Object { + /// The index of the first group's first stored frame within that group; later groups start at + /// frame zero. + pub frame_start: u64, /// Groups in strictly ascending sequence order. pub groups: Vec, } impl Object { - /// Inclusive group sequences from first to last. - pub fn bounds(&self) -> Result> { + /// An object whose first group starts at frame zero. + pub fn new(groups: Vec) -> Self { + Self { frame_start: 0, groups } + } + + /// The position of the first stored frame. + pub fn start(&self) -> Result { validate(&self.groups)?; - Ok(self.groups[0].sequence..=self.groups.last().unwrap().sequence) + Ok(Position::new(self.groups[0].sequence, self.frame_start)) + } + + /// Require this object to hold exactly the frames `start..end` of a record, with no group + /// missing or empty. + pub fn check_span(&self, start: Position, end: Position) -> Result<()> { + if self.start()? != start { + return Err(Error::Span); + } + for (expected, group) in (start.group..).zip(&self.groups) { + if group.sequence != expected || group.frames.is_empty() { + return Err(Error::Span); + } + } + let last = self.groups.last().expect("validated as non-empty"); + let first = match self.groups.len() { + 1 => self.frame_start, + _ => 0, + }; + let stop = first.checked_add(last.frames.len() as u64).ok_or(Error::Overflow)?; + let exact = match end.frame { + 0 => end.group == last.sequence + 1, + frame => end.group == last.sequence && frame == stop, + }; + if !exact { + return Err(Error::Span); + } + Ok(()) } /// Encode the binary envelope. The first sequence is absolute; later ones are `current - previous - 1`. @@ -44,6 +78,7 @@ impl Object { let mut table = BytesMut::new(); write_varint(&mut table, VERSION)?; + write_varint(&mut table, self.frame_start)?; write_varint(&mut table, self.groups.len() as u64)?; let mut payload = BytesMut::new(); @@ -82,6 +117,7 @@ impl Object { return Err(Error::Version(version)); } + let frame_start = read_varint(&mut buf)?; let group_count = read_count(&mut buf, 2)?; if group_count == 0 { return Err(Error::Empty); @@ -157,28 +193,7 @@ impl Object { return Err(Error::Table); } - Ok(Self { groups }) - } - - /// Decode and require the table's sequences to match `range`. - pub fn decode_groups(buf: impl Buf, range: RangeInclusive) -> Result { - check_range(&range)?; - let object = Self::decode(buf)?; - object.check_bounds(range)?; - Ok(object) - } - - /// Require this object's sequences to match a range-named key. - pub fn check_bounds(&self, range: RangeInclusive) -> Result<()> { - check_range(&range)?; - let got = self.bounds()?; - if got != range { - return Err(Error::Bounds { - smallest: *got.start(), - largest: *got.end(), - }); - } - Ok(()) + Ok(Self { frame_start, groups }) } } @@ -232,7 +247,14 @@ mod tests { } fn object(groups: Vec) -> Object { - Object { groups } + Object::new(groups) + } + + fn group(sequence: u64, frames: usize) -> Group { + Group { + sequence, + frames: (0..frames as u64).map(|i| frame(i, b"x")).collect(), + } } #[test] @@ -253,8 +275,14 @@ mod tests { ]); let bytes = original.encode().unwrap(); assert_eq!(Object::decode(&bytes[..]).unwrap(), original); - assert_eq!(original.bounds().unwrap(), 0..=4); - original.check_bounds(0..=4).unwrap(); + assert_eq!(original.start().unwrap(), Position::group(0)); + + let split = Object { + frame_start: 300, + ..original + }; + assert_eq!(Object::decode(split.encode().unwrap()).unwrap(), split); + assert_eq!(split.start().unwrap(), Position::new(0, 300)); } #[test] @@ -277,7 +305,8 @@ mod tests { .unwrap(); let mut buf = &bytes[..]; - assert_eq!(read_varint(&mut buf).unwrap(), 1); // version + assert_eq!(read_varint(&mut buf).unwrap(), 2); // version + assert_eq!(read_varint(&mut buf).unwrap(), 0); // frame start assert_eq!(read_varint(&mut buf).unwrap(), 3); // group count assert_eq!(read_varint(&mut buf).unwrap(), 5); // absolute assert_eq!(read_varint(&mut buf).unwrap(), 1); // one frame @@ -296,7 +325,8 @@ mod tests { fn empty_object_is_rejected() { assert!(matches!(object(vec![]).encode(), Err(Error::Empty))); let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, 0).unwrap(); assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Empty))); } @@ -336,7 +366,8 @@ mod tests { #[test] fn reconstructed_ids_must_stay_in_range() { let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, 2).unwrap(); write_varint(&mut bytes, ID_MAX).unwrap(); write_varint(&mut bytes, 0).unwrap(); // no frames @@ -345,6 +376,40 @@ mod tests { assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Id(_)))); } + /// A table with one frameless group per delta. + fn deltas(deltas: &[u64]) -> Bytes { + let mut bytes = BytesMut::new(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); + write_varint(&mut bytes, deltas.len() as u64).unwrap(); + for &delta in deltas { + write_varint(&mut bytes, delta).unwrap(); + write_varint(&mut bytes, 0).unwrap(); + } + bytes.freeze() + } + + fn sequences(bytes: Bytes) -> Result> { + Ok(Object::decode(bytes)? + .groups + .iter() + .map(|group| group.sequence) + .collect()) + } + + #[test] + fn delta_extremes() { + let varint = (1u64 << 62) - 1; + assert_eq!(sequences(deltas(&[0, 0, 0])).unwrap(), vec![0, 1, 2]); + assert_eq!(sequences(deltas(&[0, 9, 0])).unwrap(), vec![0, 10, 11]); + assert_eq!(sequences(deltas(&[0, ID_MAX - 1])).unwrap(), vec![0, ID_MAX]); + assert_eq!(sequences(deltas(&[ID_MAX])).unwrap(), vec![ID_MAX]); + assert!(matches!(sequences(deltas(&[0, ID_MAX])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[ID_MAX + 1])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[varint])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[ID_MAX, varint])), Err(Error::Id(_)))); + } + #[test] fn timestamp_bounds() { assert!( @@ -367,10 +432,13 @@ mod tests { #[test] fn unknown_version_is_refused() { - let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 2).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Version(2)))); + // Version 1 objects held whole groups under range-named keys. + for version in [1, 3] { + let mut bytes = BytesMut::new(); + write_varint(&mut bytes, version).unwrap(); + write_varint(&mut bytes, 1).unwrap(); + assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Version(v)) if v == version)); + } } #[test] @@ -391,7 +459,8 @@ mod tests { // group_count equals remaining bytes, so a remaining-bytes check would // accept it, but each group needs at least two varints. let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, VERSION).unwrap(); + write_varint(&mut bytes, 0).unwrap(); write_varint(&mut bytes, 4).unwrap(); bytes.extend_from_slice(&[0, 0, 0, 0]); assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Table))); @@ -400,47 +469,87 @@ mod tests { #[test] fn offset_overflow_and_truncated_table() { let mut bytes = BytesMut::new(); - write_varint(&mut bytes, 1).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 1).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 0).unwrap(); - write_varint(&mut bytes, 4).unwrap(); // length 4, no payload + for value in [VERSION, 0, 1, 0, 1, 0, 0, 4] { + write_varint(&mut bytes, value).unwrap(); // length 4, no payload + } assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Table))); - assert!(matches!(Object::decode(&b"\x01"[..]), Err(Error::Table))); + assert!(matches!(Object::decode(&b"\x02"[..]), Err(Error::Table))); } #[test] - fn filename_bounds_must_match_the_table() { - let object = object(vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 7, - frames: vec![frame(1, b"b")], - }, - ]); - let bytes = object.encode().unwrap(); - assert!(Object::decode_groups(&bytes[..], 5..=7).is_ok()); - assert!(matches!( - Object::decode_groups(&bytes[..], 6..=7), - Err(Error::Bounds { - smallest: 5, - largest: 7 - }) - )); - let (smallest, largest) = (7, 5); - assert!(matches!( - Object::decode_groups(&bytes[..], smallest..=largest), - Err(Error::Bounds { - smallest: 7, - largest: 5 - }) - )); + fn frame_offsets_must_tile_the_payload() { + // One group of two frames at (offset, length), followed by `payload` bytes. + let table = |frames: [(u64, u64); 2], payload: usize| { + let mut bytes = BytesMut::new(); + for value in [VERSION, 0, 1, 0, 2] { + write_varint(&mut bytes, value).unwrap(); + } + for (offset, length) in frames { + for value in [0, offset, length] { + write_varint(&mut bytes, value).unwrap(); + } + } + bytes.extend_from_slice(&vec![b'x'; payload]); + Object::decode(bytes.freeze()) + }; + assert!(table([(0, 1), (1, 2)], 3).is_ok()); + assert!(matches!(table([(0, 1), (0, 2)], 3), Err(Error::Table)), "overlap"); + assert!(matches!(table([(0, 1), (2, 1)], 3), Err(Error::Table)), "gap"); + assert!(matches!(table([(1, 1), (2, 1)], 3), Err(Error::Table)), "late start"); + assert!(matches!(table([(0, 1), (1, 3)], 3), Err(Error::Table)), "past the end"); + let varint = (1u64 << 62) - 1; + assert!( + matches!(table([(0, 1), (1, varint)], 3), Err(Error::Table)), + "huge length" + ); + } + + #[test] + fn a_span_must_match_the_record() { + let whole = object(vec![group(5, 2), group(6, 1)]); + whole.check_span(Position::group(5), Position::group(7)).unwrap(); + assert_eq!( + whole.check_span(Position::group(5), Position::group(8)), + Err(Error::Span) + ); + assert_eq!( + whole.check_span(Position::group(4), Position::group(7)), + Err(Error::Span) + ); + assert_eq!( + whole.check_span(Position::group(5), Position::new(6, 2)), + Err(Error::Span), + "a partial end names its frame count" + ); + + // A frame split: frames 3..5 of group 7. + let split = Object { + frame_start: 3, + groups: vec![group(7, 2)], + }; + split.check_span(Position::new(7, 3), Position::new(7, 5)).unwrap(); + assert_eq!( + split.check_span(Position::new(7, 3), Position::new(7, 4)), + Err(Error::Span) + ); + + // A split's tail continues into whole groups. + let tail = Object { + frame_start: 3, + groups: vec![group(7, 2), group(8, 4)], + }; + tail.check_span(Position::new(7, 3), Position::new(8, 4)).unwrap(); + tail.check_span(Position::new(7, 3), Position::group(9)).unwrap(); + + // Gaps and empty groups never appear inside a record. + let gap = object(vec![group(5, 1), group(7, 1)]); + assert_eq!(gap.check_span(Position::group(5), Position::group(8)), Err(Error::Span)); + let empty = object(vec![group(5, 1), group(6, 0)]); + assert_eq!( + empty.check_span(Position::group(5), Position::group(7)), + Err(Error::Span) + ); } #[test] diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 3a9092703e4..dfc1c16bfa4 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -1,5 +1,4 @@ use std::num::NonZeroUsize; -use std::ops::RangeInclusive; use bytes::Bytes; use futures::StreamExt; @@ -25,7 +24,7 @@ use crate::{Error, Result}; /// /// use moq_archive::store::list::Query; /// -/// let query = Query::segments("timeline.z")? +/// let query = Query::segments("video.timeline.z")? /// .page_size(NonZeroUsize::new(100).unwrap()); /// # Ok::<(), moq_archive::Error>(()) /// ``` @@ -76,12 +75,7 @@ pub mod list { Ok(Self::new().prefix(path::track_prefix(track)?)) } - /// List group objects for one track. - pub fn groups(track: &str) -> Result { - Ok(Self::new().prefix(path::groups_prefix(&Path::ROOT, track)?)) - } - - /// List timeline segment objects for one track. + /// List segment objects for one track. pub fn segments(track: &str) -> Result { Ok(Self::new().prefix(path::segments_prefix(&Path::ROOT, track)?)) } @@ -91,11 +85,6 @@ pub mod list { Ok(self.offset(key.path(&Path::ROOT)?)) } - /// List group objects beginning at the exclusive lexical offset for `group`. - pub fn groups_from(track: &str, group: u64) -> Result { - Ok(Self::groups(track)?.offset(path::groups_offset(&Path::ROOT, track, group)?)) - } - pub(crate) fn next(&self, page_token: String) -> Self { let mut next = self.clone(); next.page_token = Some(page_token); @@ -189,32 +178,48 @@ impl Store { Info::decode(&self.get_bytes(&path).await?) } - /// Create a range-named groups object. A collision is accepted only when the bytes match. - pub async fn put_groups(&self, track: &str, object: &Object) -> Result { - let key = Key::groups(track, object.bounds()?)?; - self.put_segment(&key, object.encode()?).await?; - Ok(key) - } - - /// Fetch a groups object and require its table to match the filename bounds. - pub async fn get_groups(&self, track: &str, range: RangeInclusive) -> Result { - let path = self.path(&Key::groups(track, range.clone())?)?; - Object::decode_groups(self.get_bytes(&path).await?, range) - } - - /// Create a timeline object at `segments/`. A collision is accepted only when the bytes match. + /// Create the object at `segments/`. A collision is accepted only when the bytes match. pub async fn put_segments(&self, track: &str, segment: u64, object: &Object) -> Result { let key = Key::segments(track, segment)?; self.put_segment(&key, object.encode()?).await?; Ok(key) } - /// Fetch and validate a timeline object. + /// Fetch and decode the object at `segments/`. pub async fn get_segments(&self, track: &str, segment: u64) -> Result { let path = self.path(&Key::segments(track, segment)?)?; Object::decode(self.get_bytes(&path).await?) } + /// Each recorded track's timeline track, found by listing the track directories a [`Writer`] + /// creates: a timeline is named by [`hang::timeline::default_name`]. One listing of the + /// prefix's directories plus a HEAD per timeline, so following a long recording doesn't list + /// its objects. A timeline whose `.info` isn't stored yet, such as one still enrolling, is left + /// out until it is. + /// + /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. + /// + /// [`Writer`]: crate::Writer + pub async fn timelines(&self) -> Result> { + let listed = self.inner.list_with_delimiter(self.list_path(None).as_ref()).await?; + let mut timelines = std::collections::BTreeMap::new(); + for directory in listed.common_prefixes { + let encoded = directory + .filename() + .ok_or_else(|| Error::Directory(directory.to_string()))?; + let track = crate::path::decode_track(encoded)?; + let Some(indexed) = track.strip_suffix(hang::timeline::SUFFIX) else { + continue; + }; + match self.inner.head(&self.path(&Key::info(&track)?)?).await { + Ok(_) => timelines.insert(indexed.to_string(), track), + Err(object_store::Error::NotFound { .. }) => continue, + Err(err) => return Err(err.into()), + }; + } + Ok(timelines) + } + /// Delete the object at `key`. pub async fn delete(&self, key: &Key) -> Result<()> { let path = self.path(key)?; @@ -343,113 +348,16 @@ mod tests { use std::num::NonZeroUsize; use futures::TryStreamExt; - use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; + use object_store::ObjectStoreExt; use object_store::memory::InMemory; use object_store::path::Path; - use object_store::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, - PutOptions, PutPayload, PutResult, - }; use super::*; use crate::ID_MAX; + use crate::mock::Mock; use crate::path::encode_track; use crate::segment::{Frame, Group}; - /// In-memory store with a trivial offset-based paginated listing implementation. - /// Page tokens are decimal indexes into the filtered, sorted key list. - #[derive(Debug, Clone)] - struct PaginatedMemory { - inner: InMemory, - } - - impl std::fmt::Display for PaginatedMemory { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "PaginatedMemory") - } - } - - #[async_trait::async_trait] - impl ObjectStore for PaginatedMemory { - async fn put_opts( - &self, - location: &Path, - payload: PutPayload, - opts: PutOptions, - ) -> object_store::Result { - self.inner.put_opts(location, payload, opts).await - } - - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOptions, - ) -> object_store::Result> { - self.inner.put_multipart_opts(location, opts).await - } - - async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: BoxStream<'static, object_store::Result>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { - self.inner.list(prefix) - } - - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } - } - - #[async_trait::async_trait] - impl PaginatedListStore for PaginatedMemory { - async fn list_paginated( - &self, - prefix: Option<&str>, - opts: PaginatedListOptions, - ) -> object_store::Result { - let mut metas: Vec = self.inner.list(None).try_collect().await?; - metas.sort_by(|a, b| a.location.as_ref().cmp(b.location.as_ref())); - if let Some(prefix) = prefix { - metas.retain(|meta| meta.location.as_ref().starts_with(prefix)); - } - if let Some(offset) = opts.offset.as_deref() { - metas.retain(|meta| meta.location.as_ref() > offset); - } - let start: usize = match opts.page_token { - Some(token) => token.parse().map_err(|err| object_store::Error::Generic { - store: "PaginatedMemory", - source: Box::new(err), - })?, - None => 0, - }; - let remaining = &metas[start.min(metas.len())..]; - let take = opts.max_keys.unwrap_or(remaining.len()).min(remaining.len()); - let objects = remaining[..take].to_vec(); - let next = start.saturating_add(take); - let page_token = (next < metas.len()).then(|| next.to_string()); - Ok(PaginatedListResult { - result: ListResult { - objects, - common_prefixes: Vec::new(), - extensions: Default::default(), - }, - page_token, - }) - } - } - fn memory() -> Store { Store::new(InMemory::new(), "rec") } @@ -462,27 +370,23 @@ mod tests { } fn one_group(sequence: u64, payload: &'static [u8]) -> Object { - Object { - groups: vec![Group { - sequence, - frames: vec![frame(sequence, payload)], - }], - } + Object::new(vec![Group { + sequence, + frames: vec![frame(sequence, payload)], + }]) } fn two_groups() -> Object { - Object { - groups: vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"a")], - }, - Group { - sequence: 7, - frames: vec![frame(1, b"bb")], - }, - ], - } + Object::new(vec![ + Group { + sequence: 5, + frames: vec![frame(0, b"a")], + }, + Group { + sequence: 6, + frames: vec![frame(1, b"bb")], + }, + ]) } async fn names(store: &Store, query: &Query) -> BTreeSet { @@ -506,7 +410,7 @@ mod tests { .inner() .put( &path, - br#"{ "timescale": 1000, "priority": 1, "version": 1 }"#.as_ref().into(), + br#"{ "timescale": 1000, "priority": 1, "version": 2 }"#.as_ref().into(), ) .await .unwrap(); @@ -517,7 +421,31 @@ mod tests { .bytes() .await .unwrap(); - assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 1 }"#); + assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 2 }"#); + } + + #[tokio::test] + async fn malformed_or_unsupported_existing_info_is_refused_and_kept() { + let store = memory(); + let info = Info::new(0, 1_000).unwrap(); + for (track, existing, check) in [ + ( + "v1", + &br#"{"version":1,"priority":0,"timescale":1000}"#[..], + (|err| matches!(err, Error::Version(1))) as fn(&Error) -> bool, + ), + ("junk", b"not json", |err| matches!(err, Error::Json(_))), + ("zero", br#"{"version":2,"priority":0,"timescale":0}"#, |err| { + matches!(err, Error::Timescale(0)) + }), + ] { + let path = store.path(&Key::info(track).unwrap()).unwrap(); + store.inner().put(&path, existing.to_vec().into()).await.unwrap(); + let err = store.put_info(track, &info).await.unwrap_err(); + assert!(check(&err), "{track}: {err}"); + let kept = store.inner().get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(&kept[..], existing, "{track} is not rewritten"); + } } #[tokio::test] @@ -542,38 +470,24 @@ mod tests { } #[tokio::test] - async fn groups_put_get_and_identical_collision() { + async fn segments_put_get_and_identical_collision() { let store = memory(); let object = two_groups(); - let range = object.bounds().unwrap(); - let key = store.put_groups("video", &object).await.unwrap(); - assert_eq!(key, Key::groups("video", range.clone()).unwrap()); + let key = store.put_segments("video", 3, &object).await.unwrap(); + assert_eq!(key, Key::segments("video", 3).unwrap()); assert_eq!( store.path(&key).unwrap().as_ref(), - "rec/video/groups/0000000000000000007.0000000000000000005" + "rec/video/segments/0000000000000000003" ); - assert_eq!(store.get_groups("video", range).await.unwrap(), object); - store.put_groups("video", &object).await.unwrap(); - } + assert_eq!(store.get_segments("video", 3).await.unwrap(), object); + store.put_segments("video", 3, &object).await.unwrap(); - #[tokio::test] - async fn groups_collision_with_different_bytes_fails() { - let store = memory(); - store.put_groups("video", &two_groups()).await.unwrap(); - let other = Object { - groups: vec![ - Group { - sequence: 5, - frames: vec![frame(0, b"X")], - }, - Group { - sequence: 7, - frames: vec![frame(1, b"bb")], - }, - ], - }; + let other = Object::new(vec![Group { + sequence: 5, + frames: vec![frame(0, b"X")], + }]); assert!(matches!( - store.put_groups("video", &other).await, + store.put_segments("video", 3, &other).await, Err(Error::Conflict(_)) )); } @@ -592,11 +506,11 @@ mod tests { } #[tokio::test] - async fn listing_names_build_a_range_index() { + async fn listing_names_every_object() { let store = memory(); store.put_info("video", &Info::new(0, 1).unwrap()).await.unwrap(); - store.put_groups("video", &one_group(1, b"a")).await.unwrap(); - store.put_groups("video", &one_group(3, b"b")).await.unwrap(); + store.put_segments("video", 1, &one_group(1, b"a")).await.unwrap(); + store.put_segments("video", 3, &one_group(3, b"b")).await.unwrap(); store.put_segments("timeline.z", 2, &one_group(0, b"t")).await.unwrap(); let listed: Vec<_> = store.list(&Query::new()).try_collect().await.unwrap(); @@ -607,8 +521,8 @@ mod tests { vec![ Key::segments("timeline.z", 2).unwrap(), Key::info("video").unwrap(), - Key::groups("video", 1..=1).unwrap(), - Key::groups("video", 3..=3).unwrap(), + Key::segments("video", 1).unwrap(), + Key::segments("video", 3).unwrap(), ] ); } @@ -648,7 +562,7 @@ mod tests { async fn listing_query_is_recording_relative() { let store = memory(); store.put_info("video", &Info::new(0, 1).unwrap()).await.unwrap(); - store.put_groups("video", &one_group(1, b"a")).await.unwrap(); + store.put_segments("video", 1, &one_group(1, b"a")).await.unwrap(); store.put_info("video-alt", &Info::new(0, 1).unwrap()).await.unwrap(); let query = Query::track("video").unwrap(); @@ -657,7 +571,7 @@ mod tests { paths, BTreeSet::from([ "video/.info".to_string(), - "video/groups/0000000000000000001.0000000000000000001".to_string(), + "video/segments/0000000000000000001".to_string(), ]) ); assert_eq!( @@ -702,16 +616,6 @@ mod tests { #[tokio::test] async fn paginated_query_preserves_every_supported_option() { let store = memory(); - let groups = Query::groups_from("video", 5).unwrap(); - assert_eq!( - store.paginated_prefix(groups.prefix.as_ref()).as_deref(), - Some("rec/video/groups/") - ); - assert_eq!( - store.paginated_options(&groups).offset.as_deref(), - Some("rec/video/groups/0000000000000000005") - ); - let query = Query::segments("timeline.z") .unwrap() .after(&Key::segments("timeline.z", 1).unwrap()) @@ -818,7 +722,7 @@ mod tests { #[tokio::test] async fn paginated_listing_walks_pages_and_excludes_siblings() { - let store = Store::new(PaginatedMemory { inner: InMemory::new() }, "rec"); + let store = Store::new(Mock::memory(), "rec"); for segment in 0..3 { store .put_segments("timeline.z", segment, &one_group(segment, b"t")) @@ -827,7 +731,6 @@ mod tests { } store .inner() - .inner .put( &Path::from("rec-other/video/.info"), Bytes::from_static(b"sibling").into(), @@ -896,6 +799,35 @@ mod tests { assert_eq!(paged, streamed); } + #[tokio::test] + async fn empty_prefix_lists_the_whole_store() { + let store = Store::new(Mock::memory(), ""); + store.put_info("catalog.json", &Info::new(0, 1).unwrap()).await.unwrap(); + store.put_segments("video", 4, &one_group(4, b"a")).await.unwrap(); + assert_eq!(store.paginated_prefix(None), None); + + let expected = + std::collections::HashSet::from([Key::info("catalog.json").unwrap(), Key::segments("video", 4).unwrap()]); + let streamed: std::collections::HashSet = store + .list(&Query::new()) + .map_ok(|entry| entry.key) + .try_collect() + .await + .unwrap(); + assert_eq!(streamed, expected); + let page = store.list_paginated(&Query::new()).await.unwrap(); + assert_eq!( + page.entries + .into_iter() + .map(|entry| entry.key) + .collect::>(), + expected + ); + assert!(page.next.is_none()); + let page = store.list_paginated(&Query::segments("video").unwrap()).await.unwrap(); + assert_eq!(page.entries.len(), 1); + } + #[test] fn directory_results_are_not_silently_dropped() { let store = memory(); @@ -913,9 +845,7 @@ mod tests { async fn id_endpoints_are_valid_keys() { let store = memory(); let object = one_group(ID_MAX, b"z"); - store.put_groups("v", &object).await.unwrap(); store.put_segments("t", ID_MAX, &object).await.unwrap(); - assert_eq!(store.get_groups("v", ID_MAX..=ID_MAX).await.unwrap(), object); assert_eq!(store.get_segments("t", ID_MAX).await.unwrap(), object); } @@ -927,8 +857,8 @@ mod tests { let info = Info::new(3, 90_000).unwrap(); store.put_info("audio", &info).await.unwrap(); let object = two_groups(); - store.put_groups("audio", &object).await.unwrap(); + store.put_segments("audio", 0, &object).await.unwrap(); assert_eq!(store.get_info("audio").await.unwrap(), info); - assert_eq!(store.get_groups("audio", 5..=7).await.unwrap(), object); + assert_eq!(store.get_segments("audio", 0).await.unwrap(), object); } } diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs new file mode 100644 index 00000000000..e0e4cdf8e49 --- /dev/null +++ b/rs/moq-archive/src/writer.rs @@ -0,0 +1,1988 @@ +//! Record explicitly selected tracks of a [`broadcast::Consumer`] into a [`Store`]. +//! +//! The application enrolls each track through a [`Control`]; the writer never parses a catalog. +//! Every track has its own timeline: frames feed the track's segmenter as they arrive, and for +//! each closed record the writer stores the track's object, commits the record through the +//! track's own timeline encoder, and stores that record's timeline groups before committing the +//! track's next one. A timeline therefore only advertises durable objects, and no track waits for +//! another. +//! +//! A writer started on a prefix that already holds a recording resumes it: see [`Writer::new`]. +//! +//! ```no_run +//! # async fn example(source: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { +//! use moq_archive::object_store::memory::InMemory; +//! +//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +//! let writer = moq_archive::Writer::new(store, source, Default::default()).await?; +//! let control = writer.control(); +//! let media = moq_mux::timeline::Config::default(); +//! control.track("video", media.clone()).await?; +//! control.track("audio", media.clone()).await?; +//! // Sparse data: each catalog group is its own record. +//! control.track("catalog.json", media.with_duration_min(std::time::Duration::ZERO)).await?; +//! writer.run().await?; +//! # Ok(()) +//! # } +//! ``` + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::future::Future; +use std::ops::Range; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{FutureExt, StreamExt}; +use hang::timeline::{Position, Record}; +use moq_mux::timeline::{self, Segmenter}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStore; +use tokio::sync::{mpsc, watch}; +use tokio::time::Instant; + +use crate::recover::{Resume, checkpoint, recover}; +use crate::segment::{Frame, Group, Object}; +use crate::{Error, Info, Key, Result, Store}; + +/// Subscribers ask for every cached group; the publisher clamps this to its own max age. +const REPLAY: Duration = Duration::from_secs(u32::MAX as u64); + +/// How a [`Writer`] retains its recording. Each track's cutting is chosen when it enrolls. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct Config { + /// Expire old records (a DVR), or keep everything when `None` (an archive). + pub retention: Option, +} + +impl Config { + /// Set [`retention`](Self::retention). + pub fn with_retention(mut self, retention: impl Into>) -> Self { + self.retention = retention.into(); + self + } +} + +/// A DVR retention policy. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Retention { + /// Keep at least this much of each track's content, measured from its timeline records. + pub window: Duration, + /// How long expired objects outlive the timeline that stopped advertising them, so readers + /// holding an older timeline can finish their GETs. + pub grace: Duration, +} + +impl Retention { + /// Keep `window` of content and delete expired objects `grace` after they leave the timeline. + pub fn new(window: Duration, grace: Duration) -> Self { + Self { window, grace } + } +} + +/// Records enrolled tracks until the source broadcast closes. +/// +/// Enroll tracks through [`control`](Self::control), then drive the recording with +/// [`run`](Self::run). +pub struct Writer { + control: Control, + commands: mpsc::UnboundedReceiver>, + /// Deadlines for deleting expired or orphaned objects, oldest first. + deletions: VecDeque<(Instant, Expired)>, +} + +/// Enrolls, cuts, and removes tracks on a [`Writer`]. +pub struct Control { + shared: Arc>, + commands: mpsc::UnboundedSender>, +} + +impl Clone for Control { + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + commands: self.commands.clone(), + } + } +} + +struct Shared { + store: Store, + source: broadcast::Consumer, + retention: Option, + /// Owns the recording's timeline tracks, read back to store their groups. + timelines: broadcast::Producer, + /// Each track's retained timeline from a resumed recording, taken when it enrolls again. + recovered: Mutex>, + /// Every name ever enrolled. A name is never reused, so its records stay consecutive. + enrolled: Mutex>, + /// Enrollments between their first check and their command, so the recording does not end + /// under one that is about to arrive. + enrolling: watch::Sender, +} + +/// One [`Control::track`] in flight: drops its count, and its name unless it enrolled, however +/// the call ends. +struct Enrolling<'a, S> { + shared: &'a Shared, + name: &'a str, + enrolled: bool, +} + +impl Drop for Enrolling<'_, S> { + fn drop(&mut self) { + self.shared.enrolling.send_modify(|count| *count -= 1); + if !self.enrolled { + self.shared.enrolled.lock().unwrap().remove(self.name); + } + } +} + +enum Command { + Enroll(Box>), + Cut(Timestamp), + Remove(String), + Stop, +} + +impl Writer { + /// Start a recording under `store`'s prefix, reading tracks from `source`, or resume the one + /// already there. + /// + /// Resuming replays each track's retained timeline, and a re-enrolled track continues at its + /// next record, refusing anything at or before the end of its newest committed one. A source + /// whose group sequences restarted therefore needs a new prefix. Objects a crash left before + /// their commit are deleted now, since the resumed records reuse their keys. A DVR also + /// deletes, one grace period after recovery, every object its retained records do not + /// reference, such as interrupted expirations, and every timeline object older than the + /// checkpoint it recovered from. The writer must own the prefix exclusively. + /// Fails, deleting nothing, when the recording cannot be listed or a timeline cannot be + /// replayed. + pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { + let recovery = recover(&store, config.retention.is_some()).await?; + for key in &recovery.uncommitted { + store.delete(key).await?; + } + + let mut deletions = VecDeque::new(); + let stale = Expired { + objects: recovery.orphans, + timelines: recovery.stale, + }; + if let Some(retention) = &config.retention + && !stale.is_empty() + { + deletions.push_back((Instant::now() + retention.grace, stale)); + } + + let shared = Arc::new(Shared { + store, + source, + retention: config.retention, + timelines: broadcast::Info::new().produce(), + recovered: Mutex::new(recovery.tracks), + enrolled: Mutex::new(HashSet::new()), + enrolling: watch::Sender::new(0), + }); + let (commands, receiver) = mpsc::unbounded_channel(); + Ok(Self { + control: Control { shared, commands }, + commands: receiver, + deletions, + }) + } + + /// A handle that enrolls, cuts, and removes tracks, before or during [`run`](Self::run). + pub fn control(&self) -> Control { + self.control.clone() + } + + /// Record until the source broadcast closes and every enrolled track ends, or [`Control::stop`]. + /// + /// Then flush each track's final record and finish its timeline. Fails when a timeline cannot + /// be committed or stored: the recording stops at each track's last durable timeline object. + /// Also fails when an enrolled track delivers a frame the recording cannot represent, but only + /// after stopping every track and committing what each already reported, so the recording + /// stays readable up to the bad frame. Returns the source's error, after finishing, when the + /// broadcast aborted. + pub async fn run(self) -> Result<()> { + let Self { + control, + mut commands, + mut deletions, + } = self; + let shared = control.shared.clone(); + let source = shared.source.clone(); + let grace = shared.retention.as_ref().map(|retention| retention.grace); + let mut enrolling = shared.enrolling.subscribe(); + drop(control); + // Each timeline's oldest segment that may still be stored; pruning deletes upward from it. + let mut pruned = HashMap::new(); + + let mut tracks: HashMap>> = HashMap::new(); + let mut reads = FuturesUnordered::new(); + let mut commits: FuturesUnordered> = FuturesUnordered::new(); + let mut closed = false; + // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. + let mut accepting = true; + // Every track was stopped, by `Control::stop` or malformed input, so `run` ends once flushed. + let mut stopped = false; + // Malformed source input, returned once every track flushed. + let mut failed: Option = None; + // The tracks the last event touched. An event touches at most one, except a failure, so the + // loop never scans the whole table in steady state. + let mut touched: Vec = Vec::new(); + + loop { + for name in std::mem::take(&mut touched) { + let Some(track) = tracks.get_mut(&name) else { + continue; + }; + if let Some(commit) = track.commit()? { + commits.push(commit); + } + if track.finish() { + tracks.remove(&name); + } + } + + let idle = (closed || stopped) && tracks.is_empty() && commits.is_empty(); + if idle && *enrolling.borrow_and_update() == 0 { + // Refuse late commands, so an enrollment racing the end fails instead of vanishing. + commands.close(); + if !accepting { + break; + } + } + + let deadline = deletions.front().map(|(deadline, _)| *deadline); + tokio::select! { + biased; + command = commands.recv(), if accepting => match command { + None => accepting = false, + Some(Command::Enroll(mut track)) => { + let subscriber = track.subscriber.take().expect("an enrolling track carries its subscriber"); + // A stopped recording takes no new content; the track just finishes its timeline. + if stopped { + track.remove(); + } + reads.push(guard(track.cancelled.clone(), recv(track.name.clone(), subscriber)).boxed()); + touched.push(track.name.clone()); + tracks.insert(track.name.clone(), track); + } + Some(Command::Cut(pts)) => { + for track in tracks.values_mut() { + track.segmenter.cut(pts); + } + } + Some(Command::Remove(name)) => { + if let Some(track) = tracks.get_mut(&name) { + track.remove(); + touched.push(name); + } + } + Some(Command::Stop) => { + stopped = true; + touched.extend(stop(&mut tracks)); + commands.close(); + } + }, + Some(read) = reads.next(), if !reads.is_empty() => match handle(read, &mut tracks, &mut reads) { + Ok(name) => touched.extend(name), + Err(err) => { + tracing::error!(%err, "malformed source input; flushing every track and stopping"); + failed.get_or_insert(err); + stopped = true; + touched.extend(stop(&mut tracks)); + commands.close(); + } + }, + Some((name, committer, result)) = commits.next(), if !commits.is_empty() => { + if let Some(track) = tracks.get_mut(&name) { + track.committer = Some(committer); + touched.push(name); + } + let expired = result?; + if let Some(grace) = grace && !expired.is_empty() { + deletions.push_back((Instant::now() + grace, expired)); + } + } + _ = source.closed(), if !closed => closed = true, + // Also once commands closed: an enrollment in flight still settles its count. + _ = enrolling.changed(), if idle => {} + _ = tokio::time::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { + let (_, expired) = deletions.pop_front().unwrap(); + delete(&shared.store, &mut pruned, expired).await; + } + } + } + + // Complete expirations already committed to the timelines; nothing new expires after the + // final records. + for (deadline, expired) in deletions { + tokio::time::sleep_until(deadline).await; + delete(&shared.store, &mut pruned, expired).await; + } + + if let Some(err) = failed { + return Err(err); + } + if stopped { + return Ok(()); + } + // A broadcast end carries no cause. + source.closed().await; + Ok(()) + } +} + +impl Control { + /// Enroll the track `name`, its records cut by `config`. + /// + /// Media keeps the default minimum; sparse data such as a catalog uses zero, so each group is + /// stored as soon as it finishes. Subscribes to the track and creates its `.info` and its + /// timeline's before accepting any group. Fails when the name was already enrolled, names a + /// timeline, or `.info` conflicts. + pub async fn track(&self, name: &str, config: timeline::Config) -> Result<()> { + if name.ends_with(hang::timeline::SUFFIX) || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { + return Err(Error::Enrolled(name.to_string())); + } + self.shared.enrolling.send_modify(|count| *count += 1); + // Settles the count and the name even if this future is dropped mid-enrollment. + let mut enrolling = Enrolling { + shared: &self.shared, + name, + enrolled: false, + }; + self.subscribe(name, config).await?; + enrolling.enrolled = true; + Ok(()) + } + + /// Stop recording every track: store what already arrived, finish each timeline, and end + /// [`run`](Writer::run) without waiting for the source. Later commands are refused. + pub fn stop(&self) -> Result<()> { + self.send(Command::Stop) + } + + /// Declare a boundary at `pts` on every enrolled track; see [`Segmenter::cut`]. + pub fn cut(&self, pts: Timestamp) -> Result<()> { + self.send(Command::Cut(pts)) + } + + /// Stop recording `name`, storing what already arrived and dropping groups that could not be + /// indexed yet. The name cannot be enrolled again. + pub fn remove(&self, name: &str) -> Result<()> { + self.send(Command::Remove(name.to_string())) + } + + async fn subscribe(&self, name: &str, config: timeline::Config) -> Result<()> { + let shared = &self.shared; + let replay = track::Subscription::default().with_max_age(REPLAY); + let subscriber = shared + .source + .track(name) + .map_err(source_error)? + .subscribe(replay.clone()) + .await + .map_err(source_error)?; + let timescale = subscriber.info().timescale; + let info = Info::new(subscriber.info().priority, timescale.as_u64())?; + shared.store.put_info(name, &info).await?; + + let timeline = hang::timeline::default_name(name); + let output = shared + .timelines + .create_track(timeline.as_str(), timeline::Producer::info()) + .map_err(timeline_error_net)?; + let groups = shared + .timelines + .consume() + .track(&timeline) + .map_err(timeline_error_net)? + .subscribe(replay) + .await + .map_err(timeline_error_net)? + .ordered(); + let info = Info::new(groups.info().priority, groups.info().timescale.as_u64())?; + shared.store.put_info(&timeline, &info).await?; + + let resume = shared.recovered.lock().unwrap().remove(name); + let (output, segmenter, window, checkpoints, offset, floor) = match resume { + Some(resume) => { + let floor = resume.floor(); + let output = timeline::Producer::resume(output, &resume.checkpoint).map_err(timeline_error)?; + let segmenter = Segmenter::new(config).with_sequence(resume.checkpoint.range.end); + let window = resume.checkpoint.records.into(); + (output, segmenter, window, resume.checkpoints, resume.sequence, floor) + } + None => ( + timeline::Producer::new(output), + Segmenter::new(config), + VecDeque::new(), + VecDeque::new(), + 0, + None, + ), + }; + + let (cancel, cancelled) = watch::channel(()); + let track = Track { + name: name.to_string(), + timescale, + segmenter, + floor, + largest: None, + reported: None, + accepted: BTreeMap::new(), + frames: VecDeque::new(), + records: VecDeque::new(), + subscriber: Some(Box::new(subscriber)), + subscribed: true, + closed: false, + committer: Some(Committer { + shared: shared.clone(), + name: name.to_string(), + timeline, + output, + groups, + window, + checkpoints, + offset, + }), + cancel: Some(cancel), + cancelled, + }; + self.send(Command::Enroll(Box::new(track))) + } + + fn send(&self, command: Command) -> Result<()> { + self.commands.send(command).map_err(|_| Error::Closed) + } +} + +/// A group accepted from the source, not yet fully reported. +#[derive(Default)] +struct Incoming { + /// The index of the next frame to arrive. + next: u64, + /// Frames that arrived but were not reported, with their indices. + pending: Vec<(u64, Frame)>, + /// No more frames will arrive. + finished: bool, +} + +/// One enrolled track: its source reads, its segmenter, and its committer. +struct Track { + name: String, + timescale: Timescale, + segmenter: Segmenter, + /// A resumed track's committed end; earlier content is already stored. + floor: Option, + /// The newest accepted group; later groups must exceed it. + largest: Option, + /// The first-frame timestamp of the newest reported group; later groups must not precede it. + reported: Option, + /// Accepted groups in sequence order. Reported front first, so a later group's frames wait + /// for every earlier accepted group to finish. + accepted: BTreeMap, + /// Reported frames waiting for the record that holds them. + frames: VecDeque<(Position, Frame)>, + /// Closed records waiting to be committed, oldest first. + records: VecDeque, + /// Moved into the first read once enrolled. + subscriber: Option>, + /// The subscription is still delivering groups. + subscribed: bool, + /// The segmenter was closed: every remaining record is queued. + closed: bool, + /// `None` while a commit is in flight. + committer: Option>, + /// Dropping the sender cancels every read for this track. + cancel: Option>, + cancelled: watch::Receiver<()>, +} + +impl Track { + /// Report every frame the segmenter can place: the front group's arrivals, then each later + /// group once the ones before it finish. + /// + /// Fails on a group that starts before the previous one, which the timeline cannot place. + fn report(&mut self) -> Result<()> { + while let Some(mut entry) = self.accepted.first_entry() { + let sequence = *entry.key(); + let incoming = entry.get_mut(); + for (index, frame) in incoming.pending.drain(..) { + if index == 0 { + if let Some(reported) = self.reported + && frame.timestamp < reported + { + return Err(malformed( + &self.name, + sequence, + format!("timestamp {} precedes {reported}", frame.timestamp), + )); + } + self.reported = Some(frame.timestamp); + } + // Frame timestamps were validated while reading, so this conversion succeeds. + let pts = Timestamp::new(frame.timestamp, self.timescale).map_err(|_| Error::Id(frame.timestamp))?; + let position = Position::new(sequence, index); + self.segmenter + .frame(position, pts, index == 0) + .map_err(|err| malformed(&self.name, sequence, err))?; + self.frames.push_back((position, frame)); + } + if !incoming.finished { + break; + } + entry.remove(); + self.segmenter.finish_group(sequence); + } + Ok(()) + } + + /// Stop reading: drop the groups that were never reported and close the segmenter. + fn remove(&mut self) { + self.cancel = None; + self.subscribed = false; + self.accepted.clear(); + } + + /// Start committing the next closed record, if the committer is idle. + fn commit(&mut self) -> Result>> { + if !self.subscribed && self.accepted.is_empty() && !self.closed { + self.segmenter.close(); + self.closed = true; + } + self.records.extend(self.segmenter.by_ref()); + if self.committer.is_none() { + return Ok(None); + } + let Some(record) = self.records.pop_front() else { + return Ok(None); + }; + + let mut groups: Vec = Vec::new(); + while let Some((position, _)) = self.frames.front() + && *position < record.end + { + let (position, frame) = self.frames.pop_front().expect("checked above"); + match groups.last_mut() { + Some(group) if group.sequence == position.group => group.frames.push(frame), + _ => groups.push(Group { + sequence: position.group, + frames: vec![frame], + }), + } + } + let object = Object { + frame_start: record.start.frame, + groups, + }; + object.check_span(record.start, record.end)?; + + let committer = self.committer.take().expect("checked above"); + Ok(Some(committer.commit(record, object).boxed())) + } + + /// Finish the timeline once the track ended and every record is durable, returning whether the + /// track is done. + fn finish(&mut self) -> bool { + if !self.closed || !self.records.is_empty() { + return false; + } + let Some(committer) = self.committer.as_mut() else { + return false; + }; + if let Err(err) = committer.output.finish() { + tracing::warn!(track = %self.name, %err, "failed to finish the timeline"); + } + true + } +} + +// Each read is already boxed as a future, so the variant sizes cost nothing extra. +#[allow(clippy::large_enum_variant)] +enum Read { + Group { + name: String, + subscriber: Box, + result: moq_net::Result>, + }, + Frame { + name: String, + group: Box, + result: moq_net::Result>, + }, + Cancelled, +} + +async fn recv(name: String, mut subscriber: Box) -> Read { + let result = subscriber.recv_group().await; + Read::Group { + name, + subscriber, + result, + } +} + +async fn frame(name: String, mut group: Box) -> Read { + let result = group.read_frame().await; + Read::Frame { name, group, result } +} + +/// Stop reading every track, returning their names so each flushes and finishes. +fn stop(tracks: &mut HashMap>>) -> Vec { + tracks + .iter_mut() + .map(|(name, track)| { + track.remove(); + name.clone() + }) + .collect() +} + +/// Resolve to [`Read::Cancelled`] once the track stops reading. +async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { + tokio::select! { + read = read => read, + _ = cancelled.changed() => Read::Cancelled, + } +} + +/// Returns the track that may have records to commit or be finished. +/// +/// Fails on malformed source input; a group the network aborted keeps the frames that arrived. +fn handle( + read: Read, + tracks: &mut HashMap>>, + reads: &mut FuturesUnordered>, +) -> Result> { + match read { + Read::Cancelled => Ok(None), + Read::Group { + name, + subscriber, + result, + } => { + let Some(track) = tracks.get_mut(&name) else { + return Ok(None); + }; + match result { + Ok(Some(group)) => { + let sequence = group.sequence; + let stored = track.floor.is_some_and(|floor| Position::group(sequence + 1) <= floor); + if track.largest.is_some_and(|largest| sequence <= largest) || stored { + tracing::debug!(track = %name, sequence, "refusing a duplicate, decreasing, or stored group"); + } else { + crate::path::check_id(sequence).map_err(|err| malformed(&name, sequence, err))?; + track.largest = Some(sequence); + track.accepted.insert(sequence, Incoming::default()); + reads.push(guard(track.cancelled.clone(), frame(name.clone(), Box::new(group))).boxed()); + } + reads.push(guard(track.cancelled.clone(), recv(name, subscriber)).boxed()); + // An accepted group holds no frames yet. + Ok(None) + } + Ok(None) => { + track.subscribed = false; + Ok(Some(name)) + } + Err(err) => { + tracing::warn!(track = %name, %err, "track ended"); + track.subscribed = false; + Ok(Some(name)) + } + } + } + Read::Frame { name, group, result } => { + let Some(track) = tracks.get_mut(&name) else { + return Ok(None); + }; + let sequence = group.sequence; + let Some(incoming) = track.accepted.get_mut(&sequence) else { + return Ok(None); + }; + let touched = name.clone(); + match result { + Ok(Some(frame)) => { + let index = incoming.next; + incoming.next += 1; + let timestamp = frame + .timestamp + .convert(track.timescale) + .map_err(|_| malformed(&name, sequence, Error::Overflow))? + .value(); + crate::path::check_id(timestamp).map_err(|err| malformed(&name, sequence, err))?; + // A resumed track already stored the head of the group it stopped inside. + if track.floor.is_none_or(|floor| Position::new(sequence, index) >= floor) { + let frame = Frame { + timestamp, + payload: frame.payload, + }; + incoming.pending.push((index, frame)); + } + reads.push(guard(track.cancelled.clone(), self::frame(name, group)).boxed()); + } + Ok(None) => incoming.finished = true, + Err(err) => { + tracing::warn!(track = %name, sequence, %err, "keeping the frames of an incomplete group"); + incoming.finished = true; + } + } + track.report()?; + Ok(Some(touched)) + } + } +} + +/// An in-flight record commit, returning the track, its committer, and the expired objects. +type Commit = BoxFuture<'static, (String, Committer, Result)>; + +/// Objects the timelines stopped needing, deleted once the grace elapses. +#[derive(Default)] +struct Expired { + /// Media objects no retained record references. + objects: Vec, + /// Timeline segments no recovery reads, by timeline. + timelines: Vec<(String, Range)>, +} + +impl Expired { + fn is_empty(&self) -> bool { + self.objects.is_empty() && self.timelines.is_empty() + } +} + +/// Commits one track's records in order: the object, the record, retention, then the timeline +/// object. +struct Committer { + shared: Arc>, + name: String, + /// The track's timeline track. + timeline: String, + output: timeline::Producer, + /// The timeline track read back, to store its complete groups. + groups: track::Ordered, + /// Committed records still in the timeline window, oldest first. + window: VecDeque, + /// A DVR's stored timeline segments that recovery still reads, oldest first, each with the + /// first record index its opening checkpoint restates. + checkpoints: VecDeque<(u64, u64)>, + /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. + offset: u64, +} + +impl Committer { + /// Commit `record` with its `object`. + async fn commit(mut self, record: Record, object: Object) -> (String, Self, Result) { + let result = self.commit_inner(record, object).await; + (self.name.clone(), self, result) + } + + async fn commit_inner(&mut self, mut record: Record, object: Object) -> Result { + let shared = self.shared.clone(); + let store = &shared.store; + + // Number by the window rather than the segmenter, so a record dropped below leaves no hole. + record.sequence = self.output.range().end; + if let Err(err) = store.put_segments(&self.name, record.sequence, &object).await { + // The timeline only advertises durable objects, so this span is lost. The key may have + // landed anyway; clear it so the next record can take it. + tracing::warn!(track = %self.name, sequence = record.sequence, %err, "dropping a record that was not stored"); + let _ = store.delete(&Key::segments(self.name.clone(), record.sequence)?).await; + return Ok(Expired::default()); + } + + self.output.push(&record).map_err(timeline_error)?; + let pts = Timestamp::new(record.pts, TIMESCALE).map_err(|_| Error::Id(record.pts))?; + self.window.push_back(record.clone()); + + let expired = self.expire(); + if !expired.is_empty() { + self.output.pop(expired.len() as u64).map_err(timeline_error)?; + } + self.output.flush().map_err(timeline_error)?; + + let object = self.read_timeline(pts)?; + store.put_segments(&self.timeline, record.sequence, &object).await?; + + let objects = expired + .iter() + .map(|record| Key::segments(self.name.clone(), record.sequence)) + .collect::>()?; + let segments = self.prune(record.sequence, &object)?; + let timelines = match segments.is_empty() { + true => Vec::new(), + false => vec![(self.timeline.clone(), segments)], + }; + Ok(Expired { objects, timelines }) + } + + /// Track the newly stored timeline `object`, returning the older segments no recovery reads. + /// + /// Recovery walks back from the newest object to the first whose checkpoint restates the + /// newest one's offset, so every object before that one is no longer needed. + fn prune(&mut self, segment: u64, object: &Object) -> Result> { + if self.shared.retention.is_none() { + return Ok(segment..segment); + } + let (offset, start) = checkpoint(object)?; + self.checkpoints.push_back((segment, start)); + // Checkpoint starts never decrease, so a binary search finds the last one at or before + // `offset` without walking a long DVR window on every commit. + let keep = self + .checkpoints + .partition_point(|&(_, start)| start <= offset) + .saturating_sub(1); + let first = self.checkpoints[0].0; + self.checkpoints.drain(..keep); + Ok(first..self.checkpoints[0].0) + } + + /// Pop the oldest records while the rest still cover the retention window. The newest record + /// always stays, such as a catalog that never changes. + fn expire(&mut self) -> Vec { + let Some(retention) = self.shared.retention.as_ref().map(|retention| retention.window) else { + return Vec::new(); + }; + let target = (retention.as_micros() * TIMESCALE.as_u64() as u128).div_ceil(1_000_000); + let mut expired = Vec::new(); + while self.window.len() > 1 { + let newest = self.window.back().expect("non-empty"); + let end = newest.pts as u128 + newest.duration as u128; + if end.saturating_sub(self.window[1].pts as u128) < target { + break; + } + expired.push(self.window.pop_front().expect("non-empty")); + } + expired + } + + /// Collect the timeline groups completed since the last record, stamped at `pts`. + /// + /// The live timeline track stamps frames with the wall clock; storing the record's content time + /// instead keeps a recording's bytes a function of its content alone. + fn read_timeline(&mut self, pts: Timestamp) -> Result { + let waiter = kio::Waiter::noop(); + let timescale = self.groups.info().timescale; + let mut groups = Vec::new(); + while let Poll::Ready(result) = self.groups.poll_next_group(&waiter) { + let Some(mut group) = result.map_err(timeline_error_net)? else { + break; + }; + let mut frames = Vec::new(); + loop { + match group.poll_read_frame(&waiter) { + Poll::Ready(Ok(Some(frame))) => { + let timestamp = pts.convert(timescale).map_err(|_| Error::Overflow)?.value(); + frames.push(Frame { + timestamp: crate::path::check_id(timestamp)?, + payload: frame.payload, + }); + } + Poll::Ready(Ok(None)) => break, + Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), + // Flushing closed every group, so an open one is a bug. + Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), + } + } + let sequence = group.sequence.checked_add(self.offset).ok_or(Error::Overflow)?; + groups.push(Group { + sequence: crate::path::check_id(sequence)?, + frames, + }); + } + Ok(Object::new(groups)) + } +} + +/// The timescale of record `pts` and `duration`, as the catalog section advertises it. +const TIMESCALE: Timescale = Timescale::MILLI; + +/// Delete `expired`'s media objects, and each timeline's objects from its `pruned` cursor up to +/// its expired segments' end. +/// +/// Timeline objects go oldest first and stop at the first failure, so the stored segments stay +/// consecutive, as recovery requires, and a later deletion retries from there. +async fn delete(store: &Store, pruned: &mut HashMap, expired: Expired) { + let objects = async { + let deletes = expired + .objects + .iter() + .map(|key| async move { (key, store.delete(key).await) }); + for (key, result) in futures::future::join_all(deletes).await { + // An object that outlives its expiry advertises nothing; recovery cleans it up. + if let Err(err) = result { + tracing::warn!(?key, %err, "failed to delete an expired object"); + } + } + }; + let timelines = async { + for (timeline, segments) in expired.timelines { + let cursor = pruned.entry(timeline.clone()).or_insert(segments.start); + while *cursor < segments.end { + let result = match Key::segments(timeline.clone(), *cursor) { + Ok(key) => store.delete(&key).await, + Err(err) => Err(err), + }; + match result { + Ok(()) | Err(Error::NotFound(_)) => *cursor += 1, + Err(err) => { + tracing::warn!(%timeline, segment = *cursor, %err, "failed to delete a pruned timeline object"); + break; + } + } + } + } + }; + futures::join!(objects, timelines); +} + +fn timeline_error(err: moq_mux::Error) -> Error { + Error::Timeline(err.to_string()) +} + +fn timeline_error_net(err: moq_net::Error) -> Error { + Error::Timeline(err.to_string()) +} + +fn source_error(err: moq_net::Error) -> Error { + Error::Source(err.to_string()) +} + +fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { + Error::Source(format!("track {track} group {sequence}: {err}")) +} + +#[cfg(test)] +mod tests { + use futures::TryStreamExt; + use object_store::memory::InMemory; + + use super::*; + use crate::mock::Mock; + use crate::store::list::Query; + + fn ms(v: u64) -> Timestamp { + Timestamp::from_millis(v).unwrap() + } + + /// Media cut at 1s, so one-second test groups get a record each. + fn media() -> timeline::Config { + timeline::Config::default().with_duration_min(Duration::from_secs(1)) + } + + /// Sparse data: every group is its own record. + fn sparse() -> timeline::Config { + timeline::Config::default().with_duration_min(Duration::ZERO) + } + + /// An append-only log, split every three seconds. + fn append_only() -> timeline::Config { + sparse().with_duration_max(Duration::from_secs(3)) + } + + fn timeline(track: &str) -> String { + hang::timeline::default_name(track) + } + + /// A millisecond track that keeps an hour of history, so enrollment sees every group. + fn track(broadcast: &broadcast::Producer, name: &str) -> track::Producer { + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + broadcast.create_track(name, info).unwrap() + } + + /// Write one complete group with a frame at each timestamp. + fn group(track: &track::Producer, sequence: u64, timestamps: &[u64]) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for ×tamp in timestamps { + group + .write_frame(ms(timestamp), format!("{sequence}@{timestamp}")) + .unwrap(); + } + group.finish().unwrap(); + } + + /// The stored timeline segments of `track`, ascending. + async fn segments(store: &Store, track: &str) -> Vec { + let entries: Vec<_> = store + .list(&Query::segments(&timeline(track)).unwrap()) + .try_collect() + .await + .unwrap(); + let mut segments: Vec = entries + .into_iter() + .map(|entry| match entry.key { + Key::Segments { segment, .. } => segment, + key => panic!("unexpected {key:?}"), + }) + .collect(); + segments.sort(); + segments + } + + /// Replay every stored timeline object of `track`, returning the records still in its window. + async fn window(store: &Store, track: &str) -> Vec { + let timeline = timeline(track); + let segments = segments(store, track).await; + if let (Some(first), Some(last)) = (segments.first(), segments.last()) { + assert_eq!( + segments, + (*first..=*last).collect::>(), + "timeline objects are consecutive" + ); + } + + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + let mut records = BTreeMap::new(); + for segment in segments { + let object = store.get_segments(&timeline, segment).await.unwrap(); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } + } + while let Some(event) = decoder.next_event() { + match event { + moq_json::window::Event::Push { index, value } => { + assert_eq!(index, value.sequence); + records.insert(index, value); + } + moq_json::window::Event::Pop(range) | moq_json::window::Event::Skip(range) => { + for index in range { + records.remove(&index); + } + } + _ => unreachable!(), + } + } + } + records.into_values().collect() + } + + /// Each record's groups, as `(first, last)`. + fn groups(records: &[Record]) -> Vec<(u64, u64)> { + records + .iter() + .map(|record| (*record.groups().start(), *record.groups().end())) + .collect() + } + + fn sequences(records: &[Record]) -> Vec { + records.iter().map(|record| record.sequence).collect() + } + + /// Every record resolves to one stored object holding exactly its frames. + async fn check_objects(store: &Store, track: &str, records: &[Record]) { + for record in records { + let object = store.get_segments(track, record.sequence).await.unwrap(); + object.check_span(record.start, record.end).unwrap(); + } + } + + /// Every stored media object, across all tracks. + async fn stored(store: &Store) -> HashSet { + store + .list(&Query::new()) + .try_filter_map(|entry| async move { + let media = + matches!(&entry.key, Key::Segments { track, .. } if !track.ends_with(hang::timeline::SUFFIX)); + Ok(media.then_some(entry.key)) + }) + .try_collect() + .await + .unwrap() + } + + /// The media objects the retained records reference. + fn referenced(track: &str, records: &[Record]) -> HashSet { + records + .iter() + .map(|record| Key::segments(track, record.sequence).unwrap()) + .collect() + } + + #[tokio::test] + async fn records_selected_tracks() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let catalog = track(&source, "catalog.json"); + let _ignored = track(&source, "ignored"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + control.track("catalog.json", sparse()).await.unwrap(); + + group(&catalog, 0, &[0]); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + catalog.finish().unwrap(); + source.close(); + + tokio::spawn(writer.run()).await.unwrap().unwrap(); + + let records = window(&store, "video").await; + assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, "video", &records).await; + let catalog = window(&store, "catalog.json").await; + assert_eq!(groups(&catalog), vec![(0, 0)]); + check_objects(&store, "catalog.json", &catalog).await; + assert!(window(&store, "ignored").await.is_empty()); + + let object = store.get_segments("video", 2).await.unwrap(); + assert_eq!(object.groups[0].frames[1].timestamp, 2500); + assert_eq!(object.groups[0].frames[1].payload, "2@2500"); + assert_eq!(store.get_info("video").await.unwrap(), Info::new(127, 1000).unwrap()); + store.get_info(&timeline("video")).await.unwrap(); + assert!(store.get_info("ignored").await.is_err()); + } + + #[tokio::test] + async fn tracks_cut_independently() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", timeline::Config::default()).await.unwrap(); + control.track("audio", timeline::Config::default()).await.unwrap(); + + // Two second GOPs, 400ms audio groups: with the default 2s minimum, audio packs by its own + // boundaries, not video's GOPs. + for sequence in 0..3 { + group(&video, sequence, &[sequence * 2000, sequence * 2000 + 1000]); + } + for sequence in 0..15 { + group(&audio, sequence, &[sequence * 400]); + } + video.finish().unwrap(); + audio.finish().unwrap(); + source.close(); + writer.run().await.unwrap(); + + let video = window(&store, "video").await; + assert_eq!(groups(&video), vec![(0, 0), (1, 1), (2, 2)]); + let audio = window(&store, "audio").await; + assert_eq!(groups(&audio), vec![(0, 4), (5, 9), (10, 14)]); + check_objects(&store, "audio", &audio).await; + } + + #[tokio::test] + async fn an_append_only_group_spans_several_objects() { + let source = broadcast::Info::new().produce(); + let log = track(&source, "log"); + + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default(); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("log", append_only()).await.unwrap(); + let run = tokio::spawn(writer.run()); + + // One group that never closes, a frame a second. + let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); + for second in 0..7 { + open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); + } + // The group is still open, yet its first six seconds are already durable. + while window(&store, "log").await.len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + let records = window(&store, "log").await; + assert_eq!( + records + .iter() + .map(|record| (record.start, record.end)) + .collect::>(), + vec![ + (Position::new(0, 0), Position::new(0, 3)), + (Position::new(0, 3), Position::new(0, 6)), + ] + ); + check_objects(&store, "log", &records).await; + assert_eq!(store.get_segments("log", 1).await.unwrap().frame_start, 3); + + open.finish().unwrap(); + log.finish().unwrap(); + source.close(); + run.await.unwrap().unwrap(); + + let records = window(&store, "log").await; + assert_eq!(records.last().unwrap().start, Position::new(0, 6)); + assert_eq!(records.last().unwrap().end, Position::group(1)); + check_objects(&store, "log", &records).await; + } + + #[tokio::test] + async fn a_failed_put_omits_only_that_track() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let mock = Mock::memory(); + mock.fail_puts("/audio/segments/"); + let store = Store::new(mock, "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + control.track("audio", media()).await.unwrap(); + + for sequence in 0..3 { + group(&video, sequence, &[sequence * 1000]); + group(&audio, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + audio.finish().unwrap(); + source.close(); + + writer.run().await.unwrap(); + + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); + check_objects(&store, "video", &records).await; + assert!(window(&store, "audio").await.is_empty()); + } + + #[tokio::test] + async fn retention_pops_then_deletes_expired_objects() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let catalog = track(&source, "catalog.json"); + + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + writer.control().track("catalog.json", sparse()).await.unwrap(); + + // A static catalog published once, then six seconds of video. + group(&catalog, 0, &[0]); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + catalog.finish().unwrap(); + source.close(); + + writer.run().await.unwrap(); + + // Records 3 and 4 hold two seconds; the partial final record 5 is always kept. + let records = window(&store, "video").await; + assert_eq!(sequences(&records), vec![3, 4, 5]); + check_objects(&store, "video", &records).await; + + // The catalog's newest record outlives the video it was published with. + let catalog = window(&store, "catalog.json").await; + assert_eq!(sequences(&catalog), vec![0]); + check_objects(&store, "catalog.json", &catalog).await; + + let expected = &referenced("video", &records) | &referenced("catalog.json", &catalog); + assert_eq!(stored(&store).await, expected); + // Every checkpoint restates the whole window, so only the newest timeline object is needed. + assert_eq!(segments(&store, "video").await, vec![5]); + assert_eq!(segments(&store, "catalog.json").await, vec![0]); + } + + #[tokio::test] + async fn commands_are_refused_once_the_recording_ends() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + // A long grace keeps `run` waiting on deletions after the timeline finishes. + let config = + Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + source.close(); + + let run = tokio::spawn(writer.run()); + while window(&store, "video").await.last().map(|record| record.sequence) != Some(5) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!run.is_finished()); + assert_eq!(control.remove("video"), Err(Error::Closed)); + run.abort(); + } + + #[tokio::test] + async fn accepted_groups_may_complete_out_of_order() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().track("video", media()).await.unwrap(); + let run = tokio::spawn(writer.run()); + + let mut first = video.create_group(group::Info { sequence: 0 }).unwrap(); + first.write_frame(ms(0), "0@0").unwrap(); + group(&video, 1, &[1000]); + group(&video, 2, &[2000]); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(window(&store, "video").await.is_empty(), "group 1 waits for group 0"); + + first.write_frame(ms(500), "0@500").unwrap(); + first.finish().unwrap(); + video.finish().unwrap(); + source.close(); + run.await.unwrap().unwrap(); + + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); + check_objects(&store, "video", &records).await; + let object = store.get_segments("video", 0).await.unwrap(); + let frames: Vec<_> = object.groups[0].frames.iter().map(|f| f.timestamp).collect(); + assert_eq!(frames, vec![0, 500]); + } + + #[tokio::test] + async fn decreasing_arrivals_are_refused() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().track("video", media()).await.unwrap(); + + group(&video, 0, &[0]); + group(&video, 2, &[1000]); + group(&video, 1, &[500]); + group(&video, 3, &[2000]); + video.finish().unwrap(); + source.close(); + + writer.run().await.unwrap(); + + // The skipped sequence closes a record, so no record holds a gap. + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (2, 2), (3, 3)]); + check_objects(&store, "video", &records).await; + } + + #[tokio::test] + async fn an_unrepresentable_group_fails_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + + group(&video, 0, &[0]); + // One past the recording's largest group ID. + group(&video, 1 << 53, &[1000]); + video.finish().unwrap(); + source.close(); + + assert_eq!( + writer.run().await, + Err(Error::Source(format!( + "track video group {}: {}", + 1u64 << 53, + Error::Id(1 << 53) + ))) + ); + } + + #[tokio::test] + async fn a_decreasing_timestamp_fails_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + + group(&video, 0, &[1000]); + group(&video, 1, &[500]); + video.finish().unwrap(); + source.close(); + + assert_eq!( + writer.run().await, + Err(Error::Source("track video group 1: timestamp 500 precedes 1000".into())) + ); + } + + /// A malformed group fails the recording only after committing everything reported before + /// it, including the open record, so the archive stays readable up to the bad frame. + #[tokio::test] + async fn a_failed_recording_flushes_what_was_reported() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().track("video", media()).await.unwrap(); + + group(&video, 0, &[0]); + group(&video, 1, &[1000]); + // Under the minimum, so groups 1 and 2 share the record still open at the failure. + group(&video, 2, &[1500]); + group(&video, 3, &[1200]); + + assert_eq!( + writer.run().await, + Err(Error::Source( + "track video group 3: timestamp 1200 precedes 1500".into() + )) + ); + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(0, 0), (1, 2)]); + check_objects(&store, "video", &records).await; + } + + /// Stopping a recording whose source is still live finishes every timeline and ends `run`. + #[tokio::test] + async fn stopping_ends_without_the_source() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + group(&video, 0, &[0]); + control.stop().unwrap(); + + writer.run().await.unwrap(); + let records = window(&store, "video").await; + check_objects(&store, "video", &records).await; + assert!( + control.track("audio", media()).await.is_err(), + "a stopped recording refuses enrollment" + ); + drop(source); + } + + #[tokio::test] + async fn a_stalled_track_holds_only_itself() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + control.track("audio", media()).await.unwrap(); + let run = tokio::spawn(writer.run()); + + group(&audio, 0, &[0]); + // The audio group never completes, so its record never closes on its own. + let mut stalled = audio.create_group(group::Info { sequence: 1 }).unwrap(); + stalled.write_frame(ms(1000), "stalled").unwrap(); + group(&audio, 2, &[2000]); + for sequence in 0..4 { + group(&video, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + source.close(); + + // Video commits without waiting for the stalled audio. + while window(&store, "video").await.len() < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!run.is_finished(), "a stalled track holds the recording open"); + + control.remove("audio").unwrap(); + run.await.unwrap().unwrap(); + + // What arrived of the stalled group is stored; the group queued behind it is dropped. + let records = window(&store, "audio").await; + assert_eq!( + records + .iter() + .map(|record| (record.start, record.end)) + .collect::>(), + vec![ + (Position::group(0), Position::group(1)), + (Position::group(1), Position::new(1, 1)) + ] + ); + check_objects(&store, "audio", &records).await; + drop(stalled); + } + + #[tokio::test] + async fn enrollment_is_refused_for_duplicates_and_timelines() { + let source = broadcast::Info::new().produce(); + let _video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + control.track("video", media()).await.unwrap(); + assert_eq!( + control.track("video", sparse()).await, + Err(Error::Enrolled("video".into())) + ); + let timeline = timeline("video"); + assert_eq!(control.track(&timeline, media()).await, Err(Error::Enrolled(timeline))); + } + + #[tokio::test] + async fn a_conflicting_info_fails_enrollment() { + let source = broadcast::Info::new().produce(); + let _video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + store.put_info("video", &Info::new(7, 1000).unwrap()).await.unwrap(); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + assert_eq!( + control.track("video", media()).await, + Err(Error::Priority { + existing: 7, + intended: 127 + }) + ); + } + + /// Record `video` groups `sequences`, one per second, until the source ends. + async fn record(store: &Store, config: Config, sequences: std::ops::Range) { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + for sequence in sequences { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.close(); + writer.run().await.unwrap(); + } + + fn orphan(sequence: u64) -> Object { + Object::new(vec![Group { + sequence, + frames: vec![Frame { + timestamp: sequence * 1000, + payload: "orphan".into(), + }], + }]) + } + + #[tokio::test] + async fn a_restarted_writer_resumes_the_recording() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..3).await; + // The source's cache replays groups the recording already holds. + record(&store, Config::default(), 0..6).await; + + let records = window(&store, "video").await; + assert_eq!(sequences(&records), (0..6).collect::>()); + assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, "video", &records).await; + + // The resumed timeline groups continue the stored numbering. + let mut numbers = Vec::new(); + for segment in 0..6 { + let object = store.get_segments(&timeline("video"), segment).await.unwrap(); + numbers.extend(object.groups.iter().map(|group| group.sequence)); + } + assert!(numbers.windows(2).all(|pair| pair[0] < pair[1]), "{numbers:?}"); + } + + #[tokio::test] + async fn a_resumed_append_only_group_continues_mid_group() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default(); + + // The first run stores frames 0..6 of a group that never closes, then stops. + let source = broadcast::Info::new().produce(); + let log = track(&source, "log"); + let writer = Writer::new(store.clone(), source.consume(), config.clone()) + .await + .unwrap(); + writer.control().track("log", append_only()).await.unwrap(); + let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); + for second in 0..7 { + open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); + } + let run = tokio::spawn(writer.run()); + while window(&store, "log").await.len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + run.abort(); + let _ = run.await; + drop(open); + + // The restarted source replays the whole group; the stored head is skipped. + let source = broadcast::Info::new().produce(); + let log = track(&source, "log"); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("log", append_only()).await.unwrap(); + let mut replayed = log.create_group(group::Info { sequence: 0 }).unwrap(); + for second in 0..8 { + replayed + .write_frame(ms(second * 1000), format!("line {second}")) + .unwrap(); + } + replayed.finish().unwrap(); + log.finish().unwrap(); + source.close(); + writer.run().await.unwrap(); + + let records = window(&store, "log").await; + assert_eq!( + records + .iter() + .map(|record| (record.start, record.end)) + .collect::>(), + vec![ + (Position::new(0, 0), Position::new(0, 3)), + (Position::new(0, 3), Position::new(0, 6)), + (Position::new(0, 6), Position::group(1)), + ] + ); + check_objects(&store, "log", &records).await; + } + + #[tokio::test] + async fn a_restarted_dvr_deletes_unreferenced_objects() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + record(&store, config.clone(), 0..6).await; + + // An interrupted expiration, an uncommitted upload, and a track no timeline names. + store.put_segments("video", 1, &orphan(1)).await.unwrap(); + store.put_segments("video", 6, &orphan(7)).await.unwrap(); + store.put_segments("audio", 0, &orphan(0)).await.unwrap(); + + record(&store, config, 6..10).await; + + let records = window(&store, "video").await; + assert_eq!(sequences(&records), vec![7, 8, 9]); + assert_eq!(groups(&records), vec![(7, 7), (8, 8), (9, 9)]); + check_objects(&store, "video", &records).await; + assert_eq!(stored(&store).await, referenced("video", &records)); + store.get_info("video").await.unwrap(); + store.get_info(&timeline("video")).await.unwrap(); + assert_eq!(segments(&store, "video").await, vec![9]); + } + + #[tokio::test] + async fn a_dvr_crash_between_pop_and_delete_is_cleaned_on_restart() { + let store = Store::new(InMemory::new(), "rec"); + + // A grace longer than the test keeps every expired object past the crash. + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let config = + Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + let run = tokio::spawn(writer.run()); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + // Record 5 stays open, so the newest durable timeline object is record 4. + while window(&store, "video").await.last().map(|record| record.sequence) != Some(4) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + run.abort(); + let _ = run.await; + + let retained = window(&store, "video").await; + assert_eq!(sequences(&retained), vec![3, 4]); + let expired: HashSet<_> = (0..3).map(|s| Key::segments("video", s).unwrap()).collect(); + assert_eq!(stored(&store).await, &referenced("video", &retained) | &expired); + assert_eq!(segments(&store, "video").await, (0..=4).collect::>()); + // An upload the crash left uncommitted. + store.put_segments("video", 5, &orphan(5)).await.unwrap(); + + let grace = Duration::from_millis(200); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let started = Instant::now(); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + assert!( + stored(&store).await.is_superset(&expired), + "expired objects outlive the grace, for readers holding the old timeline" + ); + assert!( + !stored(&store).await.contains(&Key::segments("video", 5).unwrap()), + "the uncommitted upload is cleared before its key is reused" + ); + assert_eq!(segments(&store, "video").await, (0..=4).collect::>()); + + writer.control().track("video", media()).await.unwrap(); + // The source replays groups the recording already holds; they are refused. + for sequence in 4..9 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.close(); + writer.run().await.unwrap(); + assert!(started.elapsed() >= grace); + + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(6, 6), (7, 7), (8, 8)]); + check_objects(&store, "video", &records).await; + assert_eq!(stored(&store).await, referenced("video", &records)); + assert_eq!(segments(&store, "video").await, vec![8]); + } + + #[tokio::test] + async fn an_archive_restart_clears_uncommitted_objects() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..3).await; + // Media stored after the last timeline commit: a crash before its record. + store.put_segments("video", 3, &orphan(3)).await.unwrap(); + + record(&store, Config::default(), 3..6).await; + let records = window(&store, "video").await; + assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, "video", &records).await; + assert_eq!(stored(&store).await, referenced("video", &records)); + } + + #[tokio::test] + async fn a_failed_recovery_deletes_nothing() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + record(&store, config.clone(), 0..6).await; + store.put_segments("video", 1, &orphan(1)).await.unwrap(); + let before = stored(&store).await; + + let failing = mock.fork(); + failing.fail_lists(); + let source = broadcast::Info::new().produce(); + let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; + assert!(matches!(result, Err(Error::Store(_)))); + + // A gap in the timeline keys leaves the retained window unrecoverable. + let newest = store.get_segments(&timeline("video"), 5).await.unwrap(); + store.put_segments(&timeline("video"), 3, &newest).await.unwrap(); + let result = Writer::new(store.clone(), source.consume(), config).await; + assert!(matches!(result, Err(Error::Timeline(_)))); + assert_eq!(segments(&store, "video").await, vec![3, 5]); + + assert_eq!(stored(&store).await, before); + } + + #[tokio::test] + async fn a_dvr_window_longer_than_one_checkpoint_is_recovered() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(280), Duration::ZERO)); + record(&store, config, 0..300).await; + + let recovery = recover(&store, true).await.unwrap(); + let resume = &recovery.tracks["video"]; + let records = window(&store, "video").await; + assert!(records.len() > 256, "the window outgrows one checkpoint"); + assert_eq!(resume.checkpoint.records, records); + assert_eq!(resume.checkpoint.range.end, 300); + assert!(recovery.orphans.is_empty()); + assert!(recovery.uncommitted.is_empty()); + assert_eq!(resume.floor(), Some(Position::group(300))); + + // The oldest stored timeline object is exactly the one recovery starts from. + let stored = segments(&store, "video").await; + assert!(stored.len() > 1, "the newest checkpoint omits part of the window"); + assert!(stored[0] > 0, "earlier timeline objects are pruned"); + assert_eq!(resume.checkpoints.front().unwrap().0, stored[0]); + assert!(recovery.stale.is_empty()); + } + + /// Yield to the writer until `check` holds. + async fn until>(mut check: impl FnMut() -> F) { + while !check().await { + tokio::task::yield_now().await; + } + } + + #[tokio::test(start_paused = true)] + async fn a_dvr_prunes_timeline_objects_one_grace_after_they_are_unneeded() { + let store = Store::new(InMemory::new(), "rec"); + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let grace = Duration::from_secs(10); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().track("video", media()).await.unwrap(); + let run = tokio::spawn(writer.run()); + + for sequence in 0..4 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + // Record 3 stays open, so record 2 is the newest timeline object. + until(|| async { segments(&store, "video").await.last() == Some(&2) }).await; + assert_eq!( + segments(&store, "video").await, + vec![0, 1, 2], + "unneeded timeline objects outlive the grace, for readers that listed them" + ); + + tokio::time::sleep(grace).await; + until(|| async { segments(&store, "video").await == [2] }).await; + + for sequence in 4..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.close(); + run.await.unwrap().unwrap(); + + assert_eq!(segments(&store, "video").await, vec![5]); + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(3, 3), (4, 4), (5, 5)]); + check_objects(&store, "video", &records).await; + let recovery = recover(&store, true).await.unwrap(); + assert_eq!(recovery.tracks["video"].checkpoint.records, records); + } + + #[tokio::test] + async fn a_failed_timeline_delete_keeps_the_keys_consecutive() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let failed = Key::segments(timeline("video"), 1).unwrap(); + mock.fail_deletes(store.path(&failed).unwrap().as_ref()); + record(&store, config.clone(), 0..6).await; + + // Pruning stops at the failed object instead of deleting past it. + assert_eq!(segments(&store, "video").await, (1..=5).collect::>()); + window(&store, "video").await; + + // A restart retries from the oldest stored object. + mock.heal(); + record(&store, config, 6..8).await; + assert_eq!(segments(&store, "video").await, vec![7]); + let records = window(&store, "video").await; + assert_eq!(groups(&records), vec![(5, 5), (6, 6), (7, 7)]); + check_objects(&store, "video", &records).await; + } + + #[tokio::test] + async fn a_timeline_that_does_not_end_at_the_next_segment_fails_recovery() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..6).await; + let before = stored(&store).await; + + // `segments/5` still decodes, but it restates an earlier window, so resuming would write the + // next record on top of it. + let older = store.get_segments(&timeline("video"), 0).await.unwrap(); + store + .delete(&Key::segments(timeline("video"), 5).unwrap()) + .await + .unwrap(); + store.put_segments(&timeline("video"), 5, &older).await.unwrap(); + + let source = broadcast::Info::new().produce(); + match Writer::new(store.clone(), source.consume(), Config::default()).await { + Err(Error::Timeline(message)) => assert!(message.contains("not segment 6"), "{message}"), + Err(err) => panic!("expected a timeline error, got {err}"), + Ok(_) => panic!("expected recovery to fail"), + } + assert_eq!(stored(&store).await, before); + } + + /// An enrollment dropped mid-flight still settles its count, so stopping the recording ends it. + #[tokio::test(start_paused = true)] + async fn a_cancelled_enrollment_does_not_block_the_end() { + let source = broadcast::Info::new().produce(); + let mut audio = track(&source, "audio"); + + let mock = Mock::memory(); + mock.hold_puts("audio/.info"); + let store = Store::new(mock.clone(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + let run = tokio::spawn(writer.run()); + let enrolling = { + let control = control.clone(); + tokio::spawn(async move { control.track("audio", media()).await }) + }; + assert!( + audio.subscription_changed().await.unwrap().is_some(), + "enrollment subscribed" + ); + enrolling.abort(); + let _ = enrolling.await; + + control.stop().unwrap(); + tokio::time::timeout(Duration::from_secs(5), run) + .await + .expect("the recording ends") + .unwrap() + .unwrap(); + // The cancelled name was never enrolled, so it isn't taken. + mock.release(); + drop(source); + } + + /// The publisher already has the subscriber, and the broadcast ends while `.info` is written. + /// + /// The enroll command is queued only after that write. The recording stays up across the gap, + /// so the track is kept instead of failing with the writer closed. + #[tokio::test] + async fn a_track_that_ends_during_enrollment_is_kept() { + let source = broadcast::Info::new().produce(); + let mut audio = track(&source, "audio"); + group(&audio, 0, &[0, 500]); + + let mock = Mock::memory(); + mock.hold_puts("audio/.info"); + let store = Store::new(mock.clone(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + let run = tokio::spawn(writer.run()); + let enrolling = tokio::spawn(async move { control.track("audio", media()).await }); + + let subscription = audio.subscription_changed().await.unwrap(); + assert!(subscription.is_some(), "enrollment subscribed"); + audio.finish().unwrap(); + source.close(); + mock.release(); + + enrolling.await.unwrap().unwrap(); + run.await.unwrap().unwrap(); + + let records = window(&store, "audio").await; + assert_eq!(groups(&records), vec![(0, 0)]); + } +} diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 3430ba5b525..6fe8f9ce282 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2278,7 +2278,7 @@ mod tests { fixture.finish().await; } - /// A recording replays what the live edge published: the archive's segment records + /// A recording replays what the live edge published: the archive's records /// carry the live timestamps across an idle restart, with the idle gap left in. #[tokio::test] async fn retained_archive_playback_keeps_the_live_timestamps() { @@ -2303,19 +2303,19 @@ mod tests { .archive .expect("the audio track enrolls an archive"); timeline = Some( - moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion) + moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion, "audio") .await .unwrap(), ); } drop(track); wait_for(&mut fixture.publication, Status::Waiting).await; - // Idle past the minimum segment, so each run is archived as its own segment. + // Idle past the minimum segment, so each run is archived as its own record. tokio::time::sleep(moq_mux::timeline::DEFAULT_DURATION_MIN + Duration::from_millis(100)).await; } let (catalog, _consumer) = fixture.finish().await; - catalog.timeline().finish().unwrap(); + catalog.timeline().finish(); let mut timeline = timeline.unwrap(); let mut archived = Vec::new(); while let Some(event) = timeline.next().await.unwrap() { @@ -2325,16 +2325,15 @@ mod tests { } } - assert_eq!(archived.len(), live.len(), "one segment per capture run: {archived:?}"); + assert_eq!(archived.len(), live.len(), "one record per capture run: {archived:?}"); for (entry, live) in archived.iter().zip(&live) { // The archive keeps millisecond precision. assert_eq!(entry.pts.as_micros() / 1000, u128::from(*live / 1000), "{archived:?}"); - assert!(entry.tracks.contains_key("audio"), "{archived:?}"); } let first = &archived[0]; assert!( archived[1].pts.as_micros() >= first.pts.as_micros() + first.duration.as_micros(), - "the resumed segment overlaps the one before it: {archived:?}" + "the resumed record overlaps the one before it: {archived:?}" ); } } diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 4ddd9cc2058..be8d840f4e9 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,7 +26,7 @@ path = "src/main.rs" doc = false [features] -default = ["iroh", "cluster-lan", "noq", "websocket", "nvidia"] +default = ["iroh", "cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). cluster-lan = ["moq-relay/cluster-lan"] @@ -37,6 +37,11 @@ noq = ["moq-tokio/noq"] # congestion-control testing. qlog = ["moq-tokio/qlog"] websocket = ["moq-tokio/websocket"] +# The cloud object stores `export archive` and `import archive` reach (`s3://`, +# `gs://`, `az://`). Local `file://` URLs need none of them. +s3 = ["object_store/aws"] +gcs = ["object_store/gcp"] +azure = ["object_store/azure"] # Device capture (camera + microphone) + encode/publish. Off by default because # it pulls in moq-video + moq-audio capture: on Linux the microphone needs # ALSA/libasound at build time through cpal (the camera needs nothing, moq-v4l @@ -97,10 +102,13 @@ base64 = { workspace = true } bytes = { workspace = true } hang = { workspace = true } humantime = { workspace = true } +moq-archive = { workspace = true } moq-audio = { workspace = true, optional = true, features = ["aac"] } moq-auth = { workspace = true, features = ["serve"] } # `server` enables the HTTP egress server for `moq export hls`; the importer is always available. moq-hls = { workspace = true, features = ["server"] } +# Names the MSF catalog track `export archive` records. +moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the # CLI's own feature set (iroh, noq, cluster-lan) is what selects them. @@ -112,6 +120,7 @@ moq-srt = { workspace = true } moq-tokio = { workspace = true, default-features = false, features = ["aws-lc-rs", "tcp", "uds"] } moq-transcode = { workspace = true, optional = true, features = ["openh264"] } moq-video = { workspace = true, optional = true, features = ["openh264"] } +object_store = { workspace = true, features = ["fs"] } pollster = { workspace = true, optional = true } reqwest = { workspace = true, features = ["rustls", "json"] } rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } @@ -129,6 +138,8 @@ winit = { version = "0.30.13", optional = true } sd-notify = { workspace = true } [dev-dependencies] +# Decodes the timeline the archive round-trip test checks. +moq-json = { path = "../moq-json" } # `test_relay` stands up a real relay for the `fetch` and `announced` tests to read through. moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } # Writes the PAT and PMTs, CRCs included, of the two-program input `import ts --program all` splits. diff --git a/rs/moq-cli/README.md b/rs/moq-cli/README.md index d5c49a1b371..469f1bc195a 100644 --- a/rs/moq-cli/README.md +++ b/rs/moq-cli/README.md @@ -23,7 +23,7 @@ Multi-arch images (`linux/amd64` and `linux/arm64`) are published to [Docker Hub ## Usage -`moq-cli` routes endpoints onto a shared MoQ Origin: `moq `. The MoQ side (before the verb) dials with `--connect `, self-hosts QUIC/WebTransport with `--listen `, or self-hosts raw qmux with `--listen-tcp-bind ` / `--listen-unix-bind ` (Unix only). `import` moves media into MoQ, `export` moves it out. The endpoint is a container format (`fmp4`, `ts`, `flv`, ... read from stdin / written to stdout), or a gateway (`hls`, `rtmp`, `srt`, `rtc`). A build with the `play` feature can also render a broadcast locally with `moq play`. +`moq-cli` routes endpoints onto a shared MoQ Origin: `moq `. The MoQ side (before the verb) dials with `--connect `, self-hosts QUIC/WebTransport with `--listen `, or self-hosts raw qmux with `--listen-tcp-bind ` / `--listen-unix-bind ` (Unix only). `import` moves media into MoQ, `export` moves it out. The endpoint is a container format (`fmp4`, `ts`, `flv`, ... read from stdin / written to stdout), or a gateway (`hls`, `rtmp`, `srt`, `rtc`, `archive`). A build with the `play` feature can also render a broadcast locally with `moq play`. Separate additional stages with `--` to bridge several broadcasts (or both directions) over one connection, each naming its own `--broadcast`: diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs new file mode 100644 index 00000000000..2863a7f403b --- /dev/null +++ b/rs/moq-cli/src/archive.rs @@ -0,0 +1,497 @@ +//! Archive endpoints: record one broadcast into an object store (export), or +//! republish a recording from one (import), through `moq-archive`. + +use std::collections::HashSet; +use std::time::Duration; + +use anyhow::Context; +use hang::moq_net; +use moq_archive::writer::{Control, Retention}; +use moq_mux::catalog::{CatalogFormat, Stream as _}; +use object_store::ObjectStore; +use url::Url; + +use crate::moq::notify_ready; + +/// How long expired objects outlive the timeline that dropped them, unless +/// `--retention-grace` says otherwise. +const GRACE: Duration = Duration::from_secs(30); + +/// `export archive` args. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct ExportArgs { + /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. + pub store: Url, + + /// Keep only this much recent content (a DVR), deleting older records. Unset keeps everything. + #[usage(long)] + pub retention: Option, + + /// How long expired objects outlive the timeline that dropped them, so readers can finish + /// their downloads. Defaults to 30s; needs `--retention`. + #[usage(long)] + pub retention_grace: Option, +} + +/// `import archive` args. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct ImportArgs { + /// The recording to replay, as `export archive` wrote it. + pub store: Url, + + /// Keep following a recording that is still growing, checking for new segments at this + /// interval. Unset replays what is stored now and ends the timeline there. + #[usage(long)] + pub follow: Option, +} + +/// Record the broadcast `name` into `args.store` until the broadcast ends. +/// +/// Reads the broadcast's own catalog: every track gets its own timeline, cut at group boundaries +/// between 2s and 10s, while the catalog and data tracks use a zero minimum and record each group +/// as it finishes. +pub async fn export( + origin: moq_net::origin::Consumer, + name: String, + format: CatalogFormat, + args: ExportArgs, +) -> anyhow::Result<()> { + let retention = match (args.retention, args.retention_grace) { + (Some(window), grace) => Some(Retention::new( + window.into_std(), + grace.map_or(GRACE, crate::duration::Duration::into_std), + )), + (None, Some(_)) => anyhow::bail!("`--retention-grace` needs `--retention`"), + (None, None) => None, + }; + let catalog_track = catalog_track(format)?; + let store = open(&args.store)?; + + let broadcast = origin + .routed_broadcast(&name) + .await + .with_context(|| format!("broadcast `{name}` is unavailable"))?; + let config = moq_archive::writer::Config::default().with_retention(retention); + let writer = moq_archive::Writer::new(store, broadcast.clone(), config) + .await + .with_context(|| format!("failed to start recording into {}", args.store))?; + let control = writer.control(); + control.track(catalog_track, sparse()).await?; + let catalog = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; + + tracing::info!(%name, store = %args.store, "recording"); + notify_ready(); + + // The writer decides when the recording is over; the catalog only feeds it tracks. + let run = writer.run(); + tokio::pin!(run); + tokio::select! { + result = &mut run => result.context("recording failed"), + Err(err) = enroll(control.clone(), catalog) => { + // Store what already arrived before failing, so the recording stays readable. + control.stop()?; + if let Err(stopped) = run.await { + tracing::warn!(%stopped, "recording failed while stopping"); + } + Err(err) + } + } +} + +/// Republish the recording at `args.store` as the broadcast `name`. +/// +/// Each track's timeline replays as a live track and every other track's groups are served on +/// request. +pub async fn import(origin: moq_net::origin::Producer, name: String, args: ImportArgs) -> anyhow::Result<()> { + let store = open(&args.store)?; + // A second handle lists the recording's timelines while the reader owns the first. + let listing = open(&args.store)?; + let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; + let mut timelines = store + .timelines() + .await + .with_context(|| format!("failed to list the recording at {}", args.store))?; + let config = moq_archive::reader::Config::new(timelines.clone()); + let mut reader = moq_archive::Reader::open(store, &broadcast, config) + .await + .with_context(|| format!("no readable recording at {}", args.store))?; + let serve = reader.serve(); + broadcast + .announce(Default::default()) + .context("failed to announce broadcast")?; + + tracing::info!(%name, store = %args.store, "replaying"); + notify_ready(); + + // The store holds no end marker, so only the caller can say the recording is complete. + let Some(interval) = args.follow.map(crate::duration::Duration::into_std) else { + reader.finish()?; + serve.await; + return Ok(()); + }; + let refresh = async { + loop { + tokio::time::sleep(interval).await; + // A rendition the recording enrolled after the replay started. + for (track, timeline) in listing.timelines().await? { + if let std::collections::btree_map::Entry::Vacant(entry) = timelines.entry(track) { + reader.track(entry.key(), &timeline).await?; + entry.insert(timeline); + } + } + reader.refresh().await?; + } + }; + tokio::select! { + () = serve => Ok(()), + result = refresh => result, + } +} + +/// Open the object store `url` names, taking cloud credentials from the `AWS_*`, `GOOGLE_*`, +/// and `AZURE_*` environment variables. +fn open(url: &Url) -> anyhow::Result>> { + let credentials = std::env::vars_os() + .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) + .filter(|(key, _)| ["AWS_", "GOOGLE_", "AZURE_"].iter().any(|cloud| key.starts_with(cloud))); + let (store, prefix) = + object_store::parse_url_opts(url, credentials).with_context(|| format!("unsupported archive store {url}"))?; + Ok(moq_archive::Store::new(store, prefix)) +} + +/// How sparse data, such as a catalog, is cut: every group is its own record. +fn sparse() -> moq_mux::timeline::Config { + moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO) +} + +/// The track carrying the catalog `format` reads. +fn catalog_track(format: CatalogFormat) -> anyhow::Result<&'static str> { + Ok(match format { + CatalogFormat::Hang => hang::Catalog::DEFAULT_NAME, + CatalogFormat::HangZ => hang::Catalog::COMPRESSED_NAME, + CatalogFormat::Msf => moq_msf::DEFAULT_NAME, + other => anyhow::bail!("`export archive` cannot record a {other:?} catalog"), + }) +} + +/// Enroll each track as the catalog lists it, until the catalog ends. +async fn enroll( + control: Control, + mut catalog: moq_mux::catalog::Consumer<()>, +) -> anyhow::Result<()> { + let mut tracks = Tracks::default(); + while let Some(snapshot) = catalog.next().await? { + for change in tracks.update(&snapshot)? { + match change { + Change::Media(name) => control.track(&name, moq_mux::timeline::Config::default()).await?, + Change::Sparse(name) => control.track(&name, sparse()).await?, + Change::Remove(name) => control.remove(&name)?, + } + } + } + Ok(()) +} + +/// A change to the recorded track set. +#[derive(Debug, PartialEq, Eq)] +enum Change { + /// Record a rendition, its records cut by duration. + Media(String), + /// Record a data track, each group its own record. + Sparse(String), + /// Stop recording a track the catalog dropped. + Remove(String), +} + +/// The catalog tracks being recorded, diffed against each new snapshot. +#[derive(Default)] +struct Tracks { + /// Every name ever enrolled. The writer never takes a name back. + enrolled: HashSet, + /// The names the latest snapshot listed. + listed: HashSet, +} + +impl Tracks { + /// The changes that bring the recording in line with `catalog`. + /// + /// Refuses a track served from another broadcast, since an archive holds exactly one, and a + /// track that returns after the catalog dropped it, since the writer cannot resume one. + fn update(&mut self, catalog: &hang::Catalog) -> anyhow::Result> { + let video = catalog + .video + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, true)); + let audio = catalog + .audio + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, true)); + let text = catalog + .text + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, false)); + let json = catalog.json.tracks.iter().map(|(name, c)| (name, &c.broadcast, false)); + let binary = catalog + .binary + .tracks + .iter() + .map(|(name, c)| (name, &c.broadcast, false)); + + let mut listed = HashSet::new(); + let mut changes = Vec::new(); + for (name, broadcast, media) in video.chain(audio).chain(text).chain(json).chain(binary) { + if let Some(broadcast) = broadcast { + anyhow::bail!( + "track `{name}` is served from broadcast `{broadcast}`; an archive records one broadcast" + ); + } + if !listed.insert(name.clone()) || self.listed.contains(name) { + continue; + } + anyhow::ensure!( + self.enrolled.insert(name.clone()), + "track `{name}` returned to the catalog after it was dropped; the recording cannot resume it" + ); + changes.push(match media { + true => Change::Media(name.clone()), + false => Change::Sparse(name.clone()), + }); + } + + let mut dropped: Vec<_> = self.listed.difference(&listed).cloned().collect(); + dropped.sort(); + changes.extend(dropped.into_iter().map(Change::Remove)); + self.listed = listed; + Ok(changes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use hang::catalog::{AudioCodec, AudioConfig, JsonConfig, Mode}; + use moq_net::{Timescale, Timestamp, group, track}; + + fn audio() -> AudioConfig { + AudioConfig::new(AudioCodec::Opus, 48_000, 2) + } + + /// Renditions are media, data tracks are sparse, and a later snapshot only adds what is new. + #[test] + fn renditions_are_media_and_the_rest_sparse() { + let mut tracks = Tracks::default(); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), audio()); + catalog + .json + .tracks + .insert("chat".into(), JsonConfig::new(Mode::Snapshot)); + + assert_eq!( + tracks.update(&catalog).unwrap(), + [Change::Media("audio".into()), Change::Sparse("chat".into())] + ); + + catalog.audio.renditions.insert("audio2".into(), audio()); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Media("audio2".into())]); + assert_eq!(tracks.update(&catalog).unwrap(), []); + } + + /// A dropped rendition stops recording, and coming back is refused rather than silently lost. + #[test] + fn a_dropped_track_is_removed_for_good() { + let mut tracks = Tracks::default(); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), audio()); + tracks.update(&catalog).unwrap(); + + let removed = catalog.audio.renditions.remove("audio").unwrap(); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Remove("audio".into())]); + + catalog.audio.renditions.insert("audio".into(), removed); + let err = tracks.update(&catalog).unwrap_err().to_string(); + assert!(err.contains("cannot resume"), "{err}"); + } + + /// A rendition living in a sibling broadcast would be missing from the recording. + #[test] + fn another_broadcast_is_refused() { + let mut rendition = audio(); + rendition.broadcast = Some(moq_net::path::RelativeOwned::new("./source")); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), rendition); + + let err = Tracks::default().update(&catalog).unwrap_err().to_string(); + assert!(err.contains("broadcast `source`"), "{err}"); + } + + /// Wait until the export's subscription has reached `track`. + /// + /// `.info` is written after that, while the enroll command is still being queued. + /// Finishing on the file races that command and the recording ends with "writer closed". + async fn subscribed(track: &mut track::Producer, recording: &mut tokio::task::JoinHandle>) { + tokio::select! { + subscription = track.subscription_changed() => { + let subscription = subscription.expect("the rendition stays open until the export subscribes"); + assert!(subscription.is_some(), "the export subscribed"); + } + result = &mut *recording => panic!("the export ended before it subscribed: {result:?}"), + } + } + + /// What `export archive` records, `import archive` serves back group for group, and the + /// export ends cleanly with its broadcast. + #[tokio::test] + async fn a_recording_replays_its_groups() { + // `open` reads the credential variables. + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let mut broadcast = origin.create_broadcast("live.hang").unwrap(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let mut track = broadcast.create_track("audio", info).unwrap(); + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), audio()); + }) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let args = ExportArgs { + store: url.clone(), + retention: None, + retention_grace: None, + }; + let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + + for sequence in 0..3 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for offset in [0, 500] { + let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); + group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); + } + group.finish().unwrap(); + } + + subscribed(&mut track, &mut recording).await; + track.finish().unwrap(); + catalog.finish().unwrap(); + broadcast.close(); + + tokio::time::timeout(Duration::from_secs(10), recording) + .await + .expect("the export ends with its broadcast") + .unwrap() + .expect("the recording succeeds"); + + let replay = moq_tokio::origin::spawn(); + let args = ImportArgs { + store: url, + follow: None, + }; + let serving = tokio::spawn(import(replay.clone(), "replay.hang".into(), args)); + + let consumer = replay.consume().routed_broadcast("replay.hang").await.unwrap(); + let audio = consumer.track("audio").unwrap(); + for sequence in 0..3 { + let mut group = audio.fetch_group(sequence, None).await.unwrap(); + for offset in [0, 500] { + let frame = group.read_frame().await.unwrap().expect("a frame"); + assert_eq!(frame.timestamp.as_millis(), u128::from(sequence * 1000 + offset)); + assert_eq!(frame.payload, format!("{sequence}+{offset}")); + } + assert!(group.read_frame().await.unwrap().is_none()); + } + + serving.abort(); + } + + /// Renditions already live in the opening catalog are all recorded from their first group. + /// + /// Groups are published before export starts, so the writer can drain the first rendition + /// while the next subscription is still in flight. + #[tokio::test] + async fn an_opening_snapshot_records_every_rendition() { + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let mut broadcast = origin.create_broadcast("live.hang").unwrap(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let mut first = broadcast.create_track("audio", info.clone()).unwrap(); + let mut second = broadcast.create_track("audio2", info).unwrap(); + for track in [&first, &second] { + for sequence in 0..3 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for offset in [0, 500] { + let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); + group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); + } + group.finish().unwrap(); + } + } + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), audio()); + catalog.audio.renditions.insert("audio2".into(), audio()); + }) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let args = ExportArgs { + store: url.clone(), + retention: None, + retention_grace: None, + }; + let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + + subscribed(&mut first, &mut recording).await; + subscribed(&mut second, &mut recording).await; + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + broadcast.close(); + + tokio::time::timeout(Duration::from_secs(10), recording) + .await + .expect("the export ends with its broadcast") + .unwrap() + .expect("the recording succeeds"); + + // Each rendition's own timeline opens at its first group. + let store = super::open(&url).unwrap(); + for track in ["audio", "audio2"] { + let object = store + .get_segments(&hang::timeline::default_name(track), 0) + .await + .expect("record 0"); + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } + } + let Some(moq_json::window::Event::Push { value, .. }) = decoder.next_event() else { + panic!("{track} record 0 is pushed first"); + }; + assert_eq!(value.start, hang::timeline::Position::group(0), "{track}"); + } + } +} diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index fb226dc4034..84a4d6359a8 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -19,7 +19,7 @@ //! conditional on the subcommand. //! - The endpoint is one subcommand: a container format (`ts`, `fmp4`, ... read //! from stdin on import, written to stdout on export) or a gateway (`hls`, -//! `rtmp`, `srt`, `rtc`). Exactly one per stage, so "which endpoint" is +//! `rtmp`, `srt`, `rtc`, `archive`). Exactly one per stage, so "which endpoint" is //! unambiguous and there's no silently-ignored flag. //! - `--` starts another stage on the same Origin and the same MoQ attachment, so //! one process can bridge several broadcasts (or both directions at once). Usage @@ -736,6 +736,8 @@ pub enum ImportSource { Srt(crate::srt::ImportArgs), /// WebRTC: WHEP client pulling a remote (`--connect`) or WHIP server accepting publishes (`--listen`). Rtc(crate::rtc::Args), + /// Replay a recording from an object store, serving its groups on demand. + Archive(crate::archive::ImportArgs), /// Capture a local source (camera, display, window, app, microphone) and /// encode natively. Run `moq devices` to list them. #[cfg(feature = "capture")] @@ -876,6 +878,8 @@ pub enum ExportSink { Srt(crate::srt::Args), /// WebRTC: WHIP client pushing to a remote (`--connect`) or WHEP server serving plays (`--listen`). Rtc(crate::rtc::Args), + /// Record the broadcast into an object store until it ends. + Archive(crate::archive::ExportArgs), } impl ExportSink { @@ -888,6 +892,7 @@ impl ExportSink { Self::Rtmp(_) => "rtmp", Self::Srt(_) => "srt", Self::Rtc(_) => "rtc", + Self::Archive(_) => "archive", }) } @@ -1101,6 +1106,7 @@ mod tests { &["rtmp", "--listen", "127.0.0.1:1935"], &["srt", "--listen", "127.0.0.1:9000"], &["rtc", "--listen", "127.0.0.1:8443"], + &["archive", "file:///tmp/archive"], ]; for sink in sinks { export(sink).unwrap().validate().unwrap(); diff --git a/rs/moq-cli/src/hls.rs b/rs/moq-cli/src/hls.rs index f09a193cdee..6a50170c84b 100644 --- a/rs/moq-cli/src/hls.rs +++ b/rs/moq-cli/src/hls.rs @@ -34,6 +34,7 @@ pub struct ExportArgs { /// Minimum media listed in each rendition's playlist window. Keep it within the /// relay's group-cache retention, since segments are fetched from there on request. + /// A timeline durable in a catalog-named store lists everything it retains instead. #[usage(long, default = "16s")] pub window: crate::duration::Duration, diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 1044b44aa62..db2ca882454 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -5,6 +5,7 @@ //! plus every stage's endpoint. mod announced; +mod archive; mod args; mod auth; mod complete; @@ -690,6 +691,11 @@ fn spawn_import( tasks.spawn(rtc::connect_import(target(name), url)); } } + ImportSource::Archive(args) => { + // A replay serves the retention the recording was made with. + anyhow::ensure!(max_age.is_none(), "`--max-age` does not apply to `import archive`"); + tasks.spawn(archive::import(origin.clone(), name, args)); + } #[cfg(feature = "capture")] ImportSource::Capture(capture) => { warn_if_missing_format(&name); @@ -770,6 +776,14 @@ fn spawn_export( tasks.spawn(rtc::connect_export(origin.consume(), url, name)); } } + ExportSink::Archive(args) => { + let format = export + .catalog_format + .map(Into::into) + .or_else(|| moq_mux::catalog::CatalogFormat::detect(&name)) + .unwrap_or_default(); + tasks.spawn(archive::export(origin.consume(), name, format, args)); + } _ => unreachable!("container formats are handled by stdout_format above"), } } diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index c4d52d5c22f..93840d19f56 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -36,6 +36,10 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] +async-trait = "0.1" +futures = { workspace = true } +moq-archive = { path = "../moq-archive" } +moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs new file mode 100644 index 00000000000..2afa12bf1cc --- /dev/null +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -0,0 +1,599 @@ +//! Serving a `moq-archive` recording as HLS. +//! +//! A [`moq_archive::Reader`] replays each track's timeline onto a broadcast and answers FETCH from +//! its record objects, so the exporter serves it exactly like a live broadcast. These tests pin +//! the storage traffic that composition produces: playlists read only the timelines, and a segment +//! GETs only objects of the requested rendition. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use futures::stream::BoxStream; +use hang::timeline::{Position, Record}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::object_store::path::Path; +use moq_archive::object_store::{ + self, CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use moq_archive::{Frame, Group, Info, Object, Store, reader}; +use moq_json::window; + +use super::*; + +/// In-memory store that records every GET path and counts listings. +#[derive(Debug, Clone, Default)] +struct Counting { + inner: Arc, + gets: Arc>>, + lists: Arc, + /// Fail every media GET, so nothing can quietly depend on one. + reject_media: Arc, +} + +impl Counting { + /// Every GET since the last call. + fn take(&self) -> Vec { + std::mem::take(&mut *self.gets.lock().unwrap()) + } + + /// Listings since the store was created. + fn lists(&self) -> usize { + self.lists.load(Ordering::SeqCst) + } + + fn reject_media(&self, reject: bool) { + self.reject_media.store(reject, Ordering::SeqCst); + } +} + +impl std::fmt::Display for Counting { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Counting") + } +} + +#[async_trait::async_trait] +impl ObjectStore for Counting { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.gets.lock().unwrap().push(location.to_string()); + if self.reject_media.load(Ordering::SeqCst) && is_media(location.as_ref()) { + return Err(object_store::Error::NotImplemented { + operation: "media GET".into(), + implementer: "Counting".into(), + }); + } + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.lists.fetch_add(1, Ordering::SeqCst); + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// One track's timeline encoder, as `moq_archive::Writer` drives it. +struct Encoder { + encoder: window::Encoder, + /// Next timeline group sequence. + group: u64, + /// Next record sequence. + record: u64, +} + +/// Writes recording objects the way `moq_archive::Writer` lays them out, with full control over +/// each record's timing and groups. +struct Recording { + store: Store, + encoders: HashMap, +} + +impl Recording { + async fn new(tracks: &[&str]) -> Self { + let store = Store::new(Counting::default(), "rec"); + let mut encoders = HashMap::new(); + for track in tracks { + // Legacy media is stamped in microseconds. + store.put_info(track, &Info::new(1, 1_000_000).unwrap()).await.unwrap(); + store + .put_info(&timeline(track), &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(0); + let encoder = Encoder { + encoder: window::Encoder::new(config), + group: 0, + record: 0, + }; + encoders.insert(track.to_string(), encoder); + } + Self { store, encoders } + } + + /// Store and commit `track`'s next record: `groups`, each a list of frame timestamps in micros, + /// spanning `pts..pts + duration` in milliseconds, popping `pop` older records. + async fn add(&mut self, track: &str, pts: u64, duration: u64, groups: &[(u64, &[u64])], pop: u64) { + let encoder = self.encoders.get_mut(track).unwrap(); + let sequence = encoder.record; + encoder.record += 1; + let first = groups.first().unwrap().0; + let last = groups.last().unwrap().0; + let record = Record::new( + sequence, + pts, + duration, + Position::group(first), + Position::group(last + 1), + ); + + let groups = groups + .iter() + .map(|&(sequence, frames)| Group { + sequence, + frames: frames + .iter() + .enumerate() + .map(|(index, µs)| legacy(track, micros, index == 0)) + .collect(), + }) + .collect(); + self.store + .put_segments(track, sequence, &Object::new(groups)) + .await + .unwrap(); + + let mut payloads = Vec::new(); + let pending = encoder.encoder.push(&record).unwrap(); + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = encoder.encoder.pop(pop).unwrap() { + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + let mut groups: Vec = Vec::new(); + for (keyframe, payload) in payloads { + if keyframe || groups.is_empty() { + groups.push(Group { + sequence: encoder.group, + frames: Vec::new(), + }); + encoder.group += 1; + } + let frame = Frame { + timestamp: record.pts, + payload, + }; + groups.last_mut().unwrap().frames.push(frame); + } + self.store + .put_segments(&timeline(track), sequence, &Object::new(groups)) + .await + .unwrap(); + } + + /// Every GET since the last call. + fn gets(&self) -> Vec { + self.store.inner().take() + } + + fn timelines(&self) -> std::collections::BTreeMap { + self.encoders + .keys() + .map(|track| (track.clone(), timeline(track))) + .collect() + } +} + +fn timeline(track: &str) -> String { + hang::timeline::default_name(track) +} + +/// One Legacy frame: a VP8 keyframe for video (geometry the muxer can parse), filler otherwise. +fn legacy(track: &str, micros: u64, keyframe: bool) -> Frame { + let payload: &'static [u8] = match (track.starts_with("audio"), keyframe) { + (true, _) => &[0xFC, 0xFF, 0xFE], + (false, true) => &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00], + (false, false) => &[0x31, 0x00, 0x00], + }; + let frame = hang::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + payload: Bytes::from_static(payload), + }; + let mut encoded = BytesMut::new(); + frame.encode(&mut encoded).unwrap(); + Frame { + timestamp: micros, + payload: encoded.freeze(), + } +} + +/// A live-style archive entry naming every rendition's timeline. +fn live() -> hang::catalog::Archive { + let mut archive = hang::catalog::Archive::new(); + for track in ["360p", "1080p", "audio"] { + archive.timelines.insert(track.to_string(), timeline(track)); + } + archive +} + +/// The archive entry a replay advertises: the recording's timelines, durable in its store. +fn durable() -> hang::catalog::Archive { + let mut archive = live(); + archive.store = Some("memory:///rec/".parse().unwrap()); + archive.version = Some(hang::catalog::Archive::VERSION); + archive +} + +/// The catalog an exporter is handed: every rendition's config, plus `archive`. Out-of-band +/// configs, so no init needs media. +fn catalog(archive: hang::catalog::Archive) -> hang::Catalog { + let mut catalog = hang::Catalog::default(); + catalog.archive = Some(archive); + for (name, width, height) in [("360p", 640, 360), ("1080p", 1920, 1080)] { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.coded_width = Some(width); + config.coded_height = Some(height); + catalog.video.renditions.insert(name.to_string(), config); + } + let audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + catalog.audio.renditions.insert("audio".to_string(), audio); + catalog +} + +/// A recording served back through an origin: the reader's broadcast carries the supplied +/// catalog, and a broadcaster exports it. +struct Replay { + /// Taken to supply finality. + reader: Option>, + broadcaster: Arc, + _catalog: moq_json::snapshot::Producer, + _broadcast: moq_net::broadcast::Producer, + _origin: moq_net::origin::Producer, +} + +impl Replay { + async fn open(recording: &Recording, cache: u64, archive: hang::catalog::Archive) -> Self { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + tokio::spawn(moq_net::time::run(driver)); + let broadcast = origin.create_broadcast("rec").unwrap(); + + let track = broadcast + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut json = moq_json::snapshot::Config::default(); + json.delta_ratio = 0; + let mut catalog = moq_json::snapshot::Producer::new(track, json); + catalog.update(&self::catalog(archive)).unwrap(); + + let config = reader::Config::new(recording.timelines()).with_cache(cache); + let reader = moq_archive::Reader::open(recording.store.clone(), &broadcast, config) + .await + .unwrap(); + tokio::spawn(reader.serve()); + broadcast.announce(Default::default()).unwrap(); + for _ in 0..10 { + tokio::task::yield_now().await; + } + + let source = moq_mux::Source::new(origin.consume(), "rec"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()) + .await + .expect("a rendition becomes playable"); + + Self { + reader: Some(reader), + broadcaster, + _catalog: catalog, + _broadcast: broadcast, + _origin: origin, + } + } + + fn rendition(&self, kind: Kind, name: &str) -> Arc { + self.broadcaster + .rendition(kind, name) + .expect("rendition in the catalog") + } + + async fn playlist(&self, kind: Kind, name: &str) -> String { + let rendition = self.rendition(kind, name); + tokio::time::timeout(Duration::from_secs(5), rendition.playlist(None)) + .await + .expect("playlist renders") + .unwrap() + .expect("playlist is servable") + } + + /// Wait until `name`'s playlist satisfies `ready`, then return it. + async fn playlist_until(&self, kind: Kind, name: &str, ready: impl Fn(&str) -> bool) -> String { + for _ in 0..500 { + let playlist = self.playlist(kind, name).await; + if ready(&playlist) { + return playlist; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} playlist never became ready"); + } +} + +fn is_media(path: &str) -> bool { + path.contains("/segments/") && !path.contains("timeline") +} + +/// Three 2s segments: one keyframe group per video record, four audio groups per audio record. +/// The first video rendition by name, 1080p, is the reference. +async fn three_segments() -> Recording { + segments(3).await +} + +/// `count` 2s segments, laid out like [`three_segments`]. +async fn segments(count: u64) -> Recording { + let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; + for segment in 0..count { + let pts = segment * 2_000_000; + for video in ["360p", "1080p"] { + recording + .add(video, segment * 2000, 2000, &[(segment, &[pts, pts + 1_000_000])], 0) + .await; + } + let audio: Vec<(u64, [u64; 1])> = (0..4).map(|i| (segment * 4 + i, [pts + i * 500_000])).collect(); + let audio: Vec<(u64, &[u64])> = audio + .iter() + .map(|(sequence, frames)| (*sequence, &frames[..])) + .collect(); + recording.add("audio", segment * 2000, 2000, &audio, 0).await; + } + recording +} + +#[tokio::test] +async fn playlists_read_only_timelines_and_segments_their_own_objects() { + let recording = three_segments().await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + + let master = replay.broadcaster.master_playlist(None); + assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); + + // Render and reload every playlist with media GETs refused: shared numbering, and not one + // media GET. The reference lists every segment; the others hold back the newest until their + // own timeline passes its end, or ends. + recording.store.inner().reject_media(true); + for _ in 0..2 { + for (kind, name, listed) in [ + (Kind::Video, "1080p", 3), + (Kind::Video, "360p", 2), + (Kind::Audio, "audio", 3), + ] { + let playlist = replay.playlist(kind, name).await; + for segment in 0..3 { + let line = format!("seg/29b1cd67.{segment}.m4s\n"); + assert_eq!(playlist.contains(&line), segment < listed, "{name}: {playlist}"); + } + assert!(!playlist.contains("#EXT-X-ENDLIST"), "no finality was supplied"); + } + } + let gets = recording.gets(); + assert!(gets.iter().any(|path| path.contains("timeline")), "{gets:?}"); + assert!( + !gets.iter().any(|path| is_media(path)), + "playlists must not GET media: {gets:?}" + ); + recording.store.inner().reject_media(false); + // A segment resolves its objects from the timelines: no listing, no index object. + let lists = recording.store.inner().lists(); + + // Switching renditions downloads only the selected rendition's object. + let low = replay + .rendition(Kind::Video, "360p") + .listed_segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&low[4..8], b"moof"); + assert_eq!( + recording.gets(), + ["rec/360p/.info", "rec/360p/segments/0000000000000000001"] + ); + let high = replay + .rendition(Kind::Video, "1080p") + .listed_segment(2) + .await + .unwrap() + .unwrap(); + assert_eq!(&high[4..8], b"moof"); + assert_eq!( + recording.gets(), + ["rec/1080p/.info", "rec/1080p/segments/0000000000000000002"] + ); + + // An audio segment spans four groups but still costs one object GET. + let audio = replay + .rendition(Kind::Audio, "audio") + .listed_segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + assert_eq!( + recording.gets(), + ["rec/audio/.info", "rec/audio/segments/0000000000000000001"] + ); + + // A repeated request hits the reader's cache, including the immutable `.info`. + replay + .rendition(Kind::Video, "360p") + .listed_segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(recording.gets(), Vec::::new()); + assert_eq!(recording.store.inner().lists(), lists, "segments never list"); +} + +#[tokio::test] +async fn a_bounded_cache_rereads_evicted_objects() { + let recording = three_segments().await; + // Too small for any object, so nothing stays cached. + let replay = Replay::open(&recording, 1, durable()).await; + replay.playlist(Kind::Audio, "audio").await; + recording.gets(); + + // Each of the segment's four groups misses the cache and GETs the same object again. + let audio = replay + .rendition(Kind::Audio, "audio") + .listed_segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + let gets: Vec = recording.gets().into_iter().filter(|path| is_media(path)).collect(); + assert_eq!(gets, vec!["rec/audio/segments/0000000000000000001"; 4]); +} + +#[tokio::test] +async fn missing_rendition_segments_are_gaps_and_time_jumps_are_discontinuities() { + let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; + recording.add("1080p", 0, 2000, &[(0, &[0])], 0).await; + recording.add("360p", 0, 8000, &[(0, &[0])], 0).await; + // 360p stored nothing for segment 1. + recording.add("1080p", 2000, 2000, &[(1, &[2_000_000])], 0).await; + // Content time jumps from 4s to 10s. + recording.add("1080p", 10_000, 2000, &[(2, &[10_000_000])], 0).await; + recording.add("360p", 10_000, 2000, &[(1, &[10_000_000])], 0).await; + + let mut replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + // Finality resolves each rendition's newest segment. + replay.reader.take().unwrap().finish().unwrap(); + let low = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; + let expected = concat!( + "#EXTINF:2.00000,\nseg/29b1cd67.0.m4s\n", + "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/29b1cd67.1.m4s\n", + "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/29b1cd67.2.m4s\n", + ); + assert!(low.contains(expected), "{low}"); + let high = replay.playlist(Kind::Video, "1080p").await; + assert!(!high.contains("#EXT-X-GAP"), "{high}"); + assert!( + high.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/29b1cd67.2.m4s\n"), + "{high}" + ); + + // A gap is never fetched. + recording.gets(); + let rendition = replay.rendition(Kind::Video, "360p"); + assert!(rendition.listed_segment(1).await.unwrap().is_none()); + assert!(!recording.gets().iter().any(|path| is_media(path))); + let after = rendition.listed_segment(2).await.unwrap().unwrap(); + assert_eq!(&after[4..8], b"moof"); +} + +#[tokio::test] +async fn a_growing_recording_ends_only_on_caller_finality() { + let mut recording = three_segments().await; + let mut replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + replay.playlist(Kind::Video, "1080p").await; + + // A DVR commit: segment 3 arrives and segment 0 expires. + recording.add("1080p", 6000, 2000, &[(3, &[6_000_000])], 1).await; + recording.gets(); + let mut reader = replay.reader.take().unwrap(); + reader.refresh().await.unwrap(); + let playlist = replay + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.3.m4s\n") + }) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); + assert!(!playlist.contains("seg/29b1cd67.0.m4s"), "{playlist}"); + assert!( + !playlist.contains("#EXT-X-ENDLIST"), + "a recording without finality stays live" + ); + let gets = recording.gets(); + assert!( + !gets.iter().any(|path| is_media(path)), + "following reads only the timelines: {gets:?}" + ); + + // The store holds no completion marker; the caller supplies finality. + reader.finish().unwrap(); + let playlist = replay + .playlist_until(Kind::Video, "1080p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; + assert!(playlist.contains("seg/29b1cd67.3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); +} + +/// A durable timeline lists the whole recording past the default 16s window, and DASH offers +/// the whole listed span. A live-style entry, or a `replay` path that moves the durable ranges +/// to another broadcast, keeps the window. +#[tokio::test] +async fn a_durable_timeline_lists_past_the_window() { + let recording = segments(12).await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + let playlist = replay + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.11.m4s\n") + }) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0\n"), "{playlist}"); + assert!(playlist.contains("seg/29b1cd67.0.m4s\n"), "{playlist}"); + + for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { + replay.rendition(kind, name).init().await.unwrap(); + } + let manifest = replay.broadcaster.manifest(None).expect("manifest renders"); + assert!(manifest.contains("timeShiftBufferDepth=\"PT24.000S\""), "{manifest}"); + + let mut elsewhere = durable(); + elsewhere.replay = Some(moq_net::path::RelativeOwned::new("./recording")); + for archive in [live(), elsewhere] { + let live = Replay::open(&recording, 64 * 1024 * 1024, archive).await; + let playlist = live + .playlist_until(Kind::Video, "1080p", |playlist| { + playlist.contains("seg/29b1cd67.11.m4s\n") + }) + .await; + assert!(!playlist.contains("seg/29b1cd67.0.m4s\n"), "{playlist}"); + } +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index baf792f69d9..aac8e24e0cf 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -1,12 +1,19 @@ //! Export: serve a MoQ broadcast as HLS or DASH, fetching media on demand. //! -//! A [`Broadcaster`] subscribes to one broadcast's catalog and its single timeline track (see -//! [`hang::timeline`]). That is the *only* standing traffic: playlists and manifests are a -//! pure function of the timeline records (each names a complete aligned segment with -//! per-track group ranges), and media bytes move only when an HTTP client requests a segment, -//! which FETCHes exactly the groups that segment covers from the relay cache and transmuxes -//! them to CMAF. A broadcast whose catalog advertises no timeline can't be served this way and -//! is skipped. +//! A [`Broadcaster`] subscribes to one broadcast's catalog and each rendition's timeline track +//! (see [`hang::timeline`]). That is the *only* standing traffic: playlists and manifests are a +//! pure function of the timeline records, and media bytes move only when an HTTP client requests +//! a segment, which FETCHes exactly the frames that segment covers from the relay cache and +//! transmuxes them to CMAF. A broadcast whose catalog advertises no timelines can't be served this +//! way and is skipped. A `moq-archive` recording replayed onto a broadcast is served the same way: +//! its playlists read only the stored timelines, and a segment GETs only its rendition's objects. +//! +//! Segment boundaries come from one reference rendition's records (the first video rendition, or +//! the first audio one when there is no video), numbered by record sequence, so every edge and +//! every reload agree without the publisher cutting for HLS. Every other rendition resolves each +//! segment against its own timeline. +//! An inline-parameter-set codec with no catalog `description` GETs one keyframe +//! group on the first playlist render to build its init, then caches it. //! //! The same machinery serves two kinds of consumer: //! @@ -18,10 +25,13 @@ //! yield every rendition and every finalized segment in order, for mirroring a broadcast to //! storage. +#[cfg(test)] +mod archive_tests; pub mod master; mod mpd; mod playlist; mod rendition; +mod spans; mod upstream; pub mod renditions; @@ -35,6 +45,7 @@ use rand::RngExt; pub(crate) use playlist::render_media; pub use rendition::{Kind, Rendition}; +use segments::Reference; pub(crate) use upstream::Upstream; /// Backoff bounds for the initial catalog subscription. @@ -61,6 +72,10 @@ pub struct Config { /// Minimum duration of media listed in each rendition's playlist window. Older timeline /// records are evicted once the remaining segments still cover this span; keep it within /// the relay's group-cache retention, since segments are fetched from there on request. + /// + /// A durable timeline (its catalog `archive` entry names a `store` and no `replay` path) + /// lists everything it retains instead, since its own retention already bounds it. The + /// window still caps segment `Cache-Control: max-age` for every broadcast. pub window: Duration, } @@ -84,9 +99,9 @@ pub struct Broadcaster { /// renditions themselves (see [`renditions::Producer::clear`]). Set once, right after /// construction. watcher: Mutex>>, - /// The broadcast's timeline watcher, spawned by the catalog watcher once the catalog - /// advertises the timeline track. On drop it is aborted only when no cursor still needs - /// its records (see `Drop`). + /// The reference rendition's timeline watcher, spawned by the catalog watcher once the + /// catalog advertises it. On drop it is aborted only when no cursor still needs its records + /// (see `Drop`). timeline_watcher: Arc>>>, } @@ -146,7 +161,7 @@ impl Broadcaster { /// broadcast carries now, or stop versioning with `None` (the default). /// /// A restarted publisher is spliced into the same broadcast and restarts its segment - /// numbers, so `seg/0.m4s` would name different bytes on each run. Supply a new generation + /// numbers, so `seg/{reference}.0.m4s` would name different bytes on each run. Supply a new generation /// before the new run's media flows. Replacing one drops every listed segment and every /// init built from the old run's media, while the first only labels the run already flowing. A segment URL carrying any other generation /// is refused. Init URLs need none: they carry a hash of their bytes. @@ -236,7 +251,8 @@ impl Broadcaster { if representations().all(|rep| rep.segments.is_empty()) { return None; } - let finished = representations().any(|rep| rep.ended); + // Static only once every representation is final, so none stops being refreshed early. + let finished = representations().all(|rep| rep.ended); if !finished { // A dynamic presentation needs pts 0 anchored to the wall clock. The fallback is // set whenever a record has been pushed, which listing a segment implies. @@ -333,17 +349,35 @@ async fn watch_catalog( } }; - let mut timeline_started = false; + let mut unbound = false; + let mut reference: Option = None; loop { match kio::wait(|waiter| consumer.poll_next(waiter)).await { Ok(Some(catalog)) => { + // Decide the playlists' retention before any rendition starts following its timeline. + if !unbound && catalog.archive.as_ref().is_some_and(durable) { + unbound = true; + renditions.fanout().unbound(); + } renditions.sync(&upstream, &catalog); - // The broadcast has one timeline; subscribe once it's advertised and fan its - // records out to every rendition. - if !timeline_started && let Some(archive) = catalog.archive.clone() { - timeline_started = true; - let watcher = tokio::spawn(watch_timeline(broadcast.clone(), archive, renditions.fanout())); - *timeline_watcher.lock().unwrap() = Some(watcher); + + // Follow the reference rendition's timeline; a new reference renumbers every segment. + let next = self::reference(&catalog); + if next != reference { + if let Some(watcher) = timeline_watcher.lock().unwrap().take() { + watcher.abort(); + renditions.fanout().skip(); + } + if let (Some(next), Some(archive)) = (&next, catalog.archive.clone()) { + let watcher = tokio::spawn(watch_timeline( + broadcast.clone(), + archive, + next.clone(), + renditions.fanout(), + )); + *timeline_watcher.lock().unwrap() = Some(watcher); + } + reference = next; } } Ok(None) => break, @@ -358,37 +392,66 @@ async fn watch_catalog( renditions.close(); } -/// The broadcast's timeline watcher: read the single timeline track and fan each record out -/// to every rendition's window. +/// The rendition every segment boundary comes from: the first video rendition with a timeline, +/// or the first audio one when the broadcast has no video. Renditions are ordered by name, so +/// every edge picks the same one. +fn reference(catalog: &moq_mux::catalog::hang::Catalog) -> Option { + let archive = catalog.archive.as_ref()?; + let indexed = |name: &&String| archive.timelines.contains_key(*name); + let video = catalog + .video + .renditions + .keys() + .find(indexed) + .map(|name| (Kind::Video, name)); + let audio = || { + catalog + .audio + .renditions + .keys() + .find(indexed) + .map(|name| (Kind::Audio, name)) + }; + video.or_else(audio).map(|(kind, name)| Arc::new((kind, name.clone()))) +} + +/// Whether every range `archive` advertises stays FETCHable from this broadcast until the +/// timeline pops it: a store makes the ranges durable, and no `replay` path means this +/// broadcast serves them. The catalog states this, so the playlists follow the timeline's own +/// retention rather than a window sized for relay caches. +fn durable(archive: &hang::catalog::Archive) -> bool { + archive.store.is_some() && archive.replay.is_none() +} + +/// The segment watcher: read the reference rendition's timeline and fan each record out to every +/// rendition's window as a segment. async fn watch_timeline( broadcast: moq_net::broadcast::Consumer, section: hang::catalog::Archive, + reference: Reference, renditions: renditions::Fanout, ) { - match watch(&broadcast, §ion, &renditions).await { - // The timeline finished cleanly: the publisher is done, so every window can end - // (ENDLIST) and recording cursors drain to completion. - Ok(()) => renditions.end_windows(), - // A transient error (subscription reset, relay hiccup): don't mark the windows ended; - // the serve path keeps serving the frozen windows. - Err(err) => { - tracing::warn!(track = %section.track, %err, "timeline watcher error; leaving the playlists live") - } - } - // The timeline stream is over either way: close so recording cursors terminate instead of - // parking forever (the serve path still reads the last windows). + // A failed timeline is never retried: the origin already rides out transient source + // failures, so an error here is final. It ends every playlist like a clean finish + // (`EXT-X-ENDLIST`), since a window left live would freeze players with no signal. + if let Err(err) = watch(&broadcast, §ion, &reference, &renditions).await { + tracing::error!(track = %reference.1, %err, "timeline failed; ending every playlist"); + } + // Recording cursors drain what is listed and end instead of parking. + renditions.end_windows(); renditions.close_windows(); } async fn watch( broadcast: &moq_net::broadcast::Consumer, section: &hang::catalog::Archive, + reference: &Reference, renditions: &renditions::Fanout, ) -> crate::Result<()> { - let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(broadcast, section).await?; + let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(broadcast, section, &reference.1).await?; while let Some(event) = timeline.next().await? { match event { - moq_mux::timeline::Event::Push { index, entry } => renditions.push(index, entry), + moq_mux::timeline::Event::Push { index, entry } => renditions.push(index, entry, reference), moq_mux::timeline::Event::Pop(range) => renditions.pop(range), moq_mux::timeline::Event::Skip(_) => renditions.skip(), _ => unreachable!("unknown timeline event"), @@ -508,10 +571,11 @@ mod tests { impl CatalogTestExt for moq_mux::catalog::Producer { fn enroll_test(&self, track: &str) -> moq_mux::Result { - let recorder = self.timeline().pacing_track(track)?; + let recorder = self.timeline().track(track, moq_mux::timeline::Config::default())?; + let section = self.timeline().section(); let mut producer = self.clone(); let mut catalog = producer.modify()?; - catalog.archive.get_or_insert_with(|| self.timeline().section()); + catalog.archive = Some(section); catalog.commit()?; Ok(recorder) } @@ -529,6 +593,17 @@ mod tests { } } + /// A catalog `archive` entry indexing `tracks` under their conventional timeline names. + fn archive(tracks: &[&str]) -> hang::catalog::Archive { + let mut archive = hang::catalog::Archive::new(); + for track in tracks { + archive + .timelines + .insert(track.to_string(), hang::timeline::default_name(track)); + } + archive + } + fn frame(micros: u64, keyframe: bool) -> moq_mux::container::Frame { payload_frame(micros, keyframe, &[0xDE, 0xAD, 0xBE, 0xEF]) } @@ -586,7 +661,7 @@ mod tests { let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); config.broadcast = Some(moq_net::path::Relative::new("../../source").to_owned()); let mut catalog = moq_mux::catalog::hang::Catalog::default(); - catalog.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + catalog.archive = Some(archive(&["video0", "audio0"])); catalog.video.renditions.insert("video".to_string(), config); let renditions = renditions::Producer::new(Config::default().window); @@ -614,7 +689,7 @@ mod tests { audio.bitrate = Some(96_000); let mut catalog = moq_mux::catalog::hang::Catalog::default(); - catalog.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + catalog.archive = Some(archive(&["video0", "audio0"])); catalog.video.renditions.insert("video0".to_string(), video_config()); catalog.audio.renditions.insert("audio0".to_string(), audio); catalog @@ -1075,7 +1150,7 @@ mod tests { let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let unversioned = rendition.media_playlist(None).expect("playable"); - assert!(unversioned.contains("\nseg/0.m4s\n")); + assert!(unversioned.contains("\nseg/656e3d1b.0.m4s\n")); assert!(matches!( broadcaster.set_generation(Some("run.1")), @@ -1087,15 +1162,15 @@ mod tests { let labeled = rendition .media_playlist(None) .expect("the first generation keeps the rows"); - assert!(labeled.contains("\nseg/run-1.0.m4s\n")); - assert!(labeled.contains("\nseg/run-1.1.m4s\n")); + assert!(labeled.contains("\nseg/run-1.656e3d1b.0.m4s\n")); + assert!(labeled.contains("\nseg/run-1.656e3d1b.1.m4s\n")); assert_eq!( init_hash(&labeled), init_hash(&unversioned), "the init needs no generation" ); let manifest = broadcaster.manifest(None).expect("manifest"); - assert!(manifest.contains("media=\"video/video0/seg/run-1.t$Time$.m4s\"")); + assert!(manifest.contains("media=\"video/video0/seg/run-1.656e3d1b.t$Time$.m4s\"")); broadcaster.set_generation(Some("run-2")).unwrap(); assert!( @@ -1106,7 +1181,7 @@ mod tests { media.write(vp8_frame(6_000_000, true)).unwrap(); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; let restarted = rendition.media_playlist(None).expect("the new run lists"); - assert!(restarted.contains("\nseg/run-2.2.m4s\n")); + assert!(restarted.contains("\nseg/run-2.656e3d1b.2.m4s\n")); assert!(!restarted.contains("run-1")); broadcaster.set_generation(None).unwrap(); @@ -1230,22 +1305,26 @@ mod tests { let rendered = rendition.media_playlist(None).expect("playable"); let hash = init_hash(&rendered); - assert!(rendered.contains("seg/0.m4s\n")); - assert!(rendered.contains("seg/1.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.0.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.1.m4s\n")); assert_eq!(rendition.init_versioned(hash).await.unwrap(), Some(init)); assert!(rendition.init_versioned("0000000000000000").await.unwrap().is_none()); // The same render, but carrying a credential into every child URL. let signed = rendition.media_playlist(Some("jwt=abc.def")).expect("playable"); assert!(signed.contains(&format!("#EXT-X-MAP:URI=\"init.{hash}.mp4?jwt=abc.def\"\n"))); - assert!(signed.contains("seg/0.m4s?jwt=abc.def\n")); + assert!(signed.contains("seg/656e3d1b.0.m4s?jwt=abc.def\n")); - let segment = rendition.segment(0).await.unwrap().expect("segment fetched on demand"); + let segment = rendition + .listed_segment(0) + .await + .unwrap() + .expect("segment fetched on demand"); assert_eq!(&segment[4..8], b"moof", "a fetched group transmuxes to moof+mdat"); // The live-edge group isn't a segment yet, and unknown groups miss. - assert!(rendition.segment(2).await.unwrap().is_none()); - assert!(rendition.segment(99).await.unwrap().is_none()); + assert!(rendition.listed_segment(2).await.unwrap().is_none()); + assert!(rendition.listed_segment(99).await.unwrap().is_none()); // Keep the publisher alive for the whole test. drop((media, registration, broadcast)); @@ -1285,7 +1364,11 @@ mod tests { .expect("rendition discovered from the catalog"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; - let segment = rendition.segment(0).await.unwrap().expect("segment fetched on demand"); + let segment = rendition + .listed_segment(0) + .await + .unwrap() + .expect("segment fetched on demand"); assert_eq!(&segment[4..8], b"moof"); drop((media, registration, broadcast)); } @@ -1354,7 +1437,7 @@ mod tests { ); assert!(manifest.contains("id=\"video/video0\"")); assert!(manifest.contains("id=\"audio/audio0\"")); - assert!(manifest.contains("media=\"video/video0/seg/t$Time$.m4s\"")); + assert!(manifest.contains("media=\"video/video0/seg/656e3d1b.t$Time$.m4s\"")); // Both renditions list the same aligned segment spans (the live edge is not listed). assert_eq!(manifest.matches("").count(), 2); assert_eq!(manifest.matches("").count(), 2); @@ -1365,17 +1448,21 @@ mod tests { let hash = between(&signed, "initialization=\"video/video0/init.", ".mp4?jwt=abc.def\""); let init = video_rendition.init().await.unwrap().expect("init segment"); assert_eq!(video_rendition.init_versioned(hash).await.unwrap(), Some(init)); - assert!(signed.contains("media=\"video/video0/seg/t$Time$.m4s?jwt=abc.def\"")); + assert!(signed.contains("media=\"video/video0/seg/656e3d1b.t$Time$.m4s?jwt=abc.def\"")); // $Time$ resolves to the same bytes the aligned number does; unknown times miss. let by_time = video_rendition - .segment_at(2_000) + .listed_segment_at(2_000) .await .unwrap() .expect("segment fetched by pts"); - let by_number = video_rendition.segment(1).await.unwrap().expect("segment by number"); + let by_number = video_rendition + .listed_segment(1) + .await + .unwrap() + .expect("segment by number"); assert_eq!(by_time, by_number); - assert!(video_rendition.segment_at(999).await.unwrap().is_none()); + assert!(video_rendition.listed_segment_at(999).await.unwrap().is_none()); drop((video, audio, video_registration, audio_registration, broadcast)); } @@ -1611,11 +1698,155 @@ mod tests { // The property HLS needs from the timeline rework: audio and video renditions share one // segment numbering, cut at the same boundaries. Video is one group per segment; an audio // segment packs every audio group inside the video segment's span. - // Most publishers declare no durationMax (a real-time GOP can be minutes long), so the - // exporter has to derive EXT-X-TARGETDURATION from the segments it has: a playlist whose - // EXTINF exceeds its target duration is invalid. + // Every rendition has its own timeline. The reference (the first video rendition by name) sets + // the segment boundaries; another rendition snaps them to its own keyframes, and a boundary with + // no keyframe nearby, such as before a rendition starts, is a gap. #[tokio::test] - async fn target_duration_covers_the_window_without_a_declared_bound() { + async fn renditions_snap_to_their_own_keyframes() { + let origin = produce_origin(); + let mut broadcast = origin.create_broadcast("live").expect("publish allowed"); + broadcast.announce(Default::default()).expect("publish allowed"); + settle().await; + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + + let reserved = catalog.reserve(); + let mut registrations = Vec::new(); + for name in ["video0", "video1", "video2"] { + let mut registration = catalog.test_video(name).unwrap(); + registration.set(video_config()).unwrap(); + registrations.push(registration); + } + drop(reserved); + + let mut producers = Vec::new(); + // The reference cuts every 2s; video1's GOPs sit 300ms later; video2 starts at 4s. + for (name, gops) in [ + ("video0", &[0u64, 2_000, 4_000, 6_000, 8_000][..]), + ("video1", &[300, 2_300, 4_300, 6_300, 8_300]), + ("video2", &[4_000, 6_000, 8_000]), + ] { + let track = broadcast.create_track(name, None).unwrap(); + let mut media = catalog + .media_producer( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), + ) + .unwrap(); + for ms in gops { + media.write(frame(ms * 1_000, true)).unwrap(); + } + producers.push(media); + } + + let source = moq_mux::Source::new(origin.consume(), "live"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + let listed = |name: &'static str, count: usize| { + let broadcaster = broadcaster.clone(); + async move { + for _ in 0..500 { + if let Some(rendition) = broadcaster.rendition(Kind::Video, name) { + let snapshot = rendition.snapshot(); + if snapshot.segments.len() >= count { + return snapshot; + } + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} never listed {count} segments"); + } + }; + + let reference = listed("video0", 4).await; + assert_eq!( + reference.segments.iter().map(|s| s.segment).collect::>(), + vec![0, 1, 2, 3] + ); + + // A segment is listed once the rendition's own timeline is a tolerance past its end. + let offset = listed("video1", 3).await; + assert_eq!(offset.media_sequence, reference.media_sequence, "one numbering"); + assert!(offset.segments.iter().all(|s| !s.gap), "every boundary snaps"); + let snapped = broadcaster + .rendition(Kind::Video, "video1") + .unwrap() + .listed_segment(1) + .await + .unwrap() + .expect("the snapped segment is served"); + assert_eq!(&snapped[4..8], b"moof"); + + let late = listed("video2", 3).await; + assert_eq!( + late.segments.iter().map(|s| s.gap).collect::>(), + vec![true, true, false], + "nothing to serve before the rendition starts" + ); + let video2 = broadcaster.rendition(Kind::Video, "video2").unwrap(); + assert!( + video2.listed_segment(0).await.unwrap().is_none(), + "a gap is never fetched" + ); + + drop((producers, registrations, broadcast)); + } + + // A broadcast without video takes its boundaries from its first audio rendition. + #[tokio::test] + async fn an_audio_only_broadcast_segments_on_its_audio() { + let origin = produce_origin(); + let mut broadcast = origin.create_broadcast("live").expect("publish allowed"); + broadcast.announce(Default::default()).expect("publish allowed"); + settle().await; + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + + let reserved = catalog.reserve(); + let mut registration = catalog.test_audio("audio0").unwrap(); + registration + .set(hang::catalog::AudioConfig::new( + hang::catalog::AudioCodec::Opus, + 48_000, + 2, + )) + .unwrap(); + drop(reserved); + + let track = broadcast.create_track("audio0", None).unwrap(); + let mut audio = catalog + .media_producer( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), + ) + .unwrap(); + // 500ms groups pack into records of the default 2s minimum. + for micros in (0..=6_000_000u64).step_by(500_000) { + let keyframe = audio.needs_keyframe(); + audio.write(frame(micros, keyframe)).unwrap(); + audio.cut(None).unwrap(); + } + + let source = moq_mux::Source::new(origin.consume(), "live"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; + let rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); + let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; + + let playlist = rendition.snapshot(); + assert_eq!( + playlist.segments.iter().map(|s| s.segment).collect::>(), + vec![0, 1, 2] + ); + assert_eq!(playlist.segments[0].duration, Duration::from_secs(2)); + let segment = rendition.listed_segment(1).await.unwrap().expect("audio segment"); + assert_eq!(&segment[4..8], b"moof"); + + drop((audio, registration, broadcast)); + } + + // The catalog's durationMax is the record split ceiling, far above a GOP, so the exporter + // derives EXT-X-TARGETDURATION from the segments it has: a playlist whose EXTINF exceeds its + // target duration is invalid, and one far above them delays every player. + #[tokio::test] + async fn target_duration_follows_the_observed_segments() { let origin = produce_origin(); let mut broadcast = origin.create_broadcast("live").expect("publish allowed"); broadcast.announce(Default::default()).expect("publish allowed"); @@ -1651,7 +1882,7 @@ mod tests { assert_eq!(playlist.segments[0].duration, Duration::from_secs(3)); assert_eq!( playlist.target_duration, 3, - "no bound was declared, so the target duration must still cover the 3s segment" + "the target duration covers the 3s segment, not the 10s split ceiling" ); } @@ -1728,13 +1959,13 @@ mod tests { // The same URI names the same span of content time on either rendition. let rendered = audio_rendition.media_playlist(None).expect("playable"); - assert!(rendered.contains("seg/0.m4s\n")); - assert!(rendered.contains("seg/1.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.0.m4s\n")); + assert!(rendered.contains("seg/656e3d1b.1.m4s\n")); // A video segment is one group; the matching audio segment transmuxes its four groups. - let video_segment = video_rendition.segment(1).await.unwrap().expect("video segment"); + let video_segment = video_rendition.listed_segment(1).await.unwrap().expect("video segment"); assert_eq!(&video_segment[4..8], b"moof"); - let audio_segment = audio_rendition.segment(1).await.unwrap().expect("audio segment"); + let audio_segment = audio_rendition.listed_segment(1).await.unwrap().expect("audio segment"); assert_eq!(&audio_segment[4..8], b"moof"); assert!( audio_segment.len() > video_segment.len(), @@ -1821,12 +2052,18 @@ mod tests { ) -> (Arc, tokio::task::JoinHandle<()>) { let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video0".to_string(), config.clone()); - let archive = hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME); + let archive = archive(&["video0", "audio0"]); catalog.archive = Some(archive.clone()); let renditions = renditions::Producer::new(Config::default().window); renditions.sync(upstream, &catalog); - let watcher = tokio::spawn(watch_timeline(upstream.broadcast.clone(), archive, renditions.fanout())); + let reference = Arc::new((Kind::Video, "video0".to_string())); + let watcher = tokio::spawn(watch_timeline( + upstream.broadcast.clone(), + archive, + reference, + renditions.fanout(), + )); let rendition = renditions.get(Kind::Video, "video0").expect("rendition synced"); (rendition, watcher) } @@ -1894,7 +2131,7 @@ mod tests { let (rendition, watcher) = export(&upstream, &config); let _ = tokio::time::timeout(Duration::from_secs(1), rendition.playable()).await; - let served = rendition.segment(0).await.unwrap(); + let served = rendition.listed_segment(0).await.unwrap(); if let Some(served) = &served { assert!( !contains(served, NEW), @@ -1920,7 +2157,7 @@ mod tests { .await .expect("the new publisher's timeline arrives"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the new publisher is servable on its own catalog"); @@ -2005,7 +2242,7 @@ mod tests { settle().await; let served = rendition - .segment(0) + .listed_segment(0) .await .expect("a replaced sibling is an unavailable group, not a server error"); if let Some(served) = &served { @@ -2033,7 +2270,7 @@ mod tests { .await .expect("the timeline reaches the fresh rendition"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the new publisher is servable on a freshly bound rendition"); @@ -2135,19 +2372,25 @@ mod tests { let mut snapshot = moq_mux::catalog::hang::Catalog::default(); snapshot.video.renditions.insert("video0".to_string(), config.clone()); - let archive = hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME); + let archive = archive(&["video0"]); snapshot.archive = Some(archive.clone()); let renditions = renditions::Producer::new(Config::default().window); renditions.sync(&upstream, &snapshot); let fanout = renditions.fanout(); - let watcher = tokio::spawn(watch_timeline(upstream.broadcast.clone(), archive, fanout.clone())); + let reference = Arc::new((Kind::Video, "video0".to_string())); + let watcher = tokio::spawn(watch_timeline( + upstream.broadcast.clone(), + archive, + reference, + fanout.clone(), + )); let rendition = renditions.get(Kind::Video, "video0").unwrap(); accept_sibling(&old_server, &old_media).await; tokio::time::timeout(Duration::from_secs(5), rendition.playable()) .await .expect("the timeline reaches the rendition"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the original sibling is servable"); @@ -2159,7 +2402,7 @@ mod tests { drop((old_server, old_media, _old_track)); until_empty(&rendition).await; assert!( - rendition.segment(0).await.unwrap().is_none(), + rendition.listed_segment(0).await.unwrap().is_none(), "a row listed for the old publisher answers 404 after replacement" ); @@ -2201,7 +2444,7 @@ mod tests { .expect("a non-gap row after the new bind") .segment; let served = rendition - .segment(listed) + .listed_segment(listed) .await .unwrap() .expect("the rebound sibling is servable"); @@ -2220,7 +2463,7 @@ mod tests { "cursor creation cannot reset the baseline" ); assert!( - rendition.segment(0).await.unwrap().is_none(), + rendition.listed_segment(0).await.unwrap().is_none(), "a row listed for the old publisher is not served from the replacement" ); watcher.abort(); @@ -2256,19 +2499,25 @@ mod tests { let mut snapshot = moq_mux::catalog::hang::Catalog::default(); snapshot.video.renditions.insert("video0".to_string(), config); - let archive = hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME); + let archive = archive(&["video0"]); snapshot.archive = Some(archive.clone()); let renditions = renditions::Producer::new(Config::default().window); renditions.sync(&upstream, &snapshot); let fanout = renditions.fanout(); - let watcher = tokio::spawn(watch_timeline(upstream.broadcast.clone(), archive, fanout.clone())); + let reference = Arc::new((Kind::Video, "video0".to_string())); + let watcher = tokio::spawn(watch_timeline( + upstream.broadcast.clone(), + archive, + reference, + fanout.clone(), + )); let rendition = renditions.get(Kind::Video, "video0").unwrap(); tokio::time::timeout(Duration::from_secs(5), rendition.playable()) .await .expect("the timeline reaches the rendition"); // The sibling request is unanswered, so the segment waits for its media broadcast. - let mut segment = std::pin::pin!(rendition.segment(0)); + let mut segment = std::pin::pin!(rendition.listed_segment(0)); assert!( tokio::time::timeout(Duration::from_millis(10), segment.as_mut()) .await @@ -2320,7 +2569,7 @@ mod tests { .await .expect("the timeline reaches the rendition"); let served = rendition - .segment(0) + .listed_segment(0) .await .unwrap() .expect("the sibling present at export start is servable"); @@ -2361,13 +2610,13 @@ mod tests { tokio::time::timeout(Duration::from_secs(5), rendition.playable()) .await .expect("the timeline reaches the rendition"); - assert!(rendition.segment(0).await.unwrap().is_some()); + assert!(rendition.listed_segment(0).await.unwrap().is_some()); drop((old_server, old_media, _old_track)); until_empty(&rendition).await; for _ in 0..4 { assert!(rendition.snapshot().segments.is_empty()); - assert!(rendition.segment(0).await.unwrap().is_none()); + assert!(rendition.listed_segment(0).await.unwrap().is_none()); } let new_server = origin.dynamic("media", sibling_route(11)).unwrap(); @@ -2399,12 +2648,12 @@ mod tests { .expect("a non-gap row after the new bind") .segment; let served = rendition - .segment(listed) + .listed_segment(listed) .await .unwrap() .expect("the rebound sibling is servable after an unroutable gap"); assert!(contains(&served, NEW)); - assert!(rendition.segment(0).await.unwrap().is_none()); + assert!(rendition.listed_segment(0).await.unwrap().is_none()); watcher.abort(); drop((new_media, _new_track, new_server, catalog, live)); } @@ -2429,7 +2678,7 @@ mod tests { let media = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video".to_string(), media); - catalog.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + catalog.archive = Some(archive(&["video0", "audio0"])); renditions.sync(&upstream, &catalog); let rendition = renditions.get(Kind::Video, "video").expect("rendition synced"); @@ -2437,7 +2686,7 @@ mod tests { // The catalog drops the rendition: its cursor must run dry rather than park. let mut empty = moq_mux::catalog::hang::Catalog::default(); - empty.archive = Some(hang::catalog::Archive::new(hang::timeline::DEFAULT_NAME)); + empty.archive = Some(archive(&["video0", "audio0"])); renditions.sync(&upstream, &empty); let ended = tokio::time::timeout(Duration::from_secs(5), segments.next()) .await @@ -2591,4 +2840,273 @@ mod tests { .expect("renditions cursor resolves"); assert!(ended.is_none(), "renditions cursor ends when the broadcast closes"); } + + /// A two-second GOP record: record `n` covers group `n` from `2n` seconds. + fn gop(sequence: u64) -> hang::timeline::Record { + hang::timeline::Record::new( + sequence, + sequence * 2_000, + 2_000, + hang::timeline::Position::group(sequence), + hang::timeline::Position::group(sequence + 1), + ) + } + + /// A `video0` reference and a `video1` rendition over hand-published timelines, reconciled and + /// watched the way `watch_catalog` does. Nothing serves the media, so a segment fetch is a miss. + struct Failing { + broadcast: moq_net::broadcast::Producer, + renditions: renditions::Producer, + watcher: tokio::task::JoinHandle<()>, + } + + impl Failing { + fn new() -> Self { + let broadcast = moq_net::broadcast::Info::new().produce(); + let upstream = Upstream { + source: moq_mux::Source::new(produce_origin().consume(), "live"), + broadcast: broadcast.consume(), + }; + let mut catalog = moq_mux::catalog::hang::Catalog::default(); + for name in ["video0", "video1"] { + catalog.video.renditions.insert(name.to_string(), video_config()); + } + let archive = archive(&["video0", "video1"]); + catalog.archive = Some(archive.clone()); + + let renditions = renditions::Producer::new(Config::default().window); + renditions.sync(&upstream, &catalog); + let watcher = tokio::spawn(watch_timeline( + upstream.broadcast.clone(), + archive, + reference(&catalog).unwrap(), + renditions.fanout(), + )); + Self { + broadcast, + renditions, + watcher, + } + } + + /// Publish `track`'s timeline. Returns the track, to fail it, with the timeline writing to it. + fn publish(&self, track: &str) -> (moq_net::track::Producer, moq_mux::timeline::Producer) { + let track = self + .broadcast + .create_track(hang::timeline::default_name(track), moq_mux::timeline::Producer::info()) + .unwrap(); + let timeline = moq_mux::timeline::Producer::new(track.clone()); + (track, timeline) + } + + fn rendition(&self, name: &str) -> Arc { + self.renditions.get(Kind::Video, name).unwrap() + } + + /// Wait until `name` lists segment `last`. + async fn listed(&self, name: &str, last: u64) -> playlist::Snapshot { + for _ in 0..500 { + let snapshot = self.rendition(name).snapshot(); + if snapshot.segments.last().is_some_and(|s| s.segment >= last) { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} never listed segment {last}"); + } + + /// Wait until `name`'s playlist ends. + async fn finished(&self, name: &str) -> playlist::Snapshot { + for _ in 0..500 { + let snapshot = self.rendition(name).snapshot(); + if snapshot.finished { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} never ended"); + } + + /// Drain `name`'s recording cursor, which must end rather than park. + async fn drain(&self, name: &str) { + let mut segments = self.rendition(name).segments(); + let drained = tokio::time::timeout(Duration::from_secs(5), async { + while segments.next().await.unwrap().is_some() {} + }) + .await; + assert!(drained.is_ok(), "{name}'s cursor parked"); + } + } + + impl Drop for Failing { + fn drop(&mut self) { + self.watcher.abort(); + } + } + + /// The timeline's publisher loses its session: the track and its open group fail. + fn cut(track: moq_net::track::Producer, timeline: moq_mux::timeline::Producer) { + drop(timeline); + track.abort(moq_net::Error::SessionClosed).unwrap(); + } + + fn numbers(snapshot: &playlist::Snapshot) -> Vec { + snapshot.segments.iter().map(|s| s.segment).collect() + } + + // A failed rendition timeline is not retried. Its playlist ends at the last segment its + // records cover, rather than listing the rest as gaps, while the other renditions go on. + #[tokio::test(start_paused = true)] + async fn a_rendition_timeline_error_ends_its_playlist() { + let test = Failing::new(); + let (_reference, mut video0) = test.publish("video0"); + let (track, mut video1) = test.publish("video1"); + for sequence in 0..4 { + video0.push(&gop(sequence)).unwrap(); + video1.push(&gop(sequence)).unwrap(); + } + // A segment resolves once the rendition's timeline is a snap tolerance past its end. + assert_eq!(numbers(&test.listed("video1", 2).await), [0, 1, 2]); + + cut(track, video1); + for sequence in 4..7 { + video0.push(&gop(sequence)).unwrap(); + } + let failed = test.finished("video1").await; + assert_eq!(numbers(&failed), [0, 1, 2, 3], "the covered segments stay listed"); + assert!( + failed.segments.iter().all(|s| !s.gap), + "nothing past the failure is a gap" + ); + test.drain("video1").await; + + let reference = test.listed("video0", 6).await; + assert!(!reference.finished, "the other renditions go on"); + } + + // A failed reference timeline is not retried. Every playlist ends (`EXT-X-ENDLIST`) instead + // of freezing live, and recording cursors end. + #[tokio::test(start_paused = true)] + async fn a_reference_timeline_error_ends_every_playlist() { + let test = Failing::new(); + let (track, mut video0) = test.publish("video0"); + let (_track, mut video1) = test.publish("video1"); + for sequence in 0..4 { + video0.push(&gop(sequence)).unwrap(); + video1.push(&gop(sequence)).unwrap(); + } + test.listed("video0", 3).await; + + cut(track, video0); + assert_eq!(numbers(&test.finished("video0").await), [0, 1, 2, 3]); + // video1's timeline hasn't passed the last segment yet, so ending its playlist now would + // stop players before that segment is listed. + let pending = test.rendition("video1").snapshot(); + assert_eq!(numbers(&pending), [0, 1, 2]); + assert!(!pending.finished, "a pending tail holds back EXT-X-ENDLIST"); + // DASH alike: the manifest turns static only once every representation ended. + assert!(test.rendition("video0").representation().unwrap().ended); + assert!(!test.rendition("video1").representation().unwrap().ended); + video1.push(&gop(4)).unwrap(); + assert_eq!(numbers(&test.finished("video1").await), [0, 1, 2, 3]); + test.drain("video0").await; + } + + // A rendition running far ahead of a stalled reference keeps the records its listed segments + // resolve to, so a listed segment never turns into a gap. + #[tokio::test(start_paused = true)] + async fn a_rendition_ahead_of_the_reference_keeps_listed_segments() { + let test = Failing::new(); + let (_track0, mut video0) = test.publish("video0"); + let (_track1, mut video1) = test.publish("video1"); + // The reference stalls at 20s while video1 runs on to 80s, far past the 16s window. + for sequence in 0..10 { + video0.push(&gop(sequence)).unwrap(); + } + for sequence in 0..40 { + video1.push(&gop(sequence)).unwrap(); + } + let listed = test.listed("video1", 9).await; + assert!(!listed.segments.is_empty()); + assert!( + listed.segments.iter().all(|s| !s.gap), + "every listed segment still resolves: {:?}", + numbers(&listed) + ); + } + + // A new reference numbers segments from its own records, so its URLs carry its own tag and + // never reuse one the old reference listed for other content. + #[tokio::test(start_paused = true)] + async fn a_reference_switch_never_reuses_a_segment_url() { + let mut test = Failing::new(); + let (_track0, mut video0) = test.publish("video0"); + let (_track1, mut video1) = test.publish("video1"); + for sequence in 0..4 { + video0.push(&gop(sequence)).unwrap(); + } + // video1's records are numbered from 0 but start 3s later, so its segment 0 is other content. + for sequence in 0..4 { + video1 + .push(&hang::timeline::Record::new( + sequence, + sequence * 2_000 + 3_000, + 2_000, + hang::timeline::Position::group(sequence), + hang::timeline::Position::group(sequence + 1), + )) + .unwrap(); + } + let before = test.listed("video0", 2).await; + let old = segments::tag(&(Kind::Video, "video0".to_string())); + assert!(before.segments.iter().all(|s| s.tag == old)); + + // Switch the reference the way `watch_catalog` does. + test.watcher.abort(); + test.renditions.fanout().skip(); + test.watcher = tokio::spawn(watch_timeline( + test.broadcast.consume(), + archive(&["video0", "video1"]), + Arc::new((Kind::Video, "video1".to_string())), + test.renditions.fanout(), + )); + let after = test.listed("video1", 2).await; + let new = segments::tag(&(Kind::Video, "video1".to_string())); + assert_eq!(numbers(&after)[0], 0, "the new reference numbers from its own records"); + assert!(after.segments.iter().all(|s| s.tag == new && s.tag != old)); + let video0 = test.rendition("video0"); + assert!( + video0.segment(&old, 0).await.unwrap().is_none(), + "the old URL is no longer served" + ); + } + + // A malformed reference timeline fails the same way. + #[tokio::test(start_paused = true)] + async fn a_malformed_reference_timeline_ends_every_playlist() { + let test = Failing::new(); + let (track, mut video0) = test.publish("video0"); + video0.push(&gop(0)).unwrap(); + test.listed("video0", 0).await; + + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::now(), &b"garbage"[..]).unwrap(); + test.finished("video0").await; + test.finished("video1").await; + } + + // A timeline subscription still waiting on its track ends with the broadcast, so nothing + // parks. + #[tokio::test(start_paused = true)] + async fn a_pending_timeline_subscription_ends_with_the_broadcast() { + let test = Failing::new(); + // A handler keeps the unpublished timelines' requests waiting rather than refused. + let _handler = test.broadcast.dynamic(); + tokio::time::sleep(Duration::from_secs(60)).await; + assert!(!test.rendition("video0").snapshot().finished); + + test.broadcast.close(); + test.finished("video0").await; + test.drain("video0").await; + } } diff --git a/rs/moq-hls/src/export/mpd.rs b/rs/moq-hls/src/export/mpd.rs index bea274ff80a..6a076dd2bc1 100644 --- a/rs/moq-hls/src/export/mpd.rs +++ b/rs/moq-hls/src/export/mpd.rs @@ -18,7 +18,7 @@ use std::time::{Duration, SystemTime}; use super::Kind; -/// One DASH representation: master-level metadata plus its slice of the shared timeline. +/// One DASH representation: master-level metadata plus its segment timeline. pub(crate) struct Representation { /// Rendition name (the `` in its `//...` paths). pub name: String, @@ -50,7 +50,9 @@ pub(crate) struct Representation { pub ended: bool, /// The init segment's content hash (`init.{init}.mp4`). pub init: String, - /// The publisher run every segment URL carries (`seg/{generation}.t$Time$.m4s`). + /// The [`tag`](super::segments::tag) of the reference numbering the segments. + pub tag: String, + /// The publisher run every segment URL carries (`seg/{generation}.{tag}.t$Time$.m4s`). pub generation: Option>, } @@ -61,8 +63,9 @@ pub(crate) struct Manifest { pub availability_start: Option, /// When this render happened (`MPD@publishTime`, dynamic only). pub publish: SystemTime, - /// The playlist window (`MPD@timeShiftBufferDepth`, dynamic only). - pub window: Duration, + /// The playlist window (`MPD@timeShiftBufferDepth`, dynamic only), or `None` when the + /// timeline bounds itself and the depth is the span it lists. + pub window: Option, /// The broadcast ended: render a `static` presentation instead of a `dynamic` one. pub finished: bool, /// Video representations, in catalog order. @@ -105,6 +108,23 @@ fn frame_rate(rate: f64) -> Option { } } +/// `units` of `timescale` as a [`Duration`]. +fn duration(units: u64, timescale: u32) -> Duration { + Duration::from_nanos((u128::from(units) * 1_000_000_000 / u128::from(timescale.max(1))) as u64) +} + +/// The longest span any representation lists, oldest segment start to newest segment end. +fn listed_span<'a>(representations: impl Iterator) -> Duration { + representations + .filter_map(|rep| { + let (first, _) = rep.segments.first()?; + let (last, d) = rep.segments.last()?; + Some(duration(last + d - first, rep.timescale)) + }) + .max() + .unwrap_or_default() +} + /// The largest listed segment duration in whole seconds, for `MPD@maxSegmentDuration` (and the /// update cadence). Like HLS's target duration, derived from the segments when the publisher /// declared no bound. @@ -123,6 +143,7 @@ fn render_representation(out: &mut String, rep: &Representation, suffix: &str) { let kind = rep.kind.as_str(); let name = escape(&rep.name); let init = escape(&rep.init); + let tag = escape(&rep.tag); let generation = rep .generation .as_deref() @@ -154,7 +175,7 @@ fn render_representation(out: &mut String, rep: &Representation, suffix: &str) { let _ = writeln!( out, - " ", + " ", rep.timescale.max(1) ); let _ = writeln!(out, " "); @@ -225,10 +246,7 @@ pub(crate) fn render_manifest(manifest: &Manifest, query: Option<&str>) -> Strin let duration = representations() .filter_map(|rep| { let (t, d) = rep.segments.last()?; - let timescale = rep.timescale.max(1) as u64; - Some(Duration::from_nanos( - ((t + d) as u128 * 1_000_000_000 / timescale as u128) as u64, - )) + Some(duration(t + d, rep.timescale)) }) .max() .unwrap_or_default(); @@ -242,14 +260,15 @@ pub(crate) fn render_manifest(manifest: &Manifest, query: Option<&str>) -> Strin // Reload cadence and live delay follow HLS conventions: players refresh about once // per segment and sit a few segments behind the live edge (bounded by the window). let update = Duration::from_secs(target); - let delay = Duration::from_secs(3 * target).min(manifest.window.max(update)); + let window = manifest.window.unwrap_or_else(|| listed_span(representations())); + let delay = Duration::from_secs(3 * target).min(window.max(update)); let _ = write!( out, " type=\"dynamic\" availabilityStartTime=\"{}\" publishTime=\"{}\" minimumUpdatePeriod=\"{}\" timeShiftBufferDepth=\"{}\" suggestedPresentationDelay=\"{}\"", humantime::format_rfc3339_millis(availability), humantime::format_rfc3339_millis(manifest.publish), xs_duration(update), - xs_duration(manifest.window), + xs_duration(window), xs_duration(delay), ); } @@ -287,6 +306,7 @@ mod tests { segments, ended, init: "0123abcd".into(), + tag: "ab12cd34".into(), generation: None, } } @@ -306,6 +326,7 @@ mod tests { segments, ended, init: "4567cdef".into(), + tag: "ab12cd34".into(), generation: None, } } @@ -315,7 +336,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_751_846_400_123)), publish: SystemTime::UNIX_EPOCH + Duration::from_millis(1_751_846_410_000), - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000), (2_000, 2_000)], false)], audio: vec![audio(vec![(0, 2_000), (2_000, 2_000)], false)], @@ -343,7 +364,7 @@ mod tests { "" )); assert!(out.contains( - "" + "" )); assert!(out.contains("")); assert!(out.contains("")); @@ -355,7 +376,7 @@ mod tests { let manifest = Manifest { availability_start: None, publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: true, // The window starts mid-broadcast: presentation time stays anchored at pts 0 (no // presentationTimeOffset), so the duration spans the lead-in and a live session @@ -379,7 +400,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000)], false)], audio: Vec::new(), @@ -387,7 +408,7 @@ mod tests { let out = render_manifest(&manifest, Some("jwt=abc.def&x=1")); assert!(out.contains("initialization=\"video/video0/init.0123abcd.mp4?jwt=abc.def&x=1\"")); - assert!(out.contains("media=\"video/video0/seg/t$Time$.m4s?jwt=abc.def&x=1\"")); + assert!(out.contains("media=\"video/video0/seg/ab12cd34.t$Time$.m4s?jwt=abc.def&x=1\"")); } #[test] @@ -397,7 +418,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![rep], audio: Vec::new(), @@ -405,7 +426,7 @@ mod tests { let out = render_manifest(&manifest, None); assert!(out.contains("initialization=\"video/video0/init.0123abcd.mp4\"")); - assert!(out.contains("media=\"video/video0/seg/run-7.t$Time$.m4s\"")); + assert!(out.contains("media=\"video/video0/seg/run-7.ab12cd34.t$Time$.m4s\"")); } #[test] @@ -420,7 +441,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000)], false)], audio: vec![audio(Vec::new(), false)], diff --git a/rs/moq-hls/src/export/playlist.rs b/rs/moq-hls/src/export/playlist.rs index 00f833bd20b..64eebbc4601 100644 --- a/rs/moq-hls/src/export/playlist.rs +++ b/rs/moq-hls/src/export/playlist.rs @@ -30,14 +30,16 @@ pub(crate) struct Snapshot { /// Wall-clock time of the first listed segment (`EXT-X-PROGRAM-DATE-TIME`), when the /// catalog advertises its root broadcast clock. pub program_date_time: Option, - /// The publisher run every segment URL carries (`seg/{generation}.{segment}.m4s`). + /// The publisher run every segment URL carries (`seg/{generation}.{tag}.{segment}.m4s`). pub generation: Option>, } /// One listed segment. pub(crate) struct Segment { - /// The aligned segment number; the URI is `seg/{segment}.m4s`. + /// The aligned segment number; the URI is `seg/{tag}.{segment}.m4s`. pub segment: u64, + /// The [`tag`](super::segments::tag) of the reference numbering the segment. + pub tag: String, /// `EXTINF` duration. pub duration: Duration, /// The rendition has no content for this span (`EXT-X-GAP`): the segment keeps its slot in @@ -91,7 +93,7 @@ pub(crate) fn render_media(snapshot: &Snapshot, init: &str, query: Option<&str>) let _ = writeln!(out, "#EXT-X-GAP"); } let _ = writeln!(out, "#EXTINF:{:.5},", segment.duration.as_secs_f64()); - let _ = writeln!(out, "seg/{generation}{}.m4s{suffix}", segment.segment); + let _ = writeln!(out, "seg/{generation}{}.{}.m4s{suffix}", segment.tag, segment.segment); } if snapshot.finished { @@ -115,12 +117,14 @@ mod tests { segments: vec![ Segment { segment: 10, + tag: "ab12cd34".into(), duration: Duration::from_secs(2), gap: false, discontinuity: false, }, Segment { segment: 11, + tag: "ab12cd34".into(), duration: Duration::from_millis(1960), gap: false, discontinuity: false, @@ -137,15 +141,15 @@ mod tests { assert!(out.contains("#EXT-X-MEDIA-SEQUENCE:10\n")); assert!(out.contains("#EXT-X-MAP:URI=\"init.0123abcd.mp4\"\n")); assert!(out.contains("#EXT-X-PROGRAM-DATE-TIME:2025-07-07T00:00:00.123Z\n")); - assert!(out.contains("#EXTINF:2.00000,\nseg/10.m4s\n")); - assert!(out.contains("#EXTINF:1.96000,\nseg/11.m4s\n")); + assert!(out.contains("#EXTINF:2.00000,\nseg/ab12cd34.10.m4s\n")); + assert!(out.contains("#EXTINF:1.96000,\nseg/ab12cd34.11.m4s\n")); assert!(!out.contains("#EXT-X-ENDLIST")); // A credential rides every child URL so a header-less player keeps sending it. let signed = render_media(&snapshot, "0123abcd", Some("jwt=abc.def")); assert!(signed.contains("#EXT-X-MAP:URI=\"init.0123abcd.mp4?jwt=abc.def\"\n")); - assert!(signed.contains("\nseg/10.m4s?jwt=abc.def\n")); - assert!(signed.contains("\nseg/11.m4s?jwt=abc.def\n")); + assert!(signed.contains("\nseg/ab12cd34.10.m4s?jwt=abc.def\n")); + assert!(signed.contains("\nseg/ab12cd34.11.m4s?jwt=abc.def\n")); } #[test] @@ -155,6 +159,7 @@ mod tests { media_sequence: 0, segments: vec![Segment { segment: 0, + tag: "ab12cd34".into(), duration: Duration::from_secs(2), gap: false, discontinuity: false, @@ -166,7 +171,7 @@ mod tests { let out = render_media(&snapshot, "0123abcd", Some("jwt=x")); assert!(out.contains("#EXT-X-MAP:URI=\"init.0123abcd.mp4?jwt=x\"\n")); - assert!(out.contains("\nseg/run-7.0.m4s?jwt=x\n")); + assert!(out.contains("\nseg/run-7.ab12cd34.0.m4s?jwt=x\n")); } #[test] @@ -176,6 +181,7 @@ mod tests { media_sequence: 0, segments: vec![Segment { segment: 0, + tag: "ab12cd34".into(), duration: Duration::from_secs(4), gap: false, discontinuity: false, diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 729d593556a..e90623e7dbf 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -1,5 +1,5 @@ -//! One rendition: playlists from its view of the broadcast timeline, segments fetched on -//! demand. +//! One rendition: playlists from its view of the broadcast's segments, resolved against its own +//! timeline, and segments fetched on demand. use std::sync::{Arc, Mutex, RwLock}; use std::task::Poll; @@ -12,6 +12,7 @@ use moq_mux::timeline::Entry; use sha2::{Digest, Sha256}; use super::playlist::{Segment, Snapshot}; +use super::spans::{Content, Spans}; use super::upstream::Upstream; use super::{mpd, segments}; use crate::Result; @@ -234,9 +235,15 @@ pub struct Rendition { /// The catalog's root broadcast clock: the wall anchor timings map through, after /// timescale conversion. Absent when the publisher exposes none. clock: Option, - /// This rendition's window over the broadcast timeline, fed by the catalog watcher's + /// This rendition's window over the broadcast's segments, fed by the catalog watcher's /// fan-out. live: Arc, + /// This rendition's own timeline, which resolves each segment to its frames. + spans: Arc, + /// The broadcast carrying the catalog and every timeline. + catalog: moq_net::broadcast::Consumer, + /// Feeds [`spans`](Self::spans); aborted when the rendition drops. + watcher: Mutex>>, /// The broadcast serving this rendition's media. A sibling is bound when the rendition is /// created and rebound if that publisher is replaced (see [`Upstream::bind`]). media: Media, @@ -321,6 +328,9 @@ impl Rendition { section, clock, live: Arc::new(segments::Producer::new()), + spans: Arc::new(Spans::new()), + catalog: upstream.broadcast.clone(), + watcher: Mutex::new(None), media: Media::bind(upstream, config.broadcast.as_ref())?, run: Mutex::default(), building: tokio::sync::Mutex::new(()), @@ -348,33 +358,98 @@ impl Rendition { section, clock, live: Arc::new(segments::Producer::new()), + spans: Arc::new(Spans::new()), + catalog: upstream.broadcast.clone(), + watcher: Mutex::new(None), media: Media::bind(upstream, config.broadcast.as_ref())?, run: Mutex::default(), building: tokio::sync::Mutex::new(()), }) } - /// Feed one timeline record into this rendition's window: its own ranges (empty when the - /// record carries none for it, a gap), timed by the record. - pub(crate) fn push(&self, index: u64, entry: &Entry, discontinuity: u64, window: Duration) { + /// Start following this rendition's own timeline, trimmed to the listed segments. + /// + /// A rendition the catalog indexes no timeline for resolves nothing but the reference's rows. + pub(crate) fn watch(&self) { + let Some(_) = self.section.timelines.get(&self.name) else { + self.spans.end(); + return; + }; + let watcher = tokio::spawn(watch_spans( + self.catalog.clone(), + self.section.clone(), + self.name.clone(), + self.spans.clone(), + )); + *self.watcher.lock().expect("watcher lock poisoned") = Some(watcher); + } + + /// Feed one reference timeline record into this rendition's window as a segment. With no + /// `window`, only source timeline pops trim it. + pub(crate) fn push( + &self, + index: u64, + entry: &Entry, + reference: &segments::Reference, + discontinuity: u64, + window: Option, + ) { if !self.media.admits(&self.live) { return; } let row = segments::Row { index, - segment: entry.segment, - ranges: entry.tracks.get(&self.name).cloned().unwrap_or_default(), + segment: entry.sequence, + reference: reference.clone(), + frames: entry.start..entry.end, + keyframe: entry.keyframe, duration: entry.duration, pts: entry.pts, - end: Duration::from(entry.pts) + entry.duration, + end: entry.end_time(), discontinuity, }; self.live.push(row, window); + self.trim(); + } + + /// Drop this rendition's own records that no listed segment can resolve to anymore. + fn trim(&self) { + if let Some(oldest) = self.live.oldest() { + self.spans.trim(oldest); + } + } + + /// What `row` holds on this rendition: the reference serves its own record, and every other + /// rendition resolves the span against its own timeline. + fn resolve(&self, row: &segments::Row) -> Content { + if self.is(&row.reference) { + return Content::Frames { + ranges: vec![row.frames.clone()], + filter: None, + }; + } + self.spans + .resolve(self.kind == Kind::Video, Duration::from(row.pts)..row.end) + } + + /// Poll until `row` resolves on this rendition. + pub(crate) fn poll_resolved(&self, waiter: &kio::Waiter, row: &segments::Row) -> Poll<()> { + if self.is(&row.reference) { + return Poll::Ready(()); + } + self.spans + .poll_resolved(waiter, self.kind == Kind::Video, Duration::from(row.pts)..row.end) + } + + /// Whether this rendition is `reference`. + fn is(&self, reference: &(Kind, String)) -> bool { + reference.0 == self.kind && reference.1 == self.name } /// Remove source timeline records in `range` from this rendition's window. pub(crate) fn pop(&self, range: std::ops::Range) { self.live.pop(range); + self.trim(); } /// Clear rows that can no longer be followed by a consecutive source timeline record. @@ -387,12 +462,15 @@ impl Rendition { self.run.lock().expect("run lock poisoned").generation = generation; } - /// Start a new publisher run labeled `generation`, dropping the previous run's rows and init. + /// Start a new publisher run labeled `generation`, dropping the previous run's rows, records, + /// and init. /// /// The rows go first: renders read the run before the rows, so one that sees the new label /// never lists the old run's segments under it. pub(crate) fn restart(&self, generation: Option>) { self.live.clear(); + // The old run's own records would bootstrap an inline init from its media. + self.spans.clear(); let mut run = self.run.lock().expect("run lock poisoned"); *run = Run { generation, @@ -411,7 +489,7 @@ impl Rendition { self.run().generation } - /// Mark this rendition's window ended (the timeline finished cleanly). + /// Mark this rendition's window ended (the timeline finished or failed). pub(crate) fn end(&self) { self.live.end(); } @@ -493,54 +571,63 @@ impl Rendition { self.media.sync(&self.live); let window = self.live.window(); - // EXT-X-TARGETDURATION starts from the catalog's declared bound when the publisher - // offered one: it knows its segment duration up front, so the first playlist already - // carries the real value instead of a guess grown from observation. Rounded up, since - // HLS wants whole seconds and the advertised value must never understate the bound. - let declared = self - .section - .duration_max - .map(|max| max.div_ceil((self.section.timescale as u64).max(1))) - .unwrap_or(0); - - // Most publishers can't promise a bound (a real-time GOP can be minutes long), and even - // a declared one is only a promise about content the publisher controls. HLS requires - // EXT-X-TARGETDURATION to be at least every EXTINF, so cover the window either way. - let observed = window + // HLS requires EXT-X-TARGETDURATION to be at least every EXTINF, rounded up to whole + // seconds. The catalog's `durationMax` is only the ceiling a record is split at, far above a + // typical GOP, and a target that large would push every player's live edge back by it. + let target_duration = window .segments .iter() .map(|s| s.duration.as_secs_f64().ceil() as u64) .max() - .unwrap_or(0); - let target_duration = declared.max(observed).max(1); + .unwrap_or(0) + .max(1); let program_date_time = window.segments.first().and_then(|first| self.wall_clock(first.pts)); let mut previous: Option = None; - let segments = window - .segments - .into_iter() - .map(|s| { + let segments: Vec = self + .resolved(&window.segments) + .map(|(s, content)| { let discontinuity = previous.replace(s.discontinuity).is_some_and(|p| p != s.discontinuity); Segment { segment: s.segment, + tag: segments::tag(&s.reference), duration: s.duration, - gap: s.ranges.is_empty(), + gap: content == Content::Gap, discontinuity, } }) .collect(); + // A failed timeline ends this playlist at its last resolvable segment. + let failed = window.segments.last().is_some_and(|row| self.is_failed(row)); + // An ended reference ends the playlist only once this rendition resolved every row, so a + // slower timeline's tail is listed before `EXT-X-ENDLIST` stops players polling. + let complete = segments.len() == window.segments.len(); Snapshot { target_duration, media_sequence: window.sequence, segments, - finished: window.ended, + finished: (window.ended && complete) || failed, program_date_time, generation, } } + /// `rows` with what each holds, up to the first one this rendition cannot resolve yet (or + /// ever, once its timeline failed): a segment is listed only once its content is final. + fn resolved<'a>(&'a self, rows: &'a [segments::Row]) -> impl Iterator { + rows.iter() + .map(|row| (row, self.resolve(row))) + .take_while(|(_, content)| !matches!(content, Content::Pending | Content::Failed)) + } + + /// Whether this rendition's timeline failed before reaching `row`'s end, so neither it nor + /// any later row will resolve. + pub(crate) fn is_failed(&self, row: &segments::Row) -> bool { + self.resolve(row) == Content::Failed + } + /// Whether the playlist has anything to serve yet (at least one segment, or the broadcast /// already ended). pub(crate) fn is_playable(&self) -> bool { @@ -548,8 +635,8 @@ impl Rendition { self.live.is_playable() } - /// This rendition's DASH representation: its master-level metadata plus its slice of the - /// shared timeline as `(t, d)` pairs in the timeline's own timescale (the record values + /// This rendition's DASH representation: its master-level metadata plus its track's + /// timeline as `(t, d)` pairs in the timeline's own timescale (the record values /// verbatim, so `$Time$` addressing resolves exactly). `None` until the init is known, as /// for [`media_playlist`](Self::media_playlist). pub(crate) fn representation(&self) -> Option { @@ -559,11 +646,12 @@ impl Rendition { self.media.sync(&self.live); let window = self.live.window(); let timescale = self.timescale(); - let segments = window - .segments - .iter() - .filter(|row| !row.ranges.is_empty()) - .filter_map(|row| { + // Static only once every row resolved here, like the playlist's `EXT-X-ENDLIST`. + let complete = self.resolved(&window.segments).count() == window.segments.len(); + let segments = self + .resolved(&window.segments) + .filter(|(_, content)| *content != Content::Gap) + .filter_map(|(row, _)| { let t = row.pts.as_scale(timescale) as u64; let span = row.end.saturating_sub(row.pts.into()); let d = (span.as_nanos() * timescale.as_u64() as u128 / 1_000_000_000) as u64; @@ -591,22 +679,23 @@ impl Rendition { channel_count, timescale: self.section.timescale.max(1), segments, - ended: window.ended, + ended: (window.ended && complete) || window.segments.last().is_some_and(|row| self.is_failed(row)), + tag: listed_tag(&window.segments), init: init.hash.clone(), generation: run.generation, }) } - /// Fetch and transmux the segment whose timeline `pts` is `time`, in the timeline's - /// timescale (DASH `$Time$` addressing of the same bytes [`segment`](Self::segment) serves - /// by aligned number). + /// Fetch and transmux the segment of the reference tagged `tag` whose timeline `pts` is + /// `time`, in the timeline's timescale (DASH `$Time$` addressing of the same bytes + /// [`segment`](Self::segment) serves by aligned number). #[cfg_attr(not(feature = "server"), allow(dead_code))] - pub(crate) async fn segment_at(&self, time: u64) -> Result> { + pub(crate) async fn segment_at(&self, tag: &str, time: u64) -> Result> { self.media.sync(&self.live); - let Some(segment) = self.live.segment_number_at(time, self.timescale()) else { + let Some(segment) = self.live.segment_number_at(tag, time, self.timescale()) else { return Ok(None); }; - self.segment(segment).await + self.segment(tag, segment).await } /// The timeline's timescale; a declared 0 is clamped rather than trusted. @@ -713,13 +802,16 @@ impl Rendition { let bytes = match muxer.init()? { Some(bytes) => bytes, None => { - let Some(sequence) = self.live.latest_keyframe_group() else { + let reference = (self.kind, self.name.clone()); + let latest = self.spans.latest_keyframe_group(); + let Some(sequence) = latest.or_else(|| self.live.latest_keyframe_group(&reference)) else { return Ok(None); }; let Some((broadcast, track)) = self.track(&binding).await else { return Ok(None); }; - let Ok(mut group) = fetch(&track, sequence, &broadcast).await? else { + let Ok(mut group) = fetch(&track, hang::timeline::Position::group(sequence), None, &broadcast).await? + else { return Ok(None); }; // A cache eviction mid-read leaves the init unbuildable for now, not an error. @@ -735,31 +827,37 @@ impl Rendition { Ok(Some(self.cache_init(run.epoch, bytes))) } - /// Fetch and transmux the segment numbered `segment`. + /// Fetch and transmux the segment numbered `segment` by the reference tagged `tag` (the + /// `seg/{tag}.{segment}.m4s` URI). /// - /// Fetches every group the segment's ranges cover (an audio segment packs many short - /// groups) and encodes them as a single CMAF fragment. `None` when the segment isn't in - /// the playlist window, is a gap for this rendition, or its groups already left the relay - /// cache. - pub async fn segment(&self, segment: u64) -> Result> { + /// Fetches every frame the segment resolves to on this rendition (an audio segment packs many + /// short groups) and encodes them as a single CMAF fragment. `None` when the segment isn't in + /// the playlist window, is a gap or not yet resolved for this rendition, or its groups already + /// left the relay cache. + pub async fn segment(&self, tag: &str, segment: u64) -> Result> { + self.media.sync(&self.live); + let Some(row) = self.live.row(tag, segment) else { + return Ok(None); + }; + self.fetch(&row).await + } + + /// Fetch and transmux `row`'s frames on this rendition; see [`segment`](Self::segment). + pub(crate) async fn fetch(&self, row: &segments::Row) -> Result> { let binding = self.media.sync(&self.live); - // Read the run before the rows (see `restart`), so an init decoded from them is never - // cached for a run that started while they were being fetched. + // Read the run before resolving the row (see `restart`), so an init decoded from its + // groups is never cached for a run that started while they were being fetched. let run = self.run(); - let Some(ranges) = self.live.segment_ranges(segment) else { + let Content::Frames { ranges, filter } = self.resolve(row) else { return Ok(None); }; - if ranges.is_empty() { - // A gap: the record says this rendition has no content for the span. - return Ok(None); - } // A segment must be served whole. If its group span exceeds the safety cap (a // pathologically coarse timeline), 404 rather than silently truncating it to a segment // shorter than its advertised duration. let total: u64 = ranges .iter() - .map(|r| r.end.saturating_sub(r.start).saturating_add(1)) + .map(|range| last_group(range).saturating_sub(range.start.group).saturating_add(1)) .sum(); if total > MAX_SEGMENT_GROUPS { tracing::warn!( @@ -780,8 +878,13 @@ impl Rendition { // many groups. let mut frames = Vec::new(); for range in &ranges { - for sequence in range.start..=range.end { - let miss = match fetch(&track, sequence, &broadcast).await? { + for sequence in range.start.group..=last_group(range) { + let start = match sequence == range.start.group { + true => range.start, + false => hang::timeline::Position::group(sequence), + }; + let end = (sequence == range.end.group).then_some(range.end.frame); + let miss = match fetch(&track, start, end, &broadcast).await? { Ok(mut group) => match read_group(&mut muxer, &mut group, &broadcast).await? { Ok(mut group_frames) => { frames.append(&mut group_frames); @@ -792,7 +895,7 @@ impl Rendition { Err(err) => err, }; tracing::warn!( - segment, + segment = row.segment, group = sequence, err = %miss, "abandoning an advertised segment: a group is unavailable" @@ -801,10 +904,14 @@ impl Rendition { } } + // A rendition resolved by time keeps only the frames inside the segment's span. + if let Some(filter) = filter { + frames.retain(|frame| filter.contains(&Duration::from(frame.timestamp))); + } if frames.is_empty() { return Ok(None); } - let fragment = muxer.fragment(segment as u32, &frames)?; + let fragment = muxer.fragment(row.segment as u32, &frames)?; // Inline codec metadata was already decoded from these completed groups. // Reuse it instead of fetching a newer, possibly unfinished timeline tail. if let Some(bytes) = muxer.init()? { @@ -814,15 +921,89 @@ impl Rendition { } } +/// The tag of the reference numbering `rows`, empty when none are listed. A reference switch +/// clears every row, so the listed rows share one. +fn listed_tag(rows: &[segments::Row]) -> String { + rows.first() + .map(|row| segments::tag(&row.reference)) + .unwrap_or_default() +} + +#[cfg(test)] +impl Rendition { + /// Fetch the listed segment numbered `segment`, under the tag its rows carry. + pub(crate) async fn listed_segment(&self, segment: u64) -> Result> { + let tag = listed_tag(&self.live.window().segments); + self.segment(&tag, segment).await + } + + /// Fetch the listed segment starting at `time`, under the tag its rows carry. + pub(crate) async fn listed_segment_at(&self, time: u64) -> Result> { + let tag = listed_tag(&self.live.window().segments); + self.segment_at(&tag, time).await + } +} + +/// The last group holding a frame of `range`. +fn last_group(range: &std::ops::Range) -> u64 { + match range.end.frame { + 0 => range.end.group.saturating_sub(1), + _ => range.end.group, + } +} + +/// Follow `track`'s own timeline into `spans` until it ends, then mark nothing pending. +async fn watch_spans(broadcast: moq_net::broadcast::Consumer, section: Archive, track: String, spans: Arc) { + let result: Result<()> = async { + let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&broadcast, §ion, &track).await?; + while let Some(event) = timeline.next().await? { + match event { + moq_mux::timeline::Event::Push { entry, .. } => spans.push(entry), + moq_mux::timeline::Event::Pop(range) => spans.pop(range), + moq_mux::timeline::Event::Skip(_) => spans.clear(), + _ => unreachable!("unknown timeline event"), + } + } + Ok(()) + } + .await; + match result { + Ok(()) => spans.end(), + // Never retried (the origin already rides out transient source failures), and not + // papered over with gaps: the rendition's playlist ends at its last record. + Err(err) => { + tracing::error!(%track, %err, "rendition timeline failed; ending its playlist"); + spans.fail(); + } + } +} + +impl Drop for Rendition { + fn drop(&mut self) { + if let Some(watcher) = self.watcher.lock().expect("watcher lock poisoned").take() { + watcher.abort(); + } + } +} + /// Fetch one group, mapping "no longer (or not yet) servable" to the error that says so /// rather than a bare `None`: a caller abandoning a segment can only explain itself with it. +/// +/// Reads from `start` and, when `end` is set, stops before that frame of the group. async fn fetch( track: &moq_net::track::Consumer, - sequence: u64, + start: hang::timeline::Position, + end: Option, broadcast: &moq_net::broadcast::Consumer, ) -> Result> { - match track.fetch_group(sequence, None).await { - Ok(group) => Ok(Ok(group)), + let options = moq_net::group::Fetch::default().with_frame_start(start.frame); + match track.fetch_group(start.group, options).await { + Ok(mut group) => { + if let Some(end) = end { + group.set_frames(..end); + } + Ok(Ok(group)) + } Err(err) if is_unservable(&err, broadcast) => Ok(Err(err)), Err(err) => Err(err.into()), } diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index 757b9657dfc..b5253b506a0 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -2,8 +2,9 @@ //! //! The `Producer` holds the current renditions, reconciled from the catalog by its `sync`; //! the HTTP serve path reads them synchronously (look one up, render the master playlist). It -//! is also the fan-out point for the broadcast's single timeline: the timeline watcher's -//! `Fanout` handle hands each record to every rendition, which keeps its own window over it. +//! is also the fan-out point for the broadcast's segments: the reference rendition's timeline +//! watcher hands each record to every rendition through the `Fanout` handle, and each rendition +//! keeps its own window over them. //! A [`Consumer`] is a cursor for a recorder that mirrors the *whole* broadcast: [`next`](Consumer::next) //! yields one [`Event`] at a time as renditions are added or removed, replaying the current //! set as [`Added`](Event::Added) when first consumed, and returning `None` once the source @@ -19,7 +20,7 @@ use moq_mux::timeline::Entry; use super::Upstream; use super::rendition::{Kind, Rendition}; -use super::segments::Discontinuities; +use super::segments::{Discontinuities, Reference}; /// The `(kind, name)` identity of a rendition. Video and audio are separate axes, so a video /// and an audio rendition may share a name without colliding. @@ -47,10 +48,10 @@ pub enum Event { struct Feed { /// Every rendition ever created and still alive, pruned as they drop. targets: Vec>, - /// Recent records with their discontinuity sequence, replayed into a rendition created - /// mid-broadcast so its playlist window isn't empty until the next record. Evicted with the - /// same policy as the windows. - history: VecDeque<(u64, Entry, u64)>, + /// Recent reference records with their discontinuity sequence, replayed into a rendition + /// created mid-broadcast so its playlist window isn't empty until the next record. Evicted with + /// the same policy as the windows. + history: VecDeque<(u64, Entry, Reference, u64)>, /// Stamps each record once, so every rendition agrees on where the timeline breaks. discontinuities: Discontinuities, /// The timeline ended cleanly; late-created renditions start ended (`EXT-X-ENDLIST`). @@ -65,6 +66,10 @@ struct Feed { anchor: Option, /// The publisher run every segment URL carries; late-created renditions inherit it. generation: Option>, + /// The playlist window duration applied on every push (see + /// [`Config::window`](super::Config::window)), or `None` when the source timeline is + /// authoritative and only its pops trim the playlists. + window: Option, } /// The producing side of a broadcast's rendition set. @@ -87,8 +92,6 @@ pub(crate) struct Producer { #[derive(Clone)] pub(crate) struct Fanout { feed: Arc>, - /// The playlist window duration (see [`Config::window`](super::Config::window)), applied on every push. - window: Duration, } impl Producer { @@ -104,8 +107,8 @@ impl Producer { closed: false, anchor: None, generation: None, + window: Some(window), })), - window, }, } } @@ -115,9 +118,10 @@ impl Producer { self.fanout.clone() } - /// The playlist window duration every rendition's window is trimmed to. - pub fn window(&self) -> Duration { - self.fanout.window + /// The playlist window duration every rendition's window is trimmed to, or `None` when + /// only the source timeline trims them. + pub fn window(&self) -> Option { + self.fanout.feed.lock().unwrap().window } /// The estimated wall-clock time of timeline `pts` 0 (see [`Feed::anchor`]); `None` until @@ -184,14 +188,21 @@ impl Producer { } impl Fanout { - /// Fan one timeline record out to every living rendition, and into the replay history. - pub fn push(&self, index: u64, entry: Entry) { + /// List every record the source timeline retains, trimming only on its pops, instead of + /// a window. Call before the first record. + pub fn unbound(&self) { + self.feed.lock().unwrap().window = None; + } + + /// Fan one reference timeline record out to every living rendition, and into the replay + /// history. + pub fn push(&self, index: u64, entry: Entry, reference: &Reference) { let mut feed = self.feed.lock().unwrap(); // Same eviction policy as the per-rendition windows, so a replay reconstructs the // same window a live rendition would have. - if let Some((_, back, _)) = feed.history.back() - && (Duration::from(entry.pts) < Duration::from(back.pts) || entry.segment <= back.segment) + if let Some((_, back, _, _)) = feed.history.back() + && (Duration::from(entry.pts) < Duration::from(back.pts) || entry.sequence <= back.sequence) { feed.history.clear(); feed.anchor = None; @@ -204,23 +215,26 @@ impl Fanout { } let pts = Duration::from(entry.pts); let discontinuity = feed.discontinuities.stamp(pts, pts + entry.duration); - feed.history.push_back((index, entry.clone(), discontinuity)); - while feed.history.len() >= 2 { + feed.history + .push_back((index, entry.clone(), reference.clone(), discontinuity)); + let window = feed.window; + while let Some(window) = window + && feed.history.len() >= 2 + { let newest = &feed.history.back().unwrap().1; let span = (Duration::from(newest.pts) + newest.duration).saturating_sub(Duration::from(feed.history[1].1.pts)); - if span < self.window { + if span < window { break; } feed.history.pop_front(); } - let window = self.window; feed.targets.retain(|target| { let Some(rendition) = target.upgrade() else { return false; }; - rendition.push(index, &entry, discontinuity, window); + rendition.push(index, &entry, reference, discontinuity, window); true }); } @@ -228,7 +242,7 @@ impl Fanout { /// Remove records that left the source timeline window from every rendition window. pub fn pop(&self, range: std::ops::Range) { let mut feed = self.feed.lock().unwrap(); - feed.history.retain(|(index, _, _)| !range.contains(index)); + feed.history.retain(|(index, ..)| !range.contains(index)); feed.targets.retain(|target| { let Some(rendition) = target.upgrade() else { return false; @@ -282,7 +296,7 @@ impl Fanout { }); } - /// Mark every living rendition's window ended (the timeline finished cleanly). + /// Mark every living rendition's window ended (the timeline finished or failed). pub fn end_windows(&self) { let mut feed = self.feed.lock().unwrap(); feed.ended = true; @@ -330,12 +344,14 @@ impl Producer { } /// Enroll a freshly-created rendition in the timeline feed, replaying the recent history - /// (and the ended/closed markers) so its window matches its siblings'. + /// (and the ended/closed markers) so its window matches its siblings', and start following its + /// own timeline. fn register(&self, rendition: &Arc) { let mut feed = self.fanout.feed.lock().unwrap(); rendition.label(feed.generation.clone()); - for (index, entry, discontinuity) in &feed.history { - rendition.push(*index, entry, *discontinuity, self.fanout.window); + rendition.watch(); + for (index, entry, reference, discontinuity) in &feed.history { + rendition.push(*index, entry, reference, *discontinuity, feed.window); } if feed.ended { rendition.end(); @@ -354,9 +370,9 @@ impl Producer { /// new advertised bitrate, since the publisher republishes the catalog every time its /// measured bitrate, jitter, or framerate moves. /// - /// Renditions are only servable when the catalog advertises the broadcast's timeline (its - /// root `archive` entry): without one there is nothing to render playlists from, so the - /// whole catalog is skipped with a warning. + /// Renditions are only servable when the catalog advertises its timelines (its root `archive` + /// entry): without one there is nothing to render playlists from, so the whole catalog is + /// skipped with a warning. /// /// `upstream` carries the broadcast the snapshot was read from, so each rendition serves media /// from that same broadcast rather than whatever is at the path by the time it is asked. A diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index e415ed935bf..f18aba26c66 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -1,21 +1,22 @@ -//! One rendition's view of the broadcast timeline, as a `Producer`/[`Consumer`] pair. +//! One rendition's view of the broadcast's segments, as a `Producer`/[`Consumer`] pair. //! -//! The broadcast has a single timeline track; the catalog watcher reads it and fans each -//! record out to every rendition as a row: the segment's number, timing, and this -//! rendition's group ranges (empty when the record carries no content for it, a gap). Records -//! are self-contained (the timeline only publishes a segment once its content is final on -//! every track), so every row is immediately listable and fetchable. Two things read the -//! window: +//! Segments are derived from one reference rendition's timeline: each of its records is a +//! segment, numbered by the record's sequence. The catalog watcher reads that timeline and fans +//! each record out to every rendition as a row: the segment's number and timing, plus the +//! reference record's frames. Each rendition resolves a row against its own timeline. Two things +//! read the window: //! -//! * the HTTP serve path, synchronously, to render a media playlist and look up a segment's -//! group ranges (nothing here touches media bytes on that path); and +//! * the HTTP serve path, synchronously, to render a media playlist and resolve a segment's +//! frames (nothing here touches media bytes on that path); and //! * a [`Consumer`] cursor, for a recorder that wants every segment *with its media*, in -//! order, exactly once. `next()` waits for the next row, FETCHes and transmuxes its groups -//! (via [`Rendition`]), and yields the CMAF bytes. +//! order, exactly once. `next()` waits for the next resolved row, FETCHes and transmuxes its +//! frames (via [`Rendition`]), and yields the CMAF bytes. //! -//! Rows carry the broadcast's aligned segment numbers, so a segment is addressed by that -//! number everywhere (the `seg/{segment}.m4s` URI, `EXT-X-MEDIA-SEQUENCE`, the recorder -//! cursor), and the same number names the same span of content time on every rendition. +//! A segment is addressed by its reference's tag and its number (the `seg/{tag}.{segment}.m4s` +//! URI); the number alone is its `EXT-X-MEDIA-SEQUENCE` and the recorder cursor's position. The +//! same URI names the same span of content time on every rendition, on every edge, and after +//! every reload. Each reference numbers segments by its own records, so a new reference starts a +//! new numbering under its own tag rather than reusing the old one's URIs. use std::collections::VecDeque; use std::sync::Arc; @@ -23,9 +24,9 @@ use std::task::Poll; use std::time::{Duration, SystemTime}; use bytes::Bytes; -use hang::timeline::Range; +use sha2::{Digest, Sha256}; -use super::Rendition; +use super::{Kind, Rendition}; use crate::Result; /// The producing side of a rendition's timeline window. @@ -81,16 +82,30 @@ impl Discontinuities { } } -/// One playlist segment: its aligned number, timing, and this rendition's group ranges. +/// The rendition a broadcast's segment boundaries come from. +pub(crate) type Reference = Arc<(Kind, String)>; + +/// The URI tag naming `reference`'s segment numbering: a hash of its kind and name, so every edge +/// derives the same one and it never contains a URI separator. +pub(crate) fn tag(reference: &(Kind, String)) -> String { + let digest = Sha256::digest(format!("{}/{}", reference.0.as_str(), reference.1)); + // 32 bits: the tag only has to tell apart the references one broadcast ever switches between. + digest[..4].iter().map(|b| format!("{b:02x}")).collect() +} + +/// One playlist segment: its number, timing, and the reference record's frames. #[derive(Clone, Debug, PartialEq)] pub(crate) struct Row { - /// The source timeline record index, used to mirror exact window trims. + /// The reference timeline's record index, used to mirror exact window trims. pub index: u64, - /// The aligned segment number (its URI: `seg/{segment}.m4s`), shared across renditions. + /// The segment number (its URI: `seg/{tag}.{segment}.m4s`), shared across renditions. pub segment: u64, - /// This rendition's group ranges within the segment. Empty means the rendition has no - /// content for the span (`EXT-X-GAP`). - pub ranges: Vec, + /// The rendition the boundaries come from. + pub reference: Reference, + /// The reference record's frames, which the reference rendition serves as is. + pub frames: std::ops::Range, + /// Whether the reference record starts on a keyframe. + pub keyframe: bool, /// Presentation duration. pub duration: Duration, /// The segment's starting presentation timestamp. @@ -136,14 +151,15 @@ impl State { } } - /// The first segment numbered past `after`, for a cursor. + /// The first segment past `after`, for a cursor. /// /// Rows are complete the moment they arrive. Segments evicted from the front of the /// window before the cursor reached them are skipped: the cursor resumes at the oldest - /// row still in the window. - fn next_after(&self, after: Option) -> Next { + /// row still in the window. A row from another reference starts a new numbering, so it + /// follows `after` whatever its number. + fn next_after(&self, after: Option<&(Reference, u64)>) -> Next { let next = self.rows.iter().find(|r| match after { - Some(after) => r.segment > after, + Some((reference, after)) => r.reference != *reference || r.segment > *after, None => true, }); match next { @@ -166,8 +182,9 @@ impl Producer { } } - /// Append a row, evicting the front of the window past `window`. - pub fn push(&self, row: Row, window: Duration) { + /// Append a row, evicting the front of the window past `window`. With no `window`, only + /// source timeline pops remove rows. + pub fn push(&self, row: Row, window: Option) { let Ok(mut state) = self.state.write() else { return; }; @@ -187,7 +204,9 @@ impl Producer { state.rows.push_back(row); // Evict from the front while the remaining rows still cover the window. - while state.rows.len() >= 2 { + while let Some(window) = window + && state.rows.len() >= 2 + { let span = state.rows.back().unwrap().end.saturating_sub(state.rows[1].pts.into()); if span < window { break; @@ -223,7 +242,7 @@ impl Producer { } } - /// Mark the timeline ended (the broadcast finished cleanly): the playlist gets + /// Mark the timeline ended (it finished, or failed): the playlist gets /// `EXT-X-ENDLIST` and cursors end once drained. pub fn end(&self) { if let Ok(mut state) = self.state.write() { @@ -233,23 +252,31 @@ impl Producer { /// Close the channel: no more rows will arrive. A [`Consumer`] drains the segments it /// can still see and then ends; the serve path keeps reading the frozen window. Call after - /// [`end`](Self::end) on a clean finish, or on its own when the source is lost mid-stream. + /// [`end`](Self::end) when the timeline is over, or on its own when a rendition is retired. pub fn close(&self) { let _ = self.state.close(); } + /// The start of the oldest listed segment, without copying the window. + pub fn oldest(&self) -> Option { + self.state.read().rows.front().map(|row| row.pts.into()) + } + /// Snapshot the current window (serve path). #[cfg_attr(not(feature = "server"), allow(dead_code))] pub fn window(&self) -> Window { self.state.read().window() } - /// The group ranges segment `segment` covers for this rendition, or `None` if it isn't in - /// the window. An empty vec means the segment is a gap for this rendition. - pub fn segment_ranges(&self, segment: u64) -> Option> { + /// The row of segment `segment` numbered by the reference tagged `tag`, or `None` if it isn't + /// in the window. + pub fn row(&self, tag: &str, segment: u64) -> Option { let state = self.state.read(); - let row = state.rows.iter().find(|r| r.segment == segment)?; - Some(row.ranges.clone()) + state + .rows + .iter() + .find(|r| r.segment == segment && self::tag(&r.reference) == tag) + .cloned() } /// The number of the segment whose `pts` is exactly `time` in the timeline's timescale @@ -257,26 +284,26 @@ impl Producer { /// safe because the rendered `S@t` and this lookup convert the same [`Row::pts`] the same /// way. #[cfg_attr(not(feature = "server"), allow(dead_code))] - pub fn segment_number_at(&self, time: u64, timescale: moq_net::Timescale) -> Option { + pub fn segment_number_at(&self, tag: &str, time: u64, timescale: moq_net::Timescale) -> Option { let state = self.state.read(); state .rows .iter() - .find(|row| row.pts.as_scale(timescale) == time as u128) + .find(|row| row.pts.as_scale(timescale) == time as u128 && self::tag(&row.reference) == tag) .map(|row| row.segment) } - /// The newest group known to start with a keyframe, used to bootstrap an init segment for - /// inline-parameter-set codecs. - pub fn latest_keyframe_group(&self) -> Option { + /// The newest group of `reference` a row starts on a keyframe, used to bootstrap an init + /// segment for inline-parameter-set codecs. + pub fn latest_keyframe_group(&self, reference: &(Kind, String)) -> Option { let state = self.state.read(); state .rows .iter() .rev() - .flat_map(|row| row.ranges.iter().rev()) - .find(|range| range.keyframe) - .map(|range| range.start) + .filter(|row| *row.reference == *reference) + .find(|row| row.keyframe && row.frames.start.frame == 0) + .map(|row| row.frames.start.group) } /// Whether the playlist has anything to serve yet (at least one segment, or the broadcast @@ -316,7 +343,7 @@ impl Producer { /// A segment with its transmuxed media, yielded by a [`Consumer`]. pub struct Segment { - /// The aligned segment number (also its `seg/{segment}.m4s` URI stem), shared across the + /// The aligned segment number (its URI is `seg/{reference}.{segment}.m4s`), shared across the /// broadcast's renditions. pub segment: u64, /// The transmuxed CMAF fragment (`moof`+`mdat`), fetched on demand by [`Consumer::next`]. @@ -343,8 +370,9 @@ pub struct Segment { pub struct Consumer { state: kio::Consumer, rendition: Arc, - /// Last fetched or skipped segment; errors leave it unchanged so callers can retry. - after: Option, + /// Last fetched or skipped segment, with the reference numbering it; errors leave it unchanged + /// so callers can retry. + after: Option<(Reference, u64)>, } impl Consumer { @@ -354,19 +382,25 @@ impl Consumer { self.rendition.init().await } - /// The next segment, with its media; `None` once the rendition ends. + /// The next segment, with its media; `None` once the rendition ends, including when its + /// timeline failed. /// - /// Waits for the next segment, then FETCHes and transmuxes its groups. A segment whose - /// groups already left the relay cache (or that is a gap for this rendition) is skipped, - /// resuming at the next one, rather than surfaced as an error; a real fetch/transmux - /// failure is returned, leaving the cursor to retry it on the next call. + /// Waits for the next segment to resolve on this rendition, then FETCHes and transmuxes its + /// frames. A segment whose groups already left the relay cache (or that is a gap for this + /// rendition) is skipped, resuming at the next one, rather than surfaced as an error; a real + /// fetch/transmux failure is returned, leaving the cursor to retry it on the next call. pub async fn next(&mut self) -> Result> { loop { let Some(row) = kio::wait(|waiter| self.poll_next(waiter)).await else { return Ok(None); }; - let media = self.rendition.segment(row.segment).await?; - self.after = Some(row.segment); + kio::wait(|waiter| self.rendition.poll_resolved(waiter, &row)).await; + // The rendition's timeline failed: it ends here, like its playlist. + if self.rendition.is_failed(&row) { + return Ok(None); + } + let media = self.rendition.fetch(&row).await?; + self.after = Some((row.reference.clone(), row.segment)); if let Some(media) = media { return Ok(Some(Segment { segment: row.segment, @@ -380,11 +414,13 @@ impl Consumer { } fn poll_next(&self, waiter: &kio::Waiter) -> Poll> { - let poll = self.state.poll(waiter, |state| match state.next_after(self.after) { - Next::Ready(row) => Poll::Ready(Some(row)), - Next::Ended => Poll::Ready(None), - Next::Pending => Poll::Pending, - }); + let poll = self + .state + .poll(waiter, |state| match state.next_after(self.after.as_ref()) { + Next::Ready(row) => Poll::Ready(Some(row)), + Next::Ended => Poll::Ready(None), + Next::Pending => Poll::Pending, + }); match poll { Poll::Ready(Ok(found)) => Poll::Ready(found), // The producer closed without a clean end (broadcast dropped): no more segments. @@ -403,7 +439,9 @@ mod tests { Row { index: segment, segment, - ranges: vec![Range::new(group, group)], + reference: Arc::new((Kind::Video, "video".to_string())), + frames: hang::timeline::Position::group(group)..hang::timeline::Position::group(group + 1), + keyframe: true, duration: Duration::from_millis(duration_ms), pts, end: Duration::from(pts) + Duration::from_millis(duration_ms), @@ -414,8 +452,8 @@ mod tests { #[test] fn every_row_is_listed() { let live = Producer::new(); - live.push(row(0, 0, 0, 2_000), Duration::from_secs(30)); - live.push(row(1, 1, 2_000, 2_000), Duration::from_secs(30)); + live.push(row(0, 0, 0, 2_000), Some(Duration::from_secs(30))); + live.push(row(1, 1, 2_000, 2_000), Some(Duration::from_secs(30))); let window = live.window(); assert_eq!(window.sequence, 0); @@ -431,7 +469,7 @@ mod tests { #[test] fn window_evicts_and_advances_sequence() { let live = Producer::new(); - let window = Duration::from_secs(4); + let window = Some(Duration::from_secs(4)); for i in 0..6u64 { live.push(row(i, i, i * 2_000, 2_000), window); } @@ -448,7 +486,7 @@ mod tests { #[test] fn source_window_pop_removes_playlist_rows() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); for i in 0..4u64 { let mut row = row(i, i, i * 2_000, 2_000); row.index = i + 10; @@ -472,9 +510,9 @@ mod tests { #[test] fn a_skipped_source_range_clears_rows_before_the_next_segment() { let live = Producer::new(); - live.push(row(4, 4, 8_000, 2_000), Duration::from_secs(10)); + live.push(row(4, 4, 8_000, 2_000), Some(Duration::from_secs(10))); live.clear(); - live.push(row(10, 10, 20_000, 2_000), Duration::from_secs(10)); + live.push(row(10, 10, 20_000, 2_000), Some(Duration::from_secs(10))); let snapshot = live.window(); assert_eq!(snapshot.sequence, 10); @@ -519,37 +557,38 @@ mod tests { } #[test] - fn segment_ranges_and_gaps() { + fn rows_and_the_keyframe_group() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 1_000), window); - // Segment 1 is a gap for this rendition: no ranges. live.push( Row { - index: 1, - segment: 1, - ranges: Vec::new(), - duration: Duration::from_secs(1), - pts: moq_net::Timestamp::from_millis(1_000).unwrap(), - end: Duration::from_millis(2_000), - discontinuity: 0, + keyframe: false, + ..row(1, 1, 1_000, 1_000) }, window, ); - live.push(row(2, 100, 2_000, 1_000), window); - - assert_eq!(live.segment_ranges(0), Some(vec![Range::new(0, 0)])); - assert_eq!(live.segment_ranges(1), Some(vec![]), "a gap is present but empty"); - assert_eq!(live.segment_ranges(7), None, "unknown segments miss"); - // The gap row carries no keyframe group; the bootstrap group comes from segment 2. - assert_eq!(live.latest_keyframe_group(), Some(100)); + let video = (Kind::Video, "video".to_string()); + let tag = tag(&video); + assert_eq!( + live.row(&tag, 0).unwrap().frames, + hang::timeline::Position::group(0)..hang::timeline::Position::group(1) + ); + assert_eq!(live.row(&tag, 7), None, "unknown segments miss"); + assert_eq!(live.row("00000000", 0), None, "another reference's numbering misses"); + assert_eq!( + live.latest_keyframe_group(&video), + Some(0), + "row 1 does not start on a keyframe" + ); + assert_eq!(live.latest_keyframe_group(&(Kind::Audio, "video".to_string())), None); } #[test] fn backwards_jump_resets_the_window() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 10_000, 2_000), window); live.push(row(1, 1, 12_000, 2_000), window); live.push(row(2, 2, 1_000, 2_000), window); // restart: pts rewound @@ -566,7 +605,7 @@ mod tests { #[test] fn next_after_walks_segments() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 2_000), window); live.push(row(1, 1, 2_000, 2_000), window); @@ -574,16 +613,42 @@ mod tests { panic!("expected a segment"); }; assert_eq!(first.segment, 0); - let Next::Ready(second) = live.state.read().next_after(Some(0)) else { + let after = |segment| Some((first.reference.clone(), segment)); + let Next::Ready(second) = live.state.read().next_after(after(0).as_ref()) else { panic!("expected a segment"); }; assert_eq!(second.segment, 1); assert!( - matches!(live.state.read().next_after(Some(1)), Next::Pending), + matches!(live.state.read().next_after(after(1).as_ref()), Next::Pending), "nothing further while live" ); live.end(); - assert!(matches!(live.state.read().next_after(Some(1)), Next::Ended)); + assert!(matches!(live.state.read().next_after(after(1).as_ref()), Next::Ended)); + } + + /// A new reference numbers segments from its own records, so a cursor past a higher number of + /// the old reference still yields the new reference's rows. + #[test] + fn a_new_reference_restarts_the_cursor() { + let live = Producer::new(); + let window = Some(Duration::from_secs(30)); + live.push(row(5, 5, 10_000, 2_000), window); + let Next::Ready(old) = live.state.read().next_after(None) else { + panic!("expected a segment"); + }; + + live.clear(); + let switched = Row { + reference: Arc::new((Kind::Video, "other".to_string())), + ..row(0, 0, 0, 2_000) + }; + live.push(switched, window); + let after = Some((old.reference.clone(), old.segment)); + let Next::Ready(next) = live.state.read().next_after(after.as_ref()) else { + panic!("the new reference's first row follows"); + }; + assert_eq!(next.segment, 0); + assert_ne!(tag(&next.reference), tag(&old.reference), "its URLs carry another tag"); } } diff --git a/rs/moq-hls/src/export/spans.rs b/rs/moq-hls/src/export/spans.rs new file mode 100644 index 00000000000..fcc7d17973f --- /dev/null +++ b/rs/moq-hls/src/export/spans.rs @@ -0,0 +1,405 @@ +//! One rendition's own timeline, which resolves the broadcast's segments to that rendition's frames. +//! +//! Segment boundaries come from the reference rendition's records (see [`super::segments`]). +//! Every other rendition maps each boundary onto its own timeline: +//! +//! * a video rendition snaps each boundary to its nearest record starting on a keyframe within +//! [`TOLERANCE`]; a segment with no start in range is a gap (HLS `EXT-X-GAP`), so a player +//! switching renditions lands on the next real segment; +//! * an audio rendition takes every frame whose timestamp falls in the segment's span, possibly +//! from more than one record. +//! +//! A segment resolves only once the rendition's timeline has reached past its end (or ended), so +//! every edge and every reload agree on its content. A failed timeline ends the rendition's +//! playlist at its last record instead of listing the rest as gaps. + +use std::collections::VecDeque; +use std::ops::Range; +use std::task::Poll; +use std::time::Duration; + +use hang::timeline::Position; +use moq_mux::timeline::Entry; + +/// How far a video boundary may move to land on a keyframe. +pub(crate) const TOLERANCE: Duration = Duration::from_secs(1); + +/// What one segment holds on one rendition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Content { + /// The rendition's timeline has not reached the segment's end yet. + Pending, + /// The rendition has no content for the segment. + Gap, + /// The rendition's timeline failed before reaching the segment's end, so neither it nor any + /// later segment will resolve. + Failed, + /// The frames to fetch, as position ranges, keeping only those whose timestamps fall in + /// `filter` when set. + Frames { + ranges: Vec>, + filter: Option>, + }, +} + +struct State { + /// Retained records, oldest first. + records: VecDeque, + /// No more records will arrive, so nothing is pending. + ended: bool, + /// The timeline failed: no more records will arrive, and a segment past the last one fails + /// rather than resolving as a gap. + failed: bool, +} + +impl State { + fn resolve(&self, is_video: bool, span: Range) -> Content { + if self.failed && self.records.back().is_none_or(|last| last.end_time() < span.end) { + return Content::Failed; + } + let records = self.records.iter().collect(); + let ended = self.ended || self.failed; + match is_video { + true => video(records, ended, span), + false => audio(records, ended, span), + } + } +} + +/// A rendition's own record window, fed by its timeline watcher. +pub(crate) struct Spans { + state: kio::Producer, +} + +impl Spans { + pub fn new() -> Self { + Self { + state: kio::Producer::new(State { + records: VecDeque::new(), + ended: false, + failed: false, + }), + } + } + + /// Append a record. Only [`trim`](Self::trim) and source pops remove records. + pub fn push(&self, entry: Entry) { + let Ok(mut state) = self.state.write() else { + return; + }; + if state + .records + .back() + .is_some_and(|back| entry.sequence <= back.sequence || entry.pts < back.pts) + { + // The publisher restarted its timeline; the old records describe other media. + state.records.clear(); + } + state.records.push_back(entry); + } + + /// Evict records that end before `oldest`, the start of the oldest listed segment, less the + /// snapping tolerance. Tied to the listed segments rather than this timeline's own newest + /// content, so a rendition running ahead of a stalled reference never drops a record a listed + /// segment still resolves to. + pub fn trim(&self, oldest: Duration) { + let Ok(mut state) = self.state.write() else { + return; + }; + while state.records.len() >= 2 && state.records[0].end_time() + 2 * TOLERANCE < oldest { + state.records.pop_front(); + } + } + + /// Remove the records with these sequences. + pub fn pop(&self, sequences: Range) { + if let Ok(mut state) = self.state.write() { + state.records.retain(|record| !sequences.contains(&record.sequence)); + } + } + + /// Forget every record after the source skipped some. + pub fn clear(&self) { + if let Ok(mut state) = self.state.write() { + state.records.clear(); + } + } + + /// No more records will arrive. + pub fn end(&self) { + if let Ok(mut state) = self.state.write() { + state.ended = true; + } + } + + /// The timeline failed: segments its records cover still resolve, and later ones are + /// [`Content::Failed`]. + pub fn fail(&self) { + if let Ok(mut state) = self.state.write() { + state.failed = true; + } + } + + /// Resolve the segment spanning `span`, snapping to keyframes when `video`. + pub fn resolve(&self, video: bool, span: Range) -> Content { + self.state.read().resolve(video, span) + } + + /// Poll until the segment spanning `span` stops being [`Content::Pending`]. + pub fn poll_resolved(&self, waiter: &kio::Waiter, video: bool, span: Range) -> Poll<()> { + let poll = self + .state + .poll_ref(waiter, |state| match state.resolve(video, span.clone()) { + Content::Pending => Poll::Pending, + _ => Poll::Ready(()), + }); + match poll { + Poll::Ready(_) => Poll::Ready(()), + Poll::Pending => Poll::Pending, + } + } + + /// The newest group known to start with a keyframe, used to bootstrap an init segment for + /// inline-parameter-set codecs. + pub fn latest_keyframe_group(&self) -> Option { + let state = self.state.read(); + state + .records + .iter() + .rev() + .find(|record| record.keyframe && record.start.frame == 0) + .map(|record| record.start.group) + } +} + +/// Snap `boundary` to the nearest keyframe record start within [`TOLERANCE`], preferring the +/// earlier one on a tie. +fn snap<'a>(candidates: &[&'a Entry], boundary: Duration) -> Option<&'a Entry> { + candidates + .iter() + .copied() + .filter(|record| Duration::from(record.pts).abs_diff(boundary) <= TOLERANCE) + .min_by_key(|record| Duration::from(record.pts).abs_diff(boundary)) +} + +/// Every record between two positions, merged into contiguous ranges. +fn ranges(records: &[&Entry], frames: Range) -> Vec> { + let mut out: Vec> = Vec::new(); + for record in records { + let start = record.start.max(frames.start); + let end = record.end.min(frames.end); + if start >= end { + continue; + } + match out.last_mut() { + Some(last) if last.end == start => last.end = end, + _ => out.push(start..end), + } + } + out +} + +fn video(records: Vec<&Entry>, ended: bool, span: Range) -> Content { + // A nearer keyframe can still arrive until the timeline passes the boundary plus the tolerance. + let reached = |time: Duration| ended || records.last().is_some_and(|last| last.end_time() >= time + TOLERANCE); + if !reached(span.end) { + return Content::Pending; + } + + let candidates: Vec<&Entry> = records + .iter() + .copied() + .filter(|record| record.keyframe && record.start.frame == 0) + .collect(); + let Some(first) = snap(&candidates, span.start) else { + return Content::Gap; + }; + // Without a keyframe near the end, run to the next one, so no frame falls between segments. + let end = match snap(&candidates, span.end) { + Some(record) => record.start, + None => match candidates.iter().find(|record| Duration::from(record.pts) > span.end) { + Some(record) => record.start, + None if ended => records.last().expect("a candidate exists").end, + None => return Content::Pending, + }, + }; + if first.start >= end { + return Content::Gap; + } + Content::Frames { + ranges: ranges(&records, first.start..end), + filter: None, + } +} + +fn audio(records: Vec<&Entry>, ended: bool, span: Range) -> Content { + if !ended && records.last().is_none_or(|last| last.end_time() < span.end) { + return Content::Pending; + } + let overlapping: Vec<&Entry> = records + .iter() + .copied() + .filter(|record| Duration::from(record.pts) < span.end && record.end_time() > span.start) + .collect(); + let (Some(first), Some(last)) = (overlapping.first(), overlapping.last()) else { + return Content::Gap; + }; + Content::Frames { + ranges: ranges(&overlapping, first.start..last.end), + filter: Some(span), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secs(s: f64) -> Duration { + Duration::from_secs_f64(s) + } + + fn record(sequence: u64, pts_ms: u64, duration_ms: u64, start: Position, end: Position) -> Entry { + Entry { + sequence, + pts: moq_net::Timestamp::from_millis(pts_ms).unwrap(), + duration: Duration::from_millis(duration_ms), + start, + end, + keyframe: start.frame == 0, + ext: (), + } + } + + /// One whole-group record per GOP, starting at `starts` (ms), the last one ending at `end`. + fn gops(starts: &[u64], end: u64) -> Vec { + starts + .iter() + .enumerate() + .map(|(i, &pts)| { + let next = starts.get(i + 1).copied().unwrap_or(end); + let group = i as u64; + record( + i as u64, + pts, + next - pts, + Position::group(group), + Position::group(group + 1), + ) + }) + .collect() + } + + fn frames(ranges: &[(Position, Position)], filter: Option>) -> Content { + Content::Frames { + ranges: ranges.iter().map(|&(start, end)| start..end).collect(), + filter, + } + } + + #[test] + fn video_snaps_each_boundary_to_the_nearest_gop() { + // GOPs 300ms off the reference's 2s boundaries. + let records = gops(&[300, 2_300, 4_300, 6_300], 8_300); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!( + video(refs.clone(), false, secs(2.0)..secs(4.0)), + frames(&[(Position::group(1), Position::group(2))], None) + ); + // The end boundary is not yet a tolerance behind the newest record. + assert_eq!(video(refs.clone(), false, secs(6.0)..secs(8.0)), Content::Pending); + assert_eq!( + video(refs, true, secs(6.0)..secs(8.0)), + frames(&[(Position::group(3), Position::group(4))], None) + ); + } + + #[test] + fn video_without_a_nearby_start_is_a_gap() { + // GOPs never coincide with the reference: 3.5s cadence against 2s boundaries. + let records = gops(&[0, 3_500, 7_000], 10_500); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!(video(refs.clone(), true, secs(2.0)..secs(4.0)), Content::Gap); + // The segment before the gap runs to the next GOP, so no frame is lost between them. + assert_eq!( + video(refs.clone(), true, secs(0.0)..secs(2.0)), + frames(&[(Position::group(0), Position::group(1))], None) + ); + assert_eq!( + video(refs, true, secs(4.0)..secs(6.0)), + frames(&[(Position::group(1), Position::group(2))], None) + ); + } + + #[test] + fn a_rendition_that_starts_later_is_a_gap_until_it_starts() { + let records = gops(&[4_000, 6_000], 8_000); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!(video(refs.clone(), false, secs(0.0)..secs(2.0)), Content::Gap); + assert_eq!( + video(refs, false, secs(4.0)..secs(6.0)), + frames(&[(Position::group(0), Position::group(1))], None) + ); + } + + #[test] + fn audio_takes_every_overlapping_record() { + // 1s audio records against 2.5s segments. + let records: Vec = (0..6) + .map(|i| record(i, i * 1_000, 1_000, Position::group(i * 2), Position::group(i * 2 + 2))) + .collect(); + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!( + audio(refs.clone(), false, secs(2.5)..secs(5.0)), + frames(&[(Position::group(4), Position::group(10))], Some(secs(2.5)..secs(5.0))) + ); + assert_eq!(audio(refs.clone(), false, secs(5.0)..secs(7.5)), Content::Pending); + assert_eq!(audio(refs, true, secs(7.0)..secs(9.0)), Content::Gap); + } + + #[test] + fn a_frame_split_record_is_not_a_video_boundary() { + let records = [ + record(0, 0, 10_000, Position::group(0), Position::new(0, 300)), + record(1, 10_000, 2_000, Position::new(0, 300), Position::group(1)), + record(2, 12_000, 2_000, Position::group(1), Position::group(2)), + ]; + let refs: Vec<&Entry> = records.iter().collect(); + assert_eq!(video(refs.clone(), true, secs(10.0)..secs(12.0)), Content::Gap); + assert_eq!( + video(refs, true, secs(0.0)..secs(12.0)), + frames(&[(Position::group(0), Position::group(1))], None), + "the split's halves merge back into one range" + ); + } + + #[test] + fn a_trim_keeps_records_a_snap_may_need() { + let spans = Spans::new(); + for entry in gops(&[0, 2_000, 4_000, 6_000, 8_000, 10_000], 12_000) { + spans.push(entry); + } + spans.trim(secs(8.0)); + let state = spans.state.read(); + assert_eq!(state.records.front().unwrap().sequence, 2); + } + + #[test] + fn a_failed_timeline_resolves_only_what_its_records_cover() { + let spans = Spans::new(); + for entry in gops(&[0, 2_000, 4_000], 6_000) { + spans.push(entry); + } + // Waiting on a keyframe that could still land near the boundary. + assert_eq!(spans.resolve(true, secs(4.0)..secs(6.0)), Content::Pending); + + spans.fail(); + assert_eq!( + spans.resolve(true, secs(4.0)..secs(6.0)), + frames(&[(Position::group(2), Position::group(3))], None) + ); + assert_eq!(spans.resolve(true, secs(6.0)..secs(8.0)), Content::Failed); + assert_eq!(spans.resolve(false, secs(6.0)..secs(8.0)), Content::Failed); + let resolved = spans.poll_resolved(&kio::Waiter::noop(), true, secs(6.0)..secs(8.0)); + assert!(resolved.is_ready(), "nothing waits on a failed timeline"); + } +} diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index ea0753f998c..c8ce1a29e77 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -49,7 +49,7 @@ const ANCHOR_SEGMENTS: usize = 3; /// How many consecutive failed steps retire a rendition (see [`TrackState::evict`]). /// /// Enough to ride out a transient fetch error or a stale playlist, short enough that a -/// genuinely dead variant stops holding the broadcast's timeline back within a few seconds. +/// genuinely dead variant leaves the catalog within a few seconds. const MAX_RENDITION_FAILURES: usize = 3; /// Configuration for the HLS import loop. @@ -364,11 +364,9 @@ impl TrackState { /// Give up on this rendition's current importer generation after repeated failures. /// - /// An enrolled track gates *every* segment record until it reports past the boundary, so a - /// rendition that stopped making progress freezes the whole broadcast's timeline rather - /// than just its own playlist. Dropping the importer closes its recorders (and retires its - /// catalog entries), letting the healthy renditions publish again; a later successful step - /// rebuilds it from the init segment. + /// Dropping the importer closes its recorders and retires its catalog entries, so players + /// stop selecting a rendition that no longer advances; a later successful step rebuilds it + /// from the init segment. fn evict(&mut self) { self.importer = None; self.map = None; @@ -724,13 +722,6 @@ impl Import { async fn step(&mut self, on_error: OnError) -> Result { self.ensure_tracks().await?; - // Reserve the timeline for the whole pass, the way each importer reserves the catalog. - // Renditions are ingested one at a time, and a record is immutable once published - // against the tracks enrolled at that moment, so a record flushed mid-pass would omit - // every rendition that hasn't loaded its init segment yet (a permanent EXT-X-GAP) and - // fold that rendition's first groups into whichever segment flushes next. - let _reserved = self.sink.catalog.timeline().reserve(); - let mut wrote_segments = 0; let mut target_duration = None; let mut failed = None; @@ -1405,10 +1396,8 @@ mod tests { assert_eq!(import.video[0].next_sequence, Some(2)); } - /// An enrolled track gates every timeline record until it reports past the boundary, so a - /// rendition that has stopped making progress would otherwise freeze the whole broadcast's - /// timeline (and every healthy rendition's playlist with it). After a few consecutive - /// failures its importer is dropped, which closes its recorders. + /// A rendition that has stopped making progress is dropped after a few consecutive + /// failures, which closes its recorders and retires its catalog entries. #[tokio::test] async fn a_persistently_failing_rendition_is_evicted() { let (init, fragments) = fmp4_parts(2); @@ -1440,10 +1429,7 @@ mod tests { } import.step(OnError::Warn).await.unwrap(); - assert!( - import.video[0].importer.is_none(), - "the dead rendition stops gating the broadcast's timeline" - ); + assert!(import.video[0].importer.is_none(), "the dead rendition is retired"); } /// A live window longer than `ANCHOR_SEGMENTS` is joined mid-playlist, which means the diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index 4e15a17b186..2dee9c18c2c 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -7,8 +7,8 @@ //! GET /{broadcast}/manifest.mpd //! GET /{broadcast}/{kind}/{rendition}/media.m3u8 //! GET /{broadcast}/{kind}/{rendition}/init.{hash}.mp4 -//! GET /{broadcast}/{kind}/{rendition}/seg/{segment}.m4s -//! GET /{broadcast}/{kind}/{rendition}/seg/t{pts}.m4s +//! GET /{broadcast}/{kind}/{rendition}/seg/{reference}.{segment}.m4s +//! GET /{broadcast}/{kind}/{rendition}/seg/{reference}.t{pts}.m4s //! ``` //! //! `{kind}` is `video` or `audio`, so a video and an audio rendition that share a @@ -17,7 +17,7 @@ //! timeline `pts` (`$Time$`). `{hash}` is a hash of the init segment's bytes, so a //! reconfigured rendition gets a new init URL. A broadcaster given a generation //! ([`Broadcaster::set_generation`](crate::export::Broadcaster::set_generation)) -//! prefixes every segment file with it (`seg/{generation}.{segment}.m4s`), and +//! prefixes every segment file with it (`seg/{generation}.{reference}.{segment}.m4s`), and //! refuses segment URLs carrying any other. //! //! Every request is served. To gate access, wrap [`Server::router`] in your own diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index 324fdd80f83..f2dd64ce2b0 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -63,7 +63,8 @@ pub enum Resource { /// The hash of the init bytes the URL names. hash: String, }, - /// `{kind}/{rendition}/seg/[{generation}.]{sequence}.m4s`: a segment by its HLS number. + /// `{kind}/{rendition}/seg/[{generation}.]{reference}.{sequence}.m4s`: a segment by its HLS + /// number. #[non_exhaustive] Segment { /// The rendition's kind. @@ -72,11 +73,13 @@ pub enum Resource { rendition: String, /// The publisher run the URL names, if the broadcaster carries one. generation: Option, + /// The tag of the reference rendition whose records number the segment. + reference: String, /// The segment's aligned number. sequence: u64, }, - /// `{kind}/{rendition}/seg/[{generation}.]t{pts}.m4s`: the same bytes, addressed by the - /// DASH timeline pts (`$Time$`). + /// `{kind}/{rendition}/seg/[{generation}.]{reference}.t{pts}.m4s`: the same bytes, addressed + /// by the DASH timeline pts (`$Time$`). #[non_exhaustive] SegmentAt { /// The rendition's kind. @@ -85,6 +88,8 @@ pub enum Resource { rendition: String, /// The publisher run the URL names, if the broadcaster carries one. generation: Option, + /// The tag of the reference rendition whose records number the segment. + reference: String, /// The segment's timeline pts. pts: u64, }, @@ -134,22 +139,26 @@ impl Route { let kind = Kind::parse(kind)?; let rendition = rendition.clone(); let stem = file.strip_suffix(".m4s")?; - // `{generation}.{segment}` when the export carries a generation, `{segment}` otherwise. - let (generation, stem) = match stem.split_once('.') { - Some((generation, stem)) => (Some(generation.to_string()), stem), - None => (None, stem), + // `{generation}.{reference}.{segment}` when the export carries a generation, + // `{reference}.{segment}` otherwise. + let (generation, reference, stem) = match stem.split('.').collect::>().as_slice() { + [reference, stem] => (None, reference.to_string(), *stem), + [generation, reference, stem] => (Some(generation.to_string()), reference.to_string(), *stem), + _ => return None, }; let resource = match stem.strip_prefix('t') { Some(pts) => Resource::SegmentAt { kind, rendition, generation, + reference, pts: pts.parse().ok()?, }, None => Resource::Segment { kind, rendition, generation, + reference, sequence: stem.parse().ok()?, }, }; @@ -201,18 +210,20 @@ impl Server { kind, rendition, generation, + reference, sequence, } => { - let at = SegmentAt::Sequence(*sequence); + let at = SegmentAt::Sequence(reference, *sequence); segment(self, broadcast, *kind, rendition, generation.as_deref(), at).await } Resource::SegmentAt { kind, rendition, generation, + reference, pts, } => { - let at = SegmentAt::Pts(*pts); + let at = SegmentAt::Pts(reference, *pts); segment(self, broadcast, *kind, rendition, generation.as_deref(), at).await } } @@ -267,11 +278,12 @@ async fn init(server: &Server, broadcast: &str, kind: Kind, rendition: &str, has media_result(rendition.init_versioned(hash).await, server) } -/// How a segment URL addresses its bytes: HLS by aligned number (`seg/0.m4s`), DASH by -/// timeline pts (`seg/t2000.m4s`, the SegmentTemplate's `$Time$`). -enum SegmentAt { - Sequence(u64), - Pts(u64), +/// How a segment URL addresses its bytes under its reference's tag: HLS by aligned number +/// (`seg/{tag}.0.m4s`), DASH by timeline pts (`seg/{tag}.t2000.m4s`, the SegmentTemplate's +/// `$Time$`). +enum SegmentAt<'a> { + Sequence(&'a str, u64), + Pts(&'a str, u64), } async fn segment( @@ -280,7 +292,7 @@ async fn segment( kind: Kind, rendition: &str, generation: Option<&str>, - at: SegmentAt, + at: SegmentAt<'_>, ) -> Response { let Some(rendition) = rendition_for(server, broadcast, kind, rendition).await else { return not_found(); @@ -292,8 +304,8 @@ async fn segment( return not_found(); } let result = match at { - SegmentAt::Sequence(sequence) => rendition.segment(sequence).await, - SegmentAt::Pts(pts) => rendition.segment_at(pts).await, + SegmentAt::Sequence(reference, sequence) => rendition.segment(reference, sequence).await, + SegmentAt::Pts(reference, pts) => rendition.segment_at(reference, pts).await, }; // A generation change while fetching may have swapped the rows under the lookup. if !current() { @@ -391,38 +403,42 @@ mod tests { }) ); assert_eq!( - resource("/project/live/video/main/seg/42.m4s"), + resource("/project/live/video/main/seg/ab12cd34.42.m4s"), Some(Resource::Segment { kind: Kind::Video, rendition: "main".to_string(), generation: None, + reference: "ab12cd34".to_string(), sequence: 42, }) ); assert_eq!( - resource("/project/live/video/main/seg/run-1.42.m4s"), + resource("/project/live/video/main/seg/run-1.ab12cd34.42.m4s"), Some(Resource::Segment { kind: Kind::Video, rendition: "main".to_string(), generation: Some("run-1".to_string()), + reference: "ab12cd34".to_string(), sequence: 42, }) ); assert_eq!( - resource("/project/live/video/main/seg/t2000.m4s"), + resource("/project/live/video/main/seg/ab12cd34.t2000.m4s"), Some(Resource::SegmentAt { kind: Kind::Video, rendition: "main".to_string(), generation: None, + reference: "ab12cd34".to_string(), pts: 2000, }) ); assert_eq!( - resource("/project/live/video/main/seg/init.t2000.m4s"), + resource("/project/live/video/main/seg/init.ab12cd34.t2000.m4s"), Some(Resource::SegmentAt { kind: Kind::Video, rendition: "main".to_string(), generation: Some("init".to_string()), + reference: "ab12cd34".to_string(), pts: 2000, }) ); @@ -431,7 +447,11 @@ mod tests { assert!(Route::parse("/project/live/data/main/media.m3u8").is_none()); assert!(Route::parse("/project/live/video/main/init.mp4").is_none()); assert!(Route::parse("/project/live/video/main/init..mp4").is_none()); - assert!(Route::parse("/project/live/video/main/seg/tx.m4s").is_none()); + assert!(Route::parse("/project/live/video/main/seg/ab12cd34.tx.m4s").is_none()); + assert!( + Route::parse("/project/live/video/main/seg/42.m4s").is_none(), + "a segment URL names its reference" + ); assert!(Route::parse("/project/live/video/main/seg/1.ts").is_none()); } @@ -615,7 +635,7 @@ mod tests { if response.status() == StatusCode::OK { let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); let body = String::from_utf8(body.to_vec()).unwrap(); - if body.contains("seg/0.m4s") { + if body.contains("seg/656e3d1b.0.m4s") { break body; } } @@ -626,7 +646,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; }; assert!(body.contains(".mp4?jwt=abc\""), "{body}"); - assert!(body.contains("seg/0.m4s?jwt=abc"), "{body}"); + assert!(body.contains("seg/656e3d1b.0.m4s?jwt=abc"), "{body}"); pair.accept.abort(); } @@ -647,14 +667,14 @@ mod tests { let server = Server::new(origin.consume(), crate::export::Config::default()); let app = server.router(); - wait_listed(&app, "/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; let broadcaster = server.broadcaster("live").await.expect("broadcaster"); broadcaster.set_generation(Some("run-1")).unwrap(); let response = oneshot(app.clone(), "/live/video/video0/media.m3u8").await; let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); let playlist = String::from_utf8(body.to_vec()).unwrap(); - assert!(playlist.contains("\nseg/run-1.0.m4s\n"), "{playlist}"); + assert!(playlist.contains("\nseg/run-1.656e3d1b.0.m4s\n"), "{playlist}"); let map = playlist.lines().find(|line| line.starts_with("#EXT-X-MAP:")).unwrap(); let init = map .strip_prefix("#EXT-X-MAP:URI=\"") @@ -672,24 +692,27 @@ mod tests { StatusCode::NOT_FOUND ); - assert_eq!(status(&app, "/live/video/video0/seg/run-1.0.m4s").await, StatusCode::OK); assert_eq!( - status(&app, "/live/video/video0/seg/run-1.t0.m4s").await, + status(&app, "/live/video/video0/seg/run-1.656e3d1b.0.m4s").await, + StatusCode::OK + ); + assert_eq!( + status(&app, "/live/video/video0/seg/run-1.656e3d1b.t0.m4s").await, StatusCode::OK ); assert_eq!( - status(&app, "/live/video/video0/seg/0.m4s").await, + status(&app, "/live/video/video0/seg/656e3d1b.0.m4s").await, StatusCode::NOT_FOUND ); assert_eq!( - status(&app, "/live/video/video0/seg/run-0.0.m4s").await, + status(&app, "/live/video/video0/seg/run-0.656e3d1b.0.m4s").await, StatusCode::NOT_FOUND ); // A new run: the previous generation's URLs are refused even for numbers it reuses. broadcaster.set_generation(Some("run-2")).unwrap(); assert_eq!( - status(&app, "/live/video/video0/seg/run-1.0.m4s").await, + status(&app, "/live/video/video0/seg/run-1.656e3d1b.0.m4s").await, StatusCode::NOT_FOUND ); } @@ -710,9 +733,9 @@ mod tests { write_three_gops(&mut media); let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); - wait_listed(&app, "/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; - let response = oneshot(app, "/live/video/video0/seg/0.m4s").await; + let response = oneshot(app, "/live/video/video0/seg/656e3d1b.0.m4s").await; assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); @@ -728,7 +751,7 @@ mod tests { write_three_gops(&mut media); let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); - wait_listed(&app, "/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; let remote = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("live")) .await @@ -739,7 +762,7 @@ mod tests { .await .expect("publisher close timed out"); - let response = oneshot(app, "/live/video/video0/seg/0.m4s").await; + let response = oneshot(app, "/live/video/video0/seg/656e3d1b.0.m4s").await; assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); @@ -773,7 +796,7 @@ mod tests { write_three_gops(&mut media); let app = Server::new(pair.sub_origin.consume(), crate::export::Config::default()).router(); - wait_listed(&app, "/room/live/video/video0/media.m3u8", "seg/0.m4s").await; + wait_listed(&app, "/room/live/video/video0/media.m3u8", "seg/656e3d1b.0.m4s").await; let remote_media = tokio::time::timeout(TIMEOUT, pair.sub_origin.consume().request_broadcast("room/source")) .await @@ -785,7 +808,7 @@ mod tests { .await .expect("sibling close timed out"); - let response = oneshot(app, "/room/live/video/video0/seg/0.m4s").await; + let response = oneshot(app, "/room/live/video/video0/seg/656e3d1b.0.m4s").await; assert_eq!(response.status(), StatusCode::NOT_FOUND); assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); diff --git a/rs/moq-json/src/window/encoder.rs b/rs/moq-json/src/window/encoder.rs index 58abe657c12..f06d52b9887 100644 --- a/rs/moq-json/src/window/encoder.rs +++ b/rs/moq-json/src/window/encoder.rs @@ -86,6 +86,15 @@ impl ProducerConfig { } } +/// A retained window to continue, such as one replayed from stored groups. +#[derive(Debug, Clone, PartialEq)] +pub struct Checkpoint { + /// Absolute index of the oldest retained record, and of the next to be pushed. + pub range: std::ops::Range, + /// The newest retained records, oldest first, ending just before `range.end`. + pub records: Vec, +} + /// One encoded frame, and the group boundary it implies. #[derive(Clone, Debug)] #[non_exhaustive] @@ -138,7 +147,7 @@ impl Pending<'_, T> { impl Drop for Pending<'_, T> { fn drop(&mut self) { if self.edit.is_some() { - self.encoder.resync(); + self.encoder.reset(); } } } @@ -216,8 +225,11 @@ impl Encoder { self.offset..self.start + self.window.len() as u64 } - /// Discard group-local state after an encoded frame did not reach the wire. - fn resync(&mut self) { + /// Discard group-local state, so the next edit opens a new group with a header. + /// + /// Call this whenever the caller closes the current group behind the encoder's back. It is also + /// how the encoder recovers after an encoded frame did not reach the wire. + pub fn reset(&mut self) { self.flate = None; self.op_bytes = 0; self.header_len = 0; @@ -288,7 +300,7 @@ impl Encoder { let encoded = self.frame(bytes)?; let group_bytes = self.header_len.saturating_add(self.op_bytes); if group_bytes > moq_net::group::MAX_CACHE_BYTES { - self.resync(); + self.reset(); Ok(None) } else { Ok(Some(encoded)) @@ -404,6 +416,30 @@ impl Encoder { } impl Encoder { + /// Create an encoder continuing `checkpoint`, so the first edit opens a group restating it. + /// + /// Fails when the records outnumber the range, or the range exceeds the safe integer range. + pub fn resume(config: ProducerConfig, checkpoint: &Checkpoint) -> Result { + let Checkpoint { range, records } = checkpoint; + if range.start > range.end || range.end > MAX_INDEX || records.len() as u64 > range.end - range.start { + return Err(Error::Json("invalid window checkpoint".into())); + } + + let mut encoder = Self::new(config); + let skip = encoder + .config + .checkpoint_records + .map(|limit| records.len().saturating_sub(limit)) + .unwrap_or_default(); + encoder.window = records[skip..] + .iter() + .map(serde_json::to_value) + .collect::>()?; + encoder.offset = range.start; + encoder.start = range.end - encoder.window.len() as u64; + Ok(encoder) + } + /// Append one record to the back of the window. /// /// Emits a push into the open group, or a header restating the window (the new record included) diff --git a/rs/moq-json/src/window/mod.rs b/rs/moq-json/src/window/mod.rs index d0c461b3b0e..51d22b18c28 100644 --- a/rs/moq-json/src/window/mod.rs +++ b/rs/moq-json/src/window/mod.rs @@ -26,8 +26,9 @@ //! //! The publisher rolls a group when the ops in it outgrow //! [`ProducerConfig::op_ratio`](ProducerConfig::op_ratio) times the header that opened it, exactly as -//! [`snapshot`](crate::snapshot) rolls on its delta budget. That is purely a compression decision: -//! there is no caller-driven cut and no age bound, and a [`Consumer`] never surfaces it. A header +//! [`snapshot`](crate::snapshot) rolls on its delta budget. That is purely a compression decision +//! with no age bound, and a [`Consumer`] never surfaces it. A caller that stores complete groups +//! can also end one with [`Producer::cut`]; the next edit opens a new group with a header. A header //! restating records a reader already has yields nothing, so however often the publisher rolls, the //! reader sees one continuous stream of [`Event`]s. [`ProducerConfig::checkpoint_records`] bounds //! the suffix repeated on each roll for a long-lived window. @@ -54,7 +55,7 @@ mod producer; pub use consumer::Consumer; pub use decoder::{ConsumerConfig, Decoder, Event, Group}; -pub use encoder::{Encoded, Encoder, Pending, ProducerConfig}; +pub use encoder::{Checkpoint, Encoded, Encoder, Pending, ProducerConfig}; pub use producer::Producer; #[cfg(test)] @@ -290,6 +291,53 @@ mod test { ); } + #[test] + fn a_resumed_encoder_continues_the_window() { + let config = ProducerConfig::default().with_op_ratio(0).with_checkpoint_records(2); + let mut original = Encoder::::new(config.clone()); + let mut decoder = Decoder::::new(ConsumerConfig::default()); + for n in 0..4 { + let frame = original.push(&rec(n)).unwrap(); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + } + let frame = original.pop(1).unwrap().unwrap(); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + std::iter::from_fn(|| decoder.next_event()).for_each(drop); + + let checkpoint = Checkpoint { + range: 1..4, + records: vec![rec(1), rec(2), rec(3)], + }; + let mut resumed = Encoder::resume(config, &checkpoint).unwrap(); + assert_eq!(resumed.range(), 1..4); + assert_eq!( + resumed.window(), + vec![rec(2), rec(3)], + "trimmed to the checkpoint bound" + ); + + let frame = resumed.push(&rec(4)).unwrap(); + assert!(frame.keyframe, "the first edit restates the window"); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + assert_eq!( + std::iter::from_fn(|| decoder.next_event()).collect::>(), + vec![Event::Push { + index: 4, + value: rec(4) + }], + "a reader that kept up sees only the new record" + ); + + let invalid = Checkpoint { + range: 3..4, + records: vec![rec(2), rec(3)], + }; + assert!(Encoder::resume(ProducerConfig::default(), &invalid).is_err()); + } + #[test] fn pops_cross_the_omitted_checkpoint_prefix() { let config = ProducerConfig::default().with_checkpoint_records(2); @@ -604,6 +652,41 @@ mod test { assert!(group.decode(br#"{"offset":0,"records":[]}"#).is_err()); } + #[test] + fn cut_finishes_the_group_and_the_next_edit_restates() { + let track = moq_net::broadcast::Info::new() + .produce() + .create_track("test", None) + .unwrap(); + let replay = moq_net::track::Subscription::default().with_max_age(std::time::Duration::from_secs(30)); + let mut groups = track.subscribe(replay); + let mut producer = Producer::::new(track, ProducerConfig::default()); + + producer.cut().unwrap(); + producer.push(&rec(0)).unwrap(); + producer.push(&rec(1)).unwrap(); + producer.cut().unwrap(); + producer.cut().unwrap(); + producer.push(&rec(2)).unwrap(); + + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(Some(mut first))) = groups.poll_recv_group(&waiter) else { + panic!("the first group is published"); + }; + assert_eq!(first.frame_count(), 2); + assert!(first.poll_finished(&waiter).is_ready(), "cut finishes the open group"); + + let Poll::Ready(Ok(Some(mut second))) = groups.poll_recv_group(&waiter) else { + panic!("the next edit opens a group"); + }; + let Poll::Ready(Ok(Some(header))) = second.poll_read_frame(&waiter) else { + panic!("the group starts with a header"); + }; + let header: Value = serde_json::from_slice(&header.payload).unwrap(); + assert_eq!(header, json!({ "offset": 0, "records": [rec(0), rec(1), rec(2)] })); + assert!(groups.poll_recv_group(&waiter).is_pending()); + } + #[test] fn trailing_data_after_a_frame_is_rejected() { let mut decoder = Decoder::::new(ConsumerConfig::default()); diff --git a/rs/moq-json/src/window/producer.rs b/rs/moq-json/src/window/producer.rs index 1bdfba06e4c..50d604bf9c0 100644 --- a/rs/moq-json/src/window/producer.rs +++ b/rs/moq-json/src/window/producer.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; use serde_json::Value; -use super::{Encoded, Encoder, ProducerConfig}; +use super::{Checkpoint, Encoded, Encoder, ProducerConfig}; use crate::Result; /// Publishes a sliding window of JSON records over a track. @@ -32,13 +32,17 @@ impl Clone for Producer { impl Producer { /// Create a producer that publishes to the given track. pub fn new(track: moq_net::track::Producer, config: ProducerConfig) -> Self { + Self::with_encoder(track, Encoder::new(config)) + } + + fn with_encoder(track: moq_net::track::Producer, encoder: Encoder) -> Self { Self { inner: Arc::new(Mutex::new(Inner { track: Track { inner: track, group: None, }, - encoder: Encoder::new(config), + encoder, finished: false, })), _marker: PhantomData, @@ -70,6 +74,14 @@ impl Producer { self.inner.lock().unwrap().pop(count) } + /// Finish the open group, leaving the next edit to open a replacement with a header. + /// + /// Idempotent: cutting when no group is open does nothing. A caller that stores complete groups + /// cuts after its edits, so every edit so far sits in a group no later frame can extend. + pub fn cut(&mut self) -> Result<()> { + self.inner.lock().unwrap().cut() + } + /// Finish the track, closing any open group. /// /// Borrows rather than consumes, matching snapshot and stream, so the handle stays @@ -81,6 +93,14 @@ impl Producer { } impl Producer { + /// Create a producer continuing `checkpoint` on the given track. + /// + /// The first edit opens a group restating the checkpoint, so a reader that already holds those + /// records sees only what follows. Fails like [`Encoder::resume`]. + pub fn resume(track: moq_net::track::Producer, config: ProducerConfig, checkpoint: &Checkpoint) -> Result { + Ok(Self::with_encoder(track, Encoder::resume(config, checkpoint)?)) + } + /// Append one record to the back of the window. pub fn push(&mut self, value: &T) -> Result<()> { self.inner.lock().unwrap().push(value) @@ -115,6 +135,17 @@ impl Inner { Ok(()) } + fn cut(&mut self) -> Result<()> { + let Some(group) = self.track.group.take() else { + return Ok(()); + }; + // The group closes either way, so reset first: a `finish` error must not leave the encoder + // appending ops to a group that is gone. + self.encoder.reset(); + group.finish()?; + Ok(()) + } + fn finish(&mut self) -> Result<()> { if self.finished { return Ok(()); diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index ae3c14e2f4e..90c90a70ad7 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -100,16 +100,18 @@ struct Reservations { published: bool, } -/// The built-in catalog's enrollment in the broadcast timeline. +/// The built-in catalog's own timeline, enrolled with the first media track. +#[derive(Default)] struct CatalogTimeline { - recorder: crate::timeline::Recorder, + recorder: Option, last_sequence: Option, } impl CatalogTimeline { - /// Report a newly published plaintext catalog group once. + /// Report a newly published plaintext catalog group once. Each catalog group is one complete + /// snapshot, so it finishes as soon as it is reported. fn record(&mut self, track: &moq_net::track::Producer) { - let Some(sequence) = track.latest() else { + let (Some(recorder), Some(sequence)) = (self.recorder.as_mut(), track.latest()) else { return; }; if self.last_sequence == Some(sequence) { @@ -117,7 +119,12 @@ impl CatalogTimeline { } self.last_sequence = Some(sequence); - self.recorder.record(sequence, moq_net::Timestamp::now(), true); + recorder.frame( + hang::timeline::Position::group(sequence), + moq_net::Timestamp::now(), + true, + ); + recorder.finish_group(sequence); } } @@ -200,10 +207,11 @@ pub struct Producer { current: Arc>>, - /// The broadcast's timeline: the shared boundary list every enrolled track's groups map - /// onto, and the track those segment records are published on. See - /// [`timeline`](Self::timeline). - timeline: crate::timeline::Producer, + /// The broadcast's per-track timelines. See [`timeline`](Self::timeline). + timeline: crate::timeline::Timelines, + /// How each media track's timeline is cut. The catalog's own uses a zero minimum. + /// See [`Config::with_timeline`]. + cutting: crate::timeline::Config, /// Retention override for the media tracks minted under this catalog, or `None` to keep /// hang's default. Fixed at construction, so every clone and every /// [`Reserved`](super::Reserved) mints tracks under one policy. See @@ -224,6 +232,7 @@ impl Clone for Producer { outputs: self.outputs.clone(), current: self.current.clone(), timeline: self.timeline.clone(), + cutting: self.cutting.clone(), max_age: self.max_age, bandwidth: self.bandwidth.clone(), baseline: self.baseline.clone(), @@ -314,7 +323,7 @@ impl Config { self } - /// Pace the broadcast's timeline with `timeline`. + /// Cut each media track's timeline with `timeline`; the catalog's own always uses a zero minimum. pub fn with_timeline(mut self, timeline: crate::timeline::Config) -> Self { self.timeline = timeline; self @@ -343,12 +352,9 @@ impl Producer { json_config.compression = moq_json::Compression::Deflate; let hangz = moq_json::snapshot::Producer::new(hangz_track, json_config); - let timeline = crate::timeline::Producer::new(broadcast, config.timeline); + let timeline = crate::timeline::Timelines::new(broadcast); #[allow(clippy::arc_with_non_send_sync)] - let catalog_timeline = Arc::new(Mutex::new(CatalogTimeline { - recorder: timeline.track(hang::Catalog::DEFAULT_NAME), - last_sequence: None, - })); + let catalog_timeline = Arc::new(Mutex::new(CatalogTimeline::default())); // The broadcast clock is advertised at the catalog root from the first snapshot, // independently of any archive timeline: a live-only publisher exposes its mapping @@ -380,6 +386,7 @@ impl Producer { estimates: Throttle::default(), })), timeline, + cutting: config.timeline, max_age: config.max_age, bandwidth: config.bandwidth, baseline: Default::default(), @@ -637,11 +644,10 @@ impl Producer { } /// Build the media [`container::Producer`](crate::container::Producer) for `track`, - /// enrolling it in the broadcast's timeline so its groups are indexed into the aligned - /// segments. + /// enrolling it in the broadcast's timelines so its groups are indexed. /// - /// The broadcast's one timeline track is created (and advertised in the catalog's root - /// `archive` entry) on first use; see [`timeline`](crate::timeline) for the whole model. + /// The track's timeline track is created and advertised in the catalog's root `archive` + /// entry; see [`timeline`](crate::timeline) for the whole model. pub(super) fn media( &self, track: moq_net::track::Producer, @@ -692,33 +698,43 @@ impl Producer { self.bandwidth.clone() } - /// Enroll `track` in the broadcast's timeline, advertising the timeline in the catalog's - /// root section the first time. + /// Enroll `track` in the broadcast's timelines, advertising its timeline in the catalog's + /// root section with the next catalog change. The first enrollment also enrolls the catalog + /// itself. /// /// The role-specific track constructors call this for you. fMP4 passthrough calls it directly /// because it writes groups by hand instead of using a [`container::Producer`](crate::container::Producer). pub(crate) fn enroll(&mut self, track: &str) -> crate::Result { - let recorder = self.timeline.pacing_track(track)?; + let recorder = self.timeline.track(track, self.cutting.clone())?; + { + let mut catalog = self.outputs.catalog_timeline.lock().unwrap(); + if catalog.recorder.is_none() { + // Sparse data: a catalog may not change again, so each group is its own record. + let cutting = self.cutting.clone().with_duration_min(std::time::Duration::ZERO); + catalog.recorder = Some(self.timeline.track(hang::Catalog::DEFAULT_NAME, cutting)?); + } + } + // Staged without publishing: the entry the track belongs to commits it, so no consumer sees a + // catalog advertising a timeline before its rendition. let section = self.timeline.section(); - let mut catalog = self.modify()?; - if catalog.archive.is_none() { - catalog.archive = Some(section); + let mut state = take(&self.current); + if let Some(err) = &state.closed { + return Err(err.clone()); + } + match &mut state.catalog.archive { + Some(archive) => archive.timelines = section.timelines, + None => state.catalog.archive = Some(section), } - catalog.commit()?; Ok(recorder) } - /// The broadcast's [`timeline::Producer`](crate::timeline::Producer): its segment index. + /// The broadcast's [`Timelines`](crate::timeline::Timelines): each enrolled track's index. /// - /// The MoQ track behind it is created (and advertised in the catalog's root `archive` - /// entry) when the first media track enrolls, so reading this costs nothing on a - /// broadcast that never segments. Use it to declare boundaries - /// ([`cut`](crate::timeline::Producer::cut)) or to hold publishing back while tracks - /// enroll ([`reserve`](crate::timeline::Producer::reserve), the timeline's counterpart to - /// this producer's own [`reserve`](Self::reserve)). - pub fn timeline(&self) -> crate::timeline::Producer { + /// A track's timeline is created (and advertised in the catalog's root `archive` entry) when + /// it enrolls, so reading this costs nothing on a broadcast that never enrolls one. + pub fn timeline(&self) -> crate::timeline::Timelines { self.timeline.clone() } @@ -813,7 +829,7 @@ impl Producer { self.outputs.hang.finish()?; self.outputs.hangz.finish()?; self.outputs.msf_track.finish()?; - self.timeline.finish()?; + self.timeline.finish(); Ok(()) } } @@ -1100,15 +1116,14 @@ mod test { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = Producer::new(&mut broadcast, Config::default()).unwrap(); - // Pacing enrollment is what mints the timeline track; a passive one publishes none. - catalog.timeline().pacing_track("video").unwrap(); + let _recorder = catalog.timeline().track("video", Default::default()).unwrap(); let consumer = broadcast.consume(); for name in [ hang::Catalog::DEFAULT_NAME, hang::Catalog::COMPRESSED_NAME, moq_msf::DEFAULT_NAME, - hang::timeline::DEFAULT_NAME, + "video.timeline.z", ] { let track = consumer.track(name).expect("track"); let info = track.query().await.expect("info"); @@ -1324,7 +1339,7 @@ mod test { let mut catalog = Producer::new(&mut broadcast, Config::default()).unwrap(); // Something else already took the name the timeline track wants. - let _taken = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _taken = broadcast.create_track("video0.timeline.z", None).unwrap(); assert!(catalog.enroll("video0").is_err()); } @@ -1336,10 +1351,15 @@ mod test { // A broadcast that never segments never advertises an archive. assert_eq!(catalog.snapshot().archive, None); let _recorder = catalog.enroll("video0").unwrap(); + let archive = catalog + .snapshot() + .archive + .expect("the root archive advertises the timelines"); + assert_eq!(archive, catalog.timeline().section()); assert_eq!( - catalog.snapshot().archive, - Some(catalog.timeline().section()), - "the root archive should advertise the timeline track" + archive.timelines.keys().collect::>(), + vec!["catalog.json", "video0"], + "the catalog indexes itself alongside its first media track" ); } diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index a1292745f57..5f304f229bf 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -1158,21 +1158,18 @@ mod tests { ); } - /// The broadcast has one timeline: every rendition's groups index into the same track, so - /// an aligned ladder (source + rung) shares it by construction. + /// Every rendition gets its own timeline, advertised together at the catalog root. #[test] - fn renditions_share_the_broadcast_timeline() { + fn renditions_get_their_own_timelines() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let mut catalog = super::super::Producer::new(&mut broadcast, super::super::Config::default()).unwrap(); - let _recorder = catalog.enroll("video0").unwrap(); - let timeline = catalog.timeline(); - assert_eq!(timeline.section().track, hang::timeline::DEFAULT_NAME); - assert_eq!( - catalog.snapshot().archive, - Some(timeline.section()), - "the one timeline is advertised at the catalog root" - ); + let _video0 = catalog.enroll("video0").unwrap(); + let _video1 = catalog.enroll("video1").unwrap(); + let section = catalog.timeline().section(); + assert_eq!(section.timelines["video0"], "video0.timeline.z"); + assert_eq!(section.timelines["video1"], "video1.timeline.z"); + assert_eq!(catalog.snapshot().archive, Some(section)); } mod custom { diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index be5a17a22a6..a2fefd9a1fd 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -114,10 +114,12 @@ async fn rendition_is_not_published_when_the_media_track_fails() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - // Squat the broadcast's timeline track, so enrolling the first rendition (and with it building - // its media producer) fails. The handle must stay alive: the broadcast tracks names weakly, so + // Squat the catalog's timeline track, so enrolling the first rendition (which enrolls the catalog + // too, and with it building its media producer) fails. The handle must stay alive: the broadcast tracks names weakly, so // dropping it frees the name. - let _squat = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _squat = broadcast + .create_track(hang::timeline::default_name(hang::Catalog::DEFAULT_NAME), None) + .unwrap(); let mut importer = Import::new(broadcast, catalog.reserve()); // A track it cannot build surfaces in the catalog rather than in this result. diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 8d35cc29dfe..b9b4eb732a4 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -66,29 +66,9 @@ pub struct Import { // open group belongs to an older one rolls, so every track segments identically. segment: u64, - // A boundary the source declared (rather than one inferred from a keyframe) still has to - // reach the timeline. - pending_timeline_cut: bool, - - // Where the current segment starts: the timestamp the FIRST track to open a group for it - // reported, which every other track then reports too. - // - // Tracks do not start a segment at the same instant. The boundary is nominal and each begins - // at its own nearest sample, so audio's first sample routinely sits a frame either side of - // video's IDR. The timeline files a group under a segment by the pts reported for it, so - // reporting each track's own first sample would file whichever track ran early under the - // previous record. That was a rounding error when an audio group was one fragment; now a - // group is a whole segment, so a 20ms skew moves a segment's entire audio into its - // predecessor and leaves the next record with none. - // - // Anchoring on the first roll rather than the earliest also keeps the declared cut on the - // source's own cadence: the timeline drops a cut landing within `duration_min` of the last - // one, so a boundary pulled 20ms earlier than a 1 second source's would be discarded and two - // segments would merge. - // - // Only the timeline report is anchored. Each fragment still carries its own timestamp on the - // wire, and `Recorder::end` still reports real content time. - segment_start: Option, + // The current segment's boundary was declared by the source (rather than inferred from a + // keyframe), so each track cuts its timeline where it opens its group for the segment. + declared: bool, } /// The catalog entry for one imported track, whichever section it lives in. @@ -118,9 +98,8 @@ struct Fmp4Track { track: moq_net::track::Producer, group: Option, - // Reports this track's group opens into the broadcast's timeline. Passthrough writes - // groups by hand (no `container::Producer`), so the recorder is fed directly at each - // keyframe fragment rather than through `with_recorder`. + // Reports this track's fragments into its timeline. Passthrough writes groups by hand (no + // `container::Producer`), so the recorder is fed directly rather than through `with_recorder`. recorder: Option, // The decode time of the last fragment, which the next one has to advance past. @@ -172,8 +151,7 @@ impl Import { buffer: BytesMut::new(), pending_cut: false, segment: 0, - pending_timeline_cut: false, - segment_start: None, + declared: false, } } @@ -181,9 +159,9 @@ impl Import { /// segmentation out of band (e.g. an HLS import following its playlist). /// /// A `styp` atom (a CMAF segment on disk) does this on its own, so an importer reading a - /// segmented file needs no help. Boundaries are broadcast-wide, but redundant ones cost - /// nothing: the timeline ignores a cut that would land inside its minimum segment duration, - /// so several renditions of one source may all declare the same boundaries. + /// segmented file needs no help. Each track's timeline then closes a record where that track + /// opens its group for the new segment, following the source's segmentation instead of the + /// minimum-duration pacing. /// /// This is where groups are drawn too: every track rolls its group at a segment boundary, so /// a group is a segment and each fragment inside it is a frame. @@ -281,16 +259,6 @@ impl Import { } // Only `finish()` releases the reservation before a moov, since a fragment needs one. let reserved = self.initial_reservation.clone().ok_or(Error::MoovAfterFinish)?; - let timeline = self.catalog.timeline(); - - // The tracks below enroll in the timeline, so advertise it in the same catalog update - // rather than publishing a second snapshot for it. - { - let mut catalog = self.catalog.modify()?; - if catalog.archive.is_none() && !moov.trak.is_empty() { - catalog.archive = Some(timeline.section()); - } - } for trak in &moov.trak { let track_id = trak.tkhd.track_id; @@ -316,10 +284,10 @@ impl Import { self.catalog.track_info(kind.priority()).with_timescale(timescale), )?; - // Enroll every track in the broadcast's timeline: passthrough writes groups by hand - // (no `container::Producer`), so the recorder is fed directly at each group open. - // The root archive entry is advertised before any rendition releases its reservation. - let recorder = timeline.pacing_track(track.name())?; + // Enroll every track in the broadcast's timelines: passthrough writes groups by hand + // (no `container::Producer`), so the recorder is fed directly at each fragment. The + // reservation holds the archive entry back until the whole track set is declared. + let recorder = self.catalog.enroll(track.name())?; // Whatever the descriptor declared (a bitrate) is authoritative; the rest is filled by // `estimate` as fragments arrive. @@ -677,9 +645,8 @@ impl Import { self.segment += 1; // A boundary the source declared is also a timeline boundary; one merely inferred // from a keyframe is not, since the default pacing already sees that group open. - self.pending_timeline_cut |= self.pending_cut; + self.declared = self.pending_cut; self.pending_cut = false; - self.segment_start = None; } let moov = self.moov.as_ref().ok_or(Error::NoMoov)?; @@ -929,6 +896,9 @@ impl Import { let mut g = if start_group { if let Some(prev) = track.group.take() { prev.finish()?; + if let Some(recorder) = track.recorder.as_mut() { + recorder.finish_group(prev.sequence); + } } track.segment = Some(self.segment); match track.pending_sequence.take() { @@ -939,28 +909,16 @@ impl Import { track.group.take().ok_or(Error::NoKeyframe)? }; + let position = hang::timeline::Position::new(g.sequence, g.frame_count() as u64); + if start_group { - // A group just opened; report it so the broadcast's timeline can index the segment - // (the timeline absorbs publish failures). Audio is always independently decodable; - // video says whether this segment really begins on an IDR, which is what an HLS - // export reads to bootstrap an init segment. A group reopened mid-segment (after a - // `seek`) reports its own timestamp: it does not start a segment. - let reported = match new_segment { - false => timestamp, - true => match self.segment_start { - Some(start) => start, - None => { - self.segment_start = Some(timestamp); - if self.pending_timeline_cut { - self.pending_timeline_cut = false; - self.catalog.timeline().cut(timestamp)?; - } - timestamp - } - }, - }; - if let Some(recorder) = track.recorder.as_mut() { - recorder.record(g.sequence, reported, contains_keyframe); + // A group opening for a declared segment is a boundary on this track's timeline. A + // group reopened mid-segment (after a `seek`) is not. + if new_segment + && self.declared + && let Some(recorder) = track.recorder.as_mut() + { + recorder.cut(timestamp); } // Close the previous group for the bitrate estimator (used only when the @@ -968,7 +926,6 @@ impl Import { track.estimator.cut(Some(timestamp)); } let fragment_len = fragment_bytes.len(); - let mut frame = g.create_frame(moq_net::frame::Info { size: fragment_bytes.len() as u64, timestamp, @@ -976,6 +933,13 @@ impl Import { frame.write(fragment_bytes)?; frame.finish()?; + // Only once published, so a rejected fragment is never indexed. Audio is always + // independently decodable; video says whether this fragment really begins on an IDR, + // which is what an HLS export reads to bootstrap an init segment. + if let Some(recorder) = track.recorder.as_mut() { + recorder.frame(position, timestamp, contains_keyframe); + } + track.group = Some(g); track.estimator.write(timestamp, fragment_len); @@ -1002,6 +966,9 @@ impl Import { track.publish_estimate()?; if let Some(g) = track.group.take() { g.finish()?; + if let Some(recorder) = track.recorder.as_mut() { + recorder.finish_group(g.sequence); + } } track.track.finish()?; } @@ -1032,6 +999,9 @@ impl Import { track.publish_estimate()?; if let Some(g) = track.group.take() { g.finish()?; + if let Some(recorder) = track.recorder.as_mut() { + recorder.finish_group(g.sequence); + } } track.pending_sequence = Some(sequence); } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index f8630de4eec..0b18cd43b0e 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -419,10 +419,34 @@ async fn test_msf_catalog_roundtrip() { assert!(matches!(audio.container, Container::Cmaf { .. })); } -/// Passthrough writes groups by hand (no `container::Producer`), so the timeline recorder is fed -/// directly. This guards that the import advertises the broadcast's timeline at the catalog root -/// and actually records both renditions' group opens into it, the index the VOD/playlist export -/// reads. Regression for the missing-timeline break that skipped VOD renditions. +/// Subscribe to `track`'s timeline as `section` advertises it. +async fn subscribe( + consumer: &moq_net::broadcast::Consumer, + section: &hang::catalog::Archive, + track: &str, +) -> crate::timeline::Consumer { + crate::timeline::Consumer::<()>::subscribe(consumer, section, track) + .await + .unwrap() +} + +/// Every record a finished timeline published, as `(start group, end group)`. +async fn groups(timeline: &mut crate::timeline::Consumer) -> Vec<(u64, u64)> { + let mut out = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let crate::timeline::Event::Push { entry, .. } = event { + assert_eq!(entry.start.frame, 0, "fragments never split a group here"); + assert_eq!(entry.end.frame, 0, "fragments never split a group here"); + out.push((entry.start.group, entry.end.group)); + } + } + out +} + +/// Passthrough writes groups by hand (no `container::Producer`), so the timeline recorders are fed +/// directly. This guards that the import advertises both renditions' timelines at the catalog root +/// and actually records their groups, the index the VOD/playlist export reads. Regression for the +/// missing-timeline break that skipped VOD renditions. #[tokio::test] async fn import_populates_the_broadcast_timeline() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -442,31 +466,21 @@ async fn import_populates_the_broadcast_timeline() { let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - // The one timeline is advertised at the catalog root, named by convention. - let section = snapshot.archive.clone().expect("the import advertises a timeline"); - assert_eq!(section.track, hang::timeline::DEFAULT_NAME); + // Each rendition's timeline is advertised at the catalog root. + let section = snapshot.archive.clone().expect("the import advertises its timelines"); + assert_eq!( + section.timelines[&video_name], + hang::timeline::default_name(&video_name) + ); - // Subscribe while the producer is alive, then finish so the timeline group closes and the - // reader terminates rather than blocking. - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + // Subscribe while the producer is alive, then finish so the timeline groups close and the + // readers terminate rather than blocking. + let mut video = subscribe(&consumer, §ion, &video_name).await; + let mut audio = subscribe(&consumer, §ion, &audio_name).await; catalog.finish().unwrap(); - // The first record indexes both renditions from their first group (sequence 0). - let event = timeline - .next() - .await - .unwrap() - .expect("a segment should be recorded in the timeline"); - let crate::timeline::Event::Push { entry: first, .. } = event else { - panic!("the first timeline event was not a segment"); - }; - assert_eq!(first.segment, 0); - let video_ranges = first.tracks.get(&video_name).expect("video is indexed"); - assert_eq!(video_ranges[0].start, 0, "the first video group is sequence 0"); - let audio_ranges = first.tracks.get(&audio_name).expect("audio is indexed"); - assert_eq!(audio_ranges[0].start, 0, "the first audio group is sequence 0"); + assert_eq!(groups(&mut video).await[0].0, 0, "the first video group is sequence 0"); + assert_eq!(groups(&mut audio).await[0].0, 0, "the first audio group is sequence 0"); } // ---- Sample-duration handling in decode() ---- @@ -786,12 +800,11 @@ async fn segmented_source_groups_per_segment() { ); } -/// The timeline indexes each segment as one group range per track, which is what makes the -/// segment servable: an HLS/DASH export fetches `start..=end` and concatenates it into one CMAF -/// segment. With audio grouping per fragment those ranges spanned many short groups, one FETCH -/// round-trip each, and the two tracks' boundaries no longer coincided. +/// Each declared segment is one group per track, and each track's timeline indexes it as one +/// record. With audio grouping per fragment those records spanned many short groups, one FETCH +/// round-trip each. #[tokio::test] -async fn segmented_source_indexes_one_group_range_per_track() { +async fn segmented_source_indexes_one_group_per_record() { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -803,12 +816,11 @@ async fn segmented_source_indexes_one_group_range_per_track() { let snapshot = catalog.snapshot(); let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); - let section = snapshot.archive.clone().expect("the import advertises a timeline"); + let section = snapshot.archive.clone().expect("the import advertises its timelines"); - // Subscribe while the producer is alive so the reader terminates on finish rather than blocking. - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + // Subscribe while the producer is alive so the readers terminate on finish rather than blocking. + let mut video = subscribe(&consumer, §ion, &video_name).await; + let mut audio = subscribe(&consumer, §ion, &audio_name).await; for segment in 0..3u64 { let base = segment * 1_000_000; @@ -833,47 +845,22 @@ async fn segmented_source_indexes_one_group_range_per_track() { fmp4.finish().unwrap(); catalog.finish().unwrap(); - let mut records = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let crate::timeline::Event::Push { entry, .. } = event { - records.push(entry); - } - } - assert_eq!(records.len(), 3, "one record per declared segment"); - - for (i, record) in records.iter().enumerate() { - assert_eq!(record.segment, i as u64); - let video = record.tracks.get(&video_name).expect("video is indexed"); - let audio = record.tracks.get(&audio_name).expect("audio is indexed"); - assert_eq!(video.len(), 1, "segment {i}: video is one contiguous group range"); - assert_eq!(audio.len(), 1, "segment {i}: audio is one contiguous group range"); - assert_eq!( - (video[0].start, video[0].end), - (i as u64, i as u64), - "segment {i}: one video group" - ); - assert_eq!( - (audio[0].start, audio[0].end), - (i as u64, i as u64), - "segment {i}: one audio group, not one per fragment" - ); - } + let expected = vec![(0, 1), (1, 2), (2, 3)]; + assert_eq!(groups(&mut video).await, expected, "one video group per record"); + assert_eq!( + groups(&mut audio).await, + expected, + "one audio group per record, not one per fragment" + ); } /// Audio and video rarely start a segment at exactly the same timestamp: the boundary is nominal -/// and each track begins at its own nearest sample. The timeline files a group under a segment by -/// the group's start, so a boundary declared later than any track's first group files that whole -/// group under the previous segment. -/// -/// That was survivable when audio groups were one fragment. Now a group is a whole segment, so a -/// 20ms skew moves an entire segment of audio into the wrong record: the previous segment gets a -/// second copy's worth and the next one opens with no audio at all. +/// and each track begins at its own nearest sample. Each track cuts its own timeline where it opens +/// its group for a declared segment, so a skew between them moves no group into another record. /// /// `video_offset_us` shifts video relative to audio, so the same source is exercised with audio -/// leading and lagging. -async fn segment_ranges_with_skew( - video_offset_us: i64, -) -> Vec<(Vec, Vec)> { +/// leading and lagging. Returns each track's records as `(start group, end group)`. +async fn records_with_skew(video_offset_us: i64) -> (Vec<(u64, u64)>, Vec<(u64, u64)>) { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -886,9 +873,8 @@ async fn segment_ranges_with_skew( let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); let section = snapshot.archive.clone().unwrap(); - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + let mut video = subscribe(&consumer, §ion, &video_name).await; + let mut audio = subscribe(&consumer, §ion, &audio_name).await; for segment in 0..3u64 { let base = segment * 1_000_000; @@ -896,7 +882,7 @@ async fn segment_ranges_with_skew( fmp4.decode(&styp()).unwrap(); // Audio first, so the segment transition is detected on the audio fragment and video's - // IDR lands after it. Two fragments each keeps the ranges unambiguous. + // IDR lands after it. Two fragments each keeps the records unambiguous. for offset in [0u64, 500_000] { let frag = super::encode_fragment( info(audio_id, audio_scale, segment as u32), @@ -917,58 +903,21 @@ async fn segment_ranges_with_skew( fmp4.finish().unwrap(); catalog.finish().unwrap(); - let mut out = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let crate::timeline::Event::Push { entry, .. } = event { - out.push(( - entry.tracks.get(&video_name).cloned().unwrap_or_default(), - entry.tracks.get(&audio_name).cloned().unwrap_or_default(), - )); - } - } - out + (groups(&mut video).await, groups(&mut audio).await) } /// Video's IDR lands 20ms AFTER audio's first sample of the segment, the normal interleave. #[tokio::test] async fn audio_leading_video_still_indexes_one_group_each() { - let records = segment_ranges_with_skew(20_000).await; - assert_eq!(records.len(), 3, "one record per segment"); - for (i, (video, audio)) in records.iter().enumerate() { - assert_eq!(video.len(), 1, "segment {i}: one video range, got {video:?}"); - assert_eq!(audio.len(), 1, "segment {i}: one audio range, got {audio:?}"); - assert_eq!( - (audio[0].start, audio[0].end), - (i as u64, i as u64), - "segment {i} audio" - ); - assert_eq!( - (video[0].start, video[0].end), - (i as u64, i as u64), - "segment {i} video" - ); - } + let expected = vec![(0, 1), (1, 2), (2, 3)]; + assert_eq!(records_with_skew(20_000).await, (expected.clone(), expected)); } /// Video's IDR lands 20ms BEFORE audio's first sample, the other side of the same skew. #[tokio::test] async fn audio_lagging_video_still_indexes_one_group_each() { - let records = segment_ranges_with_skew(-20_000).await; - assert_eq!(records.len(), 3, "one record per segment"); - for (i, (video, audio)) in records.iter().enumerate() { - assert_eq!(video.len(), 1, "segment {i}: one video range, got {video:?}"); - assert_eq!(audio.len(), 1, "segment {i}: one audio range, got {audio:?}"); - assert_eq!( - (audio[0].start, audio[0].end), - (i as u64, i as u64), - "segment {i} audio" - ); - assert_eq!( - (video[0].start, video[0].end), - (i as u64, i as u64), - "segment {i} video" - ); - } + let expected = vec![(0, 1), (1, 2), (2, 3)]; + assert_eq!(records_with_skew(-20_000).await, (expected.clone(), expected)); } /// A source that emits one leading `styp` and then never another still segments on its video @@ -1281,7 +1230,7 @@ struct Imported { /// The root clock the catalog advertised. clock: hang::catalog::Clock, arrival: std::ops::RangeInclusive, - /// The first segment the broadcast timeline recorded, the index an archive replays from. + /// The first record the video timeline published, the index an archive replays from. record: moq_net::Timestamp, /// Why a chunk was refused, if one was. refused: Option, @@ -1297,10 +1246,10 @@ async fn import_session(chunks: &[Vec]) -> Imported { let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); fmp4.decode(&init).unwrap(); - let section = catalog.snapshot().archive.expect("the import advertises a timeline"); - let mut timeline = crate::timeline::Consumer::<()>::subscribe(&consumer, §ion) - .await - .unwrap(); + let snapshot = catalog.snapshot(); + let video = snapshot.video.renditions.keys().next().unwrap().clone(); + let section = snapshot.archive.clone().expect("the import advertises its timelines"); + let mut timeline = subscribe(&consumer, §ion, &video).await; let before = std::time::SystemTime::now(); let mut after = before; @@ -1317,15 +1266,15 @@ async fn import_session(chunks: &[Vec]) -> Imported { fmp4.finish().unwrap(); catalog.finish().unwrap(); - let event = timeline.next().await.unwrap().expect("a recorded segment"); + let event = timeline.next().await.unwrap().expect("a recorded span"); let crate::timeline::Event::Push { entry, .. } = event else { - panic!("the first timeline event was not a segment"); + panic!("the first timeline event was not a record"); }; let snapshot = catalog.snapshot(); Imported { published: crate::container::test_util::published(&consumer, &snapshot).await, - video: snapshot.video.renditions.keys().next().unwrap().clone(), + video, clock: snapshot.clock.expect("the catalog advertises a clock"), arrival: before..=after, record: entry.pts, @@ -1448,7 +1397,8 @@ fn a_second_moov_is_refused() { ); } -/// A moov with every track deselected publishes at once, since no fragment will ever anchor. +/// A moov with every track deselected releases its reservation at once, since no fragment will +/// ever anchor: the catalog's next change publishes without one. #[tokio::test] async fn a_moov_with_nothing_selected_publishes() { let (init, _, _) = bbb_init(); @@ -1460,6 +1410,7 @@ async fn a_moov_with_nothing_selected_publishes() { .with_select(crate::select::Broadcast::default()); fmp4.decode(&init).unwrap(); + catalog.clone().anchor(moq_net::Timestamp::ZERO).unwrap(); assert_eq!(clocks.drain().len(), 1, "the catalog publishes without a fragment"); } diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index cbcf698cff0..4a002cfd49c 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -412,10 +412,12 @@ fn rendition_is_not_published_when_the_media_track_fails() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - // Squat the broadcast's timeline track, so building the media producer (which creates it - // on first use) fails. The handle must stay alive: the broadcast tracks names weakly, so + // Squat the catalog's timeline track, so building the media producer (whose enrollment + // enrolls the catalog too) fails. The handle must stay alive: the broadcast tracks names weakly, so // dropping it frees the name. - let _squat = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _squat = broadcast + .create_track(hang::timeline::default_name(hang::Catalog::DEFAULT_NAME), None) + .unwrap(); let mut mkv = crate::container::mkv::Import::new(broadcast, catalog.reserve()); let buf = bytes::BytesMut::from(&data[..]); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index dd039d150f6..13ae3383065 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -58,13 +58,13 @@ pub struct Producer { /// Set by [`Self::seek`] and consumed on the next group creation. pending_sequence: Option, - /// Records each group open (sequence + keyframe timestamp) into this rendition's - /// timeline track, when the producer was built with one. + /// Reports each written frame and finished group into this rendition's timeline, when the + /// producer was built with one. recorder: Option, /// The furthest presentation point written, i.e. `max(timestamp + duration)`. Reported to /// `recorder` on each [`cut`](Self::cut), since the last group of a track has no successor - /// to bound it and its segment would otherwise be published a group short. Also the base + /// to bound it and its record would otherwise end at its last frame's start. Also the base /// for a duration marker when the caller does not pass a bound. end: Option, @@ -350,11 +350,10 @@ where self } - /// Report each group open (sequence, timestamp, keyframe) through `recorder`, enrolling - /// this track in the broadcast's timeline so consumers can index the media without - /// downloading it. + /// Report each written frame and finished group through `recorder`, enrolling this track in + /// its timeline so consumers can index the media without downloading it. /// - /// Mint the recorder from the broadcast's [`timeline::Producer`](crate::timeline::Producer). + /// Mint the recorder from the broadcast's [`Timelines`](crate::timeline::Timelines). pub fn with_recorder(mut self, recorder: crate::timeline::Recorder) -> Self { self.recorder = Some(recorder); self @@ -446,13 +445,6 @@ where None => self.inner.append_group()?, }; - // Report the group the moment it opens: its start is this frame's timestamp. The - // timeline absorbs publish failures itself (it is an optional sidecar), - // so reporting can't abort the media write. - if let Some(recorder) = self.recorder.as_mut() { - recorder.record(group.sequence, frame.timestamp, frame.keyframe); - } - self.floor = self.start; self.start = Some(frame.timestamp); self.group = Some(group); @@ -462,7 +454,13 @@ where if self.buffer_duration.is_zero() { let group = self.group.as_mut().unwrap(); let (timestamp, duration, bytes) = (frame.timestamp, frame.duration, frame.payload.len()); + let (position, keyframe) = (position(group), frame.keyframe); self.container.write(group, &[frame])?; + // The timeline absorbs publish failures itself (it is an optional sidecar), so + // reporting can't abort the media write. + if let Some(recorder) = self.recorder.as_mut() { + recorder.frame(position, timestamp, keyframe); + } // Only what the container accepted is measured. A rejected frame (too large for the // group, a timestamp that won't convert) leaves the producer usable, and the estimate's @@ -523,21 +521,25 @@ where self.estimator.cut(end); self.claim(); + let tail_end = marker_at.filter(|_| !self.reordered); + self.flush_buffer(tail_end)?; + // Tell the timeline where this group's content stops: the duration marker when we - // write one, else the caller's bound, else the furthest point we wrote. + // write one, else the caller's bound, else the furthest point we wrote. After the flush, + // so a group still wholly buffered has reported its first frame and opened a record. if let Some(recorder) = self.recorder.as_mut() && let Some(end) = marker_at.or(end).max(self.end) { recorder.end(end); } - - let tail_end = marker_at.filter(|_| !self.reordered); - self.flush_buffer(tail_end)?; if let Some(group) = self.group.as_mut() { self.container.finish_group(group, tail_end)?; } if let Some(group) = self.group.take() { group.finish()?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.finish_group(group.sequence); + } } if let Some(end) = self.end { self.raise_live_edge(end); @@ -646,7 +648,7 @@ where }; let timestamp = self.live_edge.unwrap_or(moq_net::Timestamp::ZERO); if let Some(recorder) = self.recorder.as_mut() { - recorder.record(group.sequence, timestamp, false); + recorder.frame(hang::timeline::Position::group(group.sequence), timestamp, false); recorder.end(timestamp); } self.container.write( @@ -659,6 +661,11 @@ where }], )?; group.finish()?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.finish_group(group.sequence); + // The marker is contiguous, so without this the next record boundary would absorb the pause. + recorder.flush(); + } Ok(()) } @@ -700,7 +707,11 @@ where None => return Ok(()), }; + let position = position(group); self.container.write(group, &self.buffer)?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.frame(position, self.buffer[0].timestamp, self.buffer[0].keyframe); + } self.buffer.clear(); Ok(()) @@ -742,6 +753,11 @@ impl std::ops::Deref for Producer { } } +/// Where the next frame written to `group` lands. +fn position(group: &moq_net::group::Producer) -> hang::timeline::Position { + hang::timeline::Position::new(group.sequence, group.frame_count() as u64) +} + #[cfg(test)] mod tests { use super::*; @@ -1409,6 +1425,66 @@ mod tests { crate::container::fmp4::encode_fragment(info, group).unwrap(); } + /// A group shorter than the buffer reaches the timeline only when it closes, and its record + /// still runs to the group's end rather than stopping at its first frame. + #[tokio::test] + async fn a_wholly_buffered_group_records_its_end() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = crate::timeline::Timelines::new(&broadcast); + let recorder = timelines.track("video", crate::timeline::Config::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)) + .with_buffer(std::time::Duration::from_secs(10)) + .with_recorder(recorder); + + producer.write(frame(0, true)).unwrap(); + producer.write(frame(33_000, false)).unwrap(); + producer.cut(Some(Timestamp::from_micros(66_000).unwrap())).unwrap(); + drop(producer); + timelines.finish(); + + let mut timeline = + crate::timeline::Consumer::<()>::subscribe(&broadcast.consume(), &timelines.section(), "video") + .await + .unwrap(); + let Some(crate::timeline::Event::Push { entry, .. }) = timeline.next().await.unwrap() else { + panic!("expected a record"); + }; + assert_eq!(entry.duration, std::time::Duration::from_millis(66)); + } + + /// The record before a discontinuity ends at the marker, not where content resumes. + #[tokio::test] + async fn a_discontinuity_ends_the_record_at_the_marker() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = crate::timeline::Timelines::new(&broadcast); + let recorder = timelines.track("video", crate::timeline::Config::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let mut producer = + Producer::new(track, Container::Legacy(crate::container::Kind::Video)).with_recorder(recorder); + + producer.write(frame(0, true)).unwrap(); + producer.write(frame(33_000, false)).unwrap(); + producer.cut(Some(Timestamp::from_micros(66_000).unwrap())).unwrap(); + producer.discontinuity().unwrap(); + // Resumed 40 minutes later. + producer.write(frame(2_400_000_000, true)).unwrap(); + drop(producer); + timelines.finish(); + + let mut timeline = + crate::timeline::Consumer::<()>::subscribe(&broadcast.consume(), &timelines.section(), "video") + .await + .unwrap(); + let mut spans = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let crate::timeline::Event::Push { entry, .. } = event { + spans.push((entry.pts.as_micros() / 1_000, entry.duration.as_millis())); + } + } + assert_eq!(spans, vec![(0, 66), (2_400_000, 0)]); + } + #[tokio::test] async fn duration_marker_uses_cadence_not_batching_delay() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 091a24e1efd..92554636a07 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -180,29 +180,32 @@ pub enum Error { composition_time_ms: i32, }, - /// A segment ran past the [`duration_max`](crate::timeline::Config::duration_max) the - /// catalog advertised, so the timeline stopped publishing rather than contradict it. The - /// publisher declared a bound its media can't honor. - #[error("timeline segment {segment} lasted {duration:?}, over the declared maximum {duration_max:?}")] - TimelineOverrun { - /// The segment that broke the bound. - segment: u64, - /// How long it actually ran. - duration: std::time::Duration, - /// The bound the catalog advertised. - duration_max: std::time::Duration, + /// [`timeline::Producer::resume`](crate::timeline::Producer::resume) received a checkpoint + /// whose record at this window index has a different sequence. + #[error("timeline checkpoint record at index {0} has a different sequence")] + TimelineCheckpoint(u64), + + /// [`timeline::Producer::push`](crate::timeline::Producer::push) received a record out of order. + #[error("timeline record {actual} pushed where {expected} was next")] + TimelineSequence { + /// The window's next index. + expected: u64, + /// The pushed record's sequence. + actual: u64, }, - /// [`timeline::Producer::finish`](crate::timeline::Producer::finish) was called before its - /// deferred [`timeline::Segmenter`](crate::timeline::Segmenter) completed and every record - /// was committed. - #[error("finish and commit every deferred timeline record before closing the Producer")] - TimelineDeferredPending, + /// A timeline frame report did not advance past the previous one. + #[error("timeline report at {position:?} does not advance past {last:?}")] + TimelinePosition { + /// The rejected report's position. + position: hang::timeline::Position, + /// The previous report's position. + last: hang::timeline::Position, + }, - /// [`timeline::Producer::push`](crate::timeline::Producer::push) received a pending record that - /// its [`timeline::Deferred`](crate::timeline::Deferred) did not yield. - #[error("timeline segment {0} was not yielded for deferred publication")] - TimelineDeferredRecord(u64), + /// The catalog's `archive` entry indexes no timeline for this track. + #[error("no timeline for track {0}")] + TimelineMissing(String), /// Error from a muxer/demuxer that reports via `anyhow` (currently MPEG-TS). /// Boxed in an `Arc` so the enum stays `Clone` (`anyhow::Error` is not). diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs index 4af4902d1e1..7c7430e46cd 100644 --- a/rs/moq-mux/src/timeline.rs +++ b/rs/moq-mux/src/timeline.rs @@ -1,1159 +1,672 @@ -//! Timeline publish/subscribe: the broadcast's segment index HLS/DASH export is built from. +//! Per-track timelines: each track's index of spans, which HLS/DASH export and recordings use. //! -//! A broadcast has one timeline track, carrying one [`hang::timeline::Record`] per *segment*: -//! a span of content time shared by every media track, mapped to the group ranges that carry -//! it on each track. A consumer can answer "which groups cover segment N on track X" and -//! "where is the live edge" from a few bytes per segment without subscribing to media, which -//! is the primitive a playlist server (HLS/DASH), a seek bar, or a recorder index needs. +//! Every indexed track gets its own timeline track carrying one [`hang::timeline::Record`] per +//! *span*: a run of the track's frames mapped to their content time. A consumer can answer "which +//! groups cover time T on track X" and "where is the live edge" from a few bytes per span +//! without subscribing to media, which is the primitive a playlist server (HLS/DASH), a seek bar, +//! or a recorder index needs. //! -//! ## Facts up, policy down +//! ## Cutting //! -//! The write side splits into facts, policy, and publication: +//! Tracks cut independently; nothing waits for another track. //! -//! - **Tracks report facts.** Each media track enrolls via [`Producer::pacing_track`], and its -//! [`Recorder`] reports every group open (sequence, timestamp, keyframe) plus where its -//! content ends. A container import never decides where segments fall; it only states what -//! it published. -//! - **The timeline sets policy.** A segment ends at the first group boundary that gives it at -//! least [`Config::duration_min`] on every enrolled pacing track (see [`Config`] for the exact -//! rule). An application that knows its own boundaries overrides that with [`Producer::cut`]. -//! - **Pacing is explicit.** [`Producer::track`] safely enrolls a catalog or metadata track -//! without giving it control over segmentation. Continuous media opts in through -//! [`Producer::pacing_track`], which lets it vote on boundaries and gate completeness. -//! - **Records close on completeness.** A segment's record is ready only once every -//! enrolled pacing track has reported a group at or past the segment's end (or closed), proving -//! the segment's group ranges are final on every track that paces. The record is then -//! self-contained. [`Producer::reserve`] extends that across a batch -//! of enrollments, the way the catalog's own reservation does. -//! - **Publication is a commit.** [`Deferred`] yields complete records without publishing them. -//! A recorder can first persist the media groups they name, then pass the [`Pending`] handle to -//! [`Producer::push`] to make the record visible. The convenience methods on [`Producer`] commit -//! immediately for live publishers that need no asynchronous work between those steps. +//! Every track follows one rule, parameterized by its [`Config`]: //! -//! Alignment falls out of construction: every track maps its groups onto the same boundary -//! list, so segment N covers the same span of content time on every track, which is what HLS -//! requires of switchable renditions. +//! - A record ends at the first group boundary at least [`Config::duration_min`] (2s by default) +//! past its own start, so short groups (audio) pack into one record and long ones (video GOPs) +//! get one each. Only sparse data, such as a catalog that may not publish again for the rest of +//! the broadcast, uses a zero minimum: every group is then its own record, ended as soon as the +//! group finishes. +//! - A group still open [`Config::duration_max`] (10s by default) past the record's start is split +//! at the next reported frame, so an append-only group that never closes is still indexed as it +//! grows. Frames farther apart than the maximum leave nothing to split at, so the record's +//! duration stops at the maximum and the rest of the gap is uncovered. +//! - A skipped group sequence always closes the record at its newest reported content time, so a +//! record's groups are contiguous and a pause is not absorbed into its duration. +//! - [`Segmenter::cut`] adds an application boundary, such as a video keyframe cutting audio so a +//! derived segment needs fewer objects. The first cut takes over from minimum-duration pacing. //! //! ## Wiring //! -//! [`catalog::Producer`](crate::catalog::Producer) owns the broadcast's timeline (the catalog -//! is what owns the broadcast's shape) and wires all of this up: -//! The catalog's role-specific track constructors enroll the track, which -//! creates the timeline track on first use and advertises it in the catalog's root -//! [`hang::catalog::Archive`] entry. A broadcast that never enrolls a track publishes no -//! timeline at all: segmentation is opt-in per broadcast, never per track. +//! [`catalog::Producer`](crate::catalog::Producer) owns the broadcast's [`Timelines`] and advertises +//! them in the catalog's root [`hang::catalog::Archive`] entry. Its role-specific track +//! constructors enroll each media track, and the catalog enrolls itself with a zero minimum. //! -//! On the read side, [`Consumer::subscribe`] reads the timeline straight from the catalog's -//! [`hang::catalog::Archive`] entry (so the track name and timescale can't be mismatched) -//! and yields decoded [`Event`]s. On the wire the track is a DEFLATE-compressed -//! [`moq_json::window`], so a DVR can trim old records while an unbounded timeline simply never -//! pops them (see [`hang::timeline`] for the record schema). - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::ops::Deref; +//! A [`Segmenter`] builds records from a track's frame reports without publishing them, and a +//! [`Producer`] publishes records onto a timeline track. A [`Recorder`] pairs the two for live +//! publishing; a recording writer drives them separately so it can store the media a record names +//! before publishing it. On the read side, [`Consumer::subscribe`] reads one track's timeline from +//! the catalog's [`Archive`] entry and yields decoded [`Event`]s. On the wire each timeline is a +//! DEFLATE-compressed [`moq_json::window`], so a DVR can trim old records while an unbounded +//! timeline simply never pops them. + +use std::collections::{BTreeMap, VecDeque}; use std::sync::{Arc, Mutex}; use std::task::Poll; use std::time::Duration; use hang::catalog::Archive; -use hang::timeline::{DEFAULT_NAME, Range, Record, RecordExt}; +use hang::timeline::{Position, Record, RecordExt}; +use moq_json::window::Checkpoint; use moq_net::{Timescale, Timestamp}; -/// The conventional [`Config::duration_min`] (1 second), for callers with no opinion. -pub const DEFAULT_DURATION_MIN: Duration = Duration::from_secs(1); +/// The default [`Config::duration_min`] (2 seconds). Only sparse data, such as a catalog, uses zero. +pub const DEFAULT_DURATION_MIN: Duration = Duration::from_secs(2); -/// Recent segment records repeated when the Window track rolls to a new group. +/// The default [`Config::duration_max`] (10 seconds). +pub const DEFAULT_DURATION_MAX: Duration = Duration::from_secs(10); + +/// Recent records repeated when the window track rolls to a new group. const CHECKPOINT_RECORDS: usize = 256; -/// How a [`Producer`] paces its segments. +/// How a track's records are cut. +/// +/// Every track follows one rule: a record ends at the first group boundary at least +/// [`duration_min`](Self::duration_min) past its start, and a group still open +/// [`duration_max`](Self::duration_max) past it is split between frames. #[derive(Clone, Debug)] #[non_exhaustive] pub struct Config { - /// The shortest a segment may be. - /// - /// A segment ends at the first point that is a group boundary on every enrolled track and - /// at least this far past the segment's start, so the track with the coarsest groups paces - /// the broadcast. A 2 second GOP against a 1 second minimum yields 2 second segments; a - /// real-time encoder with a 10 minute GOP yields one 10 minute segment, because there is - /// nowhere else a segment could start and stay decodable. + /// The shortest a record may be before a group boundary ends it. /// - /// A floor rather than a target on purpose: a floor is always satisfiable (wait longer), - /// while a ceiling is not (a single group longer than it can't be split). + /// A floor rather than a target on purpose: a floor is always satisfiable (wait for the next + /// group), so no group is split to honor it. Zero, for sparse data such as a catalog, makes + /// every group its own record, ended as soon as the group finishes. pub duration_min: Duration, - /// The longest a segment may be, advertised in the catalog when set. + /// The longest a record may run before a frame inside a group splits it. /// - /// Set it only when the publisher can actually promise one, i.e. it controls the encoder's - /// keyframe cadence. Consumers that need a bound up front use it (an HLS exporter's - /// `EXT-X-TARGETDURATION`), so it is a contract rather than a hint: a segment that would - /// exceed it fails the timeline instead of publishing a record that contradicts the - /// catalog. Leave it `None` when the media decides, which is the common case for real-time - /// and for anything importing a source it doesn't control. - pub duration_max: Option, + /// The longest enrolled maximum is advertised in the catalog as `durationMax`. + pub duration_max: Duration, +} + +impl Config { + /// Set [`duration_min`](Self::duration_min). + pub fn with_duration_min(mut self, duration: Duration) -> Self { + self.duration_min = duration; + self + } + + /// Set [`duration_max`](Self::duration_max). + pub fn with_duration_max(mut self, duration: Duration) -> Self { + self.duration_max = duration; + self + } } impl Default for Config { fn default() -> Self { Self { duration_min: DEFAULT_DURATION_MIN, - duration_max: None, + duration_max: DEFAULT_DURATION_MAX, } } } -/// One enrolled track's report state. -#[derive(Default)] -struct TrackState { - /// Group opens reported and not yet flushed into a record: (sequence, pts, keyframe). - pending: VecDeque<(u64, Timestamp, bool)>, - /// The newest reported timestamp: everything earlier is known, which is what lets a - /// segment ending at or before it flush. Advanced by a group open (the group starts there) - /// and by [`Recorder::end`] (the content stops there). - frontier: Option, - /// The recorder was dropped; this track no longer paces boundaries or gates completeness. - closed: bool, - /// Whether this track paces boundaries and gates completeness while open. - pacing: bool, +/// The wire timescale for record `pts`/`duration`: the catalog section's default, milliseconds. +const TIMESCALE: Timescale = Timescale::MILLI; + +/// A duration in wire units, rounded up so a bound never understates itself. +fn units(duration: Duration) -> u64 { + (duration.as_micros() * TIMESCALE.as_u64() as u128).div_ceil(1_000_000) as u64 } -impl TrackState { - /// The first unflushed group starting at or after `threshold` micros: this track's vote for - /// where the open segment can end. - fn candidate(&self, threshold: u128) -> Option { - self.pending - .iter() - .map(|&(_, pts, _)| pts) - .find(|pts| pts.as_micros() >= threshold) - } +fn wire(pts: Timestamp) -> u64 { + pts.as_scale(TIMESCALE) as u64 } -/// The state one [`Segmenter`] guards. -struct State { +/// The record being built. +struct Open { + start: Position, + pts: Timestamp, + keyframe: bool, +} + +/// Builds one track's records from reports of the frames it published. +/// +/// Reports arrive in position order: [`frame`](Self::frame) for each frame that may start a record +/// (every frame, or at least the first of each group), [`finish_group`](Self::finish_group) once a +/// group can gain no more frames, and [`end`](Self::end) wherever the content is known to stop. +/// Closed records queue until [`Iterator::next`] takes them. +pub struct Segmenter { config: Config, - /// Complete records waiting for the caller to commit. - ready: VecDeque, - /// Deferred records handed to the caller but not yet committed to the visible timeline. - committing: BTreeSet, - /// Persisted deferred records waiting for every earlier segment to be persisted too. - committed: BTreeMap, - /// Where the open (unflushed) segment starts; `None` until the first report. - start: Option, - /// Explicit [`Producer::cut`] boundaries not yet reached, in order. + /// The number the next closed record gets. + sequence: u64, + open: Option, + /// The newest reported frame position. + last: Option, + /// Whether the group of `last` has finished. + finished: bool, + /// The newest reported content time in the open record. + frontier: Option, + /// Application boundaries not yet reached, in order. cuts: VecDeque, - /// A [`Producer::cut`] arrived, so the application owns the boundaries from here on and the - /// `duration_min` pacing stops. Without this the pacing races ahead of a source whose - /// segments are longer than the minimum, closing one before its real boundary is declared. + /// A cut arrived, so the application owns the group boundaries from here on. manual: bool, - /// The number the next flushed record gets. - next_segment: u64, - /// Every enrolled track, keyed by media track name. - tracks: BTreeMap, - /// Live [`Reserved`] handles: while any exists, no record flushes (more tracks are still - /// enrolling). Mirrors the catalog's own reservation gate. - reservers: usize, - /// A segment overran [`Config::duration_max`]: `(segment, duration)`. The timeline stops - /// publishing, since the catalog's promise is already broken. - overrun: Option<(u64, Duration)>, - /// The terminal flush ran; surviving clones and recorders can no longer mutate segmentation. - finished: bool, - /// The owning [`Producer`] stopped auto-publishing ready records. - deferred: bool, - /// The wire timescale for `pts`/`duration` (the catalog section's default: milliseconds). - timescale: Timescale, + ready: VecDeque, + closed: bool, } -impl State { - /// Return an abandoned deferred record to its segment-ordered ready queue. - fn requeue(&mut self, record: Record) { - self.committing.remove(&record.segment); - let index = self - .ready - .iter() - .position(|ready| ready.segment > record.segment) - .unwrap_or(self.ready.len()); - self.ready.insert(index, record); - } - - /// A group open reported by `name`: record the fact and close whatever became complete. - fn report(&mut self, name: &str, sequence: u64, pts: Timestamp, keyframe: bool) { - if self.finished { - return; - } - let Some(track) = self.tracks.get_mut(name) else { - return; - }; - track.pending.push_back((sequence, pts, keyframe)); - self.advance(name, pts); - } - - /// `name`'s content ends at `pts`, without a group opening there. This is what gives the - /// final segment an honest duration, since its end is not a boundary anybody cut. - fn report_end(&mut self, name: &str, pts: Timestamp) { - if self.finished { - return; - } - if self.tracks.contains_key(name) { - self.advance(name, pts); +impl Segmenter { + /// A segmenter numbering its records from zero. + pub fn new(config: Config) -> Self { + Self { + config, + sequence: 0, + open: None, + last: None, + finished: false, + frontier: None, + cuts: VecDeque::new(), + manual: false, + ready: VecDeque::new(), + closed: false, } } - /// Raise `name`'s frontier to `pts`, then close whatever that completed. - fn advance(&mut self, name: &str, pts: Timestamp) { - if let Some(track) = self.tracks.get_mut(name) - && track.frontier.is_none_or(|f| pts.as_micros() > f.as_micros()) - { - track.frontier = Some(pts); - } - - // The first thing a pacing track reports anchors the first segment, so content produced - // before any boundary exists belongs to the oldest segment rather than to nowhere. A - // non-pacing track must not anchor it: a catalog published while the encoder is still warming - // up would otherwise stretch segment 0 across the whole startup gap. - if self.start.is_none() && self.tracks.get(name).is_some_and(|t| t.pacing) { - self.start = Some(pts); - } - - self.pump(false); + /// Number the next record `sequence`, continuing a resumed timeline. + pub fn with_sequence(mut self, sequence: u64) -> Self { + self.sequence = sequence; + self } - /// A track's recorder was dropped: stop gating completeness on it. - fn close(&mut self, name: &str) { - if self.finished { - return; - } - if let Some(track) = self.tracks.get_mut(name) { - track.closed = true; - } - self.pump(false); + /// The number the next closed record gets. + pub fn sequence(&self) -> u64 { + self.sequence } - /// Close every segment the media has finalized. + /// Report the frame at `position`, presented at `pts`. /// - /// `finished` is the terminal pass: no track will report again, so a track that never - /// reached a boundary stops voting instead of holding the timeline open forever. - fn pump(&mut self, finished: bool) { - // With nothing enrolled there is nothing to describe. A record is immutable once - // published, so a caller that knows more tracks are still enrolling withholds it (see - // [`Producer::reserve`]), and an overrun has already broken the catalog's promise. - if self.finished || self.tracks.is_empty() || self.reservers > 0 || self.overrun.is_some() { - return; - } - - while self.close_segment(finished) { - // A segment that broke the declared maximum ends the timeline; every later record - // would inherit the same broken promise. - if self.overrun.is_some() { - break; - } - } - } - - /// Close the open segment if the media has finalized it, returning whether it did. - fn close_segment(&mut self, finished: bool) -> bool { - let Some(start) = self.start else { - return false; - }; - - // Discard boundaries the timeline has already reached. Only the front is ever consulted, - // so leaving a spent one there would block every later cut behind it and silently drop - // the caller back to `duration_min` pacing. - while self.cuts.front().is_some_and(|c| c.as_micros() <= start.as_micros()) { - self.cuts.pop_front(); - } - - let Some((end, cut)) = self.boundary(start, finished) else { - return false; - }; - - // Every open pacing track has to have reported at or past the boundary, proving its ranges - // for this segment are final. The track that voted for `end` has by construction; a track - // with shorter groups can still be behind it. A non-pacing track is excluded: it may publish - // rarely or never again, so waiting on it would stall the timeline for good. - let complete = finished - || self - .tracks - .values() - .all(|t| !t.pacing || t.closed || t.frontier.is_some_and(|f| f.as_micros() >= end.as_micros())); - if !complete { - return false; - } - - self.flush_segment(start, Some(end)); - if cut { - self.cuts.pop_front(); + /// Fails with [`TimelinePosition`](crate::Error::TimelinePosition) when `position` is at or + /// before the previous report: records never overlap, so that frame could never be indexed. + pub fn frame(&mut self, position: Position, pts: Timestamp, keyframe: bool) -> crate::Result<()> { + if self.closed { + return Ok(()); } - self.start = Some(end); - true - } - - /// Where the segment starting at `start` ends, once the media says so: an explicit cut when - /// one is registered, otherwise the first group boundary shared by every track that gives - /// the segment its minimum duration. The bool reports which. - fn boundary(&self, start: Timestamp, finished: bool) -> Option<(Timestamp, bool)> { - if let Some(&cut) = self.cuts.front() - && cut.as_micros() > start.as_micros() + if let Some(last) = self.last + && position <= last { - return Some((cut, true)); - } - - // The application declared a boundary at some point, so it owns them all: pacing here - // would close a segment the caller is about to cut somewhere else. - if self.manual { - return None; - } - - let threshold = start.as_micros() + self.config.duration_min.as_micros(); - - let mut end: Option = None; - for track in self.tracks.values().filter(|t| t.pacing) { - match track.candidate(threshold) { - // The latest vote wins: it is a group boundary on the coarsest track, and every - // finer track assigns its groups by start, so no group is split. A closed track - // still votes: it can't report more, but the groups it did report are boundaries - // like any other, and without them a backlog would collapse into one segment. - Some(pts) => { - if end.is_none_or(|e| pts.as_micros() > e.as_micros()) { - end = Some(pts); - } + return Err(crate::Error::TimelinePosition { position, last }); + } + + if let (Some(start), Some(last)) = (self.open.as_ref().map(|open| open.pts), self.last) { + let elapsed = pts.as_micros().saturating_sub(start.as_micros()); + let end = if position.group != last.group { + let contiguous = position.group == last.group + 1 && position.frame == 0; + if !contiguous { + // A skipped sequence or a group joined mid-way always closes, so a record's groups are + // contiguous and the reader never has to guess what lies between. The skipped time is + // not content, so the record ends at its newest report rather than at this frame. + let frontier = self.frontier.expect("an open record has a frontier"); + Some((Position::group(last.group + 1), frontier)) + } else if elapsed >= self.config.duration_max.as_micros() || self.boundary(start, pts, elapsed) { + Some((Position::group(last.group + 1), pts)) + } else { + None } - // This track has produced nothing past the minimum yet, so ending the segment - // would strand it. A closed track never will, and neither does anything on the - // terminal pass, so neither one blocks. - None if finished || track.closed => continue, - None => return None, + } else { + (elapsed >= self.config.duration_max.as_micros()).then_some((position, pts)) + }; + if let Some((end, pts)) = end { + self.emit(end, pts); } } - end.map(|end| (end, false)) - } - - /// Build the record for the segment starting at `start`: drain every track's groups before - /// `end` (all of them for the final, unbounded segment) into ranges. - fn flush_segment(&mut self, start: Timestamp, end: Option) { - let pts = start.as_scale(self.timescale) as u64; - let duration = match end { - Some(end) => (end.as_scale(self.timescale) as u64).saturating_sub(pts), - // The final segment has no end boundary, so it runs to the newest thing any track - // reported: its end of content when the track reported one (a finished - // `container::Producer` does), otherwise the last group it opened, which - // undercounts that group's tail. - None => self - .tracks - .values() - .filter(|t| t.pacing) - .filter_map(|t| t.frontier) - .map(|f| f.as_scale(self.timescale) as u64) - .max() - .unwrap_or(pts) - .saturating_sub(pts), - }; + if self.open.is_none() { + self.open = Some(Open { + start: position, + pts, + keyframe, + }); + self.frontier = None; + } + self.last = Some(position); + self.finished = false; + self.advance(pts); + Ok(()) + } - // The catalog promised a bound and this segment breaks it, so the record would - // contradict what consumers were told. Fail the timeline rather than publish it: a - // declared maximum the media can't honor is a bug in the publisher, and nothing a - // consumer reading the catalog can work around. - if let Some(max) = self.config.duration_max - && duration > self.units(max) + /// Whether a group starting at `pts` ends the record that started at `start`. + fn boundary(&mut self, start: Timestamp, pts: Timestamp, elapsed: u128) -> bool { + while self + .cuts + .front() + .is_some_and(|cut| cut.as_micros() <= start.as_micros()) { - let observed = Duration::from_nanos(duration * (1_000_000_000 / self.timescale.as_u64())); - tracing::error!( - segment = self.next_segment, - duration = ?observed, - duration_max = ?max, - "segment exceeded the declared duration_max; dropping the timeline track" - ); - self.overrun = Some((self.next_segment, observed)); - return; + self.cuts.pop_front(); } - - let mut record = Record::new(self.next_segment, pts, duration); - self.next_segment += 1; - - for (name, track) in &mut self.tracks { - let mut ranges: Vec = Vec::new(); - while let Some(&(sequence, group_pts, keyframe)) = track.pending.front() { - // A pacing track is assigned by content time. A non-pacing track is assigned by - // arrival, so every group pending when the segment closes belongs to it regardless - // of the timestamp basis carried by that track. - if track.pacing && end.is_some_and(|end| group_pts.as_micros() >= end.as_micros()) { - break; - } - track.pending.pop_front(); - match ranges.last_mut() { - // Contiguous sequences extend the run; a skip starts a new range (a gap: - // groups that never existed). - Some(last) if last.end + 1 == sequence => last.end = sequence, - _ => { - let mut range = Range::new(sequence, sequence); - range.keyframe = keyframe; - ranges.push(range); - } - } - } - if !ranges.is_empty() { - record.tracks.insert(name.clone(), ranges); + if self.cuts.front().is_some_and(|cut| cut.as_micros() <= pts.as_micros()) { + while self.cuts.front().is_some_and(|cut| cut.as_micros() <= pts.as_micros()) { + self.cuts.pop_front(); } + return true; } - - self.ready.push_back(record); + !self.manual && elapsed >= self.config.duration_min.as_micros() } - /// A duration in the wire timescale's units, rounded up so a bound never understates itself. - fn units(&self, duration: Duration) -> u64 { - (duration.as_micros() * self.timescale.as_u64() as u128).div_ceil(1_000_000) as u64 - } - - /// The terminal flush: close what the media finalized, then the open tail. - fn finish(&mut self) { - if self.finished { + /// Report that group `group` can gain no more frames. + /// + /// With a zero [`Config::duration_min`] this closes the record, so a sparse track's newest + /// group is indexed without waiting for the next one. + pub fn finish_group(&mut self, group: u64) { + if self.closed || self.last.is_none_or(|last| last.group != group) { return; } - - // Nothing more will enroll, so an outstanding reservation has nothing left to wait for. - self.reservers = 0; - self.pump(true); - - // Skip an empty tail: a boundary with no content after it describes nothing. - if self.overrun.is_none() - && let Some(start) = self.start.take() - && self.tracks.values().any(|t| !t.pending.is_empty()) - { - self.flush_segment(start, None); - } - self.finished = true; + if self.config.duration_min.is_zero() && self.open.is_some() { + let pts = self.frontier.expect("an open record has a frontier"); + self.emit(Position::group(group + 1), pts); + } } - /// The error a segment overrunning [`Config::duration_max`] left behind, if any. - fn failure(&self) -> Option { - let (segment, duration) = self.overrun?; - Some(crate::Error::TimelineOverrun { - segment, - duration, - duration_max: self.config.duration_max.unwrap_or_default(), - }) - } - - /// The catalog section advertising this timeline. + /// Report that the content extends to `pts`, without a frame starting there. /// - /// Carries the track name, timescale, and duration bound. Wall-clock mapping is the - /// catalog root clock's job ([`Clock`](crate::Clock)), not this section's. - fn section(&self) -> Archive { - let mut section = Archive::new(DEFAULT_NAME); - section.timescale = self.timescale.as_u64() as u32; - section.duration_max = self.config.duration_max.map(|max| self.units(max)); - section - } -} - -/// Builds complete segment records without publishing them. -/// -/// Use this directly to build records without a broadcast. An archive writer instead obtains a -/// cancellation-safe [`Deferred`] from [`Producer::deferred`], stores the groups named by each -/// [`Pending`] record, and passes that handle to [`Producer::push`]. `Clone` shares the -/// segmentation state. -#[derive(Clone)] -pub struct Segmenter { - state: Arc>, -} - -impl Segmenter { - /// Create a segmenter paced by `config`. - pub fn new(config: Config) -> Self { - // The contents are `Send + Sync` natively; on wasm moq-net's handles are - // `Rc`-backed, so clippy sees a pointlessly atomic `Arc`. Keeping one type for - // both targets is worth the unused atomics on the single-threaded one. - #[allow(clippy::arc_with_non_send_sync)] - Self { - state: Arc::new(Mutex::new(State { - config, - ready: VecDeque::new(), - committing: BTreeSet::new(), - committed: BTreeMap::new(), - start: None, - cuts: VecDeque::new(), - manual: false, - next_segment: 0, - tracks: BTreeMap::new(), - reservers: 0, - overrun: None, - finished: false, - deferred: false, - timescale: Timescale::MILLI, - })), + /// A frame report says where content starts; the last record has no successor to bound it, so + /// its duration would otherwise stop at its last frame's start. + pub fn end(&mut self, pts: Timestamp) { + if !self.closed && self.open.is_some() { + self.advance(pts); } } - /// Enroll `name` without letting it influence segmentation. - pub fn track(&self, name: &str) -> Recorder { - self.enroll(name, false, None) - } - - /// Enroll `name` as a pacing track. - pub fn pacing_track(&self, name: &str) -> Recorder { - self.enroll(name, true, None) - } - - fn enroll(&self, name: &str, pacing: bool, output: Option>>) -> Recorder { - let mut state = self.state.lock().unwrap(); - let output = if state.finished { - None - } else { - state.tracks.insert( - name.to_string(), - TrackState { - pacing, - ..Default::default() - }, - ); - output - }; - drop(state); - - Recorder { - segmenter: self.clone(), - output, - name: name.to_string(), + fn advance(&mut self, pts: Timestamp) { + if self + .frontier + .is_none_or(|frontier| pts.as_micros() > frontier.as_micros()) + { + self.frontier = Some(pts); } } - /// Declare a segment boundary at `pts`, overriding automatic minimum-duration pacing. - pub fn cut(&self, pts: Timestamp) -> crate::Result<()> { - let mut state = self.state.lock().unwrap(); - if let Some(err) = state.failure() { - return Err(err); - } - if state.finished { - return Err(moq_net::Error::Closed.into()); + /// Declare a boundary at `pts`: the record closes at the first group starting at or after it. + /// + /// The first cut takes over from [`Config::duration_min`] pacing for good, since pacing would + /// otherwise close a record just before the caller declares where it really ends. A cut at or + /// before an earlier one is ignored, so several producers declaring the same boundaries (the + /// renditions of one import) cost nothing. + pub fn cut(&mut self, pts: Timestamp) { + if self.closed { + return; } - - state.manual = true; - let floor = state - .cuts - .back() - .copied() - .or(state.start) - .map(|since| since.as_micros() + state.config.duration_min.as_micros()); - if floor.is_none_or(|floor| pts.as_micros() >= floor) { - state.cuts.push_back(pts); - state.pump(false); + self.manual = true; + if self.cuts.back().is_none_or(|back| pts.as_micros() > back.as_micros()) { + self.cuts.push_back(pts); } - - Ok(()) } - /// Withhold complete records while a batch of tracks enrolls. - pub fn reserve(&self) -> Reserved { - let mut state = self.state.lock().unwrap(); - if !state.finished { - state.reservers += 1; - } - Reserved { - segmenter: self.clone(), - output: None, + /// Close the open record now, at the end of the newest reported frame. + /// + /// The record ends after the newest group when that group finished, otherwise after the newest + /// reported frame. Later reports start a new record. + pub fn flush(&mut self) { + let (Some(last), Some(frontier)) = (self.last, self.frontier) else { + return; + }; + if self.open.is_none() { + return; } + let end = match self.finished { + true => Position::group(last.group + 1), + false => Position::new(last.group, last.frame + 1), + }; + self.emit(end, frontier); } - /// The catalog section describing records built by this segmenter. - pub fn section(&self) -> Archive { - self.state.lock().unwrap().section() - } - - /// Flush the final open segment and return a handle that can only drain records. - pub fn finish(self) -> Drain { - let mut state = self.state.lock().unwrap(); - state.finish(); - drop(state); - Drain { segmenter: self } + /// Flush, then ignore every later report. + pub fn close(&mut self) { + self.flush(); + self.closed = true; } - /// Take the next complete segment record, if one is ready. - pub fn next(&self) -> Option { - self.state.lock().unwrap().ready.pop_front() + /// Close the open record at `end`, its content running to `pts`. + fn emit(&mut self, end: Position, pts: Timestamp) { + let Some(open) = self.open.take() else { + return; + }; + let start = wire(open.pts); + // Frames farther apart than the maximum leave nothing to split at, so the record stops at the + // advertised bound and the rest of the gap is uncovered. + let duration = wire(pts).saturating_sub(start).min(units(self.config.duration_max)); + let mut record = Record::new(self.sequence, start, duration, open.start, end); + record.keyframe = open.keyframe; + self.sequence += 1; + self.ready.push_back(record); } } -/// A finished segmenter that can only drain its final complete records. -pub struct Drain { - segmenter: Segmenter, -} - -impl Drain { - /// Report whether terminal segmentation exceeded the declared duration bound. - pub fn result(&self) -> crate::Result<()> { - result(&self.segmenter) - } +/// Yields each closed record once; `None` means none is ready yet, not that none ever will be. +impl Iterator for Segmenter { + type Item = Record; - /// Take the next complete segment record, if one is ready. - pub fn next(&self) -> Option { - self.segmenter.next() + fn next(&mut self) -> Option { + self.ready.pop_front() } } -/// A deferred segmenter whose records must be persisted before publication. +/// Publishes one track's records onto its timeline track. /// -/// [`next`](Self::next) returns a cancellation-safe [`Pending`] handle. Dropping the handle puts -/// the record back at the front of the ready queue so another storage task can retry it. -#[derive(Clone)] -pub struct Deferred { - segmenter: Segmenter, +/// Records must be pushed in sequence order, starting at the window's next index. +pub struct Producer { + sink: moq_json::window::Producer, } -impl Deferred { - /// Enroll `name` without letting it influence segmentation. - pub fn track(&self, name: &str) -> Recorder { - self.segmenter.track(name) - } - - /// Enroll `name` as a pacing track. - pub fn pacing_track(&self, name: &str) -> Recorder { - self.segmenter.pacing_track(name) - } - - /// Declare a segment boundary at `pts`, overriding automatic minimum-duration pacing. - pub fn cut(&self, pts: Timestamp) -> crate::Result<()> { - self.segmenter.cut(pts) - } - - /// Withhold complete records while a batch of tracks enrolls. - pub fn reserve(&self) -> Reserved { - self.segmenter.reserve() +impl Producer { + /// The properties a timeline track is created with. + pub fn info() -> moq_net::track::Info { + moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog) } - /// The catalog's root `archive` entry describing records built by this segmenter. - pub fn section(&self) -> Archive { - self.segmenter.section() + /// Publish onto `track`, starting at record zero. + pub fn new(track: moq_net::track::Producer) -> Self { + Self { + sink: moq_json::window::Producer::new(track, Self::config()), + } } - /// Flush the final open segment and return a handle that can only drain pending records. - pub fn finish(self) -> DeferredDrain { - let drain = self.segmenter.finish(); - DeferredDrain { - segmenter: drain.segmenter, + /// Publish onto `track`, continuing `checkpoint`, such as a window recovered from storage. + /// + /// The track's first group restates the checkpoint, and the next record is sequence + /// `checkpoint.range.end`. Fails when a checkpoint record is not the sequence at its index or the + /// checkpoint is malformed. + pub fn resume(track: moq_net::track::Producer, checkpoint: &Checkpoint) -> crate::Result { + let sink = moq_json::window::Producer::resume(track, Self::config(), checkpoint).map_err(json)?; + // The encoder accepted the checkpoint, so the records fit before `range.end`. + let start = checkpoint.range.end - checkpoint.records.len() as u64; + for (index, record) in (start..).zip(&checkpoint.records) { + if record.sequence != index { + return Err(crate::Error::TimelineCheckpoint(index)); + } } + Ok(Self { sink }) } - /// Take the next complete record for storage, if one is ready. - pub fn next(&self) -> Option { - pending(&self.segmenter) + fn config() -> moq_json::window::ProducerConfig { + moq_json::window::ProducerConfig::default() + .with_compression(true) + .with_checkpoint_records(CHECKPOINT_RECORDS) } -} - -/// A finished deferred segmenter that can only drain its final records for storage. -pub struct DeferredDrain { - segmenter: Segmenter, -} -impl DeferredDrain { - /// Report whether terminal segmentation exceeded the declared duration bound. - pub fn result(&self) -> crate::Result<()> { - result(&self.segmenter) + /// The window indices currently retained. + pub fn range(&self) -> std::ops::Range { + self.sink.range() } - /// Take the next complete record for storage, if one is ready. - pub fn next(&self) -> Option { - pending(&self.segmenter) + /// Append `record`, which must be the window's next sequence. + pub fn push(&mut self, record: &Record) -> crate::Result<()> { + let next = self.sink.range().end; + if record.sequence != next { + return Err(crate::Error::TimelineSequence { + expected: next, + actual: record.sequence, + }); + } + self.sink.push(record).map_err(json) } -} -fn result(segmenter: &Segmenter) -> crate::Result<()> { - match segmenter.state.lock().unwrap().failure() { - Some(err) => Err(err), - None => Ok(()), + /// Remove up to `count` oldest records from the window. + pub fn pop(&mut self, count: u64) -> crate::Result<()> { + self.sink.pop(count).map_err(json) } -} - -fn pending(segmenter: &Segmenter) -> Option { - let mut state = segmenter.state.lock().unwrap(); - let record = state.ready.pop_front()?; - state.committing.insert(record.segment); - drop(state); - Some(Pending { - segmenter: segmenter.clone(), - record: Some(record), - }) -} - -/// A complete deferred record being stored before timeline publication. -/// -/// Dereference it to inspect or serialize the [`Record`]. Pass ownership to [`Producer::push`] -/// only after storage succeeds. Dropping it requeues the record for another attempt. -pub struct Pending { - segmenter: Segmenter, - record: Option, -} -impl Pending { - /// Convert a record whose media could not be stored into a gap. + /// Close the track's open group, so every push and pop so far sits in a complete group. /// - /// Its segment number and timing stay intact, so committing it unblocks later segments without - /// advertising media objects that are not durable. - pub fn gap(mut self) -> Self { - self.record - .as_mut() - .expect("a pending record is present until commit") - .tracks - .clear(); - self + /// A recorder stores those groups before committing the next record. The next edit opens a + /// new group restating the window. + pub fn flush(&mut self) -> crate::Result<()> { + self.sink.cut().map_err(json) } -} -impl Deref for Pending { - type Target = Record; - - fn deref(&self) -> &Self::Target { - self.record.as_ref().expect("a pending record is present until commit") + /// Finish the timeline track. + pub fn finish(&mut self) -> crate::Result<()> { + self.sink.finish().map_err(json) } } -impl Drop for Pending { - fn drop(&mut self) { - let Some(record) = self.record.take() else { - return; - }; - let mut state = self.segmenter.state.lock().unwrap(); - state.requeue(record); +/// Surface a transport failure as one, rather than wrapped in the JSON layer. +fn json(err: moq_json::Error) -> crate::Error { + match err { + moq_json::Error::Net(err) => err.into(), + err => err.into(), } } -struct Output { - broadcast: moq_net::broadcast::Producer, - sink: Option>, - closed: bool, +/// A live track's segmenter and the timeline it publishes to. +struct Live { + segmenter: Segmenter, + /// `None` once a publish failed: the timeline is an optional sidecar, so it stops instead. + output: Option, + /// Whether a [`Recorder`] owns the segmenter, so a second enrollment can't share it. + recording: bool, } -impl Output { - fn prepare(&mut self) -> crate::Result<()> { - if self.sink.is_none() && !self.closed { - let info = moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog); - let net = self.broadcast.create_track(DEFAULT_NAME, info)?; - let config = moq_json::window::ProducerConfig::default() - .with_compression(true) - .with_checkpoint_records(CHECKPOINT_RECORDS); - self.sink = Some(moq_json::window::Producer::new(net, config)); - } - Ok(()) - } - - fn push(&mut self, record: &Record) -> crate::Result<()> { - self.prepare()?; - let Some(sink) = self.sink.as_mut() else { - return Err(moq_net::Error::Closed.into()); - }; - match sink.push(record) { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), +impl Live { + fn publish(&mut self) { + for record in self.segmenter.by_ref() { + let Some(output) = self.output.as_mut() else { + continue; + }; + if let Err(err) = output.push(&record) { + tracing::warn!(%err, "timeline publish failed; dropping the timeline track"); + let _ = output.finish(); + self.output = None; + } } } - fn pop(&mut self, count: u64) -> crate::Result<()> { - let Some(sink) = self.sink.as_mut() else { - return Ok(()); - }; - match sink.pop(count) { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), + fn finish(&mut self) { + self.segmenter.close(); + self.publish(); + if let Some(output) = self.output.as_mut() { + let _ = output.finish(); } } +} - fn finish(&mut self) -> crate::Result<()> { - self.closed = true; - let Some(sink) = self.sink.as_mut() else { - return Ok(()); - }; - // Keep this handle alive after closing so the finished track remains discoverable for - // subscribers. - match sink.finish() { - Ok(()) => Ok(()), - Err(moq_json::Error::Net(err)) => Err(err.into()), - Err(err) => Err(err.into()), - } - } +struct Registry { + /// Each enrolled track's timeline, by track name. Kept after its recorder drops, so a finished + /// span stays discoverable and a re-enrolled track continues its numbering. + tracks: BTreeMap>)>, + finished: bool, } -/// The broadcast's timeline: a segmenter plus the track committed records are published on. +/// A broadcast's timelines, one per enrolled track. /// -/// Its enrollment methods commit complete records immediately. Use [`deferred`](Self::deferred) -/// when media must be stored before the corresponding record becomes visible. +/// Each enrollment creates the track's timeline track and returns the [`Recorder`] its producer +/// reports through. `Clone` shares the set. #[derive(Clone)] -pub struct Producer { - segmenter: Segmenter, - output: Arc>, +pub struct Timelines { + broadcast: moq_net::broadcast::Producer, + registry: Arc>, } -impl Producer { - /// A timeline for `broadcast`, paced by `config`. - pub fn new(broadcast: &moq_net::broadcast::Producer, config: Config) -> Self { - // Like Segmenter, the output is Send + Sync natively and Rc-backed on single-threaded wasm. +impl Timelines { + /// Timelines for tracks of `broadcast`. + pub fn new(broadcast: &moq_net::broadcast::Producer) -> Self { + // The contents are `Send + Sync` natively; on wasm moq-net's handles are `Rc`-backed, so + // clippy sees a pointlessly atomic `Arc`. One type for both targets is worth it. #[allow(clippy::arc_with_non_send_sync)] - let output = Arc::new(Mutex::new(Output { - broadcast: broadcast.clone(), - sink: None, - closed: false, + let registry = Arc::new(Mutex::new(Registry { + tracks: BTreeMap::new(), + finished: false, })); Self { - segmenter: Segmenter::new(config), - output, - } - } - - /// Enroll `name` without letting it influence segmentation, returning its [`Recorder`]. - /// - /// Its groups are recorded into whichever segment is open when they arrive, but the track - /// never votes on where a boundary falls and never holds a record back. This safe default is - /// for a catalog that emits only when renditions change, or an application's metadata track. - /// - /// Placement is by arrival rather than by content time: nothing waits for the track, so a group - /// that arrives after its segment flushed is recorded in the next one. Frames still carry - /// their own timestamps. A group that never closes (a `moq_json::stream` log) is recorded once, - /// in the segment its group opened in. - /// - /// This does not create the timeline track, so enrolling only non-pacing tracks publishes no - /// timeline. One recorder per track: enrolling the same name again resets its state. - pub fn track(&self, name: &str) -> Recorder { - self.segmenter.enroll(name, false, Some(self.output.clone())) - } - - /// Enroll the media track `name` as a pacing track, returning the [`Recorder`] it reports - /// through. - /// - /// The segment records key ranges by this name, and the track votes on boundaries and gates - /// completeness until its recorder drops. Enroll it when it is about to produce: an enrolled - /// but silent pacing track holds every record back, by design, since a segment is not complete - /// until every pacing track's content is known. - /// - /// This opt-in is for continuously publishing media. A sparse track belongs in - /// [`track`](Self::track), otherwise it can stall the timeline when it goes quiet. - /// - /// Creates the timeline track on first use, which errors if the broadcast cannot because - /// something else already took the name. - pub fn pacing_track(&self, name: &str) -> crate::Result { - self.output.lock().unwrap().prepare()?; - Ok(self.segmenter.enroll(name, true, Some(self.output.clone()))) - } - - /// Declare a segment boundary at `pts`, overriding the [`Config::duration_min`] pacing. - /// - /// For applications that know their own boundaries (an HLS import following the source - /// playlist, CMAF segments on disk, an encoder placing keyframes). Cutting ahead of the - /// media is fine: the segment's record still waits for every track's groups. A cut that - /// would make a segment shorter than [`Config::duration_min`] is ignored, so several - /// producers declaring the same boundaries (the renditions of one import) cost nothing. - /// - /// The first call takes over for good: [`Config::duration_min`] pacing stops, since it would - /// otherwise close a segment just before the caller declares where it really ends. Segments - /// then last exactly as long as the caller says, and the final one runs to the end of the - /// media. - /// - /// Errors if a segment already overran [`Config::duration_max`]. - pub fn cut(&self, pts: Timestamp) -> crate::Result<()> { - self.segmenter.cut(pts)?; - self.publish_ready(); - Ok(()) - } - - /// Begin reserving the track set, returning a clonable [`Reserved`]. - /// - /// The counterpart to [`catalog::Producer::reserve`](crate::catalog::Producer::reserve), for - /// the same reason: while any `Reserved` clone is alive the track set may still grow, so - /// records are withheld from the broadcast. A record is immutable once published and its - /// completeness is judged against the tracks enrolled *at that moment*, so a segment that - /// flushes while a sibling rendition is still enrolling omits it for good, and that - /// rendition's earlier groups then land in whatever segment flushes next. - /// - /// Hand it (or clones) to whatever brings the tracks up, so an importer that enrolls its - /// renditions one at a time publishes nothing until they are all in. Unlike the catalog's, - /// this gate is not one-shot: the catalog is a snapshot, so only its *first* publish needs - /// protecting, while every timeline record is an immutable log entry. Take a fresh - /// reservation around every batch. - pub fn reserve(&self) -> Reserved { - let mut state = self.segmenter.state.lock().unwrap(); - let output = if state.finished { - None - } else { - state.reservers += 1; - Some(self.output.clone()) - }; - Reserved { - segmenter: self.segmenter.clone(), - output, + broadcast: broadcast.clone(), + registry, } } - /// The catalog's root `archive` entry advertising this timeline. - pub fn section(&self) -> Archive { - self.segmenter.section() - } - - /// Create a handle that closes segments without publishing them automatically. + /// Enroll the track `name`, its records cut by `config`. /// - /// Drain it with [`Deferred::next`], perform any asynchronous storage, then commit the final - /// record with [`push`](Self::push). Preparing the timeline track here makes that later commit - /// infallible with respect to track-name ownership. This switches the shared producer for good: - /// recorders and reservations minted before this call also stop auto-publishing. - pub fn deferred(&self) -> crate::Result { - // Match publish_ready's output-then-state lock order. Once this returns, no auto-publisher can - // still be between dequeuing a record and making it visible. - let mut output = self.output.lock().unwrap(); - let mut state = self.segmenter.state.lock().unwrap(); - if state.finished { + /// Creates its timeline track, named by [`hang::timeline::default_name`], on first enrollment. + /// Enrolling a name again after its recorder dropped continues its timeline's numbering for a new + /// producer, cut by the new `config`. Errors when the name's recorder is still alive, the timeline + /// track name is taken, or the timelines finished. + pub fn track(&self, name: &str, config: Config) -> crate::Result { + let mut registry = self.registry.lock().unwrap(); + if registry.finished { return Err(moq_net::Error::Closed.into()); } - output.prepare()?; - state.deferred = true; - Ok(Deferred { - segmenter: self.segmenter.clone(), - }) - } - - /// Reacquire the drain for a finished deferred segmenter. - /// - /// This is cancellation recovery for the task that owned [`DeferredDrain`]. It is rejected - /// until deferred segmentation has completed. - pub fn drain(&self) -> crate::Result { - let state = self.segmenter.state.lock().unwrap(); - if !state.deferred || !state.finished { - return Err(crate::Error::TimelineDeferredPending); - } - drop(state); - Ok(DeferredDrain { - segmenter: self.segmenter.clone(), - }) - } - - /// Mark one complete record committed, publishing it once every earlier record is committed. - pub fn push(&self, mut pending: Pending) -> crate::Result<()> { - if !Arc::ptr_eq(&self.segmenter.state, &pending.segmenter.state) { - return Err(crate::Error::TimelineDeferredRecord(pending.segment)); - } - - let mut output = self.output.lock().unwrap(); - let mut state = self.segmenter.state.lock().unwrap(); - if !state.deferred || !state.committing.contains(&pending.segment) { - return Err(crate::Error::TimelineDeferredRecord(pending.segment)); - } - - let record = pending.record.take().expect("a pending record is present until commit"); - state.committing.remove(&record.segment); - state.committed.insert(record.segment, record); - - while let Some((&segment, record)) = state.committed.first_key_value() { - let earlier_ready = state.ready.front().is_some_and(|record| record.segment < segment); - let earlier_committing = state.committing.first().is_some_and(|pending| *pending < segment); - if earlier_ready || earlier_committing { - break; - } - - let record = record.clone(); - if let Err(err) = output.push(&record) { - let record = state.committed.remove(&segment).unwrap(); - state.requeue(record); - return Err(err); + if let Some((_, existing)) = registry.tracks.get(name) { + // A re-enrolled track is a new producer whose group sequences may restart, so it gets a + // fresh segmenter continuing the record numbering. + let mut live = existing.lock().unwrap(); + if live.recording { + return Err(moq_net::Error::Duplicate.into()); } - state.committed.remove(&segment); + live.recording = true; + live.segmenter.flush(); + live.publish(); + let sequence = live.segmenter.sequence(); + live.segmenter = Segmenter::new(config).with_sequence(sequence); + drop(live); + return Ok(Recorder { live: existing.clone() }); } - Ok(()) + let timeline = hang::timeline::default_name(name); + let track = self.broadcast.create_track(timeline.as_str(), Producer::info())?; + #[allow(clippy::arc_with_non_send_sync)] + let live = Arc::new(Mutex::new(Live { + segmenter: Segmenter::new(config), + output: Some(Producer::new(track)), + recording: true, + })); + registry.tracks.insert(name.to_string(), (timeline, live.clone())); + Ok(Recorder { live }) } - /// Remove up to `count` oldest records from the visible timeline window. - pub fn pop(&self, count: u64) -> crate::Result<()> { - self.output.lock().unwrap().pop(count) + /// The catalog's root `archive` entry advertising every enrolled track's timeline, bounded by + /// the longest maximum any of them is cut by. + pub fn section(&self) -> Archive { + let registry = self.registry.lock().unwrap(); + let mut section = Archive::new(); + section.timescale = TIMESCALE.as_u64() as u32; + section.duration_max = registry + .tracks + .values() + .map(|(_, live)| units(live.lock().unwrap().segmenter.config.duration_max)) + .max(); + section.timelines = registry + .tracks + .iter() + .map(|(name, (timeline, _))| (name.clone(), timeline.clone())) + .collect(); + section } - fn publish_ready(&self) { - // Serialize the dequeue and publish steps together. Recorders can report from different - // tasks, and taking one record before locking the output would let a later record win the - // output lock and become visible first. - let mut output = self.output.lock().unwrap(); - if self.segmenter.state.lock().unwrap().deferred { - return; - } - while let Some(record) = self.segmenter.next() { - if let Err(err) = output.push(&record) { - tracing::warn!(%err, "timeline publish failed; dropping the timeline track"); - let _ = output.finish(); - return; - } - } - if self.segmenter.state.lock().unwrap().failure().is_some() { - let _ = output.finish(); + /// Flush every open record and finish every timeline track. Later enrollments fail. + pub fn finish(&self) { + let mut registry = self.registry.lock().unwrap(); + registry.finished = true; + for (_, live) in registry.tracks.values() { + live.lock().unwrap().finish(); } } - - /// Flush the final (still open) segment and finish the track. - /// - /// In deferred mode, first call [`Deferred::finish`] and use its returned drain handle to store - /// and [`push`](Self::push) every remaining record, then call this to close the timeline track. - /// Calling this before the deferred segmenter finishes or while any record remains uncommitted - /// is rejected without changing state. - /// - /// Also errors if a segment overran [`Config::duration_max`], or if the track can't be finished. - pub fn finish(&mut self) -> crate::Result<()> { - // Serialize both the mode decision and terminal flush with deferred(), which takes these - // locks in the same order. Either deferred mode wins before finishing starts, or finishing - // closes the output and a later deferred() is rejected. - let mut output = self.output.lock().unwrap(); - let mut state = self.segmenter.state.lock().unwrap(); - if state.deferred { - if !state.finished || !state.ready.is_empty() || !state.committing.is_empty() || !state.committed.is_empty() - { - return Err(crate::Error::TimelineDeferredPending); - } - if let Some(err) = state.failure() { - drop(state); - let _ = output.finish(); - return Err(err); - } - drop(state); - return output.finish(); - } - - state.finish(); - if let Some(err) = state.failure() { - drop(state); - let _ = output.finish(); - return Err(err); - } - while let Some(record) = state.ready.pop_front() { - if let Err(err) = output.push(&record) { - drop(state); - let _ = output.finish(); - return Err(err); - } - } - drop(state); - output.finish() - } } -/// A clonable reservation withholding [`Producer`] records while tracks are still enrolling. +/// Reports one track's frames into its live timeline. /// -/// Made via [`Producer::reserve`], mirroring [`catalog::Reserved`](crate::catalog::Reserved). -/// Whatever became complete meanwhile flushes once the last clone drops. -pub struct Reserved { - segmenter: Segmenter, - output: Option>>, +/// Move-only: it is the track's reporting handle, minted by [`Timelines::track`] and held by a +/// rendition's [`container::Producer`](crate::container::Producer). Dropping it flushes the open +/// record; the timeline track stays open until [`Timelines::finish`]. +pub struct Recorder { + live: Arc>, } -impl Clone for Reserved { - fn clone(&self) -> Self { - let mut state = self.segmenter.state.lock().unwrap(); - if !state.finished { - state.reservers += 1; - } - Self { - segmenter: self.segmenter.clone(), - output: self.output.clone(), - } +impl Recorder { + fn report(&mut self, f: impl FnOnce(&mut Segmenter)) { + let mut live = self.live.lock().unwrap(); + f(&mut live.segmenter); + live.publish(); } -} -impl Drop for Reserved { - fn drop(&mut self) { - let mut state = self.segmenter.state.lock().unwrap(); - state.reservers = state.reservers.saturating_sub(1); - state.pump(false); - drop(state); - if let Some(output) = &self.output { - Producer { - segmenter: self.segmenter.clone(), - output: output.clone(), - } - .publish_ready(); + /// Report the frame at `position`, presented at `pts`; see [`Segmenter::frame`]. + /// + /// A report the timeline cannot place ends the timeline track with an error rather than + /// publishing a timeline that silently omits frames. Media publishing is unaffected. + pub fn frame(&mut self, position: Position, pts: Timestamp, keyframe: bool) { + let mut live = self.live.lock().unwrap(); + if let Err(err) = live.segmenter.frame(position, pts, keyframe) { + tracing::warn!(%err, "timeline report rejected; dropping the timeline track"); + live.segmenter.close(); + // Dropped without a finish, so subscribers see the timeline fail rather than end. + live.output = None; } + live.publish(); } -} -/// Reports one media track's group opens into the shared [`Producer`]. -/// -/// Move-only: it is the track's single reporting handle, and dropping it closes the track's -/// enrollment (segments stop waiting on it). Minted by [`Producer::track`] and held by a -/// rendition's [`container::Producer`](crate::container::Producer). -pub struct Recorder { - segmenter: Segmenter, - output: Option>>, - name: String, -} + /// Report that group `group` can gain no more frames; see [`Segmenter::finish_group`]. + pub fn finish_group(&mut self, group: u64) { + self.report(|segmenter| segmenter.finish_group(group)); + } -impl Recorder { - /// Report that group `sequence` opened at presentation time `pts`, `keyframe` stating - /// whether its first frame is one (i.e. whether a player could join here). - /// - /// Reports must be in group order with monotonic timestamps; this is the fact the timeline - /// builds ranges, boundaries and completeness from. - pub fn record(&mut self, sequence: u64, pts: Timestamp, keyframe: bool) { - self.segmenter - .state - .lock() - .unwrap() - .report(&self.name, sequence, pts, keyframe); - self.publish_ready(); - } - - /// Report that this track's content extends to `pts`, without a group opening there. - /// - /// A group open says where content *starts*; the last group of a broadcast has no successor - /// to bound it, so its segment would otherwise be published a group short (zero for a - /// segment that is a single group). Report the end whenever you know it: closing a group, - /// finishing a track. + /// Report that the content extends to `pts`; see [`Segmenter::end`]. pub fn end(&mut self, pts: Timestamp) { - self.segmenter.state.lock().unwrap().report_end(&self.name, pts); - self.publish_ready(); + self.report(|segmenter| segmenter.end(pts)); } - fn publish_ready(&self) { - if let Some(output) = &self.output { - Producer { - segmenter: self.segmenter.clone(), - output: output.clone(), - } - .publish_ready(); - } + /// Declare a boundary at `pts` on this track; see [`Segmenter::cut`]. + pub fn cut(&mut self, pts: Timestamp) { + self.report(|segmenter| segmenter.cut(pts)); + } + + /// Close the open record now; see [`Segmenter::flush`]. + pub(crate) fn flush(&mut self) { + self.report(|segmenter| segmenter.flush()); } } impl Drop for Recorder { fn drop(&mut self) { - self.segmenter.state.lock().unwrap().close(&self.name); - self.publish_ready(); + let mut live = self.live.lock().unwrap(); + live.segmenter.flush(); + live.publish(); + live.recording = false; } } -/// One decoded timeline entry: a complete aligned segment with real [`Timestamp`]s and the -/// group ranges each track contributes. +/// One decoded timeline record, with real [`Timestamp`]s. #[derive(Debug, Clone, PartialEq)] pub struct Entry { - /// The segment's number, consecutive within the broadcast. - pub segment: u64, + /// The record's number, consecutive within its track's timeline. + pub sequence: u64, - /// The segment's start. + /// The span's start. pub pts: Timestamp, - /// The segment's duration. The next entry starts at `pts + duration` unless content time - /// itself jumped (a discontinuity). + /// The span's duration. The next entry starts at `pts + duration` unless content time itself + /// jumped (a discontinuity). pub duration: Duration, - /// The group ranges each participating media track contributes, keyed by track name. A - /// track absent from the map has no content in this span (HLS `EXT-X-GAP`). - pub tracks: BTreeMap>, + /// The first frame of the span, inclusive. + pub start: Position, + + /// The end of the span, exclusive. + pub end: Position, + + /// Whether the span's first frame is a keyframe. + pub keyframe: bool, /// The record's application extension (nothing for the default `()`). pub ext: E, } +impl Entry { + /// The span's exclusive end time. + pub fn end_time(&self) -> Duration { + Duration::from(self.pts) + self.duration + } +} + /// One change to the visible timeline window. #[derive(Debug, Clone, PartialEq)] #[non_exhaustive] pub enum Event { - /// A complete segment record became visible at this window index. + /// A record became visible at this window index, which equals its sequence. Push { /// Absolute window index assigned to the record. index: u64, - /// The decoded segment entry. + /// The decoded record. entry: Entry, }, /// These previously visible window indices were removed. @@ -1162,7 +675,7 @@ pub enum Event { Skip(std::ops::Range), } -/// Reads a broadcast's timeline, yielding decoded [`Event`]s in window order. +/// Reads one track's timeline, yielding decoded [`Event`]s in window order. /// /// Generic over the record extension `E` (see [`RecordExt`]). pub struct Consumer { @@ -1171,20 +684,27 @@ pub struct Consumer { } impl Consumer { - /// Subscribe to the timeline advertised by the catalog's [`Archive`] entry. + /// Subscribe to `track`'s timeline as advertised by the catalog's [`Archive`] entry. /// - /// The section supplies both the track name and the timescale, so a reader can't pair the - /// wrong scale with the track. Errors if the section declares a timescale that isn't - /// representable. - pub async fn subscribe(broadcast: &moq_net::broadcast::Consumer, section: &Archive) -> crate::Result { - let track = broadcast.track(§ion.track)?.subscribe(None).await?; - + /// The section supplies both the timeline name and the timescale, so a reader can't pair the + /// wrong scale with the track. Errors if the section indexes no such track or declares a + /// timescale that isn't representable. + pub async fn subscribe( + broadcast: &moq_net::broadcast::Consumer, + section: &Archive, + track: &str, + ) -> crate::Result { + let name = section + .timelines + .get(track) + .ok_or_else(|| crate::Error::TimelineMissing(track.to_string()))?; + let timescale = + Timescale::new(section.timescale as u64).map_err(|_| crate::Error::InvalidTimescale(section.timescale))?; + let track = broadcast.track(name)?.subscribe(None).await?; let config = moq_json::window::ConsumerConfig::default().with_compression(true); - Ok(Self { inner: moq_json::window::Consumer::new(track, config), - timescale: Timescale::new(section.timescale as u64) - .map_err(|_| crate::Error::InvalidTimescale(section.timescale))?, + timescale, }) } @@ -1196,10 +716,12 @@ impl Consumer { let scale = self.timescale.as_u64() as u128; let nanos = (record.duration as u128) * 1_000_000_000 / scale; Ok(Entry { - segment: record.segment, + sequence: record.sequence, pts: Timestamp::new(record.pts, self.timescale)?, duration: Duration::from_nanos(nanos as u64), - tracks: record.tracks, + start: record.start, + end: record.end, + keyframe: record.keyframe, ext: record.ext, }) } @@ -1238,873 +760,452 @@ impl Consumer { mod test { use super::*; + /// Sparse data: every group is its own record. + fn sparse() -> Config { + Config::default().with_duration_min(Duration::ZERO) + } + fn ms(v: u64) -> Timestamp { Timestamp::from_millis(v).unwrap() } - /// Build an Entry the tests compare against. - fn entry(segment: u64, pts_ms: u64, duration_ms: u64, tracks: &[(&str, &[(u64, u64)])]) -> Entry { - Entry { - segment, - pts: ms(pts_ms), - duration: Duration::from_millis(duration_ms), - tracks: tracks - .iter() - .map(|(name, ranges)| { - ( - name.to_string(), - ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(), - ) - }) - .collect(), - ext: (), - } + fn at(group: u64) -> Position { + Position::group(group) } - /// Drain a finished timeline track by subscribing to the producer's advertised section. - async fn drain(broadcast: &moq_net::broadcast::Producer, producer: &Producer) -> Vec { - let mut consumer = Consumer::subscribe(&broadcast.consume(), &producer.section()) - .await - .unwrap(); - drain_consumer(&mut consumer) + /// A record's `(pts, duration, start, end)` in wire units, for compact comparisons. + fn span(record: &Record) -> (u64, u64, Position, Position) { + (record.pts, record.duration, record.start, record.end) } - fn drain_consumer(consumer: &mut Consumer<()>) -> Vec { - let waiter = kio::Waiter::noop(); + fn drain(segmenter: &mut Segmenter) -> Vec<(u64, u64, Position, Position)> { let mut out = Vec::new(); - while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { - if let Event::Push { entry, .. } = event { - out.push(entry); + let mut sequence = None; + for record in segmenter.by_ref() { + if let Some(previous) = sequence { + assert_eq!(record.sequence, previous + 1, "records are numbered consecutively"); } + sequence = Some(record.sequence); + out.push(span(&record)); } out } - fn setup() -> (moq_net::broadcast::Producer, Producer) { - setup_with(Config::default()) - } - - fn setup_with(config: Config) -> (moq_net::broadcast::Producer, Producer) { - let broadcast = moq_net::broadcast::Info::new().produce(); - let timeline = Producer::new(&broadcast, config); - (broadcast, timeline) - } - - #[tokio::test] - async fn deferred_records_are_invisible_until_committed() { - let (broadcast, timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - let record = segmenter.next().expect("the complete segment is ready for storage"); - assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - - timeline.push(record).unwrap(); - let Poll::Ready(Ok(Some(Event::Push { index, entry: actual }))) = consumer.poll_next(&waiter) else { - panic!("the committed segment was not visible"); - }; - assert_eq!(index, 0); - assert_eq!(actual, entry(0, 0, 2_000, &[("video0", &[(0, 0)])])); - - timeline.pop(1).unwrap(); - assert!(matches!( - consumer.poll_next(&waiter), - Poll::Ready(Ok(Some(Event::Pop(range)))) if range == (0..1) - )); - - video.end(ms(4_000)); - drop(video); - segmenter.finish().result().unwrap(); - } - - #[tokio::test] - async fn deferred_disables_existing_auto_publishing_recorders() { - let (broadcast, timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let segmenter = timeline.deferred().unwrap(); - let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - let record = segmenter.next().expect("the complete segment is ready for storage"); - assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - - timeline.push(record).unwrap(); - assert!(matches!( - consumer.poll_next(&waiter), - Poll::Ready(Ok(Some(Event::Push { index: 0, .. }))) - )); + /// One frame per group, one group every `step` ms. + fn groups(segmenter: &mut Segmenter, groups: std::ops::Range, step: u64) { + for group in groups { + segmenter.frame(at(group), ms(group * step), true).unwrap(); + segmenter.finish_group(group); + } } #[test] - fn dropping_a_pending_record_requeues_it() { - let (_broadcast, timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - - let pending = segmenter.next().expect("the complete segment is ready for storage"); - assert_eq!(pending.segment, 0); - drop(pending); - - let retry = segmenter.next().expect("the abandoned segment is ready to retry"); - assert_eq!(retry.segment, 0); - timeline.push(retry).unwrap(); - } + fn long_groups_get_a_record_each() { + let mut video = Segmenter::new(Config::default()); + groups(&mut video, 0..3, 2_000); + video.end(ms(6_000)); + video.close(); - #[test] - fn ordinary_producer_rejects_a_deferred_record() { - let (_ordinary_broadcast, ordinary) = setup(); - let (_deferred_broadcast, deferred) = setup(); - let segmenter = deferred.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - - let pending = segmenter.next().expect("the complete segment is ready for storage"); - assert!(matches!( - ordinary.push(pending), - Err(crate::Error::TimelineDeferredRecord(0)) - )); assert_eq!( - segmenter - .next() - .expect("the rejected segment returns to its producer") - .segment, - 0 + drain(&mut video), + vec![ + (0, 2_000, at(0), at(1)), + (2_000, 2_000, at(1), at(2)), + (4_000, 2_000, at(2), at(3)), + ] ); } #[test] - fn deferred_mode_is_rejected_after_finishing_starts() { - let (_broadcast, mut timeline) = setup(); - timeline.finish().unwrap(); - assert!(matches!( - timeline.deferred(), - Err(crate::Error::Moq(moq_net::Error::Closed)) - )); - } - - #[tokio::test] - async fn deferred_records_publish_in_segment_order_before_finish() { - let (broadcast, mut timeline) = setup(); - let segmenter = timeline.deferred().unwrap(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - drop(video); - let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap(); - let waiter = kio::Waiter::noop(); - - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - let drain = segmenter.finish(); - drain.result().unwrap(); - drop(drain); - let drain = timeline.drain().expect("a cancelled terminal drain is recoverable"); - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - // Storage failed for the first segment, so publish its timing as a gap. It must still - // unblock the successfully stored segment behind it. - let first = drain.next().expect("the first record is ready for storage").gap(); - let second = drain.next().expect("the final record is ready for storage"); - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - - timeline.push(second).unwrap(); - assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - assert!(matches!(timeline.finish(), Err(crate::Error::TimelineDeferredPending))); - - timeline.push(first).unwrap(); - timeline.finish().unwrap(); + fn short_groups_pack_up_to_the_minimum() { + let mut audio = Segmenter::new(Config::default().with_duration_min(Duration::from_millis(1_500))); + groups(&mut audio, 0..8, 500); + audio.close(); assert_eq!( - drain_consumer(&mut consumer), + drain(&mut audio), vec![ - entry(0, 0, 2_000, &[]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), + (0, 1_500, at(0), at(3)), + (1_500, 1_500, at(3), at(6)), + (3_000, 500, at(6), at(8)), ] ); } #[test] - fn segmenter_finish_is_terminal() { - let segmenter = Segmenter::new(Config::default()); - let late_segmenter = segmenter.clone(); - let mut video = segmenter.pacing_track("video0"); - video.record(0, ms(0), true); - video.end(ms(2_000)); - - let drain = segmenter.finish(); - drain.result().unwrap(); - assert!(drain.next().is_some(), "the terminal flush keeps its final record"); - - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - let mut late = late_segmenter.pacing_track("late"); - late.record(0, ms(2_000), true); - late.end(ms(4_000)); - - assert!(drain.next().is_none(), "surviving recorders cannot reopen the timeline"); - assert!(matches!( - late_segmenter.cut(ms(4_000)), - Err(crate::Error::Moq(moq_net::Error::Closed)) - )); - } - - // The coarsest track paces: video GOPs are longer than duration_min, so segments are GOPs. - #[tokio::test] - async fn the_coarsest_track_paces_the_segments() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - // Video keyframes every 2s, audio groups every 500ms, minimum 1s. - video.record(0, ms(0), true); - for (seq, t) in [(0u64, 0u64), (1, 500), (2, 1_000), (3, 1_500)] { - audio.record(seq, ms(t), true); - } - video.record(1, ms(2_000), true); - for (seq, t) in [(4u64, 2_000u64), (5, 2_500), (6, 3_000), (7, 3_500)] { - audio.record(seq, ms(t), true); + fn a_long_group_splits_by_frame_at_the_maximum() { + let mut video = Segmenter::new(Config::default().with_duration_max(Duration::from_secs(3))); + for frame in 0..8 { + video + .frame(Position::new(0, frame), ms(frame * 1_000), frame == 0) + .unwrap(); } - video.record(2, ms(4_000), true); - audio.record(8, ms(4_000), true); - drop(video); - drop(audio); - timeline.finish().unwrap(); + video.close(); - // Audio's own candidate (1s) loses to video's (2s), so no segment splits a GOP. Every - // record is self-contained: start AND end groups, explicit duration. + let records: Vec = std::iter::from_fn(|| video.next()).collect(); assert_eq!( - drain(&broadcast, &timeline).await, + records.iter().map(span).collect::>(), vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 3)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)]), ("audio0", &[(4, 7)])]), - entry(2, 4_000, 0, &[("video0", &[(2, 2)]), ("audio0", &[(8, 8)])]), + (0, 3_000, Position::new(0, 0), Position::new(0, 3)), + (3_000, 3_000, Position::new(0, 3), Position::new(0, 6)), + (6_000, 1_000, Position::new(0, 6), Position::new(0, 8)), ] ); + assert!(records[0].keyframe); + assert!( + !records[1].keyframe, + "a split inside a group does not start on a keyframe" + ); } - // A real-time encoder's GOP can dwarf the minimum. Nothing is violated: there is nowhere - // else a segment could start and stay decodable, so the segment is simply long. - #[tokio::test] - async fn a_gop_longer_than_the_minimum_is_one_segment() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); + #[test] + fn a_sparse_track_records_each_group_when_it_finishes() { + let mut catalog = Segmenter::new(sparse()); + catalog.frame(at(0), ms(0), true).unwrap(); + assert!(catalog.next().is_none(), "the group may still grow"); + catalog.finish_group(0); + assert_eq!(drain(&mut catalog), vec![(0, 0, at(0), at(1))]); - video.record(0, ms(0), true); - video.record(1, ms(30_000), true); - video.end(ms(60_000)); - drop(video); - timeline.finish().unwrap(); + catalog.frame(at(1), ms(60_000), true).unwrap(); + catalog.finish_group(1); + assert_eq!(drain(&mut catalog), vec![(60_000, 0, at(1), at(2))]); + } + #[test] + fn an_append_only_group_is_indexed_as_it_grows() { + // A never-closing log: one frame a second, split every three seconds. + let mut log = Segmenter::new(sparse().with_duration_max(Duration::from_secs(3))); + for frame in 0..7 { + log.frame(Position::new(0, frame), ms(frame * 1_000), true).unwrap(); + } assert_eq!( - drain(&broadcast, &timeline).await, + drain(&mut log), vec![ - entry(0, 0, 30_000, &[("video0", &[(0, 0)])]), - entry(1, 30_000, 30_000, &[("video0", &[(1, 1)])]), + (0, 3_000, Position::new(0, 0), Position::new(0, 3)), + (3_000, 3_000, Position::new(0, 3), Position::new(0, 6)), ] ); - } - - // Groups shorter than the minimum pack into one segment rather than each becoming one. - #[tokio::test] - async fn short_groups_pack_up_to_the_minimum() { - let (broadcast, mut timeline) = setup_with(Config { - duration_min: Duration::from_millis(1_500), - ..Default::default() - }); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - for seq in 0..8u64 { - audio.record(seq, ms(seq * 500), true); - } - drop(audio); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - // The first group at or past 1500ms ends segment 0, so it holds groups 0..=2. - assert_eq!(entries[0], entry(0, 0, 1_500, &[("audio0", &[(0, 2)])])); - assert_eq!(entries[1], entry(1, 1_500, 1_500, &[("audio0", &[(3, 5)])])); - } - // A track that hasn't reached the minimum yet holds the segment open, so a record never - // omits a rendition that was about to contribute to it. - #[tokio::test] - async fn a_segment_waits_for_every_track() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - video.record(1, ms(2_000), true); - // Video alone would close segment 0 here, but audio hasn't crossed 2s. - assert!(drain(&broadcast, &timeline).await.is_empty()); - - audio.record(1, ms(2_000), true); - let entries = drain(&broadcast, &timeline).await; + // The tail ends after the newest frame, since the group never finished. + log.close(); assert_eq!( - entries, - vec![entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 0)])])] - ); - drop(video); - drop(audio); - timeline.finish().unwrap(); - } - - // An application that knows its own boundaries overrides the pacing. - #[tokio::test] - async fn explicit_cuts_override_the_pacing() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - // Keyframes every second, cut every three: the segments follow the cuts, not the GOPs. - timeline.cut(ms(3_000)).unwrap(); - timeline.cut(ms(6_000)).unwrap(); - for seq in 0..7u64 { - video.record(seq, ms(seq * 1_000), true); - } - drop(video); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries[0], entry(0, 0, 3_000, &[("video0", &[(0, 2)])])); - assert_eq!(entries[1], entry(1, 3_000, 3_000, &[("video0", &[(3, 5)])])); - } - - // Every rendition of one import declares the same boundaries, so a cut that would make a - // segment shorter than the minimum is dropped rather than producing a stray segment. - #[tokio::test] - async fn a_cut_below_the_minimum_is_ignored() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - video.record(0, ms(0), true); - timeline.cut(ms(2_000)).unwrap(); - // A sibling rendition's duplicate, and a boundary too close to be a segment. - timeline.cut(ms(2_000)).unwrap(); - timeline.cut(ms(2_500)).unwrap(); - timeline.cut(ms(4_000)).unwrap(); - video.record(1, ms(2_000), true); - video.record(2, ms(4_000), true); - drop(video); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries.len(), 3, "the duplicate and the 500ms cut are both dropped"); - assert_eq!(entries[0], entry(0, 0, 2_000, &[("video0", &[(0, 0)])])); - assert_eq!(entries[1], entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])])); - } - - // The declared maximum is a contract with consumers (an HLS EXT-X-TARGETDURATION), so a - // segment that breaks it fails the timeline rather than publishing a record that - // contradicts the catalog. - #[tokio::test] - async fn exceeding_the_declared_maximum_fails_the_timeline() { - let (broadcast, mut timeline) = setup_with(Config { - duration_min: Duration::from_secs(1), - duration_max: Some(Duration::from_secs(3)), - ..Default::default() - }); - let mut video = timeline.pacing_track("video0").unwrap(); - - let mut consumer = { - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - Consumer::<()>::subscribe(&broadcast.consume(), &timeline.section()) - .await - .unwrap() - }; - - // A 4s GOP against a declared 3s maximum. - video.record(2, ms(6_000), true); - drop(video); - - let err = timeline.finish().unwrap_err(); - assert!( - matches!(err, crate::Error::TimelineOverrun { segment: 1, .. }), - "unexpected error: {err}" + drain(&mut log), + vec![(6_000, 0, Position::new(0, 6), Position::new(0, 7))] ); - - // The record that was still true published; the one that would have contradicted the - // catalog did not, and the track ended there. - let waiter = kio::Waiter::noop(); - let mut entries = Vec::new(); - while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { - if let Event::Push { entry, .. } = event { - entries.push(entry); - } - } - assert_eq!(entries, vec![entry(0, 0, 2_000, &[("video0", &[(0, 0)])])]); } #[test] - fn a_deferred_overrun_is_reported_when_the_output_finishes() { - let (_broadcast, mut timeline) = setup_with(Config { - duration_min: Duration::from_secs(1), - duration_max: Some(Duration::from_secs(3)), - ..Default::default() - }); - let deferred = timeline.deferred().unwrap(); - let mut video = deferred.pacing_track("video0"); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - - video.record(2, ms(6_000), true); - drop(video); - let drain = deferred.finish(); - assert!(matches!( - drain.result(), - Err(crate::Error::TimelineOverrun { segment: 1, .. }) - )); - // The overrun does not consume the only path to the valid prefix. - timeline.push(drain.next().unwrap()).unwrap(); - assert!(matches!( - timeline.finish(), - Err(crate::Error::TimelineOverrun { segment: 1, .. }) - )); - } - - #[tokio::test] - async fn an_undeclared_maximum_is_omitted_from_the_catalog() { - let (_broadcast, timeline) = setup(); - assert_eq!(timeline.section().duration_max, None); - - let (_broadcast, timeline) = setup_with(Config { - duration_max: Some(Duration::from_millis(2_500)), - ..Default::default() - }); - assert_eq!(timeline.section().duration_max, Some(2_500)); - } - - // The last group of a broadcast has no successor to bound it, so without a reported end the - // final segment's duration collapses to zero (an HLS EXTINF:0). - #[tokio::test] - async fn the_final_segment_runs_to_the_reported_end() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - // A finished `container::Producer` reports where its content stops. - video.end(ms(4_000)); - drop(video); - timeline.finish().unwrap(); + fn a_skipped_sequence_closes_the_record() { + let mut audio = Segmenter::new(Config::default().with_duration_min(Duration::from_secs(1))); + audio.frame(at(0), ms(0), true).unwrap(); + audio.frame(at(1), ms(300), true).unwrap(); + audio.frame(at(5), ms(600), true).unwrap(); + audio.frame(at(6), ms(1_700), true).unwrap(); + audio.close(); assert_eq!( - drain(&broadcast, &timeline).await, + drain(&mut audio), vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), + (0, 300, at(0), at(2)), + (600, 1_100, at(5), at(6)), + (1_700, 0, at(6), Position::new(6, 1)) ] ); } - // A record is immutable and judged against the tracks enrolled when it flushes, so a - // producer that enrolls its tracks one batch at a time (an HLS import walking its - // renditions) holds flushing back until they are all in. - #[tokio::test] - async fn a_reservation_defers_flushing_until_every_track_enrolls() { - let (broadcast, mut timeline) = setup(); - let reserved = timeline.reserve(); - - // The primary rendition runs a whole batch of segments through before its sibling has - // even loaded an init segment. - let mut first = timeline.pacing_track("video0").unwrap(); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - first.record(seq, ms(t), true); - } - - let mut second = timeline.pacing_track("video1").unwrap(); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - second.record(seq, ms(t), true); - } - - drop(reserved); - drop(first); - drop(second); - timeline.finish().unwrap(); + #[test] + fn a_skipped_run_ends_at_the_previous_frontier() { + // A data track paused for 40 minutes, skipping a sequence as its discontinuity marker. + let mut data = Segmenter::new(Config::default()); + groups(&mut data, 0..2, 1_000); + data.end(ms(2_000)); + data.frame(at(3), ms(2_400_000), true).unwrap(); + data.close(); - // Both renditions are indexed from segment 0. Without the reservation, segments 0 and 1 would - // have flushed knowing only video0 (a permanent gap for video1), and video1's first two - // groups would have folded into segment 2. - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries[0], - entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("video1", &[(0, 0)])]) - ); assert_eq!( - entries[1], - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)]), ("video1", &[(1, 1)])]) + drain(&mut data), + vec![(0, 2_000, at(0), at(2)), (2_400_000, 0, at(3), Position::new(3, 1))], + "the pause is not absorbed into the record before it" ); } - // A track that races ahead of the others still lands in the segment its content falls in. - #[tokio::test] - async fn groups_before_the_first_boundary_join_the_first_segment() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - // Audio races ahead of video's first keyframe (the startup race): its early group - // belongs to segment 0, which starts where the earliest content does. - audio.record(0, ms(0), true); - video.record(0, ms(30), true); - for (seq, t) in [(1u64, 500u64), (2, 1_000), (3, 1_500), (4, 2_000)] { - audio.record(seq, ms(t), true); - } - video.record(1, ms(2_030), true); - audio.record(5, ms(2_500), true); - drop(video); - drop(audio); - timeline.finish().unwrap(); + #[test] + fn a_record_never_exceeds_the_maximum() { + // A quiet log: nothing between its frames to split at. + let mut log = Segmenter::new(sparse().with_duration_max(Duration::from_secs(10))); + log.frame(Position::new(0, 0), ms(0), true).unwrap(); + log.frame(Position::new(0, 1), ms(20_000), true).unwrap(); + log.frame(at(1), ms(45_000), true).unwrap(); + log.close(); - let entries = drain(&broadcast, &timeline).await; assert_eq!( - entries[0], - entry(0, 0, 2_030, &[("video0", &[(0, 0)]), ("audio0", &[(0, 4)])]), + drain(&mut log), + vec![ + (0, 10_000, Position::new(0, 0), Position::new(0, 1)), + (20_000, 10_000, Position::new(0, 1), at(1)), + (45_000, 0, at(1), Position::new(1, 1)), + ] ); } - #[tokio::test] - async fn sequence_gaps_split_ranges() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - // Elemental-style gap: audio groups 2..=4 never existed inside segment 0. - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - audio.record(1, ms(300), true); - audio.record(5, ms(1_500), true); - video.record(1, ms(2_000), true); - audio.record(6, ms(2_100), true); - drop(video); - drop(audio); - timeline.finish().unwrap(); + #[test] + fn explicit_cuts_override_the_pacing() { + let mut video = Segmenter::new(Config::default()); + // Keyframes every second, cut every three: the records follow the cuts, not the GOPs. + video.cut(ms(3_000)); + video.cut(ms(3_000)); + video.cut(ms(6_000)); + groups(&mut video, 0..7, 1_000); - let entries = drain(&broadcast, &timeline).await; assert_eq!( - entries[0], - entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 1), (5, 5)])]), + drain(&mut video), + vec![(0, 3_000, at(0), at(3)), (3_000, 3_000, at(3), at(6))] ); } - // A track with nothing to contribute is a whole-segment gap: the record simply omits it - // (an HLS exporter renders EXT-X-GAP). Only a *closed* track produces one, since a live - // track that has merely gone quiet still gets to say where the boundary is. - #[tokio::test] - async fn a_closed_track_leaves_a_whole_segment_gap() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - drop(audio); - video.record(1, ms(2_000), true); - video.record(2, ms(4_000), true); - drop(video); - timeline.finish().unwrap(); + #[test] + fn a_cut_on_the_first_group_does_not_poison_later_cuts() { + let mut video = Segmenter::new(Config::default()); + video.cut(ms(0)); + for group in 0..10u64 { + if group % 3 == 0 { + video.cut(ms(group * 1_000)); + } + video.frame(at(group), ms(group * 1_000), true).unwrap(); + } - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries[1], entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])])); + assert_eq!(drain(&mut video)[0], (0, 3_000, at(0), at(3))); } - #[tokio::test] - async fn a_non_keyframe_range_start_is_flagged() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - video.record(0, ms(0), true); + #[test] + fn a_non_keyframe_start_is_flagged() { + let mut video = Segmenter::new(Config::default()); + video.frame(at(0), ms(0), true).unwrap(); // A mid-stream join: the group doesn't open on an IDR. - video.record(1, ms(2_000), false); - video.record(2, ms(4_000), true); - drop(video); - timeline.finish().unwrap(); + video.frame(at(1), ms(2_000), false).unwrap(); + video.frame(at(2), ms(4_000), true).unwrap(); - let entries = drain(&broadcast, &timeline).await; - let range = entries[1].tracks["video0"][0]; - assert!(!range.keyframe, "a range whose first group isn't an IDR says so"); + let records: Vec = std::iter::from_fn(|| video.next()).collect(); + assert!(records[0].keyframe); + assert!(!records[1].keyframe); } - // A dead track would otherwise hold every later segment: dropping its recorder is what - // says it will never report again. - #[tokio::test] - async fn a_closed_track_stops_gating() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut audio = timeline.pacing_track("audio0").unwrap(); - - video.record(0, ms(0), true); - audio.record(0, ms(0), true); - video.record(1, ms(2_000), true); - assert!(drain(&broadcast, &timeline).await.is_empty(), "audio still gates"); - - drop(audio); - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries, - vec![entry(0, 0, 2_000, &[("video0", &[(0, 0)]), ("audio0", &[(0, 0)])])] - ); - drop(video); - timeline.finish().unwrap(); + #[test] + fn a_report_that_does_not_advance_is_refused() { + let mut video = Segmenter::new(Config::default()); + video.frame(at(3), ms(0), true).unwrap(); + for position in [at(2), at(3)] { + assert!(matches!( + video.frame(position, ms(2_000), true), + Err(crate::Error::TimelinePosition { last, .. }) if last == at(3) + )); + } + video.frame(at(4), ms(2_000), true).unwrap(); + assert_eq!(drain(&mut video), vec![(0, 2_000, at(3), at(4))]); } - // The timeline track (and its catalog section) exist only once a media track enrolls. - #[tokio::test] - async fn the_track_is_created_on_first_enrollment() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let timeline = Producer::new(&broadcast, Config::default()); - assert!( - broadcast.create_track(DEFAULT_NAME, None).is_ok(), - "nothing took the name yet" - ); + #[test] + fn a_resumed_segmenter_continues_the_numbering() { + let mut video = Segmenter::new(Config::default()).with_sequence(7); + groups(&mut video, 10..12, 2_000); + assert_eq!(video.next().unwrap().sequence, 7); + } - let broadcast = moq_net::broadcast::Info::new().produce(); - let timeline2 = Producer::new(&broadcast, Config::default()); - let _recorder = timeline2.pacing_track("video0").unwrap(); - assert!( - broadcast.create_track(DEFAULT_NAME, None).is_err(), - "the timeline took the name once a track enrolled" - ); - drop(timeline); + /// Drain a finished timeline track. + async fn read(broadcast: &moq_net::broadcast::Producer, section: &Archive, track: &str) -> Vec { + let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), section, track) + .await + .unwrap(); + let waiter = kio::Waiter::noop(); + let mut out = Vec::new(); + while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { + if let Event::Push { index, entry } = event { + assert_eq!(index, entry.sequence); + out.push(entry); + } + } + out } #[tokio::test] - async fn section_advertises_track_without_wall() { - // Wall mapping moved to the catalog root clock: the timeline section carries the track, - // timescale, and duration bound, and nothing else time-anchoring. - let (_broadcast, timeline) = setup(); - let section = timeline.section(); - assert_eq!(section.track, DEFAULT_NAME); - assert_eq!(section.timescale, 1000); - assert_eq!(section.duration_max, None); - } - - // Clones nest like the catalog's, so the first batch to finish doesn't publish records the - // others are still filling in. - #[tokio::test] - async fn reservations_nest() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - let outer = timeline.reserve(); - let inner = outer.clone(); + async fn tracks_publish_independent_timelines() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut video = timelines.track("video0", Config::default()).unwrap(); + let mut catalog = timelines.track("catalog.json", sparse()).unwrap(); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - video.record(seq, ms(t), true); + catalog.frame(at(0), ms(0), true); + catalog.finish_group(0); + for group in 0..3 { + video.frame(at(group), ms(group * 2_000), true); + video.finish_group(group); } + video.end(ms(6_000)); + drop(video); + timelines.finish(); - drop(inner); - assert!( - drain(&broadcast, &timeline).await.is_empty(), - "the other clone still gates" - ); - - drop(outer); - assert_eq!( - drain(&broadcast, &timeline).await.len(), - 2, - "the last clone dropped flushes" - ); + let section = timelines.section(); + assert_eq!(section.timelines["video0"], "video0.timeline.z"); + assert_eq!(section.timelines["catalog.json"], "catalog.json.timeline.z"); + assert_eq!(section.duration_max, Some(10_000)); - drop(video); - timeline.finish().unwrap(); + let video = read(&broadcast, §ion, "video0").await; + assert_eq!(video.len(), 3); + assert_eq!(video[2].end, at(3), "the terminal flush ends after the finished group"); + let catalog = read(&broadcast, §ion, "catalog.json").await; + assert_eq!(catalog.len(), 1, "the catalog does not wait for the video"); } - // A reservation outstanding when the broadcast ends must not strand the terminal flush. #[tokio::test] - async fn finish_overrides_an_outstanding_reservation() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let reserved = timeline.reserve(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - drop(video); - - timeline.finish().unwrap(); - assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] - ); + async fn a_sparse_record_publishes_before_the_track_ends() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut catalog = timelines.track("catalog.json", sparse()).unwrap(); + catalog.frame(at(0), ms(0), true); + catalog.finish_group(0); - // Dropping it afterwards is a no-op rather than an underflow. - drop(reserved); + assert_eq!(read(&broadcast, &timelines.section(), "catalog.json").await.len(), 1); + drop(catalog); } #[tokio::test] - async fn rejects_an_invalid_timescale() { - let (broadcast, timeline) = setup(); - let _recorder = timeline.pacing_track("video0").unwrap(); - let mut section = timeline.section(); - section.timescale = 0; - let err = Consumer::<()>::subscribe(&broadcast.consume(), §ion).await; - assert!(matches!(err, Err(crate::Error::InvalidTimescale(0)))); - } + async fn a_rejected_report_fails_the_timeline() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut video = timelines.track("video0", Config::default()).unwrap(); + let mut consumer = Consumer::<()>::subscribe(&broadcast.consume(), &timelines.section(), "video0") + .await + .unwrap(); - // A cut registered before the media, landing exactly on the first reported group (what the - // fMP4 importer does: it cuts at the keyframe fragment's timestamp, then records that group). - #[tokio::test] - async fn a_cut_on_the_first_group_does_not_poison_later_cuts() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - - // Source segments every 3s, keyframes every 1s: 3s segments, not the 1s the - // duration_min pacing would produce on its own. - timeline.cut(ms(0)).unwrap(); - video.record(0, ms(0), true); - for seq in 1..10u64 { - if seq % 3 == 0 { - timeline.cut(ms(seq * 1_000)).unwrap(); + video.frame(at(1), ms(0), true); + video.frame(at(0), ms(2_000), true); + loop { + match consumer.next().await { + Ok(Some(_)) => continue, + Ok(None) => panic!("the timeline ended cleanly despite omitting a frame"), + Err(_) => break, } - video.record(seq, ms(seq * 1_000), true); } - drop(video); - timeline.finish().unwrap(); - - let entries = drain(&broadcast, &timeline).await; - assert_eq!( - entries[0], - entry(0, 0, 3_000, &[("video0", &[(0, 2)])]), - "the source's 3s boundaries should be reproduced, not duration_min pacing" - ); } - // A catalog publishes a group only when the renditions change, so it can go quiet for the - // rest of the broadcast. Enrolled as a pacing track it would stall the timeline for good; - // non-pacing, it rides along in whichever segment is open. #[tokio::test] - async fn a_non_pacing_track_neither_paces_nor_gates() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut catalog = timeline.track("catalog.json"); - - catalog.record(0, ms(0), true); - for (seq, t) in [(0u64, 0u64), (1, 2_000), (2, 4_000)] { - video.record(seq, ms(t), true); - } - video.end(ms(6_000)); - timeline.finish().unwrap(); + async fn re_enrolling_continues_the_timeline() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut first = timelines.track("video0", Config::default()).unwrap(); + first.frame(at(0), ms(0), true); + drop(first); + let mut second = timelines.track("video0", Config::default()).unwrap(); + // The new producer restarts its group sequences. + second.frame(at(0), ms(2_000), true); + drop(second); + timelines.finish(); - let entries = drain(&broadcast, &timeline).await; + let entries = read(&broadcast, &timelines.section(), "video0").await; assert_eq!( - entries, - vec![ - entry(0, 0, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - entry(2, 4_000, 2_000, &[("video0", &[(2, 2)])]), - ], - "the silent catalog should neither hold records back nor move a boundary" + entries.iter().map(|entry| entry.sequence).collect::>(), + vec![0, 1] ); } - // Nothing waits for a non-pacing track, so a group arriving after its segment already flushed - // is recorded in the next one. Placement is by arrival; the frames still carry their own - // timestamps. + /// Two live recorders would share one segmenter, each resetting or closing the other's + /// timeline, so a name is enrolled once at a time. #[tokio::test] - async fn a_late_non_pacing_group_lands_in_the_next_segment() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut catalog = timeline.track("catalog.json"); - - video.record(0, ms(0), true); - // Flushes segment 0, which the catalog has contributed nothing to. - video.record(1, ms(2_000), true); + async fn enrolling_an_active_track_is_refused() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let mut first = timelines.track("video0", Config::default()).unwrap(); + assert!(timelines.track("video0", Config::default()).is_err()); - // The update happened a second in, but only reaches the timeline now. - catalog.record(0, ms(1_000), true); + // The live recorder is untouched by the refusal. + first.frame(at(0), ms(0), true); + first.frame(at(1), ms(2_000), true); + drop(first); + assert!(timelines.track("video0", Config::default()).is_ok()); + } - video.record(2, ms(4_000), true); - video.end(ms(6_000)); - timeline.finish().unwrap(); + #[test] + fn a_taken_timeline_name_fails_enrollment() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let _squat = broadcast.create_track("video0.timeline.z", None).unwrap(); + let timelines = Timelines::new(&broadcast); + assert!(timelines.track("video0", Config::default()).is_err()); + assert!(timelines.section().timelines.is_empty()); + } - let entries = drain(&broadcast, &timeline).await; - assert_eq!(entries[0], entry(0, 0, 2_000, &[("video0", &[(0, 0)])])); - assert_eq!( - entries[1], - entry(1, 2_000, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(1, 1)])]), - "the late group belongs to the segment that was open when it arrived" - ); + #[test] + fn enrollment_after_finish_is_refused() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + timelines.finish(); + assert!(matches!( + timelines.track("video0", Config::default()), + Err(crate::Error::Moq(moq_net::Error::Closed)) + )); } - // Non-pacing placement is by arrival rather than timestamp. A catalog can stamp snapshots from a - // wall-clock basis while imported media starts at PTS zero, so its frontier must neither hold - // the group for a much later segment nor stretch the final media segment to that timestamp. #[tokio::test] - async fn a_non_pacing_track_uses_arrival_and_does_not_extend_the_tail() { - let (broadcast, mut timeline) = setup(); - let mut video = timeline.pacing_track("video0").unwrap(); - let mut catalog = timeline.track("catalog.json"); - - video.record(0, ms(0), true); - catalog.record(0, ms(10_000), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - timeline.finish().unwrap(); + async fn a_resumed_producer_continues_the_checkpoint() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video0.timeline.z", Producer::info()).unwrap(); + let checkpoint = Checkpoint { + range: 1..3, + records: vec![ + Record::new(1, 1_000, 1_000, at(1), at(2)), + Record::new(2, 2_000, 1_000, at(2), at(3)), + ], + }; + let mut producer = Producer::resume(track, &checkpoint).unwrap(); + assert!(matches!( + producer.push(&Record::new(4, 3_000, 1_000, at(3), at(4))), + Err(crate::Error::TimelineSequence { expected: 3, actual: 4 }) + )); + producer.push(&Record::new(3, 3_000, 1_000, at(3), at(4))).unwrap(); + producer.finish().unwrap(); + let mut section = Archive::new(); + section + .timelines + .insert("video0".to_string(), "video0.timeline.z".to_string()); + let entries = read(&broadcast, §ion, "video0").await; assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] + entries.iter().map(|entry| entry.sequence).collect::>(), + vec![1, 2, 3] ); } - #[tokio::test] - async fn the_built_in_catalog_is_recorded_without_pacing() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let timeline = catalog.timeline(); - let mut video = catalog.enroll("video0").unwrap(); - - video.record(0, ms(0), true); - video.record(1, ms(2_000), true); - video.end(ms(4_000)); - drop(video); - catalog.finish().unwrap(); - - assert_eq!( - drain(&broadcast, &timeline).await, - vec![ - entry(0, 0, 2_000, &[("catalog.json", &[(0, 0)]), ("video0", &[(0, 0)])]), - entry(1, 2_000, 2_000, &[("video0", &[(1, 1)])]), - ] - ); + #[test] + fn a_checkpoint_record_must_be_the_sequence_at_its_index() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("video0.timeline.z", Producer::info()).unwrap(); + let checkpoint = Checkpoint { + range: 0..1, + records: vec![Record::new(5, 0, 1_000, at(0), at(1))], + }; + assert!(matches!( + Producer::resume(track, &checkpoint), + Err(crate::Error::TimelineCheckpoint(0)) + )); } - // Segmentation stays opt-in by pacing track: non-pacing tracks alone describe nothing, so the - // timeline track is never created and its name stays free. #[tokio::test] - async fn non_pacing_tracks_alone_publish_no_timeline() { - let (broadcast, mut timeline) = setup(); - let mut catalog = timeline.track("catalog.json"); - - catalog.record(0, ms(0), true); - timeline.finish().unwrap(); + async fn a_missing_timeline_is_an_error() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let section = Archive::new(); + assert!(matches!( + Consumer::<()>::subscribe(&broadcast.consume(), §ion, "video0").await, + Err(crate::Error::TimelineMissing(_)) + )); + } - assert!( - broadcast.create_track(DEFAULT_NAME, None).is_ok(), - "the timeline track should never have been created" - ); + #[tokio::test] + async fn rejects_an_invalid_timescale() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let timelines = Timelines::new(&broadcast); + let _recorder = timelines.track("video0", Config::default()).unwrap(); + let mut section = timelines.section(); + section.timescale = 0; + let err = Consumer::<()>::subscribe(&broadcast.consume(), §ion, "video0").await; + assert!(matches!(err, Err(crate::Error::InvalidTimescale(0)))); } } diff --git a/rs/moq-rtc/src/codec/vp8.rs b/rs/moq-rtc/src/codec/vp8.rs index 4cede5be7ef..9f458a7e0cb 100644 --- a/rs/moq-rtc/src/codec/vp8.rs +++ b/rs/moq-rtc/src/codec/vp8.rs @@ -67,7 +67,9 @@ mod tests { async fn importer_creation_failure_preserves_abort_error() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - let _collision = broadcast.create_track(hang::timeline::DEFAULT_NAME, None).unwrap(); + let _collision = broadcast + .create_track(hang::timeline::default_name(hang::Catalog::DEFAULT_NAME), None) + .unwrap(); let consumer = broadcast.consume(); let mut bridge = super::Bridge::new(broadcast, catalog).unwrap(); let mut track = consumer.track("0.vp8").unwrap().subscribe(None).await.unwrap(); diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index 8f741798cf6..559b8ac1b93 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -845,7 +845,10 @@ mod tests { .video .insert("video", source(1920, 1080, Some(6_000_000))) .unwrap(); - let mut archive = hang::catalog::Archive::new("timeline.z"); + let mut archive = hang::catalog::Archive::new(); + archive + .timelines + .insert("video".to_string(), "video.timeline.z".to_string()); archive.replay = Some(RelativeOwned::from("./recordings/clip".to_string())); archive.version = Some(hang::catalog::Archive::VERSION); child.archive = Some(archive.clone()); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index b08542be824..c3d702c733f 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1402,7 +1402,7 @@ mod tests { .await } - /// A recording replays what the live edge published: the archive's segment records + /// A recording replays what the live edge published: the archive's records /// carry the live timestamps across an idle restart, with the idle gap left in. #[tokio::test] async fn retained_archive_playback_keeps_the_live_timestamps() { @@ -1414,9 +1414,10 @@ mod tests { .snapshot() .archive .expect("the video track enrolls an archive"); - let mut timeline = moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion) - .await - .unwrap(); + let mut timeline = + moq_mux::timeline::Consumer::<()>::subscribe(&fixture.consumer, §ion, "video") + .await + .unwrap(); let mut live = Vec::new(); for _ in 0..2 { @@ -1428,12 +1429,12 @@ mod tests { fixture.assert_acquired(published, captured); live.push(published); drop(track); - // Idle past the minimum segment, so each run is archived as its own segment. + // Idle past the minimum segment, so each run is archived as its own record. tokio::time::sleep(moq_mux::timeline::DEFAULT_DURATION_MIN + Duration::from_millis(100)).await; } let (catalog, _consumer) = fixture.finish().await; - catalog.timeline().finish().unwrap(); + catalog.timeline().finish(); let mut archived = Vec::new(); while let Some(event) = timeline.next().await.unwrap() { match event { @@ -1442,16 +1443,15 @@ mod tests { } } - assert_eq!(archived.len(), live.len(), "one segment per capture run: {archived:?}"); + assert_eq!(archived.len(), live.len(), "one record per capture run: {archived:?}"); for (entry, live) in archived.iter().zip(&live) { // The archive keeps millisecond precision. assert_eq!(entry.pts.as_micros() / 1000, u128::from(*live / 1000), "{archived:?}"); - assert!(entry.tracks.contains_key("video"), "{archived:?}"); } let first = &archived[0]; assert!( archived[1].pts.as_micros() >= first.pts.as_micros() + first.duration.as_micros(), - "the resumed segment overlaps the one before it: {archived:?}" + "the resumed record overlaps the one before it: {archived:?}" ); }) .await