From f34e18e0ee429df8e52a720f71b69068dd3e6a41 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 19:06:20 -0700 Subject: [PATCH 1/4] chore: claim leading-picture recovery quest Co-Authored-By: GPT-6 From e955cc33508f4d27e79faa15779c1e48de0c9766 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:34:44 -0700 Subject: [PATCH 2/4] fix(video): trim orphan leading pictures after delivery breaks Validated both JS decode paths, seven native recovery tests, actual software WebCodecs and its disabled-filter control, and the strict generated TS roundtrip. Required broad check and all-language interop remain pending; the quest is retained until they finish. Co-Authored-By: GPT-6 # Conflicts: # test/justfile --- .github/workflows/ts.yml | 6 +- bun.lock | 1 + js/watch/src/video/decoder.test.ts | 34 ++++++ js/watch/src/video/decoder.ts | 23 +++- js/watch/src/video/recovery.ts | 22 ++++ rs/moq-cli/src/play/video.rs | 68 +++++++++++ rs/moq-mux/src/container/ts/export.rs | 15 +++ rs/moq-mux/src/container/ts/export_test.rs | 55 +++++++++ rs/moq-video/src/decode/consumer.rs | 95 ++++++++++++++- test/interop/clients/js/open-gop.browser.ts | 123 ++++++++++++++++++++ test/interop/clients/js/open-gop/index.html | 1 + test/interop/clients/js/open-gop/main.ts | 109 +++++++++++++++++ test/interop/clients/js/package.json | 1 + test/interop/clients/js/tsconfig.json | 2 + test/justfile | 5 +- test/ts/README.md | 16 ++- test/ts/open-gop.py | 115 +++++++++--------- test/ts/open-gop.test.py | 33 ++++++ test/ts/run.sh | 2 +- 19 files changed, 649 insertions(+), 77 deletions(-) create mode 100644 js/watch/src/video/recovery.ts create mode 100644 test/interop/clients/js/open-gop.browser.ts create mode 100644 test/interop/clients/js/open-gop/index.html create mode 100644 test/interop/clients/js/open-gop/main.ts create mode 100644 test/ts/open-gop.test.py diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index de682c95480..5b8fe064191 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -141,11 +141,11 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} - # The same round-trip over an open-GOP clip: every leading picture after a - # recovery point must come back, in decode order. + # The same round-trip over an open-GOP clip: recovery-group orphans are trimmed, + # continuous leading pictures stay in decode order, and all shape checks gate. - name: TS open GOP if: ${{ !cancelled() }} - run: nix develop --command just test ts --open-gop + run: nix develop --command just test ts --open-gop --strict env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} diff --git a/bun.lock b/bun.lock index a458d7ba417..85cc11a4e64 100644 --- a/bun.lock +++ b/bun.lock @@ -377,6 +377,7 @@ "test/interop/clients/js": { "name": "@moq/interop-browser", "dependencies": { + "@moq/hang": "workspace:*", "@moq/net": "workspace:*", "@moq/publish": "workspace:*", "@moq/signals": "workspace:*", diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 194901f5c47..b161c81320f 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -707,3 +707,37 @@ it("a reset drops a parked video frame even when the next anchor restores the re now.mockRestore(); } }); + +for (const kind of ["legacy", "cmaf"] as const) { + it(`${kind} trims orphan leading pictures on join and later gap, preserving continuous groups`, async () => { + const reads: Read[] = [ + sample(10, 1000), + sample(10, 800, false), + sample(10, 900, false), + sample(10, 1100, false), + sample(11, 2000), + sample(11, 1800, false), + sample(11, 1900, false), + sample(11, 2100, false), + gap, + sample(13, 4000), + sample(13, 3800, false), + sample(13, 3900, false), + sample(13, 4100, false), + { kind: "end", group: 14, end: Time.Micro(4200) }, + sample(15, 5000), + sample(15, 4800, false), + sample(15, 4900, false), + sample(15, 5100, false), + ]; + const playback = await guardedPlayback(kind, reads); + try { + await microtasks(); + expect(playback.submitted).toEqual([ + 1000, 1100, 2000, 1800, 1900, 2100, 4000, 4100, 5000, 4800, 4900, 5100, + ]); + } finally { + playback.close(); + } + }); +} diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index cc99c163c9f..42634edb0f3 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -27,6 +27,7 @@ import { supportCacheKey, } from "./config"; import { caughtUp, renditionJitter, switchJitter } from "./playhead"; +import { Recovery } from "./recovery"; import type { Source } from "./source"; // The amount of time to wait before considering the video to be buffering. @@ -553,6 +554,7 @@ class DecoderTrack { let previous: Time.Micro | undefined; let latest: number | undefined; + const recovery = new Recovery(); effect.spawn(async () => { for (;;) { @@ -562,7 +564,10 @@ class DecoderTrack { // A gap or a declared end breaks the timeline: re-anchor before decoding further, and // never bridge the buffer across it. if (next.kind !== "frame") { - if (next.kind === "gap") this.arrivals?.write(gap(next)); + if (next.kind === "gap") { + recovery.gap(); + this.arrivals?.write(gap(next)); + } if (latest !== undefined) this.#onBreak(); previous = undefined; continue; @@ -573,11 +578,12 @@ class DecoderTrack { // sequences only grow within a subscription, so a break (which a stale marker can // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; + const accepted = !stale && recovery.accept(frame); this.stats.update((current) => ({ - frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), + frameCount: (current?.frameCount ?? 0) + (accepted ? 1 : 0), bytesReceived: (current?.bytesReceived ?? 0) + frame.payload.byteLength, })); - if (stale) continue; + if (!accepted) continue; // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); @@ -638,6 +644,7 @@ class DecoderTrack { let previous: Time.Micro | undefined; let latest: number | undefined; + const recovery = new Recovery(); effect.spawn(async () => { for (;;) { @@ -646,7 +653,10 @@ class DecoderTrack { // A gap or a declared end breaks the timeline (see #runLegacy). if (next.kind !== "frame") { - if (next.kind === "gap") this.arrivals?.write(gap(next)); + if (next.kind === "gap") { + recovery.gap(); + this.arrivals?.write(gap(next)); + } if (latest !== undefined) this.#onBreak(); previous = undefined; continue; @@ -657,11 +667,12 @@ class DecoderTrack { // sequences only grow within a subscription, so a break (which a stale marker can // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; + const accepted = !stale && recovery.accept(frame); this.stats.update((current) => ({ - frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), + frameCount: (current?.frameCount ?? 0) + (accepted ? 1 : 0), bytesReceived: (current?.bytesReceived ?? 0) + frame.payload.byteLength, })); - if (stale) continue; + if (!accepted) continue; // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp); diff --git a/js/watch/src/video/recovery.ts b/js/watch/src/video/recovery.ts new file mode 100644 index 00000000000..0ddc7418b58 --- /dev/null +++ b/js/watch/src/video/recovery.ts @@ -0,0 +1,22 @@ +import type { Container } from "@moq/hang"; +import type { Time } from "@moq/net"; + +/** Refuse leading pictures whose previous GOP was not delivered to this decoder. @internal */ +export class Recovery { + #needed = true; + #floor?: Time.Micro; + + /** Arm recovery after media was lost, preserving continuous groups and declared ends. */ + gap(): void { + this.#needed = true; + } + + /** Whether this access unit has the references available at the current delivery boundary. */ + accept(frame: Container.ReceivedFrame): boolean { + if (frame.keyframe) { + this.#floor = this.#needed ? frame.timestamp : undefined; + this.#needed = false; + } + return this.#floor === undefined || frame.timestamp >= this.#floor; + } +} diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 6bc0487bcba..9f0f58bf08e 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -76,12 +76,22 @@ impl Video { // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out. let mut end = None; + // Leading pictures need the missing preceding GOP, unlike a declared encoder end. + let mut recover = true; + let mut leading_floor = None; loop { let read = track.read().await?; let mut buffer = buffer.lock().unwrap(); let before = buffer.generation; let ended = match read { Some(Read::Frame { group, frame }) => { + if frame.keyframe { + leading_floor = recover.then_some(frame.timestamp); + recover = false; + } + if leading_floor.is_some_and(|floor| frame.timestamp < floor) { + continue; + } if end.take_if(|end| *end != group).is_some() { buffer.clear(); self.presentation.lock().unwrap().video_restarted(); @@ -99,6 +109,7 @@ impl Video { } // Applied in delivery order, so the restart lands before the frame after it. Some(Read::Gap { .. }) => { + recover = true; end = None; buffer.clear(); self.presentation.lock().unwrap().video_restarted(); @@ -248,6 +259,7 @@ mod tests { }; use super::*; use hang::moq_net; + use moq_mux::container::Container as _; /// A codec whose reorder buffer holds `depth` pictures and bumps the /// earliest one out, the way a real decoder returns display order. @@ -498,6 +510,62 @@ mod tests { assert_eq!(media.output.present(&media).unwrap().as_millis(), 500); } + #[tokio::test] + async fn leading_pictures_are_trimmed_before_native_decode() { + tokio::time::pause(); + let delay = Duration::from_secs(1); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let published = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = published + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let write = |sequence, stamps: Vec| { + let mut group = published.create_group(moq_net::group::Info { sequence }).unwrap(); + let frames: Vec<_> = stamps + .into_iter() + .enumerate() + .map(|(index, ms)| frame(ms, index == 0)) + .collect(); + format.write(&mut group, &frames).unwrap(); + group.finish().unwrap(); + }; + write(10, vec![100, 80, 90, 110]); + write(11, vec![200, 180, 190, 210]); + let track = moq_mux::container::Consumer::new( + subscriber, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + let decoder = Buffered::new(0); + let decoded = decoder.decoded.clone(); + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); + for _ in 0..1300 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + assert_eq!( + decoded.lock().unwrap().len(), + 6, + "continuous media reached the codec before the later delivery hole" + ); + write(13, vec![400, 380, 390, 410]); + published.finish().unwrap(); + for _ in 0..1300 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + let got: Vec<_> = decoded.lock().unwrap().iter().map(|(at, _)| at.as_millis()).collect(); + assert_eq!(got, [100, 110, 200, 180, 190, 210, 400, 410]); + } + #[tokio::test] async fn a_declared_end_plays_out_the_pictures_before_it() { tokio::time::pause(); diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index bbd4398298f..9830cf37028 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -254,6 +254,10 @@ struct Track { descriptors: Vec, /// Authors each video frame's DTS, holding the frames until it can. clock: DecodeClock<(Pending, Option)>, + /// Loss counter at the last read, excluding declared ends. + recovery_gap: Option, + /// Leading pictures below the recovery keyframe have unavailable references. + leading_floor: Option, /// What the catalog and the SPS declare about the video's reordering. timing: Timing, /// The receiver's buffers for an audio or verbatim track; video's come from [`Timing`]. @@ -288,6 +292,8 @@ impl Track { kind, descriptors: Vec::new(), clock: DecodeClock::default(), + recovery_gap: None, + leading_floor: None, timing: Timing::default(), buffer: None, ac3_carry: None, @@ -314,6 +320,15 @@ impl Track { let Kind::Video(stream_type) = self.kind else { return self.push(name, pending, None, None, jitter); }; + // A declared endpoint increments both counters; only delivery loss arms recovery. + let gap = pending.skip.saturating_sub(pending.restart); + let recover = self.recovery_gap.replace(gap).is_none_or(|previous| previous != gap); + if pending.frame.keyframe { + self.leading_floor = recover.then_some(pending.frame.timestamp); + } + if self.leading_floor.is_some_and(|floor| pending.frame.timestamp < floor) { + return Ok(()); + } if pending.frame.keyframe { self.timing.describe(stream_type, self.source.description(), name); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index e77cd97475e..a0789cd339e 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -16,6 +16,7 @@ use mpeg2ts::pes::{PesPacketReader, ReadPesPacket}; use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use crate::catalog::hang::Container as HangContainer; +use crate::container::Container as _; use crate::container::ts::export::PCR_INTERVAL; use crate::container::ts::{Export, catalog as tscat, stats}; use crate::container::{Frame, Kind, Producer}; @@ -979,6 +980,60 @@ async fn export_bframe_video_authors_dts() { } } +#[tokio::test(start_paused = true)] +async fn export_trims_leading_pictures_after_each_delivery_break() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + catalog.modify().unwrap().video.renditions.insert( + "video".into(), + VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: true, + }), + ); + let format = HangContainer::Legacy(Kind::Video); + for (sequence, stamps) in [ + (10, vec![100, 80, 90, 110]), + (11, vec![200, 180, 190, 210]), + (13, vec![400, 380, 390, 410]), + ] { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let frames: Vec<_> = stamps + .into_iter() + .enumerate() + .map(|(index, ms)| Frame { + timestamp: Timestamp::from_millis(ms).unwrap(), + duration: None, + keyframe: index == 0, + payload: if index == 0 { + annexb(&[SPS, PPS, &[0x65, 0xab, 0xcd]]) + } else { + annexb(&[&[0x41, 0xab, 0xcd]]) + }, + }) + .collect(); + format.write(&mut group, &frames).unwrap(); + group.finish().unwrap(); + } + track.finish().unwrap(); + let ts = drain(consumer).await; + let (video, _) = collect_pes_pts(&ts); + let expected: Vec<_> = [100, 110, 200, 180, 190, 210, 400, 410] + .into_iter() + .map(|ms| ms * 90) + .collect(); + assert_eq!( + video, expected, + "continuous leading pictures survive; only join and later-hole orphans are trimmed" + ); +} + /// `(PTS, DTS)` of every video PES start, in transport (decode) order. Read raw, so a capture /// that starts before its PMT still counts. fn video_pes_timing(ts: &[u8]) -> Vec<(u64, u64)> { diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index f47c1443d41..b297be2109e 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -86,6 +86,9 @@ pub struct Consumer { pending: VecDeque, /// Whether the ended track's decoder has already been drained. drained: bool, + /// The first recovery group lacks the preceding GOP's references. + recover: bool, + leading_floor: Option, } impl Consumer { @@ -139,6 +142,8 @@ impl Consumer { track, pending: VecDeque::new(), drained: false, + recover: true, + leading_floor: None, }) } @@ -150,6 +155,8 @@ impl Consumer { /// Read the next decoded frame, or `None` after the track ends and the /// decoder's buffered tail has been drained. /// + /// Join and delivery-gap recovery trim leading pictures whose previous GOP is absent. + /// /// This inherits [`Sink`]'s cancellation contract. If a queued codec /// operation is cancelled, the next read returns a codec error: a cancelled /// mid-stream decode poisons the sink so every later read keeps returning @@ -166,9 +173,12 @@ impl Consumer { let mux_frame = match self.track.read().await? { Some(Read::Frame { frame, .. }) => Some(frame), - // The next group starts on a keyframe with parameter sets, so neither a - // gap nor a declared end flushes the decoder. - Some(Read::Gap { .. } | Read::End { .. }) => continue, + Some(Read::Gap { .. }) => { + self.recover = true; + continue; + } + // A fresh encoder has no preceding GOP to reference. + Some(Read::End { .. }) => continue, None => None, }; @@ -189,6 +199,14 @@ impl Consumer { continue; }; + if mux_frame.keyframe { + self.leading_floor = self.recover.then_some(mux_frame.timestamp); + self.recover = false; + } + if self.leading_floor.is_some_and(|floor| mux_frame.timestamp < floor) { + continue; + } + self.pending.extend( self.decoder .decode(mux_frame.payload, mux_frame.timestamp, mux_frame.keyframe) @@ -221,6 +239,7 @@ mod tests { use crate::decode::Kind; use crate::decode::backend::probe; use crate::encode::{Config as EncodeConfig, Encoder, Kind as EncodeKind, Producer as EncodeProducer}; + use moq_mux::container::Container as _; #[tokio::test] #[cfg(feature = "openh264")] @@ -512,6 +531,76 @@ mod tests { ); } + #[tokio::test] + async fn leading_pictures_require_a_continuous_previous_group() { + tokio::time::pause(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let catalog = VideoConfig::new(hang::catalog::H264 { + inline: true, + profile: 0x42, + constraints: 0, + level: 30, + }); + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "video", + Options { + decoder: Config { + kind: Kind::Named(probe::BUFFERED_NAME.into()), + ..Config::new() + }, + max_delay: std::time::Duration::from_secs(10), + ..Options::new() + }, + ) + .await + .unwrap(); + let wire = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let write = |sequence, stamps: &[u64]| { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let frames: Vec<_> = stamps + .iter() + .enumerate() + .map(|(index, stamp)| moq_mux::container::Frame { + timestamp: Timestamp::from_micros(*stamp).unwrap(), + duration: None, + payload: Bytes::from_static(b"access unit"), + keyframe: index == 0, + }) + .collect(); + wire.write(&mut group, &frames).unwrap(); + group.finish().unwrap(); + }; + write(10, &[1000, 800, 900, 1100]); + write(11, &[2000, 1800, 1900, 2100]); + let mut got = Vec::new(); + for _ in 0..5 { + got.push(consumer.read().await.unwrap().unwrap().timestamp.as_micros()); + } + assert_eq!( + got, + [1000, 1100, 2000, 1800, 1900], + "join trims, continuous delivery keeps every leading picture" + ); + consumer.track.set_max_delay(std::time::Duration::ZERO); + write(12, &[3000]); + write(13, &[4000, 3800, 3900, 4100]); + track.finish().unwrap(); + got.clear(); + while let Some(frame) = consumer.read().await.unwrap() { + got.push(frame.timestamp.as_micros()); + } + assert_eq!( + got, + [2100, 4000, 4100], + "a later latency skip must trim its recovery group too" + ); + } + /// A track ends before a decoder that reorders pictures does. The consumer /// drains the backend once and returns its tail before reporting the end. #[tokio::test] diff --git a/test/interop/clients/js/open-gop.browser.ts b/test/interop/clients/js/open-gop.browser.ts new file mode 100644 index 00000000000..c4f00b61196 --- /dev/null +++ b/test/interop/clients/js/open-gop.browser.ts @@ -0,0 +1,123 @@ +/** Drive the real player and software WebCodecs decoder through an open-GOP join and later delivery hole. @module */ +import { expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { build } from "vite"; +import { worklet } from "../../../../js/common/vite-plugin-worklet"; +import { launch } from "./harness"; +import type { Packet } from "./open-gop/main"; + +test("software video recovery trims orphan pictures after join and later gap", async () => { + const run = mkdtempSync(join(tmpdir(), "moq-open-gop-")); + const file = join(run, "fixture.ts"); + const command = (cmd: string[]) => { + const result = Bun.spawnSync(cmd, { stderr: "pipe", stdout: "pipe" }); + if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr)); + return new TextDecoder().decode(result.stdout); + }; + const browser = await launch(); + let server: ReturnType | undefined; + try { + command([ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x96:rate=30", + "-frames:v", + "72", + "-c:v", + "libx264", + "-threads", + "2", + "-profile:v", + "high", + "-level:v", + "3.1", + "-pix_fmt", + "yuv420p", + "-x264-params", + "keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3:b-adapt=0:repeat-headers=1", + "-f", + "mpegts", + file, + ]); + const probe = JSON.parse( + command([ + "ffprobe", + "-v", + "error", + "-select_streams", + "v", + "-show_packets", + "-show_data", + "-of", + "json", + file, + ]), + ) as { packets: Array<{ pts_time: string; flags: string; data: string }> }; + let group = -1; + const all: Packet[] = probe.packets.map((packet) => { + if (packet.flags.includes("K")) group++; + const hex = packet.data + .split("\n") + .filter((line) => line.includes(":")) + .map((line) => + line + .slice(line.indexOf(":") + 1) + .trimStart() + .split(" ")[0] + .replaceAll(" ", ""), + ) + .join(""); + return { + group, + timestamp: Math.round(Number(packet.pts_time) * 1_000_000), + data: Buffer.from(hex, "hex").toString("base64"), + }; + }); + const packets = all.filter((packet) => [1, 2, 4, 5].includes(packet.group)); + const floors = new Map(); + for (const packet of packets) if (!floors.has(packet.group)) floors.set(packet.group, packet.timestamp); + const orphan = (packet: Packet) => + [1, 4].includes(packet.group) && packet.timestamp < (floors.get(packet.group) ?? 0); + expect(packets.filter(orphan).length).toBeGreaterThan(0); + const expected = packets.filter((packet) => !orphan(packet)).map((packet) => packet.timestamp); + const dist = join(run, "dist"); + await build({ + configFile: false, + root: join(import.meta.dir, "open-gop"), + logLevel: "warn", + plugins: [worklet()], + build: { target: "esnext", outDir: dist, emptyOutDir: true }, + }); + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const path = new URL(req.url).pathname; + const file = Bun.file(join(dist, path === "/" ? "index.html" : path)); + return (await file.exists()) ? new Response(file) : new Response("not found", { status: 404 }); + }, + }); + const page = await browser.newPage(); + page.on("pageerror", (error) => console.error(error)); + page.on("console", (message) => { + if (message.type() === "error") console.error(message.text()); + }); + await page.goto(`http://127.0.0.1:${server.port}`); + await page.waitForFunction(() => typeof window.runRecovery === "function"); + const result = await page.evaluate((packets) => window.runRecovery(packets), packets); + expect(result.software).toBe(true); + expect(result.submitted).toEqual(expected); + expect(result.decoded).toBeGreaterThan(0); + } finally { + await browser.close(); + server?.stop(); + rmSync(run, { recursive: true, force: true }); + } +}, 60_000); diff --git a/test/interop/clients/js/open-gop/index.html b/test/interop/clients/js/open-gop/index.html new file mode 100644 index 00000000000..13bd888a0f0 --- /dev/null +++ b/test/interop/clients/js/open-gop/index.html @@ -0,0 +1 @@ +Open GOP recovery diff --git a/test/interop/clients/js/open-gop/main.ts b/test/interop/clients/js/open-gop/main.ts new file mode 100644 index 00000000000..955d9437c66 --- /dev/null +++ b/test/interop/clients/js/open-gop/main.ts @@ -0,0 +1,109 @@ +import * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import * as Moq from "@moq/net"; +import { Signal } from "@moq/signals"; +import type { Broadcast } from "../../../../../js/watch/src/broadcast"; +import { Sync } from "../../../../../js/watch/src/sync"; +import { Decoder } from "../../../../../js/watch/src/video/decoder"; +import { Source } from "../../../../../js/watch/src/video/source"; + +/** One real open-GOP access unit generated by ffmpeg. */ +export type Packet = { group: number; timestamp: number; data: string }; + +declare global { + interface Window { + runRecovery(packets: Packet[]): Promise<{ submitted: number[]; decoded: number; software: boolean }>; + } +} + +window.runRecovery = async (packets) => { + const Native = window.VideoDecoder; + const codecs: VideoDecoder[] = []; + const submitted: number[] = []; + let decoded = 0; + let software = true; + let failed: Error | undefined; + let ended!: () => void; + const inputDone = new Promise((resolve) => { + ended = resolve; + }); + const next = Container.Consumer.prototype.next; + Container.Consumer.prototype.next = async function () { + const item = await next.call(this); + if (!item) ended(); + return item; + }; + window.VideoDecoder = class extends Native { + constructor(init: VideoDecoderInit) { + super({ + output(frame) { + decoded++; + init.output(frame); + }, + error(error) { + failed = error; + ended(); + init.error(error); + }, + }); + codecs.push(this); + } + override configure(config: VideoDecoderConfig) { + const softwareConfig = { ...config, hardwareAcceleration: "prefer-software" as const }; + software &&= softwareConfig.hardwareAcceleration === "prefer-software"; + super.configure(softwareConfig); + } + override decode(chunk: EncodedVideoChunk) { + submitted.push(chunk.timestamp); + super.decode(chunk); + } + }; + const producer = new Moq.Broadcast.Producer(); + const track = producer.createTrack("video", { timescale: Moq.Time.Timescale.MICRO }); + let group: Moq.Group.Producer | undefined; + for (const packet of packets) { + if (group?.sequence !== packet.group) { + group?.close(); + group = new Moq.Group.Producer(packet.group); + track.writeGroup(group); + } + const timestamp = Moq.Time.Micro(packet.timestamp); + const payload = Uint8Array.from(atob(packet.data), (c) => c.charCodeAt(0)); + group.writeFrame({ + payload: Container.Legacy.encodeFrame(payload, timestamp), + timestamp: Moq.Time.Timestamp.fromMicros(timestamp), + }); + } + group?.close(); + track.close(); + const media = producer.consume(); + const config = Catalog.VideoConfigSchema.parse({ + codec: "avc1.64001f", + container: { kind: "legacy" }, + codedWidth: 160, + codedHeight: 96, + }); + const catalog = new Signal({ video: { renditions: { video: config } } }); + const broadcast = new Signal({ + out: { catalog, active: new Signal(media) }, + relativeBroadcast: () => media, + } as unknown as Broadcast); + const source = new Source({ broadcast, supported: async () => true }); + const sync = new Sync({ delay: Moq.Time.Milli(0), buffer: Moq.Time.Milli(10_000) }); + const decoder = new Decoder({ source, sync }); + try { + await inputDone; + if (failed) throw failed; + for (const codec of codecs) await codec.flush(); + if (failed) throw failed; + return { submitted, decoded, software }; + } finally { + decoder.close(); + source.close(); + sync.close(); + media.close(); + producer.close(); + window.VideoDecoder = Native; + Container.Consumer.prototype.next = next; + } +}; diff --git a/test/interop/clients/js/package.json b/test/interop/clients/js/package.json index 9a6aac0ce99..1f10005a24a 100644 --- a/test/interop/clients/js/package.json +++ b/test/interop/clients/js/package.json @@ -7,6 +7,7 @@ "test": "bun test --only-failures" }, "dependencies": { + "@moq/hang": "workspace:*", "@moq/net": "workspace:*", "@moq/publish": "workspace:*", "@moq/signals": "workspace:*", diff --git a/test/interop/clients/js/tsconfig.json b/test/interop/clients/js/tsconfig.json index 884d98a8dfe..a524cb1b466 100644 --- a/test/interop/clients/js/tsconfig.json +++ b/test/interop/clients/js/tsconfig.json @@ -8,6 +8,8 @@ "driver.ts", "harness.ts", "harness.browser.ts", + "open-gop.browser.ts", + "open-gop", "capture-channel-count.browser.ts", "buffering", "media.ts", diff --git a/test/justfile b/test/justfile index 9dc104843b0..e3db84a2b76 100644 --- a/test/justfile +++ b/test/justfile @@ -26,7 +26,7 @@ harness: cd .. && bun test test/interop/compat/resolve.test.ts cd .. && bun install --frozen-lockfile cd interop/clients/js && bunx playwright install chromium - cd interop/clients/js && bun test ./harness.browser.ts ./capture-channel-count.browser.ts ./src/contract.test.ts ./src/resume.test.ts + cd interop/clients/js && bun test ./harness.browser.ts ./capture-channel-count.browser.ts ./open-gop.browser.ts ./src/contract.test.ts ./src/resume.test.ts # Cross-language media interop test, built from this checkout. Stands up a # relay and runs the publisher x subscriber matrix. Default: rust only (fast @@ -128,7 +128,7 @@ drill-pr mode base="": # (skip the round-trip), `--strict` (fail on shape warnings), `--source file.ts` # (use a real capture), `--live` (grade PCR release timing and byte position off # the pipe instead, which a capture cannot carry; nightly runs this arm), -# `--open-gop` (an open-GOP clip whose leading pictures must survive), `--hrd` +# `--open-gop` (an open-GOP clip whose orphan leading pictures are trimmed), `--hrd` # (1080p video filling a broadcast-sized CPB), `--delay 2s` (the exporter's). See # ts/README.md. @@ -137,6 +137,7 @@ drill-pr mode base="": ts *args: python3 ./ts/fragment_pes.test.py python3 ./ts/pcr-timing.test.py + python3 ./ts/open-gop.test.py ./ts/run.sh "$@" # Table-placement grader: acquisition, interleave, phase, drift and missing-table controls. diff --git a/test/ts/README.md b/test/ts/README.md index 3ae16c10b81..90a535e8ac2 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -33,7 +33,7 @@ just test ts --strict # also fail on broadcast-shape warnings just test ts --with-eit # add a synthetic EPG first, report which SI survived just test ts --live # grade PCR release timing off the live pipe just test ts --pair # two exporters of one broadcast, grade table anchoring -just test ts --open-gop # open-GOP clip; its leading pictures must survive +just test ts --open-gop # open-GOP clip; orphan leading pictures are trimmed just test ts --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB just test ts --headroom # the same with four audio PIDs, muxed above the video's Rx just test ts --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak @@ -445,12 +445,18 @@ Controls run in PR CI; the real pair runs nightly and on manual TS validation. ## Open GOP (`open-gop.py`) +`just test harness` also runs the actual JS container and player against generated +open-GOP access units with Chromium's software WebCodecs decoder forced. It +checks a join, continuous GOPs, and a later delivery hole; disabling the trim +reproduces `EncodingError` instead of silently accepting orphan references. + Broadcast contribution encoders commonly send open GOP: after the first IDR, each keyframe is a non-IDR I picture flagged by a recovery-point SEI, and the B pictures that follow it in decode order but are presented before it (its leading pictures) reference the previous GOP. Only a viewer that already holds that GOP can decode -them. Dropping them is right at a tune-in, which the transport cannot see, so the -round-trip has to hand every one of them on, in decode order. +them. The consumer trims the recovery group's orphan leading pictures at a join +or after a delivery gap. Continuous groups keep every picture in decode order. +A declared end is a fresh encoder boundary, not a delivery gap. `--open-gop` swaps the generated clip for an x264 `open-gop=1` encode whose 24-frame GOP and fixed B placement put three leading pictures behind every recovery point, @@ -469,8 +475,8 @@ rebase. | Check | Severity | What it verifies | |---|---|---| | `fixture` | hard | the source has at least two non-IDR recovery points with leading pictures | -| `decode-order` | hard | the capture is one contiguous run of the source's access units from a random-access point, with DTS strictly increasing | -| `leading-pictures` | hard | every leading picture in that run is there and keeps the presentation offset the source gave it | +| `decode-order` | hard | each delivered segment starts at a random-access point, omits only its orphan leading pictures, and keeps DTS strictly increasing | +| `leading-pictures` | hard | every continuous leading picture is kept with its source presentation offset; recovery-group orphans are absent | | `random-access-indicator` | shape | the exporter flags exactly the random-access access units | | `recovery-point-sei` | shape | each recovery point keeps its SEI | | `dts-before-pts` | shape | no access unit is stamped to decode after it presents | diff --git a/test/ts/open-gop.py b/test/ts/open-gop.py index 65e921237a7..404f41ed137 100755 --- a/test/ts/open-gop.py +++ b/test/ts/open-gop.py @@ -7,16 +7,16 @@ pictures that follow it in decode order but are presented before it, its leading pictures, reference the previous GOP, so only a viewer that already holds that GOP can decode them. A viewer playing continuously needs every one of them, in decode order. -Dropping them is right only at a tune-in, which the transport cannot see, so the -round-trip has to hand them all on. +A join or delivery gap trims that recovery group's orphan leading pictures. +Every continuous group keeps all of them. Access units are matched by their slice data. That makes the comparison indifferent to the parameter sets and delimiters the round-trip may re-insert and to the timestamp -rebase, and exact about order: the capture must be one contiguous run of the source's -access units, starting at a random-access point. +rebase, and exact about order: each delivered segment starts at a random-access +point and omits only that recovery group's leading pictures. Hard checks: the source is open GOP with leading pictures (else nothing here is tested), -the capture is that contiguous run with DTS strictly increasing, and every captured +the capture follows that delivery rule with DTS strictly increasing, and every captured picture keeps the presentation offset the source gave it, which is what makes a leading picture leading. Shape checks, reported and failing only under `--strict`: whether each exported random-access point carries the random_access_indicator and keeps its @@ -199,6 +199,30 @@ def leading(units): return out +def align(source, captured, leads): + """Match every AU, omitting leading pictures only at a join or a later RAP after lost media.""" + for start in [i for i, unit in enumerate(source) if captured and unit.key == captured[0].key and is_rap(unit)]: + position = start + omitted = set(leads.get(start, [])) + indices = [] + recoveries = {0: start} + for index, unit in enumerate(captured): + while position in omitted: + position += 1 + if position >= len(source) or source[position].key != unit.key: + resume = next((i for i in range(position + 1, len(source)) if source[i].key == unit.key and is_rap(source[i])), None) + if resume is None: + break + position = resume + omitted = set(leads.get(resume, [])) + recoveries[index] = resume + indices.append(position) + position += 1 + else: + return indices, recoveries + return None + + def main(): ap = argparse.ArgumentParser(description="Grade open-GOP leading pictures across a TS round-trip.") ap.add_argument("source", help="TS that was published") @@ -223,76 +247,53 @@ def check(name, severity, ok, detail): f"{sum(len(src_leading[i]) for i in open_rps)} leading picture(s)", ) - # Where the capture starts in the source. Frames can repeat in principle, so take the - # first candidate the whole run agrees with rather than the first match on one key. - starts = [i for i, unit in enumerate(src) if cap and unit.key == cap[0].key] - start = next((i for i in starts if [u.key for u in src[i : i + len(cap)]] == [u.key for u in cap]), None) - if start is None: - detail = "the capture's first AU is not in the source" - if starts: - s, prefix = starts[0], 0 - while prefix < len(cap) and s + prefix < len(src) and src[s + prefix].key == cap[prefix].key: - prefix += 1 - want = s + prefix - kind = ( - "a leading picture" - if any(want in lead for lead in src_leading.values()) - else "a random-access point" - if want < len(src) and is_rap(src[want]) - else "a picture" - ) - detail = ( - f"capture AU {prefix} of {len(cap)} departs from the source; " - f"source AU {want} ({kind}) is missing or moved" - ) - check("decode-order", HARD, False, detail) + matched = align(src, cap, src_leading) + if matched is None: + check("decode-order", HARD, False, "capture differs from source beyond orphan leading pictures at delivery breaks") else: + indices, recoveries = matched backwards = [i for i in range(1, len(cap)) if delta(cap[i].dts, cap[i - 1].dts) <= 0] - ok = is_rap(src[start]) and not backwards - detail = f"{len(cap)} AUs, source AUs {start}..{start + len(cap) - 1} in order" - if not is_rap(src[start]): - detail += f"; but starts at source AU {start}, which is not a random-access point" + detail = f"{len(cap)} AUs in source order, {len(recoveries)} delivery recovery group(s)" if backwards: detail += f"; DTS does not advance at {len(backwards)} AU(s), first at {backwards[0]}" - check("decode-order", HARD, ok, detail) - - if start is not None: - # Leading pictures are defined by presentation offset, so check the offset of every - # captured picture rather than only counting the ones that still come out leading. - moved = [ - i - for i in range(len(cap)) - if abs(delta(cap[i].pts, cap[0].pts) - delta(src[start + i].pts, src[start].pts)) > PTS_TOLERANCE - ] - # Keyed on the source's random-access points, so an exporter that strips the SEI - # fails the SEI check below rather than hiding its leading pictures from this one. + check("decode-order", HARD, not backwards, detail) + + # A delivery break may rebase the schedule, but every retained picture + # keeps its source PTS offset within that delivered segment. + moved = [] + anchor = 0 + for i, original in enumerate(indices): + if i in recoveries: + anchor = i + if abs(delta(cap[i].pts, cap[anchor].pts) - delta(src[original].pts, src[indices[anchor]].pts)) > PTS_TOLERANCE: + moved.append(i) + positions = {original: i for i, original in enumerate(indices)} + recovery_groups = set(recoveries.values()) kept = { - i - start: [j - start for j in lead if j < start + len(cap)] - for i, lead in src_leading.items() - if lead and start <= i < start + len(cap) + positions[rap]: [positions[j] for j in lead if j in positions] + for rap, lead in src_leading.items() + if rap in positions and rap not in recovery_groups } - # A capture cut short can end inside a GOP; only a leading picture it reached is owed. - owed = sum(len(lead) for lead in kept.values()) + owed = sum(sum(j <= indices[-1] for j in src_leading[indices[i]]) for i in kept) found = sum(1 for i, lead in kept.items() for j in lead if delta(cap[j].pts, cap[i].pts) < 0) - # Only a recovery point with a captured leading picture tests anything. - crossed = len([i for i, lead in kept.items() if lead and not src[start + i].idr]) + crossed = sum(bool(lead) and not src[indices[i]].idr for i, lead in kept.items()) ok = not moved and found == owed and crossed >= MIN_RECOVERY_POINTS - detail = f"{found}/{owed} leading picture(s) across {crossed} recovery point(s)" + detail = f"{found}/{owed} continuous leading picture(s) across {crossed} recovery point(s); {len(recoveries)} orphan sets trimmed" if moved: - detail += f"; {len(moved)} AU(s) presented away from the source's offset, first at {moved[0]}" + detail += f"; {len(moved)} AU(s) presented away from the source offset, first at {moved[0]}" if crossed < MIN_RECOVERY_POINTS: - detail += f"; the capture reaches leading pictures at fewer than {MIN_RECOVERY_POINTS} recovery points" + detail += f"; fewer than {MIN_RECOVERY_POINTS} continuous recovery points" check("leading-pictures", HARD, ok, detail) - raps = [i for i in range(len(cap)) if is_rap(src[start + i])] + raps = [i for i in range(len(cap)) if is_rap(src[indices[i]])] flagged = [i for i in raps if cap[i].rai] - stray = [i for i in range(len(cap)) if cap[i].rai and not is_rap(src[start + i])] + stray = [i for i in range(len(cap)) if cap[i].rai and not is_rap(src[indices[i]])] detail = f"{len(flagged)}/{len(raps)} random-access AU(s) flagged" if stray: detail += f"; {len(stray)} non-random-access AU(s) flagged too" check("random-access-indicator", SHAPE, len(flagged) == len(raps) and not stray, detail) - rps = [i for i in raps if src[start + i].recovery] + rps = [i for i in raps if src[indices[i]].recovery] survived = [i for i in rps if cap[i].recovery] check("recovery-point-sei", SHAPE, len(survived) == len(rps), f"{len(survived)}/{len(rps)} kept the SEI") diff --git a/test/ts/open-gop.test.py b/test/ts/open-gop.test.py new file mode 100644 index 00000000000..39801a2587f --- /dev/null +++ b/test/ts/open-gop.test.py @@ -0,0 +1,33 @@ +import importlib.util +import pathlib +import sys +import unittest + +spec = importlib.util.spec_from_file_location("open_gop", pathlib.Path(__file__).with_name("open-gop.py")) +gop = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = gop +spec.loader.exec_module(gop) + +class Recovery(unittest.TestCase): + def setUp(self): + self.source = [gop.AccessUnit(bytes([group * 5 + offset]), group * 1000 + pts, group * 1000 + offset * 100, + offset == 0, False, offset == 0) + for group in range(4) for offset, pts in enumerate([1000, 800, 900, 1100, 1200])] + self.leads = gop.leading(self.source) + + def test_join_and_later_gap_trim_only_their_orphans(self): + indices = [0, 3, 4, 5, 6, 7, 8, 9, 15, 18, 19] + self.assertEqual(gop.align(self.source, [self.source[i] for i in indices], self.leads), + (indices, {0: 0, 8: 15})) + + def test_orphan_leading_picture_is_refused(self): + self.assertIsNone(gop.align(self.source, self.source, self.leads)) + indices = [0, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 18, 19] + self.assertIsNone(gop.align(self.source, [self.source[i] for i in indices], self.leads)) + + def test_continuous_leading_picture_and_other_delta_are_owed(self): + for indices in [[0, 3, 4, 5, 7, 8, 9], [0, 4, 5, 6, 7, 8, 9]]: + self.assertIsNone(gop.align(self.source, [self.source[i] for i in indices], self.leads)) + +if __name__ == "__main__": + unittest.main() diff --git a/test/ts/run.sh b/test/ts/run.sh index 8756789e816..807fb3c1285 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -18,7 +18,7 @@ # ./run.sh --with-eit # add a synthetic EPG first, report which SI survived # ./run.sh --live # grade PCR release timing off the live pipe # ./run.sh --pair # two exporters of one broadcast, grade table anchoring -# ./run.sh --open-gop # open-GOP clip; its leading pictures must survive +# ./run.sh --open-gop # open-GOP clip; orphan leading pictures are trimmed # ./run.sh --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB # ./run.sh --headroom # the same with four audio PIDs, muxed above the video's Rx # ./run.sh --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak From 45d6b39ecfe38d19be9c8d6dab3256626f9c3e3c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:52:33 -0700 Subject: [PATCH 3/4] chore: complete leading-picture recovery quest Reconciled merged TS timing changes on61496fc18. Required just check, all-language interop, software-browser regression and disabled-filter control, and strict generated TS recovery/content/PCR grading passed. No public API or wire format changes. Co-Authored-By: GPT-6 --- quest/m1/README.md | 1 - quest/m1/open-gop-leading-pictures.md | 81 ----------------------- quest/m1/ts-reliability/recovery.md | 1 - quest/m2/intra-refresh/README.md | 4 -- quest/m2/intra-refresh/consumer-warmup.md | 1 - 5 files changed, 88 deletions(-) delete mode 100644 quest/m1/open-gop-leading-pictures.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 18f6a258a58..a070a8b230a 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -146,7 +146,6 @@ above them. - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the supported T-STD buffer models, starting with a fixed `--delay` -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [CLI stdin shutdown on release](/quest/m1/cli-stdin-release.md) - backport only the runtime-teardown fix so a stopped CLI exits while its stdin writer stays open - [Draft 14-16 updates](/quest/m1/ietf-legacy-updates.md) - Rust and JS apply and answer moq-transport 14-16 request updates without ending or leaking the request - [Tail arrivals](/quest/m1/tail-arrivals.md) - a track judges its pending tail from recorded arrivals, not a cache scan, and errors a reader whose tail ends short diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md deleted file mode 100644 index 8b5b347e8bb..00000000000 --- a/quest/m1/open-gop-leading-pictures.md +++ /dev/null @@ -1,81 +0,0 @@ -# [M] Open-GOP recovery drops orphaned leading pictures after each delivery break - -## Goal - -A viewer joining or resuming an open-GOP broadcast after a gap never hands the decoder a leading picture whose references it -does not have. Continuous groups keep every frame. The rule is the same for H.264 -recovery-point keyframes with `recovery_frame_cnt = 0` (the broadcast -contribution case, and what the reference capture carries) and H.265 CRA -pictures. Gradual recovery (`recovery_frame_cnt > 0`) is out of scope: unsafe pictures -there sit at or after the keyframe timestamp, and the intra-refresh epic covers -them: players use the [dual decoder](/quest/m2/intra-refresh/dual-decoder.md) -and the transcoder decodes history per -[consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md). - -## Plan - -Leading pictures are decoded after the keyframe and presented before it, so -they are exactly the frames of a group whose timestamp is below the group's -keyframe timestamp. That makes them identifiable from the container timestamps -alone, with no POC parsing and no change to the splitter, which is why the -drop belongs on the consumer: ingest cannot know whether a given viewer has the -previous GOP, and dropping at ingest would degrade continuous playback for -everyone. - -Measured through `` in Chromium 153 on macOS, on the clip -`just test ts --open-gop` generates (three leading pictures per recovery -point) stretched to 120 s. Continuous playback decoded every leading picture -on both decoder paths, with no errors, and the two paths' frames were -identical. At a cold join the paths differ. VideoToolbox (the default there) -outputs nothing for the orphaned leading pictures and raises no error, and -every frame it does output matches the continuous decode, so a viewer merely -starts at the keyframe. The software decoder (`prefer-software`, the path a -browser without hardware H.264 takes; Linux was not measured) raises -`EncodingError` in every join, and the watch then closes its decoder, so video -stops. The same joins with the leading pictures removed from the stream decode -cleanly in software, so the error is the leading pictures, not the non-IDR -keyframe. A latency skip ("skipping slow group") orphans the next group's -leading pictures in the same way. The trim therefore has to happen before -decode, and the JS test should drive the software decoder. - -- In the JS consumer (`js/hang/src/container/consumer.ts` forces the first - sample of a group to `keyframe`, and `js/watch/src/video/decoder.ts` submits - it as `"key"`): for the first group read and the first group after a gap - item, skip delta frames stamped before that group's keyframe. `next()` - yields `{ kind: "gap" }` after a latency skip, eviction, or hole, but not - before the first frame of a subscribe; a declared end is not a gap, since a fresh encoder has no earlier GOP - to reference. A latency skip does not flush the decoder, and a viewer that - skipped into a later GOP lacks its references just like a cold join. - Every group without a gap before it is passed through untouched. -- The same rule in the Rust decode path (`moq-video` decode consumers), so - native playback and the transcoder tune in the same way, keyed on the - first frame read and the container read's `Read::Gap` item. -- The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay - export still sends a join's orphaned leading pictures, so 3 of 500 - frames on the open-GOP fixture decode after they present - (`dts-before-pts`). Trim them at tune-in from the same signal, on the - first group after each gap, and make - `just test ts --open-gop` pass under - `--strict`. Decided in the 2026-10-05 audit: the grader changes with it. - `test/ts/open-gop.py`'s decode-order check (a contiguous run of source - access units) and leading-pictures check, and the `test/ts/README.md` text - that says the round-trip must hand every leading picture on, both expect - every picture today; rewrite them to expect orphaned leading pictures - trimmed after each delivery break and every continuous group's kept. - The 2026-10-11 audit applies the same recovery rule to TS as to the - playback paths: the previous first-group-only wording missed later skips. - Rejected: dropping - this bullet and keeping the grader. -- Tests: a synthetic group with a keyframe followed by two earlier-stamped - deltas is trimmed on the first group and kept on the second; and a viewer - that plays continuously, then latency-skips into a later open GOP, has that - group's leading pictures trimmed too, so an implementation that only trims - the initial group fails. Both cases in both languages. - -Decided 2026-10-12: both container reads yield the gap item, so nothing -here waits on another quest. Decided 2026-10-13 (#5311): a join is not a gap, -so the first frame read is its own trigger. - -## Related - -- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - the `recovery_frame_cnt > 0` case this rule does not cover diff --git a/quest/m1/ts-reliability/recovery.md b/quest/m1/ts-reliability/recovery.md index edce1671009..10643afca92 100644 --- a/quest/m1/ts-reliability/recovery.md +++ b/quest/m1/ts-reliability/recovery.md @@ -43,5 +43,4 @@ and measured release timing as separate checks. ## Related - [Automated validation](/test/ts/README.md) - shared graders, profiles and known-bad controls -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims after a gap - [Gradual recovery](/quest/m2/intra-refresh/README.md) - recovery counts and per-group warmup diff --git a/quest/m2/intra-refresh/README.md b/quest/m2/intra-refresh/README.md index 8e36de2316a..bfccac1e677 100644 --- a/quest/m2/intra-refresh/README.md +++ b/quest/m2/intra-refresh/README.md @@ -56,7 +56,3 @@ make these presentation guarantees true even when sweep lengths change. - [H.264 import](/quest/m2/intra-refresh/h264-import.md) - the splitter keeps `recovery_frame_cnt` and import publishes `warmup` from it - [H.265 import](/quest/m2/intra-refresh/h265-import.md) - the splitter reads the recovery-point SEI so an HEVC intra-refresh stream forms groups and publishes `warmup` - [Export sync flags](/quest/m2/intra-refresh/export-sync-flags.md) - fmp4, MKV, and HLS stop advertising a refresh group start as a sync sample - -## Related - -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - frames stamped before the group's keyframe are the other tune-in trim diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index cd6bd15de6f..39ffe40acf2 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -59,5 +59,4 @@ The fix is history rather than a withhold rule: ## Related -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims frames stamped before the keyframe, the other tune-in case - [Native dual decoder](/quest/m2/intra-refresh/native-dual-decoder.md) - the player path, which shows stripes instead From 940d43446a86061f3241015cc9b1f1f6fe3b5548 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:41:04 -0700 Subject: [PATCH 4/4] fix(video): trim leading pictures after local drops too The native player's encoded buffer and the TS export's jitter buffer drop frames themselves (age eviction, missed deadlines, acquisition), resuming at a keyframe whose leading pictures then reached the decoder or the stream without their references. The play buffer now owns the join, gap and eviction floor and never buffers a picture below it; the jitter buffer drops a sync frame's leading frames when the track resumed at it after a drop. Also adapts the merged open-GOP harness and test to @moq/clock time units. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 2 +- js/watch/src/video/recovery.ts | 4 +- rs/moq-cli/src/play/buffer.rs | 74 +++++++++++++++++++--- rs/moq-cli/src/play/video.rs | 11 ---- rs/moq-mux/benches/jitter.rs | 1 + rs/moq-mux/src/container/flv/export.rs | 3 + rs/moq-mux/src/container/mkv/export.rs | 1 + rs/moq-mux/src/container/ts/export.rs | 11 +++- rs/moq-mux/src/container/ts/export_test.rs | 72 +++++++++++++++++++++ rs/moq-mux/src/container/ts/release.rs | 1 + rs/moq-mux/src/container/ts/stats.rs | 2 +- rs/moq-mux/src/jitter.rs | 63 +++++++++++++++++- test/interop/clients/js/open-gop/main.ts | 5 +- 13 files changed, 221 insertions(+), 29 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index e5f74cde674..56fb4e46719 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -717,7 +717,7 @@ for (const kind of ["legacy", "cmaf"] as const) { sample(13, 3800, false), sample(13, 3900, false), sample(13, 4100, false), - { kind: "end", group: 14, end: Time.Micro(4200) }, + { kind: "end", group: 14, end: Micro(4200) }, sample(15, 5000), sample(15, 4800, false), sample(15, 4900, false), diff --git a/js/watch/src/video/recovery.ts b/js/watch/src/video/recovery.ts index 0ddc7418b58..0297cb8f034 100644 --- a/js/watch/src/video/recovery.ts +++ b/js/watch/src/video/recovery.ts @@ -1,10 +1,10 @@ +import type { Micro } from "@moq/clock"; import type { Container } from "@moq/hang"; -import type { Time } from "@moq/net"; /** Refuse leading pictures whose previous GOP was not delivered to this decoder. @internal */ export class Recovery { #needed = true; - #floor?: Time.Micro; + #floor?: Micro; /** Arm recovery after media was lost, preserving continuous groups and declared ends. */ gap(): void { diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index d03bf3590c4..9c458574873 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -6,7 +6,6 @@ use std::collections::VecDeque; use std::time::Duration; /// Encoded access units stay in decode order, even when their PTS reorders. -#[derive(Default)] pub(super) struct Buffer { pub frames: VecDeque, pub bytes: usize, @@ -14,11 +13,29 @@ pub(super) struct Buffer { // Monotone minimum: PTS can reorder behind the front of decode order. oldest: VecDeque, pub generation: u64, + /// The keyframe the buffer last resumed at. A picture presenting before it references + /// media the buffer never had, so it is neither decoded nor shown. pub floor: Option, waiting_keyframe: bool, pub ended: bool, } +impl Default for Buffer { + fn default() -> Self { + Self { + frames: VecDeque::new(), + bytes: 0, + edge: None, + oldest: VecDeque::new(), + generation: 0, + floor: None, + // A join resumes at its first keyframe, like any other break. + waiting_keyframe: true, + ended: false, + } + } +} + impl Buffer { pub fn clear(&mut self) { self.frames.clear(); @@ -53,15 +70,10 @@ impl Buffer { } self.waiting_keyframe = false; } - if self.waiting_keyframe { + if self.waiting_keyframe || self.below_floor(&frame) { return; } - while self.oldest.back().is_some_and(|at| *at > frame.timestamp) { - self.oldest.pop_back(); - } - self.oldest.push_back(frame.timestamp); - self.bytes += frame.payload.len(); - self.frames.push_back(frame); + self.index(frame); let edge = self.edge.unwrap(); let stale = |oldest: &moq_net::Timestamp| edge.as_micros().saturating_sub(oldest.as_micros()) > max_delay.as_micros(); @@ -75,8 +87,30 @@ impl Buffer { self.generation += 1; self.waiting_keyframe = self.frames.is_empty(); self.floor = self.frames.front().map(|frame| frame.timestamp); + // The retained keyframe's leading pictures referenced what was just dropped. + let mut frames = std::mem::take(&mut self.frames); + frames.retain(|frame| !self.below_floor(frame)); + self.bytes = 0; + self.oldest.clear(); + for frame in frames { + self.index(frame); + } } } + + /// Append a frame, tracking its bytes and the earliest presentation time. + fn index(&mut self, frame: Frame) { + while self.oldest.back().is_some_and(|at| *at > frame.timestamp) { + self.oldest.pop_back(); + } + self.oldest.push_back(frame.timestamp); + self.bytes += frame.payload.len(); + self.frames.push_back(frame); + } + + fn below_floor(&self, frame: &Frame) -> bool { + self.floor.is_some_and(|floor| frame.timestamp < floor) + } } #[cfg(test)] @@ -113,6 +147,30 @@ mod tests { assert_eq!(buffer.bytes, 7); } + #[test] + fn eviction_drops_the_retained_keyframes_leading_pictures() { + let mut buffer = Buffer::default(); + let age = Duration::from_millis(100); + for (ms, keyframe) in [ + (0, true), + (50, false), + (100, true), + (80, false), + (90, false), + (110, false), + ] { + buffer.push(frame(ms, keyframe), age); + } + buffer.push(frame(95, false), age); + let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis()).collect(); + assert_eq!(kept, [100, 110], "80, 90 and 95 reference the evicted group"); + assert_eq!(buffer.bytes, 14); + assert_eq!(buffer.oldest().unwrap().as_millis(), 100); + buffer.push(frame(200, true), age); + buffer.push(frame(180, false), age); + assert_eq!(buffer.frames.len(), 4, "a continuous group keeps its leading pictures"); + } + #[test] fn reordered_timestamps_keep_decode_order_and_the_newest_edge() { let mut buffer = Buffer::default(); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 9f0f58bf08e..7827dad4a61 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -76,22 +76,12 @@ impl Video { // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out. let mut end = None; - // Leading pictures need the missing preceding GOP, unlike a declared encoder end. - let mut recover = true; - let mut leading_floor = None; loop { let read = track.read().await?; let mut buffer = buffer.lock().unwrap(); let before = buffer.generation; let ended = match read { Some(Read::Frame { group, frame }) => { - if frame.keyframe { - leading_floor = recover.then_some(frame.timestamp); - recover = false; - } - if leading_floor.is_some_and(|floor| frame.timestamp < floor) { - continue; - } if end.take_if(|end| *end != group).is_some() { buffer.clear(); self.presentation.lock().unwrap().video_restarted(); @@ -109,7 +99,6 @@ impl Video { } // Applied in delivery order, so the restart lands before the frame after it. Some(Read::Gap { .. }) => { - recover = true; end = None; buffer.clear(); self.presentation.lock().unwrap().video_restarted(); diff --git a/rs/moq-mux/benches/jitter.rs b/rs/moq-mux/benches/jitter.rs index 66ad1492baa..ad5ba32f003 100644 --- a/rs/moq-mux/benches/jitter.rs +++ b/rs/moq-mux/benches/jitter.rs @@ -28,6 +28,7 @@ async fn steer(tracks: u16, floors: u64) -> u64 { restart: 0, skip: 0, sync: true, + leading: false, item: (), }; buffer.push(track, arrival).unwrap(); diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 04542ec6249..3ffefcd3b11 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -363,6 +363,7 @@ impl Export { restart: track.source.markers(), skip: track.source.skips(), sync: kind == 1 || frame.keyframe, + leading: false, item: frame, } }; @@ -380,6 +381,7 @@ impl Export { restart, skip, sync, + leading, item, .. } = arrival; @@ -392,6 +394,7 @@ impl Export { restart, skip, sync, + leading, item: (item, timestamp), }, )? == crate::jitter::Push::Late diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 36176b1e8d0..963b294613b 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -382,6 +382,7 @@ impl Export { restart: track.source.markers(), skip: track.source.skips(), sync: track.kind == TrackKind::Audio || frame.keyframe, + leading: false, item: frame, } }; diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 9830cf37028..afa1d7a998e 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -258,6 +258,8 @@ struct Track { recovery_gap: Option, /// Leading pictures below the recovery keyframe have unavailable references. leading_floor: Option, + /// The PTS of the last keyframe queued, which leading pictures present before. + keyframe: Option, /// What the catalog and the SPS declare about the video's reordering. timing: Timing, /// The receiver's buffers for an audio or verbatim track; video's come from [`Timing`]. @@ -294,6 +296,7 @@ impl Track { clock: DecodeClock::default(), recovery_gap: None, leading_floor: None, + keyframe: None, timing: Timing::default(), buffer: None, ac3_carry: None, @@ -613,6 +616,11 @@ impl Track { .and_then(|ticks| Timestamp::from_scale(ticks, 90_000).ok()) .unwrap_or(frame.timestamp); let (key, pes) = self.release_key(decode, jitter.delay(), &frame)?; + let video = matches!(self.kind, Kind::Video(_)); + let leading = video && !frame.keyframe && self.keyframe.is_some_and(|keyframe| frame.timestamp < keyframe); + if video && frame.keyframe { + self.keyframe = Some(frame.timestamp); + } // Replaying anchors on the first frame, and the tracks arrive together rather than each // sent ahead by its send-ahead: judged as arriving that much earlier, as a live source // sends it, every track's first frames are on the clock whichever anchors it. @@ -626,7 +634,8 @@ impl Track { decode: key, restart, skip, - sync: frame.keyframe || !matches!(self.kind, Kind::Video(_)), + sync: frame.keyframe || !video, + leading, item: Queued { frame, decode, diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index a0789cd339e..1ac00ebca6b 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1034,6 +1034,78 @@ async fn export_trims_leading_pictures_after_each_delivery_break() { ); } +/// A frame dropped for missing its deadline takes the leading pictures of the keyframe the +/// track resumes at with it, though nothing was lost upstream. +#[tokio::test(start_paused = true)] +async fn export_trims_leading_pictures_after_a_late_drop() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + catalog.modify().unwrap().video.renditions.insert( + "video".into(), + VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: true, + }), + ); + let mut video = Producer::new(track, HangContainer::Legacy(Kind::Video)); + let mut export = open_export(crate::source::announced(&consumer).await) + .await + .unwrap() + .with_delay(Duration::from_millis(500)); + let start = tokio::time::Instant::now(); + let mut out = Vec::new(); + // (arrival, PTS) in milliseconds, in decode order. The second group acquires the clock. + // Its 160 ms picture stalls past its deadline, before the next group exists for the + // source to skip to. That group arrives on time with its 1920 and 1960 ms leading pictures. + for (at, pts, keyframe) in [ + (0, 0, true), + (40, 40, false), + (80, 80, true), + (120, 120, false), + (1000, 160, false), + (2000, 2000, true), + (2000, 1920, false), + (2000, 1960, false), + (2040, 2040, false), + (2080, 2080, true), + (2080, 2060, false), + (2120, 2120, false), + ] { + tokio::time::sleep_until(start + Duration::from_millis(at)).await; + // Finding the source empty dates the next frame's arrival. + out.extend(poll_frames(&mut export)); + video + .write(Frame { + timestamp: Timestamp::from_millis(pts).unwrap(), + duration: None, + keyframe, + payload: if keyframe { + annexb(&[SPS, PPS, &[0x65, 0xab, 0xcd]]) + } else { + annexb(&[&[0x41, 0xab, 0xcd]]) + }, + }) + .unwrap(); + out.extend(poll_frames(&mut export)); + } + video.finish().unwrap(); + out.extend(drain_frames(&mut export).await); + let bytes: Vec = out.iter().flat_map(|frame| frame.payload.iter().copied()).collect(); + let (pts, _) = collect_pes_pts(&bytes); + let expected: Vec<_> = [0, 40, 80, 120, 2000, 2040, 2080, 2060, 2120] + .into_iter() + .map(|ms| ms * 90) + .collect(); + assert_eq!(pts, expected, "only the resumed keyframe's leading pictures go"); + assert_eq!(export.dropped(), 3, "the late picture and the two it orphaned"); +} + /// `(PTS, DTS)` of every video PES start, in transport (decode) order. Read raw, so a capture /// that starts before its PMT still counts. fn video_pes_timing(ts: &[u8]) -> Vec<(u64, u64)> { diff --git a/rs/moq-mux/src/container/ts/release.rs b/rs/moq-mux/src/container/ts/release.rs index 611b9360fdb..b1c2c65295e 100644 --- a/rs/moq-mux/src/container/ts/release.rs +++ b/rs/moq-mux/src/container/ts/release.rs @@ -236,6 +236,7 @@ impl Release { // The output opens on a group, at a random access point; from then on every // object goes out on its own, so one dropped late is a gap of its packets only. sync: frame.keyframe || self.released, + leading: false, item: frame, }; if self.jitter.push((), arrival)? == Push::Late { diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index baf84a40af8..b2fdb5d5d4f 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -175,7 +175,7 @@ pub struct Export { /// One row per elementary stream PID the exporter writes, ordered by PID. pub streams: BTreeMap, /// Frames dropped for arriving after their deadline, and the frames a video track then - /// dropped waiting for its next keyframe. + /// dropped waiting for its next keyframe or leading the keyframe it resumed at. pub dropped: u64, /// How fast the source's clock runs against ours, in parts per million (positive runs /// fast), as last measured. `None` until there is a measurement. diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index a11c6deb3d2..107d5bf5160 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -26,6 +26,9 @@ pub(crate) struct Arrival { pub skip: u64, /// Whether the frame decodes without the track's earlier frames. pub sync: bool, + /// Whether it presents before the track's last sync frame, so it also references frames + /// before that one: an open GOP's leading picture. + pub leading: bool, pub item: T, } @@ -44,7 +47,7 @@ pub(crate) enum Push { Queued, /// It missed its deadline and was dropped; the track resumes at its next sync frame. Late, - /// It was dropped because the track is waiting for a sync frame. + /// It was dropped because the track is waiting for a sync frame, or leads the one it resumed at. Waiting, } @@ -305,6 +308,8 @@ struct Track { generation: u64, /// Frames are dropped until the next sync frame, and why. waiting: Option, + /// Why the track resumed at its last sync frame, whose leading frames are dropped too. + resumed: Option, } /// Why a track waits for a sync frame. @@ -528,6 +533,7 @@ impl Buffer { skip: arrival.skip, generation, waiting: None, + resumed: None, }); track.restart = arrival.restart; track.skip = arrival.skip; @@ -538,7 +544,11 @@ impl Buffer { Push::Waiting } _ if track.waiting.is_some() && !arrival.sync => Push::Waiting, + _ if track.resumed.is_some() && arrival.leading => Push::Waiting, Some(deadline) if arrival.arrived <= deadline || self.delay.is_zero() => { + if arrival.sync { + track.resumed = track.waiting; + } track.waiting = None; track .queue @@ -552,7 +562,7 @@ impl Buffer { Push::Late } }; - if push != Push::Queued && track.waiting == Some(Wait::Late) { + if push != Push::Queued && track.waiting.or(track.resumed) == Some(Wait::Late) { self.dropped += 1; } push @@ -862,7 +872,8 @@ impl Buffer { self.acquire.is_none() && self.tracks.values().all(|track| track.queue.is_empty()) } - /// How many frames were dropped, late or waiting for a sync frame after one was. + /// How many frames were dropped, late or lost with one: waiting for a sync frame after it, + /// or leading the sync frame the track resumed at. pub fn dropped(&self) -> u64 { self.dropped } @@ -943,6 +954,7 @@ mod tests { restart: 0, skip: 0, sync: true, + leading: false, item, } } @@ -1056,6 +1068,49 @@ mod tests { assert_eq!(due(&mut buffer), ["v0", "a0", "a180", "v240"]); } + /// The sync frame a track resumes at after a drop loses its leading frames too, since + /// they reference the dropped ones. The next sync frame's leading frames are kept. + #[tokio::test(start_paused = true)] + async fn a_resumed_sync_frame_drops_its_leading_frames() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY).replay(); + let video = |arrived, decode, sync, leading, item| Arrival { + sync, + leading, + ..arrival(arrived, decode, item) + }; + buffer.push(1, video(start, 0, true, false, "v0")).unwrap(); + let late = start + Duration::from_millis(200); + assert_eq!( + buffer.push(1, video(late, 40, false, false, "v40")).unwrap(), + Push::Late + ); + assert_eq!( + buffer.push(1, video(late, 200, true, false, "v200")).unwrap(), + Push::Queued + ); + assert_eq!( + buffer.push(1, video(late, 240, false, true, "v240")).unwrap(), + Push::Waiting + ); + assert_eq!( + buffer.push(1, video(late, 280, false, false, "v280")).unwrap(), + Push::Queued + ); + assert_eq!( + buffer.push(1, video(late, 320, true, false, "v320")).unwrap(), + Push::Queued + ); + assert_eq!( + buffer.push(1, video(late, 360, false, true, "v360")).unwrap(), + Push::Queued + ); + assert_eq!(buffer.dropped(), 2, "the leading frame was lost with the late one"); + + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(due(&mut buffer), ["v0", "v200", "v280", "v320", "v360"]); + } + /// A restart before anything went out keeps the clock, so the tracks stay on one. #[tokio::test(start_paused = true)] async fn a_skip_before_the_first_release_keeps_one_clock() { @@ -1377,6 +1432,7 @@ mod tests { restart: 0, skip: 0, sync: true, + leading: false, item: "v", }; let held = buffer @@ -1551,6 +1607,7 @@ mod tests { restart: 0, skip: 0, sync: true, + leading: false, item: "v", }; match buffer.push(track, frame) { diff --git a/test/interop/clients/js/open-gop/main.ts b/test/interop/clients/js/open-gop/main.ts index 955d9437c66..d851fc3dd8c 100644 --- a/test/interop/clients/js/open-gop/main.ts +++ b/test/interop/clients/js/open-gop/main.ts @@ -1,3 +1,4 @@ +import { Micro, Milli } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Moq from "@moq/net"; @@ -67,7 +68,7 @@ window.runRecovery = async (packets) => { group = new Moq.Group.Producer(packet.group); track.writeGroup(group); } - const timestamp = Moq.Time.Micro(packet.timestamp); + const timestamp = Micro(packet.timestamp); const payload = Uint8Array.from(atob(packet.data), (c) => c.charCodeAt(0)); group.writeFrame({ payload: Container.Legacy.encodeFrame(payload, timestamp), @@ -89,7 +90,7 @@ window.runRecovery = async (packets) => { relativeBroadcast: () => media, } as unknown as Broadcast); const source = new Source({ broadcast, supported: async () => true }); - const sync = new Sync({ delay: Moq.Time.Milli(0), buffer: Moq.Time.Milli(10_000) }); + const sync = new Sync({ delay: Milli(0), buffer: Milli(10_000) }); const decoder = new Decoder({ source, sync }); try { await inputDone;