From 2eae8587e223fb4d3f78cb082aa73a234e7652ce Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 10:36:50 -0800 Subject: [PATCH 1/2] quest: quarantine moq-ffi's warmup flake and plan its fix raw_media_catalog_warmup_reaches_the_subscription skipped its middle group in the merge queue for #8272. Ignore it until quest/m0/test-flakes-2/ffi-warmup-wall-clock.md fixes the cause, and merge the allowlist's three flake headers into one. Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-2/README.md | 7 ++++ .../m0/test-flakes-2/ffi-warmup-wall-clock.md | 39 +++++++++++++++++++ rs/moq-ffi/src/media.rs | 1 + sh/gh/test-inventory.allow | 5 +-- 4 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 quest/m0/test-flakes-2/ffi-warmup-wall-clock.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 199497632b4..4783e9a1d98 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -203,6 +203,12 @@ Added 2026-11-08 (maintainer, /quest-complete): `js/net`'s widened max-age re-drop test, which failed on [moq-dev/moq#8230](https://github.com/moq-dev/moq/pull/8230) with the `Cancel` reset [moq-dev/moq#8195](https://github.com/moq-dev/moq/pull/8195) intends for a reply still pending at the stream's end; the test races its finish against that reply. +Added 2026-11-08 (maintainer's standing rule that flakes join this epic): moq-ffi's +raw media warmup test, which failed in the merge queue for +[moq-dev/moq#8272](https://github.com/moq-dev/moq/pull/8272) when its reader skipped one of three +buffered groups on the FFI runtime's real clock. It is quarantined with `#[ignore]` until its +child fixes it. + Public API: none. Wire: none. ## Required @@ -211,3 +217,4 @@ Public API: none. Wire: none. - [moq-relay's forged-dial test waits for bob's endpoint](/quest/m0/test-flakes-2/forged-dial-announce.md) - `a_forged_dial_never_connects` requests bob's endpoint only after its announce reaches mallory, and its `#[ignore]` is removed - [moq-uring's qlog accounting test waits on the accounting](/quest/m0/test-flakes-2/qlog-staging-spin.md) - `an_open_trace_accounts_for_staging_and_close` waits for the accounting, not 10,000 spins - [js/net's widened max-age re-drop test ends cleanly, not with a reset](/quest/m0/test-flakes-2/widened-max-age-reset.md) - the test finishes the track only after the re-drop reply arrives, so the end is a FIN, and drops its wall-clock assert +- [moq-ffi's warmup test keeps every group under load](/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md) - `raw_media_catalog_warmup_reaches_the_subscription` keeps the group between its warmup start and the live edge, and its `#[ignore]` is removed diff --git a/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md b/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md new file mode 100644 index 00000000000..67cd28a7f14 --- /dev/null +++ b/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md @@ -0,0 +1,39 @@ +# [S] moq-ffi's warmup test keeps every group under load + +## Goal + +moq-ffi's `media::read_tests::raw_media_catalog_warmup_reaches_the_subscription` passes under load +and runs again. The reader keeps the 4.7 s group that sits between the start the warmup reaches and +the live edge. + +## Plan + +Planned 2026-11-08 (maintainer's standing rule that flakes join this epic). The test failed in the +merge queue for [moq-dev/moq#8272](https://github.com/moq-dev/moq/pull/8272), which touches no +moq-ffi read path ([job 128797584818](https://github.com/moq-dev/moq/actions/runs/42502335110/job/128797584818)). +It panicked at `rs/moq-ffi/src/media.rs:1635` with `video=false; left: [4400000, 5000000]; right: +[4400000, 4700000, 5000000]`: the audio pass started at group 3 and ended at group 5, but skipped +group 4. It is quarantined with `#[ignore]` and a `sh/gh/test-inventory.allow` entry until this quest +fixes it. + +The test writes groups 5, 4, 3 newest first, finishes the track, then reads. The read polls on the +FFI runtime, whose clock is real time, so every wall-clock rule in the read path runs on a loaded +host's clock. Group 4 lands before group 3, so a rule keyed on arrival ages group 4 out first, while +group 5 survives as the newest and group 3 as the start. Candidates in `moq-mux`'s +`container::Consumer`: the missing-sequence walk (`front.late.elapsed() >= max_delay` and its hole +timer, with a 200 ms max delay), and the start's `Startup::Earliest` arrival anchor. Also check +`moq-net`'s subscription delivery under the widened 500 ms max delay. + +Reproduce first: run the test repeatedly under a loaded `just check`, with tracing on +`skip_to` and the hole walk, to name the rule that drops group 4. Then fix the cause: + +- If the rule drops an arrived, finished group whose media lies inside the budget, that is a + `moq-mux` bug a real viewer on a slow host hits too. Fix it there, with a test on a controlled + clock beside the start-reach tests. +- If the skip is correct for that much elapsed time, drive the test's clock instead of the host's + (`kio::clock::Manual`, or build the consumer on the test runtime's clock), so elapsed time is + zero by construction. + +Then remove the `#[ignore]` and the allowlist entry, and delete this quest. + +Public API: none. Wire: none. diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 7753c9b1b3e..a4db3b680d2 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1554,6 +1554,7 @@ mod read_tests { /// runtime's, so a wait for the reach would measure real time. moq-mux's start-reach tests /// cover that wait on one clock. #[tokio::test] + #[ignore = "flaky: the 4.7 s group is sometimes skipped under load; quest/m0/test-flakes-2/ffi-warmup-wall-clock.md"] async fn raw_media_catalog_warmup_reaches_the_subscription() { use crate::consumer::{MoqDelay, MoqSubscription, moq_subscription_defaults}; diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 85303d41dd0..6a063207f97 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -36,11 +36,8 @@ ignore moq-video::render::export::dx12::tests::exports_an_nv12_texture /quest/m5 ignore moq-video::render::export::dx12::tests::media_foundation_encodes_the_export_in_place /quest/m5/video-hardware.md # Quarantined flakes, re-enabled by the quest that fixes them. - -# Flaky, disabled until its quest fixes the cause. +ignore moq-ffi::media::read_tests::raw_media_catalog_warmup_reaches_the_subscription /quest/m0/test-flakes-2/ffi-warmup-wall-clock.md ignore moq-relay::socket_auth::a_forged_dial_never_connects /quest/m0/test-flakes-2/forged-dial-announce.md # Need another implementation to speak the route wire; the quest ports it. ignore moq-net::test_interop::lite_tracks_interop /quest/m2/cluster-routing/routes/js.md - -# Quarantined flakes, until the quest that owns the flake fixes it at its cause. From 2f4f3082abd42d363193a7d5320386a33013d484 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 11:08:07 -0800 Subject: [PATCH 2/2] test(moq-ffi): keep the warmup subscription asserts in the gate Quarantining the warmup read test also dropped the only check that a catalog warmup widens the FFI subscription's max delay. Split those asserts into their own test, which reads nothing, so only the flaky read stays ignored. Co-Authored-By: Claude Opus 5.5 --- .../m0/test-flakes-2/ffi-warmup-wall-clock.md | 2 + rs/moq-ffi/src/media.rs | 126 ++++++++++-------- 2 files changed, 73 insertions(+), 55 deletions(-) diff --git a/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md b/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md index 67cd28a7f14..44bccd6fcf3 100644 --- a/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md +++ b/quest/m0/test-flakes-2/ffi-warmup-wall-clock.md @@ -15,6 +15,8 @@ It panicked at `rs/moq-ffi/src/media.rs:1635` with `video=false; left: [4400000, [4400000, 4700000, 5000000]`: the audio pass started at group 3 and ended at group 5, but skipped group 4. It is quarantined with `#[ignore]` and a `sh/gh/test-inventory.allow` entry until this quest fixes it. +Its subscription assertions moved to `raw_media_catalog_warmup_widens_the_subscription`, +which stays in the gate. The test writes groups 5, 4, 3 newest first, finishes the track, then reads. The read polls on the FFI runtime, whose clock is real time, so every wall-clock rule in the read path runs on a loaded diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index a4db3b680d2..0b1a19e294d 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1546,66 +1546,82 @@ mod read_tests { use moq_mux::container::Container as _; use std::time::Duration; - /// A catalog rendition's warmup survives conversion, widens the raw subscription past the - /// caller's max delay, keeping the min delay as asked, and starts the reader that much further - /// back. - /// - /// Reads no clock: the reader polls on the FFI runtime, whose clock is not a paused test - /// runtime's, so a wait for the reach would measure real time. moq-mux's start-reach tests - /// cover that wait on one clock. - #[tokio::test] - #[ignore = "flaky: the 4.7 s group is sometimes skipped under load; quest/m0/test-flakes-2/ffi-warmup-wall-clock.md"] - async fn raw_media_catalog_warmup_reaches_the_subscription() { + const WARMUP: Duration = Duration::from_millis(300); + const DELAY: Duration = Duration::from_millis(200); + + /// Subscribes to a rendition whose catalog warmup is [`WARMUP`], at a [`DELAY`] min and max delay. + async fn subscribe_warmup( + video: bool, + ) -> ( + moq_net::broadcast::Producer, + moq_net::track::Producer, + Arc, + moq_mux::container::Kind, + ) { use crate::consumer::{MoqDelay, MoqSubscription, moq_subscription_defaults}; - let warmup = Duration::from_millis(300); - let delay = Duration::from_millis(200); - for video in [false, true] { - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast.create_track("media", hang::container::track_info(0)).unwrap(); - let mut catalog = moq_mux::catalog::hang::Catalog::default(); - let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); - audio.warmup = Some(warmup); - catalog.audio.renditions.insert("media".into(), audio); - let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); - config.warmup = Some(warmup); - catalog.video.renditions.insert("media".into(), config); - let catalog = convert_catalog(&catalog); - let (rendition, kind) = match video { - true => ( - MoqRendition::Video { - id: "media".into(), - config: catalog.video["media"].clone(), - }, - moq_mux::container::Kind::Video, - ), - false => ( - MoqRendition::Audio { - id: "media".into(), - config: catalog.audio["media"].clone(), - }, - moq_mux::container::Kind::Audio, - ), - }; - let reader = MoqMediaTrackConsumer::subscribe( - &MoqBroadcastConsumer::new(broadcast.consume()), - MoqMediaTrackConfig { - rendition, - subscription: Some(MoqSubscription { - delay: MoqDelay { - min: Some(delay), - max: Some(delay), - }, - ..moq_subscription_defaults() - }), + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast.create_track("media", hang::container::track_info(0)).unwrap(); + let mut catalog = moq_mux::catalog::hang::Catalog::default(); + let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + audio.warmup = Some(WARMUP); + catalog.audio.renditions.insert("media".into(), audio); + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.warmup = Some(WARMUP); + catalog.video.renditions.insert("media".into(), config); + let catalog = convert_catalog(&catalog); + let (rendition, kind) = match video { + true => ( + MoqRendition::Video { + id: "media".into(), + config: catalog.video["media"].clone(), }, - ) - .await - .unwrap(); + moq_mux::container::Kind::Video, + ), + false => ( + MoqRendition::Audio { + id: "media".into(), + config: catalog.audio["media"].clone(), + }, + moq_mux::container::Kind::Audio, + ), + }; + let reader = MoqMediaTrackConsumer::subscribe( + &MoqBroadcastConsumer::new(broadcast.consume()), + MoqMediaTrackConfig { + rendition, + subscription: Some(MoqSubscription { + delay: MoqDelay { + min: Some(DELAY), + max: Some(DELAY), + }, + ..moq_subscription_defaults() + }), + }, + ) + .await + .unwrap(); + (broadcast, track, reader, kind) + } + + /// A catalog rendition's warmup survives conversion and widens the raw subscription past the + /// caller's max delay, keeping the min delay as asked. + #[tokio::test] + async fn raw_media_catalog_warmup_widens_the_subscription() { + for video in [false, true] { + let (_broadcast, track, _reader, _) = subscribe_warmup(video).await; let subscription = track.subscription().unwrap(); - assert_eq!(subscription.delay.min, Some(delay), "video={video}"); - assert_eq!(subscription.delay.max, Some(delay + warmup), "video={video}"); + assert_eq!(subscription.delay.min, Some(DELAY), "video={video}"); + assert_eq!(subscription.delay.max, Some(DELAY + WARMUP), "video={video}"); + } + } + /// The warmup starts the reader that much further back than the min delay. + #[tokio::test] + #[ignore = "flaky: the 4.7 s group is sometimes skipped under load; quest/m0/test-flakes-2/ffi-warmup-wall-clock.md"] + async fn raw_media_catalog_warmup_reaches_the_subscription() { + for video in [false, true] { + let (_broadcast, track, reader, kind) = subscribe_warmup(video).await; let write = |sequence: u64, micros| { let mut group = track.create_group(sequence.into()).unwrap(); moq_mux::catalog::hang::Container::Legacy(kind)