diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 33925471b6..3fb1e1c13b 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -218,11 +218,11 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs # The subscriber's leg congested the way real paths are, through moq-shaper: 7 Mb/s under - # the 9 Mb/s video, behind a 2 s queue, 25 ms each way. The export must stay T-STD compliant - # and continuous, shed only whole groups, and keep its latency, so the last group arrives. + # the 9 Mb/s video and four audio PIDs, behind a 2 s queue, 25 ms each way. The export must + # stay T-STD compliant and continuous, shed whole groups, and keep its latency through EOF. - name: TS congested path if: ${{ !cancelled() && github.event_name != 'pull_request' }} - run: nix develop --command just test ts --hrd --rate 7000000 --duration 120 + run: nix develop --command just test ts --headroom --rate 7000000 --duration 120 env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs @@ -231,18 +231,19 @@ jobs: # inside the export's 1 s delay, and the run fails unless the queue overflowed. - name: TS overflowing path if: ${{ !cancelled() && github.event_name != 'pull_request' }} - run: nix develop --command just test ts --hrd --rate 7000000 --queue 250ms --duration 120 + run: nix develop --command just test ts --headroom --rate 7000000 --queue 250ms --duration 120 env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs MOQ_QUIC_CONGESTION_CONTROL: loss - # The passthrough over the congested path, under its 10 Mb/s multiplex: what arrives must be the - # source's packets in place. - - name: TS passthrough congested path + # Passthrough over the overflowing path, under its 10 Mb/s multiplex: what arrives must be + # the source's packets in place, through the source's last packet. + - name: TS passthrough overflowing path if: ${{ !cancelled() && github.event_name != 'pull_request' }} - run: nix develop --command just test ts --passthrough --rate 7000000 --duration 120 + run: nix develop --command just test ts --passthrough --rate 7000000 --queue 250ms --duration 120 env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + MOQ_QUIC_CONGESTION_CONTROL: loss - name: Upload failure artifacts if: ${{ failure() }} diff --git a/quest/m1/README.md b/quest/m1/README.md index 18f4bd1d61..e822a2f8f9 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -106,7 +106,6 @@ lands before the cut. - [Transcode finishes its group](/quest/m1/transcode-open-group.md) - `moq-transcode` never aborts a group a fetch's live loop opened - [Rust request credit](/quest/m1/rs-request-credit.md) - a moq-net request fails at once without stream credit instead of waiting - [qmux no-wait opens](/quest/m1/qmux-no-wait.md) - @moq/qmux rejects an over-limit create when waitUntilAvailable is false, like Chrome -- [Random-loss audit](/quest/m1/random-loss-audit.md) - every decision random-loss gates drove is kept, reworked, or reverted on shaped paths - [Timed-out requests clean up](/quest/m1/legacy-request-timeout-cleanup.md) - on drafts 14-16, Rust and JS send PUBLISH_NAMESPACE_DONE or UNSUBSCRIBE_NAMESPACE for a timed-out request before its local reset - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [Resume reorder](/quest/m1/resume-reorder.md) - a group whose stream header arrives after its successor's still resumes on failover; only a dropped or long-gone group fails diff --git a/quest/m1/random-loss-audit.md b/quest/m1/random-loss-audit.md deleted file mode 100644 index caf37c02ff..0000000000 --- a/quest/m1/random-loss-audit.md +++ /dev/null @@ -1,56 +0,0 @@ -# [M] Audit decisions made to pass random-loss tests - -## Goal - -Every acceptance or performance decision a random-loss test drove (code, -defaults, harness choices, CI steps, and quests) is re-judged on the -bufferbloat harness and kept, reworked, or reverted, each with a recorded -verdict and its evidence. - -## Plan - -Decided 2026-10-17 (maintainer): random loss may prove correctness, but it -is not how real paths fail, so anything tuned to survive it as a quality or -performance target may be wrong. Scope is every random-loss-driven -decision, not only the 10% TS work. Small reverts land in the audit's PR; -anything bigger becomes its own quest. Verdicts are measured on the -bufferbloat harness (`just test ts --rate`, see `test/ts/README.md`), not -argued. - -Measured with that harness 2026-10-18: BBRv3 alone keeps about a round trip -queued, so a deep queue never overflows and congestion lands at the relay as -`Old` group evictions instead. A verdict that needs congestion loss, such as -BBR's loss response, needs a queue about a round trip deep or a loss-based -sender sharing the path. The nightly overflowing path (2026-10-19) runs -CUBIC alone behind a 250 ms queue, which it overflows every run. - -Leads found 2026-10-17 (no production default or congestion threshold was -changed for random loss; `LOSS_THRESH` is still 2%): - -- `quest/m2/quic-bbr-natural-drain.md` targets "BBR's response to random - loss" on a 10% profile and asks whether BBR should tell random loss from - congestion. The main live risk: it would weaken BBR's loss response. Its - premise came from #5805 moving 10% loss out of the T-STD epic. -- #4645's fixed-delay jitter buffer and skip design was partly justified by - the 10% netem rig (#4613); re-check the skip budget and acquire wait. -- The BBR fork fixes (#4193, noq 1.3.1) were accepted on a 2% random-loss - netem check; the fixes are correctness fixes, but re-check the evidence. -- Harness choices from #5805 and #5856: nightly `--hrd` because - `--headroom` loses audio at 1% loss, WebSocket fallback disabled under - `--loss`, a run that fails if the shaper dropped nothing, and - `passthrough.py --loss-ok`. -- Decided 2026-10-17 (maintainer): the audit decides whether - `quic-bbr-natural-drain.md` and `m3/quic-gcc.md` survive, since their - premise rests on random loss. -- Other quests only list random loss among their measurement inputs: - `quic-probe.md` (with noq#811 and noq#476), `quic-sim.md`, - `quic-deadline.md` and `quic-careful-resume.md` ("the impaired path - profile"), `m3/cut-through/bench.md`, and - `m1/ts-reliability/recovery.md`'s network impairment arm. Their goals - stand; the audit only moves any acceptance or performance input among - them to bufferbloat. - -Record each verdict (decision, origin PR, keep/rework/revert, evidence) in -the PR description. - -Public API: depends on the verdicts. Wire: none expected. diff --git a/quest/m1/ts-reliability/recovery.md b/quest/m1/ts-reliability/recovery.md index f28e724183..5e67266b10 100644 --- a/quest/m1/ts-reliability/recovery.md +++ b/quest/m1/ts-reliability/recovery.md @@ -21,7 +21,11 @@ nothing, in its stream row's `units` and `quiet`. Test content loss separately from IP packet loss. QUIC may retransmit IP loss; a netem percentage does not establish that any MoQ object was omitted. -Keep the real network impairment arm for transport and pacing evidence, and +Keep the real network impairment arm for transport and pacing evidence, +using a rate below the media bitrate behind a deep but finite queue with +verified overflow drops. Random loss only proves retransmission correctness, +never output quality or latency. Reuse the overflowing lane in +`test/ts/README.md`, and add deterministic fault schedules at the MoQ content boundary. Cover a cold join, a skipped group, a partially delivered group, late objects diff --git a/quest/m2/probe/native.md b/quest/m2/probe/native.md index f8f597b263..0634a731d1 100644 --- a/quest/m2/probe/native.md +++ b/quest/m2/probe/native.md @@ -40,8 +40,12 @@ Acceptance, in moq-quic's seeded simulation (`rs/moq-quic/src/tests/`, extending `Bottleneck` to step its rate) and wired into CI at least nightly: an illustrative 2 Mbit/s source seeking 4 Mbit/s while the path steps 3 to 12 to 3 Mbit/s, on identical media traces and wire budgets. Cover keyframes, idle -restart, stale high estimates, receiver credit limits, random and burst loss, -shallow queues, compressed ACKs, and competing CUBIC and BBR flows. Set the +restart, stale high estimates, receiver credit limits, compressed ACKs, and +competing CUBIC and BBR flows. Add a congested arm below the media bitrate +behind a deep but finite queue for impairment acceptance and performance. +Verify actual overflow drops, adding competing loss-based traffic if BBR +alone will not fill it. Sweep queue depth separately. Random or scripted burst loss may +prove recovery correctness, but never gates performance. Set the deadline, overhead, and fairness budgets before tuning. Report validated upward adaptation, downward reaction, estimate age, frame deadlines, queue delay, unique goodput, and redundant bytes, against a no-probing baseline. A diff --git a/quest/m2/quic-bbr-natural-drain.md b/quest/m2/quic-bbr-natural-drain.md index df4d0e5a17..134c89dc16 100644 --- a/quest/m2/quic-bbr-natural-drain.md +++ b/quest/m2/quic-bbr-natural-drain.md @@ -8,8 +8,10 @@ ProbeRTT, recognizing bandwidth growth within a round, and precautionary bandwidth probing. Reduce demonstrated frame deadline interference without stale minimum RTT, persistent queues, or unfairness. Application limitation alone is not evidence that the path drained, and Google parity is not -assumed to be an improvement. Grade them, and BBR's response to random -loss, on a 10 % random-loss media profile as well. +assumed to be an improvement. Acceptance and performance use a rate below +the media bitrate behind a deep but finite queue with verified overflow +drops. Random loss may prove recovery correctness, but never grades media +quality or performance. ## Plan @@ -42,8 +44,10 @@ path change. Blanket skipping is only a diagnostic control. Retain the simpler baseline if no material interference is demonstrated. Use identical media traces and seeds. Start with short and long RTT paths, -then test capacity changes, base-RTT changes, competing bulk traffic, shallow -queues, loss bursts, and compressed or delayed ACKs. Record deadlines and +then test capacity changes, base-RTT changes, competing bulk traffic, queue +depths, and compressed or delayed ACKs. Use loss-based cross traffic when +BBR alone does not fill the deep queue; verify actual overflow drops. Keep +scripted loss bursts as recovery correctness checks only. Record deadlines and frame completion, withheld useful bytes, sender backlog separately from network queueing, RTT-baseline error, sample labels, and mode residence. Set latency and fairness acceptance limits before tuning. Persist the @@ -78,38 +82,30 @@ BBR3 state machine. and [Google QUICHE](https://github.com/google/quiche/blob/535a2730e77d47e0dc03746555cc9c34b17bc9e9/quiche/quic/core/congestion_control/bbr3_sender.cc#L1079) stop precautionarily when probing reaches a previously lossy inflight bound, then accelerate a later probe if feedback is clean. Noq lacks this - state and transition. Compare shallow queues, capacity increases, random - loss, and competing flows. + state and transition. Compare queue depths, capacity increases, and + competing flows on shaped paths with verified overflow drops. Report throughput, queue delay, loss, convergence time, and fairness with pinned code and configurations. Keep transport differences and QUICHE flags explicit and compare each algorithm change separately. A simulation result is not an end-to-end network measurement. -### Random loss - -Moved from the finished T-STD export epic (2026-10-16), whose nightly -stays at 1 %. -`just test ts --hrd --loss ` drops that share of the relay's -datagrams to the subscriber at random through `moq-shaper`, 25 ms each way, -and grades the export with T-STD, PCR accuracy and `holes.py`. Measured on a -loaded host (rerun before deciding): - -| Clip | Loss | Result | -|---|---:|---| -| default (low media rate) | 10 % | passes | -| `--hrd` (9 Mb/s video, 10 Mb/s mux) | 1 %, 3 % | passes, no holes | -| `--hrd` | 5 %, 7 %, 10 % | groups evicted `Old`, export fails | -| `--hrd`, `LOSS_THRESH` 20 % (experiment) | 10 % | T-STD and PCR pass; 1.56 s of video misses the 1 s delay | -| `--hrd`, CUBIC | 10 % | no output | - -BBRv3 takes a round with more than 2 % loss (`LOSS_THRESH`) as congestion -and cuts its inflight bound, so random loss above it holds the leg below the -media rate. Decide whether and how BBR should tell random loss from -congestion without giving up fairness against loss-based flows, and how that -interacts with the precautionary probe stop under Google differences. +### Congestion evidence + +Use the overflowing and deep-queue lanes in `test/ts/README.md` as the +end-to-end baseline. A BBR-only run that sheds groups at the relay but never +overflows the network queue cannot judge the loss response. A CUBIC run +that overflows establishes that the harness exercises recovery; it cannot +establish whether changing BBR improves it. + +Keep the current 2% `LOSS_THRESH` and baseline controller policy unless a +candidate improves the shaped-path comparison, including queues and +fairness. Surviving an independent random-loss percentage is not a reason +to weaken the congestion response or precautionary probe stop. This study +still decides natural-drain credit, growth recognition, and probe behavior +separately; it does not adopt any of them in advance. ## Related -- [TS validation](/test/ts/README.md) - the `--loss` rig, `holes.py` and the full measurements +- [TS validation](/test/ts/README.md) - shaped paths, whole-group shedding, and measured baselines - [Native probing](/quest/m2/probe/native.md) - preserving an estimate and discovering spare capacity are separate problems diff --git a/quest/m2/quic-deadline.md b/quest/m2/quic-deadline.md index 3d259f5608..9bae16af5e 100644 --- a/quest/m2/quic-deadline.md +++ b/quest/m2/quic-deadline.md @@ -44,7 +44,11 @@ Implement in `moq-quic`. existing group-expired code on the MoQ wire, so a viewer sees the same signal it sees for a relay-side expiry today. -Measure on the impaired path profile: delivery latency p95 and p99, bytes +Measure on bufferbloat: a rate below the media bitrate behind a deep but +finite queue whose overflow demonstrably drops packets, with competing +loss-based traffic if needed to fill it. Random loss may prove recovery +correctness, but never grades latency or throughput. Report delivery latency +p95 and p99, bytes retransmitted past their deadline (must be zero), spurious resets under reordering, and probe overhead versus the default PTO. A proactive probe that raises loss or latency under any profile stays off by default. diff --git a/quest/m2/quic-outage-probe.md b/quest/m2/quic-outage-probe.md index 0fc5e633ca..0c2d0c082f 100644 --- a/quest/m2/quic-outage-probe.md +++ b/quest/m2/quic-outage-probe.md @@ -17,7 +17,7 @@ Decided 2026-10-19 by the maintainer (#5953): - Declined from #5953: a congestion controller that holds a fixed rate through random loss, exposing the reorder thresholds, and a named provisioned-path profile. Random loss is not a quality target - ([random-loss audit](/quest/m1/random-loss-audit.md)), and the reorder + (see [shaped-path grading](/test/ts/README.md)), and the reorder evidence came from random netem reorder only. Revisit the thresholds with a bufferbloat or real-path measurement. - Test on mocked time: a link that drops everything for 5 s, then returns. diff --git a/quest/m2/quic-sim.md b/quest/m2/quic-sim.md index 24260ffd8e..9869acedc3 100644 --- a/quest/m2/quic-sim.md +++ b/quest/m2/quic-sim.md @@ -16,6 +16,11 @@ driver. Start from moq-shaper's in-memory `UdpSocket` (`rs/moq-shaper/src/mem.rs BBR's controller-level `Sim` in `bbr3/mod.rs` stays separate unless this replaces it cleanly. +Keep random loss for recovery correctness. Any acceptance or performance +consumer uses a rate below its media bitrate behind a deep but finite queue, +and checks that the queue overflowed. A stalled sender that never filled the +queue is not evidence about congestion loss. + ## Related - [Per-stream deadlines](/quest/m2/quic-deadline.md) - a consumer diff --git a/quest/m3/cut-through/bench.md b/quest/m3/cut-through/bench.md index 196d8e2555..2fbc05e5ad 100644 --- a/quest/m3/cut-through/bench.md +++ b/quest/m3/cut-through/bench.md @@ -13,11 +13,17 @@ Decided 2026-09-30: real stacks, not a mock. The loss recovery being measured belongs to the QUIC stack, so a mock transport with a made-up retransmit model would partly measure its own assumptions. -- Publisher, then a seeded `moq-shaper` (loss and delay), then `moq-relay` over - QUIC, then a clean or rate-limited hop to the viewer. Reuse the shaper wiring - from `rs/moq-relay/tests/drills.rs`. -- Sweep loss rate, frame size (audio-sized through a large I-frame), and egress - headroom (egress rate relative to the media rate). +- Publisher, then `moq-shaper` with an ingress rate below the media bitrate + and a deep but finite queue whose overflow drops packets, then `moq-relay` + over QUIC, then a clean or rate-limited hop to the viewer. Reuse the shaper + wiring from `rs/moq-relay/tests/drills.rs` and the overflowing bufferbloat + profiles described in `test/ts/README.md`. Verify that the measured path + actually overflows its queue. +- Sweep ingress rate, queue depth, frame size (audio-sized through a large + I-frame), and egress headroom (egress rate relative to the media rate). + The go or no-go verdict uses these bufferbloat runs. Seeded random loss + may check correctness, such as the payload-loss and header-loss cases + below, but never gates acceptance or performance. - Report per-frame completion at the viewer, from publish to last byte, p50 and p99. The opportunity is the drain time on this same two-hop path: per frame, the time from the relay's ingress hole filling to the frame completing at the diff --git a/quest/m3/quic-careful-resume.md b/quest/m3/quic-careful-resume.md index 7f405fb34e..a99abf4ff4 100644 --- a/quest/m3/quic-careful-resume.md +++ b/quest/m3/quic-careful-resume.md @@ -28,8 +28,12 @@ start on the first loss or a mismatched RTT. `failover.rs`) seeds a redial from the session it replaces, and the relay's cluster peers seed from the previous session to the same peer. -Measure time to the encoder's target rate after a reconnect on the impaired -path profile, plus loss and latency during the jump. Ship it on by default +Measure time to the encoder's target rate after a reconnect on bufferbloat: +a rate below the media bitrate behind a deep but finite queue, with actual +overflow drops, plus loss and latency during the jump. Include competing +loss-based traffic when BBR alone does not fill the queue. Random loss may +prove fallback correctness, but never grades the jump's performance. +Ship it on by default only when the jump never makes the first second worse than slow start. ## Related diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index c3892e71c4..d0f2079516 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1148,10 +1148,11 @@ where /// ride is laid, and each slot goes out the delay after its own time on the source's /// clock rather than a delay later still. /// - /// The clock is acquired before anything goes out: frames are held until a track starts - /// a new group, or for the delay at most, and the freshest of them against its release - /// time sets the clock, so a joiner runs at the delay from its first output and every - /// frame has the whole delay to arrive in. A source that sends its video its send-ahead + /// The clock is acquired before anything goes out: frames are held until every expected + /// track has delivered (two delays at most) and a track starts a new group, or for a delay + /// at most once all have. The track sent latest against its release time sets the clock, + /// so a joiner runs at the delay from its first output and every frame has the whole delay + /// to arrive in. A source that sends its video its send-ahead /// early, as a TS mux does, is trailed by the delay; one that sends every track at its /// decode time, by the delay and the video's send-ahead. The output then keeps the /// source's pace and muxes every track in `(release time, PID)` order whatever the diff --git a/test/ts/README.md b/test/ts/README.md index 60f8fc2339..f6ebd0c1a6 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -586,7 +586,10 @@ rather than let it grow with the queue. Nothing bounds how much it sheds. The generated clips go through `holes.py --shed`, which matches the capture's frames to the source's by presentation timestamp, since the export derives each -picture's decode timestamp and a group cut short moves it. Of each source group, +picture's decode timestamp and a group cut short moves it. Verbatim MP2, AC-3 +and E-AC-3 timestamps are compared after their exact legacy microsecond wire +conversion back to 90 kHz, rather than allowing an arbitrary timestamp error. +Of each source group, the capture keeps a prefix in decode order (all, part, or none) and never resumes mid-group, so it also joins on a sync frame, and a frame reordered, repeated, or not in the source fails. The source's last group must arrive whole: a stall, or a latency grown @@ -611,7 +614,7 @@ retransmit behind a full queue (about twice 300 ms) still lands inside the 1 s delay. A loss-based run fails unless the queue overflowed. ```bash -MOQ_QUIC_CONGESTION_CONTROL=loss just test ts --hrd --rate 7000000 --queue 250ms +MOQ_QUIC_CONGESTION_CONTROL=loss just test ts --headroom --rate 7000000 --queue 250ms ``` Measured on 2026-10-18 and 2026-10-19 with other test runs sharing the host: @@ -630,6 +633,9 @@ Measured on 2026-10-18 and 2026-10-19 with other test runs sharing the host: | `--hrd --rate 7000000`, CUBIC | 500 ms | 60-120 s, three runs | passes; 541-625 overflowed, 79-229 units missed their deadline | | `--hrd --rate 7000000`, CUBIC | 1 s | 120 s | fails: 919 overflowed, 107 s shed and the last group lost | | `--hrd --rate 7000000`, CUBIC | 2 s | 60 s | fails: 918 overflowed, then the clock-drift refusal | +| `--headroom --rate 7000000`, BBRv3 | 2 s | 120 s, saved capture regraded | passes; none overflowed, 62.5 s video and 3 AC-3 frames shed, no MP2 holes | +| `--headroom --rate 7000000`, CUBIC | 250 ms | 120 s | passes; 427 overflowed, 74 units missed their deadline, 64.5 s video and 3 audio frames shed | +| `--passthrough --rate 7000000`, CUBIC | 250 ms | 120 s | passes; 428 overflowed, 116 gaps, no missing tail; 458 continuity errors reported | ```bash ./holes.py capture.ts source.ts [--shed] @@ -651,8 +657,9 @@ on purpose belongs to the [recovery tests](/quest/m1/ts-reliability/recovery.md) if the shaper dropped none. Real paths do not lose datagrams independently of their load, so this proves correctness only: QUIC retransmits what the path drops, so a capture with a hole or a gap fails as on a clean path. It does not judge the -export's quality or latency, and no CI step runs it; combined with `--rate`, the -congested path's grading applies. +export's quality or latency, and no CI step runs it. It cannot be combined with +`--rate` or `--live`. Use `--rate` without random loss when grading quality or +release timing on an impaired path. ## Passthrough (`--passthrough`, `passthrough.py`) @@ -848,9 +855,10 @@ open-GOP, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Eac All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, -`just test ts --live --duration 120`, `just test ts --hrd --rate 7000000 --duration 120`, +`just test ts --live --duration 120`, `just test ts --headroom --rate 7000000 --duration 120`, the same with `--queue 250ms` and `MOQ_QUIC_CONGESTION_CONTROL=loss`, -and `just test ts --passthrough --rate 7000000 --duration 120`. +and `just test ts --passthrough --rate 7000000 --queue 250ms --duration 120` +with `MOQ_QUIC_CONGESTION_CONTROL=loss`. The pair exercises late-joining subscribers; the live window distinguishes ongoing clock drift from initial buffer filling; the congested paths judge what the export does when the path cannot carry the media. diff --git a/test/ts/holes.py b/test/ts/holes.py index 798c499bb6..1e5dd51a70 100755 --- a/test/ts/holes.py +++ b/test/ts/holes.py @@ -17,8 +17,9 @@ but only by whole groups: of each group of the source, from a sync frame to the next, the capture keeps a prefix in decode order (all of it, part of it, or none) and never resumes mid-group, so it also joins on a sync frame. Frames are matched by presentation timestamp, since the export derives each -picture's decode timestamp from the frames around it and a group cut short moves it, and in -capture order: a frame reordered, repeated, or not in the source fails. The +picture's decode timestamp from the frames around it and a group cut short moves it. Legacy +audio's source timestamps undergo its microsecond wire conversion before comparison. Matching +keeps capture order: a frame reordered, repeated, or not in the source fails. The source's last group still has to arrive whole, so a stream that stalled, or whose latency grew past the exporter's delay, fails. @@ -169,7 +170,12 @@ def main() -> None: span = (f"from {float(got.span[0]):.3f}s to {float(got.span[1]):.3f}s " f"of {float(src.span[0]):.3f}s to {float(src.span[1]):.3f}s") if args.shed: - graded = shed(src.pts, src.sync, got.pts) + expected = src.pts + if src.info["codec_name"] in ("mp2", "ac3", "eac3"): + # Verbatim audio uses Hang's legacy microsecond prefix, then TS floors + # back to 90 kHz. Match that exact conversion, not a timestamp tolerance. + expected = [t * 1_000_000 // 90_000 * 90_000 // 1_000_000 for t in expected] + graded = shed(expected, src.sync, got.pts) reasons = [f"resumes mid-group at {float(t * base):.3f}s" for t in graded.mid_group[:5]] if graded.unmatched: reasons.append(f"{len(graded.unmatched)} frames from {float(graded.unmatched[0] * base):.3f}s " diff --git a/test/ts/holes.test.py b/test/ts/holes.test.py index d455f0944b..f41307243a 100644 --- a/test/ts/holes.test.py +++ b/test/ts/holes.test.py @@ -1,9 +1,13 @@ +import contextlib import importlib.util +import io import pathlib import subprocess +import sys import tempfile import unittest from fractions import Fraction +from unittest.mock import patch spec = importlib.util.spec_from_file_location("holes", pathlib.Path(__file__).with_name("holes.py")) holes = importlib.util.module_from_spec(spec) @@ -110,6 +114,43 @@ def test_lost_prefix(self): self.assertEqual(len(holes.cuts(self.SOURCE, capture, self.FRAME)), 1) +class LegacyTimestamps(unittest.TestCase): + def grade(self, codec, source, capture): + def stream(pts): + return holes.Stream( + {"codec_name": codec, "time_base": "1/90000"}, pts, pts, [True] * len(pts), + tuple(Fraction(t, 90_000) for t in (pts[0], pts[-1], pts[-1])), + ) + with patch.object(holes, "probe", side_effect=[{"0x101": stream(capture)}, {"0x101": stream(source)}]), \ + patch.object(sys, "argv", ["holes.py", "capture.ts", "source.ts", "--shed"]), \ + contextlib.redirect_stdout(io.StringIO()), self.assertRaises(SystemExit) as exit: + holes.main() + return exit.exception.code + + def test_legacy_microseconds_keep_the_source_frames(self): + # Legacy audio keeps its cadence, but its microsecond timestamps return + # one 90 kHz tick early unless they already land on the microsecond grid. + for codec, first, step, shift in [ + ("mp2", 132298, 2160, 1), ("ac3", 132720, 2880, 1), + ("mp2", 132300, 2160, 0), ("ac3", 132723, 2880, 0), + ]: + with self.subTest(codec=codec, first=first): + source = [first + i * step for i in range(5)] + self.assertEqual(self.grade(codec, source, [t - shift for t in source]), 0) + + def test_a_shift_beyond_legacy_quantization_fails(self): + source = [132298 + i * 2160 for i in range(5)] + self.assertEqual(self.grade("mp2", source, [t - 2 for t in source]), 1) + + def test_a_lost_legacy_tail_still_fails(self): + source = [132298 + i * 2160 for i in range(5)] + self.assertEqual(self.grade("mp2", source, [t - 1 for t in source[:-1]]), 1) + + def test_raw_video_does_not_use_legacy_quantization(self): + source = [133200 + i * 3600 for i in range(5)] + self.assertEqual(self.grade("h264", source, [t - 1 for t in source]), 1) + + class Probe(unittest.TestCase): def test_audio_starting_past_the_default_probe(self): # Intra-only video ahead of an MP2 stream that starts 3 s in, past FFmpeg's default 5 MB diff --git a/test/ts/process.test.sh b/test/ts/process.test.sh index 06965ec56c..f0f4aefd85 100644 --- a/test/ts/process.test.sh +++ b/test/ts/process.test.sh @@ -38,3 +38,33 @@ for code in 0 42 124; do } done echo "PASS process status: partial output cannot hide a failure or timeout" + +# Invalid grading modes must fail before a build or any network process starts. +mkdir -p "$HARNESS_RUN/bin" +cat >"$HARNESS_RUN/bin/cargo" <<'SH' +#!/usr/bin/env bash +echo "unexpected cargo invocation" >&2 +exit 73 +SH +chmod +x "$HARNESS_RUN/bin/cargo" +for mode in --rate --live; do + args=(--loss 1 "$mode") + [[ "$mode" != --rate ]] || args+=(7000000) + if PATH="$HARNESS_RUN/bin:$PATH" bash "$DIR/run.sh" "${args[@]}" >"$HARNESS_RUN/${mode#--}.log" 2>&1; then + echo "accepted random-loss quality grading with $mode" >&2 + exit 1 + fi + if ! grep -Fq 'error: --loss cannot be combined with --rate or --live' "$HARNESS_RUN/${mode#--}.log"; then + cat "$HARNESS_RUN/${mode#--}.log" >&2 + echo "did not refuse unsupported random-loss grading with $mode" >&2 + exit 1 + fi +done +status=0 +PATH="$HARNESS_RUN/bin:$PATH" bash "$DIR/run.sh" --loss 1 >"$HARNESS_RUN/correctness.log" 2>&1 || status=$? +if [[ "$status" -ne 73 ]]; then + cat "$HARNESS_RUN/correctness.log" >&2 + echo "standalone random-loss correctness did not reach the guarded build" >&2 + exit 1 +fi +echo "PASS grading modes: random loss cannot grade congestion or live timing" diff --git a/test/ts/run.sh b/test/ts/run.sh index 0fb1d4b656..e47824e715 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -275,6 +275,10 @@ fi # fail and is what judges the export; `--loss` drops datagrams at random (moq-dev/moq#4613), # which real paths do not, so it only proves QUIC retransmission keeps the output whole. SHAPED="$LOSS$RATE" +if [[ -n "$LOSS" && -n "$RATE$LIVE" ]]; then + echo "error: --loss cannot be combined with --rate or --live; use --rate without random loss for quality grading" >&2 + exit 1 +fi if [[ -n "$SHAPED" && -n "$ANALYZE_ONLY$PAIR" ]]; then echo "error: --loss and --rate impair the round-trip's one subscriber and cannot be combined with --analyze-only or --pair" >&2 exit 1