From fb4cbe34bb81e067ff051fef9238e54a22d1b2b7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 21:50:11 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m1/ietf-extension/README.md Quest-Claim: 26ad77ae414c5d55 From d96126c65a52027820884cee7cb9741e79fb674b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 22:46:36 -0700 Subject: [PATCH 2/4] Pin Moxygen conformance and document relay contract conflicts Co-Authored-By: GPT-6 --- .github/workflows/ietf-conformance.yml | 35 +++++ .github/workflows/nightly.yml | 4 + flake.nix | 4 + nix/moxygen-sources.nix | 44 ++++++ nix/moxygen.nix | 101 ++++++++++++ quest/m1/ietf-extension/README.md | 55 ++++++- .../tests/integration/ietf_conformance.rs | 86 ++++++++++ rs/moq-net/tests/integration/main.rs | 1 + test/README.md | 1 + test/ietf-conformance/README.md | 69 ++++++++ test/ietf-conformance/prepare.py | 40 +++++ test/ietf-conformance/run.sh | 147 ++++++++++++++++++ test/ietf-conformance/selection.test.py | 67 ++++++++ test/justfile | 4 + 14 files changed, 655 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ietf-conformance.yml create mode 100644 nix/moxygen-sources.nix create mode 100644 nix/moxygen.nix create mode 100644 rs/moq-net/tests/integration/ietf_conformance.rs create mode 100644 test/ietf-conformance/README.md create mode 100644 test/ietf-conformance/prepare.py create mode 100755 test/ietf-conformance/run.sh create mode 100644 test/ietf-conformance/selection.test.py diff --git a/.github/workflows/ietf-conformance.yml b/.github/workflows/ietf-conformance.yml new file mode 100644 index 00000000000..e6c6a1ab75a --- /dev/null +++ b/.github/workflows/ietf-conformance.yml @@ -0,0 +1,35 @@ +name: IETF conformance + +permissions: + contents: read + +on: + workflow_call: + +jobs: + moxygen: + runs-on: ubuntu-24.04 + environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + - uses: ./.github/actions/rust-cache + with: + access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + - name: Pinned Moxygen through single and clustered relays + run: nix develop --command just test ietf-conformance + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/ietf-conformance + MOQ_TEST_KEEP: 1 + - name: Keep conformance reports and endpoint logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: ietf-conformance + path: ${{ runner.temp }}/ietf-conformance/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index edca5a31179..1a7bfbe26af 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -52,6 +52,10 @@ jobs: uses: ./.github/workflows/features.yml secrets: inherit + ietf-conformance: + uses: ./.github/workflows/ietf-conformance.yml + secrets: inherit + runner-vm: uses: ./.github/workflows/runner-vm.yml diff --git a/flake.nix b/flake.nix index 4d6d27e5ff8..90f27fc8504 100644 --- a/flake.nix +++ b/flake.nix @@ -574,6 +574,10 @@ legacyPackages = { inherit (pkgs) gst_all_1; + # Nightly conformance needs both pinned Moxygen endpoints. Keep this + # C++ dependency build out of the default shell and flake checks. + moxygen = pkgs.callPackage ./nix/moxygen.nix { }; + runner-userns-profile = import ./nix/runner/unshare.nix { inherit pkgs; }; # `nix develop .#obs`: the default shell plus obs-studio, which linking diff --git a/nix/moxygen-sources.nix b/nix/moxygen-sources.nix new file mode 100644 index 00000000000..4de3c2a895b --- /dev/null +++ b/nix/moxygen-sources.nix @@ -0,0 +1,44 @@ +{ + moxygen = { + owner = "facebookexperimental"; + repo = "moxygen"; + rev = "c27ae6358df62a1195da479b2e07c50dd8174279"; + hash = "sha256-ZbTb3kOyFYL0MfALDgoQC50NX+OR6geC4iUXc/phSik="; + }; + folly = { + owner = "facebook"; + repo = "folly"; + rev = "e89841cb61bf8f5283cda05b1bd35de99557a557"; + hash = "sha256-PMOouxfCqzzkdzIC3lO0iwSi82YjwQ5empXOUSV1j8E="; + }; + mvfst = { + owner = "facebook"; + repo = "mvfst"; + rev = "00fb3329dc43c6095668083d92a626031aa1eead"; + hash = "sha256-tTDiMMGZ82THJnrZc5V4xgScm+RaQiAgYUXbGLtY3qs="; + }; + proxygen = { + owner = "facebook"; + repo = "proxygen"; + rev = "05d1c685b865a2c90f6ab13aef6f01ccd53c7931"; + hash = "sha256-ogSop9Vc1Arfa9FOyhxo1F8n2QQ+ytBmlB7SK7Am0xE="; + }; + wangle = { + owner = "facebook"; + repo = "wangle"; + rev = "a39ad0756413158253824d08fa3d212dcfac4e42"; + hash = "sha256-OpmCdhEokiH0kTzsDGchHMtwHsvE/2ZjZ61ph3SeF0Y="; + }; + fizz = { + owner = "facebookincubator"; + repo = "fizz"; + rev = "327e167726bdde385a4fbb368ea148aeb2641acb"; + hash = "sha256-tQC/2apy2BS5+Ckb7U3DRwwsMs3Zbv5sUvW/g+wzywo="; + }; + fast_float = { + owner = "fastfloat"; + repo = "fast_float"; + rev = "v8.0.0"; + hash = "sha256-shP+me3iqTRrsPGYrvcbnJNRZouQbW62T24xfkEgGSE="; + }; +} diff --git a/nix/moxygen.nix b/nix/moxygen.nix new file mode 100644 index 00000000000..57aa83e55ea --- /dev/null +++ b/nix/moxygen.nix @@ -0,0 +1,101 @@ +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + ninja, + pkg-config, + gperf, + python3, + perl, + openssl, + glog, + gflags, + fmt, + double-conversion, + boost, + libevent, + libsodium, + zstd, + zlib, + c-ares, + libunwind, +}: +let + pins = import ./moxygen-sources.nix; + sources = lib.mapAttrs (_: pin: fetchFromGitHub pin) pins; +in +stdenv.mkDerivation { + pname = "moxygen-conformance"; + version = "unstable-2026-10-23"; + src = sources.moxygen; + cmakeDir = "../standalone"; + nativeBuildInputs = [ + cmake + ninja + pkg-config + gperf + python3 + perl + ]; + buildInputs = [ + openssl + glog + gflags + fmt + double-conversion + boost + libevent + libsodium + zstd + zlib + c-ares + libunwind + ]; + # The standalone client uses blockingWait, which needs Folly's concrete executor. + postPatch = + lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: source: '' + cp -R ${source} .vendor-${name} + chmod -R u+w .vendor-${name} + patchShebangs .vendor-${name} + '') (builtins.removeAttrs sources [ "moxygen" ]) + ) + + '' + substituteInPlace moxygen/moqtest/CMakeLists.txt \ + --replace-fail 'moqtest_client PUBLIC' \ + 'moqtest_client PUBLIC Folly::folly_executors_manual_executor' + ''; + preConfigure = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: _: '' + cmakeFlagsArray+=("-DFETCHCONTENT_SOURCE_DIR_${lib.toUpper name}=$PWD/.vendor-${name}") + '') (builtins.removeAttrs sources [ "moxygen" ]) + ); + cmakeFlags = [ + "-DBUILD_TESTS=OFF" + "-DBUILD_MOQTEST=ON" + "-DBUILD_SAMPLES=OFF" + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" + ]; + ninjaFlags = [ + "moqtest_client" + "moqtest_server" + ]; + installPhase = '' + runHook preInstall + mkdir -p $out/bin $out/libexec/moxygen/moqtest $out/share/moxygen + for executable in moqtest_client moqtest_server; do + install -m755 moxygen/moqtest/$executable $out/libexec/moxygen/moqtest/$executable + ln -s $out/libexec/moxygen/moqtest/$executable $out/bin/$executable + done + install -m755 $src/moxygen/moqtest/conformance_test.sh $out/share/moxygen/conformance_test.sh + cp ${./moxygen-sources.nix} $out/share/moxygen/sources.nix + runHook postInstall + ''; + meta = { + description = "Pinned Moxygen client and origin for relay conformance"; + homepage = "https://github.com/facebookexperimental/moxygen"; + license = lib.licenses.asl20; + platforms = lib.platforms.linux; + }; +} diff --git a/quest/m1/ietf-extension/README.md b/quest/m1/ietf-extension/README.md index c52db0029f8..54a72bbc160 100644 --- a/quest/m1/ietf-extension/README.md +++ b/quest/m1/ietf-extension/README.md @@ -76,12 +76,61 @@ Work owned by this README, after the children land: END_OF_GROUP status object, and a FIRST_OBJECT bit that differs. Every child must preserve these byte for byte. - Check routing first. Sections 1-7 send a SUBSCRIBE for a 16-field namespace - under the announced `moq-test-00` prefix. Whether `origin.dynamic` routes it - is unverified. If it does not, bring the routing decision to the maintainer - before working around it. + under the announced `moq-test-00` prefix. Mock-clock integration tests now + prove that path on drafts 16 and 18, through one relay and through two relays + joined by lite-07; the live harness reaches the publisher too. - FETCH cases are skipped upstream. Section 8 stays out while PUBLISH is declined. + +### Remaining conformance decision + +The original Goal above is unmet. The pinned harness is in +[`test/ietf-conformance`](../../../test/ietf-conformance/README.md), with Moxygen +`c27ae6358df62a1195da479b2e07c50dd8174279` and its matching dependency revisions. +It keeps all selected case arguments and deadlines, excludes only the agreed +nonzero-subgroup cases, and fails on client crashes or publisher reconnects. +The nightly job is a diagnostic checkpoint, not a passing conformance claim. + +The direct-origin controls pass 29/29 cases on draft 16 and 28/29 on draft 18. +The draft-18 large-scale case fails even without a relay with +`Group Mismatch: Actual=1 Expected=0`; its provenance must be resolved upstream. +A different single-relay draft-18 case, "End of group markers with single object", also reports `(1, expected 0)`; +its relationship to the direct-control failure is not established. +Every relay cell fails 29/29. Single-relay cells expose the conflicts below; +cluster cells fail before media because their TRACK_INFO query needs upstream +TRACK_STATUS, and the pinned MoQTestPublisher inherits NOT_SUPPORTED. Preserve +that refusal: changing query-only demand into a SUBSCRIBE is not an authorized +fallback here. [Range widening](/quest/m1/compress/ietf-range-widen.md), claimed +by [moq-dev/moq#6465](https://github.com/moq-dev/moq/pull/6465), owns that separate +cold-metadata boundary. The recommended prerequisite is an upstream publisher +that answers metadata queries, plus a validated direct-origin control. + +The suite's assumptions conflict with two settled behaviors: + +- Its subgroup-zero verifier requires `publisherPriorityForGroup(group)` + (200 on even groups, 201 on odd groups). This Plan explicitly ranks egress + by track priority, so a track with priority 200 remains 200. +- It rejects END_OF_TRACK. The relay keeps that status, as decided on + 2026-10-22 in [the runner decision](https://github.com/moq-dev/moq/blob/5f09861634b793056f840408a589f89a8c464fb0/quest/m1/interop-coverage/runner-cases.md). + Its subgroup-header verifier also applies ordinary payload-group expectations + to that extra terminal stream. A standalone marker at `(last_group + 1, 0)` + does not claim END_OF_GROUP for a payload group that never existed. + +Retain the hard Goal until a maintainer chooses a compatible verification +contract. Recommended: keep the approved relay behavior and agree an upstream +conformance profile that understands track-level priority and legitimate +END_OF_TRACK, while still checking forwarded object IDs, properties, +END_OF_GROUP and FIRST_OBJECT. That changes the Goal's current requirement that +this selected unmodified suite passes, so it needs approval. The alternative is +to revisit the egress-priority and terminal-status decisions and explicitly +scope their API/wire and scheduling costs. Do not silently alter those decisions, +weaken assertions, or exclude every failing case. + +After that decision, rerun both direct-origin controls and all four relay cells, +fix any remaining independently justified forwarding failures at source, and +make the nightly gate pass before deleting this README. + ## Related - [#4926](https://github.com/moq-dev/moq/issues/4926) - the declined request for an object-level API; this carries the fields for relays only diff --git a/rs/moq-net/tests/integration/ietf_conformance.rs b/rs/moq-net/tests/integration/ietf_conformance.rs new file mode 100644 index 00000000000..163ae649fe2 --- /dev/null +++ b/rs/moq-net/tests/integration/ietf_conformance.rs @@ -0,0 +1,86 @@ +#![cfg(feature = "async")] + +//! Moxygen announces one prefix, then serves its sixteen-field test namespaces. + +use crate::support::harness::{MockConnectOptions, connect_mock, spawn}; +use moq_net::{Hop, Version, broadcast, origin}; +use std::sync::Arc; + +// Moxygen c27ae6358df62a1195da479b2e07c50dd8174279, moqtest/Utils.cpp. +const PREFIX: &str = "moq-test-00"; +const NAMESPACE: &str = "moq-test-00/0/0/0/1/2/2/1024/100/50/1/1/1/-1/-1/0"; + +fn origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + spawn(driver); + producer +} + +async fn routing(version: &str, cluster: bool) { + let version: Version = version.parse().unwrap(); + let publisher = origin(1); + let relay = origin(2); + let reader = origin(4); + let prefix = Arc::new(publisher.dynamic_route(PREFIX, Default::default()).unwrap()); + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("test", moq_net::track::Info::default()).unwrap(); + let serving = prefix.clone(); + let answer = broadcast.consume(); + moq_clock::sim::spawn(async move { + let request = serving.requested_broadcast().await.unwrap(); + assert_eq!(request.path().as_str(), NAMESPACE); + request.accept(answer); + }); + + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(relay.clone()); + let mut links = vec![connect_mock(options).await]; + let downstream = if cluster { + let second = origin(3); + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse().unwrap()); + options.server_publish = Some(relay.consume()); + options.client_subscribe = Some(second.clone()); + options.client_extensions.ietf = true; + options.server_extensions.ietf = true; + links.push(connect_mock(options).await); + second + } else { + relay.clone() + }; + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(downstream.consume()); + options.client_subscribe = Some(reader.clone()); + links.push(connect_mock(options).await); + + reader.consume().routed(NAMESPACE).await.unwrap(); + let remote = reader.consume().request_broadcast(NAMESPACE, None).await.unwrap(); + let mut subscription = remote.track("test").unwrap().subscribe(None).await.unwrap(); + crate::support::harness::subscribed(&track).await; + let mut group = track.append_group().unwrap(); + group.write_frame(None, &b"conformance"[..]).unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + let mut received = subscription.recv_group().await.unwrap().unwrap(); + assert_eq!(received.sequence, 0); + assert_eq!( + received.read_frame().await.unwrap().unwrap().payload.as_ref(), + b"conformance" + ); + assert!(received.read_frame().await.unwrap().is_none()); + drop((links, prefix, broadcast, track)); +} + +#[moq_clock::test] +async fn conformance_namespaces_route_through_one_relay() { + for version in ["moq-transport-16", "moq-transport-18"] { + routing(version, false).await; + } +} + +#[moq_clock::test] +async fn conformance_namespaces_route_through_a_lite_cluster() { + for version in ["moq-transport-16", "moq-transport-18"] { + routing(version, true).await; + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 99e9941f634..7669be7f7b2 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -34,6 +34,7 @@ mod head_fetch; mod history_groups; mod idle_cancel_refetch; mod idle_session; +mod ietf_conformance; mod late_lower_group; mod lazy_discovery; mod mesh_withdraw; diff --git a/test/README.md b/test/README.md index f6e975981c6..cc54e379e65 100644 --- a/test/README.md +++ b/test/README.md @@ -10,6 +10,7 @@ questions to a harness. For builds and CI selection, see the | --- | --- | --- | | [interop](interop/README.md) | `just test interop --all` | clients built from this checkout interoperate; without `--all`, only Rust runs | | [wire compatibility](interop/README.md) | `just test wire-compat` | this checkout exchanges data with published versions | +| [IETF conformance](ietf-conformance/README.md) | `just test ietf-conformance` | pinned Moxygen sections 1-7 preserve subgroup-zero objects through one relay and a lite-07 cluster | | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | diff --git a/test/ietf-conformance/README.md b/test/ietf-conformance/README.md new file mode 100644 index 00000000000..6e5194639f2 --- /dev/null +++ b/test/ietf-conformance/README.md @@ -0,0 +1,69 @@ +# IETF relay conformance + +Run `nix develop --command just test ietf-conformance` on Linux. The harness +builds the Moxygen revision and matching Meta dependencies pinned in +[`nix/moxygen-sources.nix`](../../nix/moxygen-sources.nix), then runs its +`conformance_test.sh` through this checkout's relay. This is separate from the +published-package smoke matrix. + +Sections 1-7 run on both shared drafts, 16 and 18, through one relay and through +two relays joined with native lite-07. Each cell runs 29 upstream SUBSCRIBE +cases. A direct-origin control runs the same cases once per draft, separating +suite or pin failures from relay behavior. FETCH is skipped using upstream's `SKIP_FETCH`; section 8 uses PUBLISH +and is outside the relay's support. Cases using forwarding modes 1 or 2 send +nonzero subgroups and are explicitly excluded. Modes 0 and 3 are retained. +The staged script preserves every selected case's arguments and deadline. +A client that crashes after printing SUCCESS still fails. Selection tests prove +these exclusions and failure sensitivity, and refuse a changed script anchor. + +The publisher advertises the `moq-test-00` prefix. The client subscribes to +16-field namespaces below it. Mock-clock integration tests in +[`ietf_conformance.rs`](../../rs/moq-net/tests/integration/ietf_conformance.rs) +prove this routing path on both drafts and topologies before the live suite. +The live harness waits for that prefix at the subscriber relay before starting +cases and rejects upstream publisher reconnects, so a recovered session cannot +hide a failure. + +The nightly workflow runs this recipe. `MOQ_TEST_KEEP=1` retains the upstream +reports, client and server logs, relay logs, configs, and endpoints. All children +and port reservations belong to the shared harness and are reaped on exit. +The pinned C++ build is opt-in under `legacyPackages..moxygen`; ordinary +checks and development shells do not build it. Updating the pins also requires +reviewing the selected cases and rerunning the full matrix. The build recipe +adds the client's missing Folly ManualExecutor link dependency and patches +dependency script interpreters for Nix; it changes no protocol code. + +## Contract decision + +The full command currently fails against the relay. Its verdict is preserved: +the upstream client requires a priority that alternates by group, while this +relay deliberately ranks egress by the track's priority. It also rejects the +relay's END\_OF\_TRACK status. Keeping that status is a settled decision in +[the runner decision](https://github.com/moq-dev/moq/blob/5f09861634b793056f840408a589f89a8c464fb0/quest/m1/interop-coverage/runner-cases.md). + +The [passthrough quest](../../quest/m1/ietf-extension/README.md) retains the unmet +Goal and the remaining conformance-profile decision. This draft does not change +protocol behavior or hide these differences behind extra exclusions. The nightly +job will remain red until that conflict is resolved. + +The two-relay path also needs metadata from TRACK\_STATUS before subscribing. +The pinned MoQTestPublisher inherits NOT\_SUPPORTED for that request. The relay +preserves that refusal, rather than widening query-only demand into SUBSCRIBE. +The [range-widening quest](../../quest/m1/compress/ietf-range-widen.md) owns that +separate design boundary. + +Measured against relay base `5f09861634`: + +| Draft | Direct origin | One relay | Two relays | +| --- | --- | --- | --- | +| 16 | 29 pass, 0 fail | 0 pass, 29 fail | 0 pass, 29 fail | +| 18 | 28 pass, 1 fail | 0 pass, 29 fail | 0 pass, 29 fail | + +The direct draft-18 failure is upstream's "Large scale: Many groups and objects" +case: `Group Mismatch: Actual=1 Expected=0`. One single-relay draft-18 case also +stops on that diagnostic. The other single-relay failures report the priority +and terminal-status differences above; END\_OF\_GROUP header mismatches name only +the added terminal stream at `last_group + 1`. Neither run reports a missing +FIRST\_OBJECT signal. Both cluster cells refuse the metadata query before media +flows. These results establish prerequisites and a contract conflict, not a +passing data-preservation result or a completed quest. diff --git a/test/ietf-conformance/prepare.py b/test/ietf-conformance/prepare.py new file mode 100644 index 00000000000..34f2bb08c5e --- /dev/null +++ b/test/ietf-conformance/prepare.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Stage the pinned upstream suite with the relay's explicit subgroup scope.""" + +import sys +from pathlib import Path + + +def prepare(source: str) -> str: + selection = ' local args=("$@") # SC2124: assign as array\n\n section_enabled || return 0\n' + if source.count(selection) != 1: + raise ValueError("upstream run_test selection changed") + source = source.replace(selection, selection + ''' # These upstream cases send subgroups other than zero, outside relay support. + for argument in "${args[@]}"; do + case "$argument" in + --forwarding_preference=1|--forwarding_preference=2) + echo "EXCLUDED (nonzero subgroup): $test_name" + return 0 + ;; + --forwarding_preference=0|--forwarding_preference=3) ;; + --forwarding_preference=*) + echo "unknown upstream forwarding preference: $argument" >&2 + exit 1 + ;; + esac + done +''', 1) + status = ' local exit_code="$2"\n' + if source.count(status) != 1: + raise ValueError("upstream verdict status changed") + return source.replace(status, status + ''' + # A success line before a crash is still a failed client. + if [ "$exit_code" -ne 0 ]; then + echo "Client exited with status $exit_code" + return 0 + fi +''', 1) + + +if __name__ == "__main__": + Path(sys.argv[2]).write_text(prepare(Path(sys.argv[1]).read_text())) diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh new file mode 100755 index 00000000000..6e346ef697c --- /dev/null +++ b/test/ietf-conformance/run.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Pinned upstream conformance, subgroup zero, single relay and lite-07 cluster. +set -euo pipefail + +CONFORMANCE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$CONFORMANCE_DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$CONFORMANCE_DIR/../lib/harness.sh" + +system=$(nix eval --impure --raw --expr builtins.currentSystem) +package=$(nix build --cores 4 --no-link --print-out-paths "$WORKSPACE#legacyPackages.$system.moxygen") +python3 "$CONFORMANCE_DIR/selection.test.py" "$package/share/moxygen/conformance_test.sh" +cargo build --locked -p moq-relay +relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" +# Cargo accepts an absolute target directory too. +[[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" + +harness_begin ietf-conformance "$(harness_env MOQ_TEST_RESULTS)just test ietf-conformance" +python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" +# LibreSSL in the dev shell has no system-wide openssl.cnf. +cat >"$HARNESS_RUN/openssl.cnf" <<'EOF' +[req] +prompt = no +distinguished_name = dn +x509_extensions = extensions +[dn] +CN = localhost +[extensions] +subjectAltName = DNS:localhost,IP:127.0.0.1 +EOF +openssl req -config "$HARNESS_RUN/openssl.cnf" -x509 -newkey rsa:2048 -nodes \ + -keyout "$HARNESS_RUN/key.pem" -out "$HARNESS_RUN/cert.pem" -days 1 \ + >"$HARNESS_RUN/certificate.log" 2>&1 + +start_relay() { + local label="$1" version="$2" seed="${3:-}" + harness_port "$label" + local port="$HARNESS_PORT" config="$HARNESS_RUN/$label.toml" + cat >"$config" <>"$config" <&2 + return 1 +} + +conformance_client() { + cd "$HARNESS_RUN" + SECTIONS="1 2 3 4 5 6 7" SKIP_FETCH=1 MOXYGEN_DIR="$package/libexec" \ + bash ./conformance.sh "$@" Q +} + +run_suite() { + local name="$1" url="$2" status=0 + harness_endpoint "$name-subscriber" "$url" + echo "=== $name / draft $version / sections 1-7 / subgroup 0 ===" + # Upstream owns each case's deadline. No retry or replacement verdict. + harness_spawn "$name-client" "$HARNESS_RUN/$name-client.log" \ + conformance_client "$url" "$version" + harness_wait "$HARNESS_PID" || status=$? + cat "$HARNESS_RUN/$name-client.log" + if ((status != 0)) || ! publisher_healthy; then + harness_result fail suite=ietf-conformance case="$name" version="moq-transport-$version" transport=quic + failed=1 + else + harness_result pass suite=ietf-conformance case="$name" version="moq-transport-$version" transport=quic + fi +} + +# The pinned MoQVersions.h supports drafts 16 and 18; this relay supports both. +failed=0 +for version in 16 18; do + for topology in single cluster; do + label="$topology-$version" + start_relay "$label-origin" "$version" + ingress="$HARNESS_PORT" + ingress_pid="$HARNESS_PID" + egress="$ingress" + egress_pid="" + if [[ "$topology" == cluster ]]; then + start_relay "$label-edge" "$version" "$ingress" + egress="$HARNESS_PORT" + egress_pid="$HARNESS_PID" + fi + harness_port "$label-publisher" + publisher_port="$HARNESS_PORT" + harness_spawn "$label-publisher" "$HARNESS_RUN/$label-publisher.log" \ + "$package/bin/moqtest_server" --port="$HARNESS_PORT" \ + --cert="$HARNESS_RUN/cert.pem" --key="$HARNESS_RUN/key.pem" \ + --relay_url="https://127.0.0.1:$ingress/" --versions="$version" --transport=quic --logging=INFO + publisher_pid="$HARNESS_PID" + namespace_ready + # A direct-origin control distinguishes suite/pin failures from relay behavior. + if [[ "$topology" == single ]]; then + run_suite "reference-$version" "https://127.0.0.1:$publisher_port/test" + fi + run_suite "$label" "https://127.0.0.1:$egress/" + harness_reap "$publisher_pid" + [[ -z "$egress_pid" ]] || harness_reap "$egress_pid" + harness_reap "$ingress_pid" + done +done +if ((failed != 0)); then + echo "Moxygen conformance failed; see each cell's client and endpoint logs" >&2 + exit 1 +fi +echo "Moxygen conformance: all shared drafts and both topologies passed" diff --git a/test/ietf-conformance/selection.test.py b/test/ietf-conformance/selection.test.py new file mode 100644 index 00000000000..746c3d2d933 --- /dev/null +++ b/test/ietf-conformance/selection.test.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Prove scoped upstream selection and failure sensitivity without a network.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from prepare import prepare + +UPSTREAM = Path(sys.argv.pop(1)).read_text() + + +class Selection(unittest.TestCase): + def run_suite(self, exit_code=0): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / "conformance.sh" + script.write_text(prepare(UPSTREAM)) + client = root / "moxygen/moqtest/moqtest_client" + client.parent.mkdir(parents=True) + client.write_text('''#!/usr/bin/env python3 +import json, os, sys +with open(os.environ["CONFORMANCE_CALLS"], "a") as log: + log.write(json.dumps(sys.argv[1:]) + "\\n") +print("MoQTest verification result: SUCCESS") +sys.exit(int(os.environ["CONFORMANCE_EXIT"])) +''') + client.chmod(0o755) + calls = root / "calls.jsonl" + env = dict(os.environ, SECTIONS="1 2 3 4 5 6 7", SKIP_FETCH="1", + MOXYGEN_DIR=str(root), CONFORMANCE_CALLS=str(calls), + CONFORMANCE_EXIT=str(exit_code)) + result = subprocess.run(["bash", str(script), "http://127.0.0.1:1", "16", "Q"], + cwd=root, env=env, capture_output=True, text=True) + return result, [json.loads(line) for line in calls.read_text().splitlines()] + + def test_every_selected_case_uses_subscribe_and_supported_forwarding(self): + result, calls = self.run_suite() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(calls), 29) + preferences = set() + for call in calls: + self.assertIn("--request=subscribe", call) + preference = next(arg for arg in call if arg.startswith("--forwarding_preference=")) + self.assertIn(preference, ["--forwarding_preference=0", "--forwarding_preference=3"]) + preferences.add(preference) + self.assertEqual(len(preferences), 2) + self.assertIn("EXCLUDED (nonzero subgroup)", result.stdout) + print(f"selected {len(calls)} upstream cases") + + def test_a_client_crash_after_success_still_fails_the_suite(self): + result, calls = self.run_suite(5) + self.assertTrue(calls) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Client exited with status 5", result.stdout) + + def test_an_upstream_selection_change_is_refused(self): + with self.assertRaises(ValueError): + prepare(UPSTREAM.replace("local args=", "local renamed=")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/justfile b/test/justfile index a783bd33f9f..95843da1938 100644 --- a/test/justfile +++ b/test/justfile @@ -66,6 +66,10 @@ interop-report results out *args: wire-compat *args: ./interop/compat.sh "$@" +# Pinned Moxygen sections 1-7, subgroup zero, through one relay and a lite-07 cluster. +ietf-conformance: + ./ietf-conformance/run.sh + # Installs each client from its public registry, inside test/smoke's own Nix # shell rather than this repo's toolchain. The first argument picks a smoke # recipe and the rest pass through, e.g. `just test smoke full` or `just test From 9cfeb323b05fe744a75c46b542d1d235a15b3248 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 09:39:11 -0700 Subject: [PATCH 3/4] test(ietf): verify Moxygen conformance under the relay profile The pinned client now accepts the track's advertised priority on every group and one END_OF_TRACK directly after the last object, and keeps every other upstream check. Both single-relay cells pass on drafts 16 and 18. The quest's Goal moves to that profile; the cluster's metadata query and the direct draft-18 control remain. Co-Authored-By: Claude Opus 5.5 --- nix/moxygen-profile.patch | 116 ++++++++++++++++++++++++++++++ nix/moxygen.nix | 3 + quest/m1/README.md | 2 +- quest/m1/ietf-extension/README.md | 81 ++++++++++----------- test/README.md | 2 +- test/ietf-conformance/README.md | 57 +++++++-------- 6 files changed, 188 insertions(+), 73 deletions(-) create mode 100644 nix/moxygen-profile.patch diff --git a/nix/moxygen-profile.patch b/nix/moxygen-profile.patch new file mode 100644 index 00000000000..a5efe587b17 --- /dev/null +++ b/nix/moxygen-profile.patch @@ -0,0 +1,116 @@ +--- a/moxygen/moqtest/MoQTestClient.cpp ++++ b/moxygen/moqtest/MoQTestClient.cpp +@@ -554,6 +554,10 @@ + XLOG(DBG1) << "MoQTest DEBUGGING: calling onObjectStatus"; + + ObjectHeader header = objHeader; ++ if (header.status == ObjectStatus::END_OF_TRACK) { ++ validateEndOfTrack(state, header); ++ return; ++ } + // Validate the received data + if (header.status != ObjectStatus::END_OF_GROUP) { + XLOG(ERR) +@@ -594,6 +598,35 @@ + } + } + ++void MoQTestClient::validateEndOfTrack( ++ const ReceiveState& state, ++ const ObjectHeader& header) { ++ // A subscription that runs to the track's end may learn where it ended from ++ // an END_OF_TRACK placed directly after the last object, either inline or ++ // opening the next group. It carries no payload and ticks off nothing. ++ const AbsoluteLocation location{header.group, header.id}; ++ const AbsoluteLocation inlineEnd{window_.last.group, window_.last.object + 1}; ++ const AbsoluteLocation nextGroup{window_.last.group + 1, 0}; ++ if (state.type != ReceivingType::SUBSCRIBE || !window_.endOfTrack || ++ endOfTrackReceived_ || ++ (location != inlineEnd && location != nextGroup)) { ++ recordSemanticsFailure( ++ folly::to( ++ "Unexpected End of Track at group=", ++ header.group, ++ " id=", ++ header.id, ++ " Expected group=", ++ nextGroup.group, ++ " id=0 or group=", ++ inlineEnd.group, ++ " id=", ++ inlineEnd.object)); ++ return; ++ } ++ endOfTrackReceived_ = true; ++} ++ + void MoQTestClient::onEndOfStream() { + XLOG(DBG1) << "MoQTest DEBUGGING: calling onEndOfStream"; + } +@@ -793,6 +826,16 @@ + publishDoneReceived_ = true; + } + ++namespace { ++// The origin alternates priority by group, but advertises the even-group value ++// as the track's publisher priority. A relay that ranks egress by track ++// priority forwards that value for every group; anything else still fails. ++bool priorityMatches(uint64_t groupID, uint8_t priority) { ++ return priority == publisherPriorityForGroup(groupID) || ++ priority == kMoQTestPublisherPriority; ++} ++} // namespace ++ + void MoQTestClient::validateSubgroupHeader( + const ReceiveState& state, + uint64_t groupID, +@@ -811,8 +854,11 @@ + return; + } + ++ // A relay may end the track on a stream of its own, past the last group. ++ // No payload group exists there to end; onObjectStatus checks its contents. ++ const bool terminal = groupID > window_.last.group; + auto expectEndOfGroup = subgroupCarriesLastObject(params_, subgroupID); +- if (options.containsLastInGroup != expectEndOfGroup) { ++ if (!terminal && options.containsLastInGroup != expectEndOfGroup) { + recordSemanticsFailure( + folly::to( + "End of Group Signal Mismatch for group=", +@@ -839,7 +885,7 @@ + // The publisher may elide the priority from the wire, but the value the + // subscriber ends up with must still be the one the publisher chose. + auto expectedPriority = publisherPriorityForGroup(groupID); +- if (priority != expectedPriority) { ++ if (!priorityMatches(groupID, priority)) { + recordSemanticsFailure( + folly::to( + "Subgroup Priority Mismatch for group=", +@@ -876,7 +922,7 @@ + } + + auto expectedPriority = publisherPriorityForGroup(header.group); +- if (header.priority != expectedPriority) { ++ if (!header.priority || !priorityMatches(header.group, *header.priority)) { + recordSemanticsFailure( + folly::to( + "Datagram Priority Mismatch for group=", +--- a/moxygen/moqtest/MoQTestClient.h ++++ b/moxygen/moqtest/MoQTestClient.h +@@ -254,6 +254,7 @@ + ReceiveState& state, + const std::optional& trackAlias, + const ObjectHeader& objHeader); ++ void validateEndOfTrack(const ReceiveState& state, const ObjectHeader& header); + void onEndOfStream(); + void onError(ReceiveState& state, ResetStreamErrorCode); + void onPublishDone(); +@@ -318,6 +319,8 @@ + // Last group the joining FETCH half covers; unset when there is no join. + std::optional fetchHalfLastGroup_; + bool publishDoneReceived_{false}; ++ // A track ends at most once. ++ bool endOfTrackReceived_{false}; + // A datagram that arrives after its deadline is as good as dropped. + uint64_t datagramDrops_{0}; + diff --git a/nix/moxygen.nix b/nix/moxygen.nix index 57aa83e55ea..ef8879bc552 100644 --- a/nix/moxygen.nix +++ b/nix/moxygen.nix @@ -30,6 +30,9 @@ stdenv.mkDerivation { version = "unstable-2026-10-23"; src = sources.moxygen; cmakeDir = "../standalone"; + # The relay conformance profile: the client accepts track-level priority and + # an END_OF_TRACK directly after the last object. Every other check stays. + patches = [ ./moxygen-profile.patch ]; nativeBuildInputs = [ cmake ninja diff --git a/quest/m1/README.md b/quest/m1/README.md index 0d3574d8ffe..47fd0782b09 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -86,7 +86,7 @@ tracks, so per-group stats stop costing cluster announces. - [Video preference in bindings](/quest/m1/ffi-video-preference.md) - every binding's video catalog record carries hang's rendition `preference` - [Binding encoder config](/quest/m1/binding-encoder-config.md) - binding encoders take an exact rational framerate, and the publication track name moves to options only the producers take - [FFI shutdown requests the drain synchronously](/quest/m1/ffi-shutdown-sync.md) - `shutdown()` becomes a sync drain request awaited with `closed()`, so a dropped future can no longer cancel -- [moq-transport passthrough](/quest/m1/ietf-extension/README.md) - object IDs, properties, end status, datagram objects, and unknown track properties survive moq-transport -> moq-relay (and a relay cluster) -> moq-transport; moxygen's moq-test sections 1-7 pass nightly, except cases that use a subgroup other than 0 +- [moq-transport passthrough](/quest/m1/ietf-extension/README.md) - object IDs, properties, end status, datagram objects, and unknown track properties survive moq-transport -> moq-relay (and a relay cluster) -> moq-transport; moxygen's moq-test sections 1-7 pass nightly under the relay conformance profile, except cases that use a subgroup other than 0 - [Announcement takeover](/quest/m1/announce-takeover.md) - a new independent route explicitly continues instances served through its predecessor, with shared Rust/JS semantics and lite-07 wire - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH; merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - a warm relay serves a reader with a wider max age the cached groups a cold relay would, with no wire change diff --git a/quest/m1/ietf-extension/README.md b/quest/m1/ietf-extension/README.md index 2f26f7bb74c..6e694199f14 100644 --- a/quest/m1/ietf-extension/README.md +++ b/quest/m1/ietf-extension/README.md @@ -10,7 +10,10 @@ Tracks advertising DYNAMIC_GROUPS relay NEW_GROUP_REQUEST back to the original publisher, with redundant requests coalesced. Across the cluster, refresh demand carries them on native lite-07. moxygen's moq-test conformance (sections 1-7, except cases that use a -subgroup other than 0) passes through moq-relay in a nightly CI job. +subgroup other than 0) passes through moq-relay in a nightly CI job under the +relay conformance profile. The profile accepts the track's publisher priority +on every group and an END_OF_TRACK directly after the last object. It still +checks forwarded object IDs, properties, END_OF_GROUP and FIRST_OBJECT. Boundaries: @@ -82,54 +85,46 @@ Work owned by this README, after the children land: - FETCH cases are skipped upstream. Section 8 stays out while PUBLISH is declined. +### Conformance profile and remaining work -### Remaining conformance decision +Decided with the maintainer on 2026-10-23: keep the relay's track-level egress +priority and its END_OF_TRACK, and verify under a relay conformance profile +instead of the literal upstream suite. Changing per-group priority transport or +terminal representation to satisfy one verifier would cost scheduling and wire +complexity for no behavioral gain. -The original Goal above is unmet. The pinned harness is in +The pinned harness is in [`test/ietf-conformance`](../../../test/ietf-conformance/README.md), with Moxygen `c27ae6358df62a1195da479b2e07c50dd8174279` and its matching dependency revisions. It keeps all selected case arguments and deadlines, excludes only the agreed nonzero-subgroup cases, and fails on client crashes or publisher reconnects. -The nightly job is a diagnostic checkpoint, not a passing conformance claim. - -The direct-origin controls pass 29/29 cases on draft 16 and 28/29 on draft 18. -The draft-18 large-scale case fails even without a relay with -`Group Mismatch: Actual=1 Expected=0`; its provenance must be resolved upstream. -A different single-relay draft-18 case, "End of group markers with single object", also reports `(1, expected 0)`; -its relationship to the direct-control failure is not established. -Every relay cell fails 29/29. Single-relay cells expose the conflicts below; -cluster cells fail before media because their TRACK_INFO query needs upstream -TRACK_STATUS, and the pinned MoQTestPublisher inherits NOT_SUPPORTED. Preserve -that refusal: changing query-only demand into a SUBSCRIBE is not an authorized -fallback here. [Range widening](/quest/m1/compress/ietf-range-widen.md), claimed -by [moq-dev/moq#6465](https://github.com/moq-dev/moq/pull/6465), owns that separate -cold-metadata boundary. The recommended prerequisite is an upstream publisher -that answers metadata queries, plus a validated direct-origin control. - -The suite's assumptions conflict with two settled behaviors: - -- Its subgroup-zero verifier requires `publisherPriorityForGroup(group)` - (200 on even groups, 201 on odd groups). This Plan explicitly ranks egress - by track priority, so a track with priority 200 remains 200. -- It rejects END_OF_TRACK. The relay keeps that status, as decided on - 2026-10-22 in [the runner decision](https://github.com/moq-dev/moq/blob/5f09861634b793056f840408a589f89a8c464fb0/quest/m1/interop-coverage/runner-cases.md). - Its subgroup-header verifier also applies ordinary payload-group expectations - to that extra terminal stream. A standalone marker at `(last_group + 1, 0)` - does not claim END_OF_GROUP for a payload group that never existed. - -Retain the hard Goal until a maintainer chooses a compatible verification -contract. Recommended: keep the approved relay behavior and agree an upstream -conformance profile that understands track-level priority and legitimate -END_OF_TRACK, while still checking forwarded object IDs, properties, -END_OF_GROUP and FIRST_OBJECT. That changes the Goal's current requirement that -this selected unmodified suite passes, so it needs approval. The alternative is -to revisit the egress-priority and terminal-status decisions and explicitly -scope their API/wire and scheduling costs. Do not silently alter those decisions, -weaken assertions, or exclude every failing case. - -After that decision, rerun both direct-origin controls and all four relay cells, -fix any remaining independently justified forwarding failures at source, and -make the nightly gate pass before deleting this README. +`nix/moxygen-profile.patch` applies the profile to the client verifier: it also +accepts the track's advertised priority on every group, and one END_OF_TRACK +directly after the last object, whose stream ends no payload group. All other +upstream checks stay, including object IDs, properties, END_OF_GROUP and +FIRST_OBJECT. + +Measured on the profile: the direct-origin controls and both single-relay cells +pass 29/29 on drafts 16 and 18. Both cluster cells fail 29/29 before media. +Remaining, in order: + +- **A publisher that answers metadata queries.** Cluster cells fail because the + edge's TRACK_INFO query becomes TRACK_STATUS upstream, and the pinned + MoQTestPublisher inherits NOT_SUPPORTED. Preserve that refusal: widening + query-only demand into a SUBSCRIBE is not an authorized fallback. + [Range widening](/quest/m1/compress/ietf-range-widen.md), claimed by + [moq-dev/moq#6465](https://github.com/moq-dev/moq/pull/6465), owns that + separate cold-metadata boundary. Needs a Moxygen revision whose test + publisher answers TRACK_STATUS; then bump the pins and rebase the profile. +- **The direct draft-18 control.** One checkpoint run failed upstream's "Large + scale: Many groups and objects" with `Group Mismatch: Actual=1 Expected=0` + without any relay; the profile run passed it. The verifier expects groups in + order although each arrives on its own stream, which its own comment calls a + spurious failure. Resolve it at the verifier, not with a retry. + +Then rerun both direct-origin controls and all four relay cells, fix any +independently justified forwarding failures at source, and make the nightly +gate pass before deleting this README. ## Related diff --git a/test/README.md b/test/README.md index cc54e379e65..e082a353c88 100644 --- a/test/README.md +++ b/test/README.md @@ -10,7 +10,7 @@ questions to a harness. For builds and CI selection, see the | --- | --- | --- | | [interop](interop/README.md) | `just test interop --all` | clients built from this checkout interoperate; without `--all`, only Rust runs | | [wire compatibility](interop/README.md) | `just test wire-compat` | this checkout exchanges data with published versions | -| [IETF conformance](ietf-conformance/README.md) | `just test ietf-conformance` | pinned Moxygen sections 1-7 preserve subgroup-zero objects through one relay and a lite-07 cluster | +| [IETF conformance](ietf-conformance/README.md) | `just test ietf-conformance` | pinned Moxygen sections 1-7, under the relay conformance profile, preserve subgroup-zero objects through one relay and a lite-07 cluster | | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | diff --git a/test/ietf-conformance/README.md b/test/ietf-conformance/README.md index 6e5194639f2..357424772b5 100644 --- a/test/ietf-conformance/README.md +++ b/test/ietf-conformance/README.md @@ -29,41 +29,42 @@ reports, client and server logs, relay logs, configs, and endpoints. All childre and port reservations belong to the shared harness and are reaped on exit. The pinned C++ build is opt-in under `legacyPackages..moxygen`; ordinary checks and development shells do not build it. Updating the pins also requires -reviewing the selected cases and rerunning the full matrix. The build recipe -adds the client's missing Folly ManualExecutor link dependency and patches -dependency script interpreters for Nix; it changes no protocol code. +reviewing the selected cases, rebasing the profile patch, and rerunning the full +matrix. The build recipe adds the client's missing Folly ManualExecutor link +dependency and patches dependency script interpreters for Nix. -## Contract decision +## Conformance profile -The full command currently fails against the relay. Its verdict is preserved: -the upstream client requires a priority that alternates by group, while this -relay deliberately ranks egress by the track's priority. It also rejects the -relay's END\_OF\_TRACK status. Keeping that status is a settled decision in +The client runs under a relay conformance profile, applied by +[`nix/moxygen-profile.patch`](../../nix/moxygen-profile.patch). The test origin +alternates priority by group and never ends the track in-band. This relay ranks +egress by the track's priority and keeps END\_OF\_TRACK, as decided in [the runner decision](https://github.com/moq-dev/moq/blob/5f09861634b793056f840408a589f89a8c464fb0/quest/m1/interop-coverage/runner-cases.md). +The profile therefore accepts: -The [passthrough quest](../../quest/m1/ietf-extension/README.md) retains the unmet -Goal and the remaining conformance-profile decision. This draft does not change -protocol behavior or hide these differences behind extra exclusions. The nightly -job will remain red until that conflict is resolved. +- the track's advertised publisher priority (200) on any group, besides the + per-group value; +- one END\_OF\_TRACK on the subscription, directly after the last object, + either inline or opening the next group. That terminal stream skips the + END\_OF\_GROUP header expectation, since it ends no payload group. -The two-relay path also needs metadata from TRACK\_STATUS before subscribing. -The pinned MoQTestPublisher inherits NOT\_SUPPORTED for that request. The relay -preserves that refusal, rather than widening query-only demand into SUBSCRIBE. -The [range-widening quest](../../quest/m1/compress/ietf-range-widen.md) owns that -separate design boundary. +Every other check is upstream's: forwarded object IDs, properties, payloads, +END\_OF\_GROUP status objects and header bits, and FIRST\_OBJECT. Any other +priority, a second END\_OF\_TRACK, or one anywhere else still fails. -Measured against relay base `5f09861634`: +Measured on this branch: | Draft | Direct origin | One relay | Two relays | | --- | --- | --- | --- | -| 16 | 29 pass, 0 fail | 0 pass, 29 fail | 0 pass, 29 fail | -| 18 | 28 pass, 1 fail | 0 pass, 29 fail | 0 pass, 29 fail | +| 16 | 29 pass | 29 pass | 0 pass, 29 fail | +| 18 | 29 pass | 29 pass | 0 pass, 29 fail | -The direct draft-18 failure is upstream's "Large scale: Many groups and objects" -case: `Group Mismatch: Actual=1 Expected=0`. One single-relay draft-18 case also -stops on that diagnostic. The other single-relay failures report the priority -and terminal-status differences above; END\_OF\_GROUP header mismatches name only -the added terminal stream at `last_group + 1`. Neither run reports a missing -FIRST\_OBJECT signal. Both cluster cells refuse the metadata query before media -flows. These results establish prerequisites and a contract conflict, not a -passing data-preservation result or a completed quest. +The command still exits 1, so the nightly job stays red until the +[passthrough quest](../../quest/m1/ietf-extension/README.md)'s remaining work +lands. The two-relay path needs metadata from TRACK\_STATUS before subscribing, +and the pinned MoQTestPublisher inherits NOT\_SUPPORTED for that request. The +relay preserves that refusal, rather than widening query-only demand into +SUBSCRIBE. Separately, the direct draft-18 control has failed the "Large scale: +Many groups and objects" case with `Group Mismatch: Actual=1 Expected=0`; the +upstream verifier expects groups in order although each arrives on its own +stream. From 2808a9a78ff274d73fcbb34243fa6980c45e3883 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 10:45:03 -0700 Subject: [PATCH 4/4] test(ietf): gate the nightly on cells that can pass Two-relay cells fail every case until a pinned test publisher answers TRACK_STATUS, which would keep the whole nightly red and mask unrelated regressions. Run the direct-origin controls and the single-relay cells, which pass under the profile, and track the cluster cell in the quest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ietf-conformance.yml | 2 +- quest/m1/ietf-extension/README.md | 26 +++++----- test/README.md | 2 +- test/ietf-conformance/README.md | 40 +++++++-------- test/ietf-conformance/run.sh | 68 +++++++++----------------- test/justfile | 2 +- 6 files changed, 61 insertions(+), 79 deletions(-) diff --git a/.github/workflows/ietf-conformance.yml b/.github/workflows/ietf-conformance.yml index e6c6a1ab75a..afe1f63e61b 100644 --- a/.github/workflows/ietf-conformance.yml +++ b/.github/workflows/ietf-conformance.yml @@ -20,7 +20,7 @@ jobs: with: access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} - - name: Pinned Moxygen through single and clustered relays + - name: Pinned Moxygen through the relay under the conformance profile run: nix develop --command just test ietf-conformance env: MOQ_TEST_RUNS: ${{ runner.temp }}/ietf-conformance diff --git a/quest/m1/ietf-extension/README.md b/quest/m1/ietf-extension/README.md index 6e694199f14..0d2c380a11e 100644 --- a/quest/m1/ietf-extension/README.md +++ b/quest/m1/ietf-extension/README.md @@ -104,9 +104,10 @@ directly after the last object, whose stream ends no payload group. All other upstream checks stay, including object IDs, properties, END_OF_GROUP and FIRST_OBJECT. -Measured on the profile: the direct-origin controls and both single-relay cells -pass 29/29 on drafts 16 and 18. Both cluster cells fail 29/29 before media. -Remaining, in order: +Measured on the profile over six runs: the direct-origin controls and the +single-relay cells pass 29/29 on drafts 16 and 18, and the nightly job gates on +them. Two-relay cells failed 29/29 before media, so the harness does not run +them yet. Remaining, in order: - **A publisher that answers metadata queries.** Cluster cells fail because the edge's TRACK_INFO query becomes TRACK_STATUS upstream, and the pinned @@ -115,16 +116,19 @@ Remaining, in order: [Range widening](/quest/m1/compress/ietf-range-widen.md), claimed by [moq-dev/moq#6465](https://github.com/moq-dev/moq/pull/6465), owns that separate cold-metadata boundary. Needs a Moxygen revision whose test - publisher answers TRACK_STATUS; then bump the pins and rebase the profile. + publisher answers TRACK_STATUS; then bump the pins, rebase the profile, and + add the two-relay cell back to `test/ietf-conformance/run.sh` (it ran in + [d96126c65a](https://github.com/moq-dev/moq/commit/d96126c65a52027820884cee7cb9741e79fb674b)). - **The direct draft-18 control.** One checkpoint run failed upstream's "Large scale: Many groups and objects" with `Group Mismatch: Actual=1 Expected=0` - without any relay; the profile run passed it. The verifier expects groups in - order although each arrives on its own stream, which its own comment calls a - spurious failure. Resolve it at the verifier, not with a retry. - -Then rerun both direct-origin controls and all four relay cells, fix any -independently justified forwarding failures at source, and make the nightly -gate pass before deleting this README. + without any relay; it has not recurred in six runs since. The verifier + expects groups in order although each arrives on its own stream, which its + own comment calls a spurious failure. Resolve it at the verifier, not with a + retry. + +Then run both direct-origin controls and all four relay cells, fix any +independently justified forwarding failures at source, and keep the nightly +gate passing before deleting this README. ## Related diff --git a/test/README.md b/test/README.md index e082a353c88..c565adaffcb 100644 --- a/test/README.md +++ b/test/README.md @@ -10,7 +10,7 @@ questions to a harness. For builds and CI selection, see the | --- | --- | --- | | [interop](interop/README.md) | `just test interop --all` | clients built from this checkout interoperate; without `--all`, only Rust runs | | [wire compatibility](interop/README.md) | `just test wire-compat` | this checkout exchanges data with published versions | -| [IETF conformance](ietf-conformance/README.md) | `just test ietf-conformance` | pinned Moxygen sections 1-7, under the relay conformance profile, preserve subgroup-zero objects through one relay and a lite-07 cluster | +| [IETF conformance](ietf-conformance/README.md) | `just test ietf-conformance` | pinned Moxygen sections 1-7, under the relay conformance profile, preserve subgroup-zero objects through one relay | | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | diff --git a/test/ietf-conformance/README.md b/test/ietf-conformance/README.md index 357424772b5..a252809a15d 100644 --- a/test/ietf-conformance/README.md +++ b/test/ietf-conformance/README.md @@ -6,11 +6,11 @@ builds the Moxygen revision and matching Meta dependencies pinned in `conformance_test.sh` through this checkout's relay. This is separate from the published-package smoke matrix. -Sections 1-7 run on both shared drafts, 16 and 18, through one relay and through -two relays joined with native lite-07. Each cell runs 29 upstream SUBSCRIBE -cases. A direct-origin control runs the same cases once per draft, separating -suite or pin failures from relay behavior. FETCH is skipped using upstream's `SKIP_FETCH`; section 8 uses PUBLISH -and is outside the relay's support. Cases using forwarding modes 1 or 2 send +Sections 1-7 run on both shared drafts, 16 and 18, through one relay. Each cell +runs 29 upstream SUBSCRIBE cases. A direct-origin control runs the same cases +once per draft, separating suite or pin failures from relay behavior. FETCH is +skipped using upstream's `SKIP_FETCH`; section 8 uses PUBLISH and is outside the +relay's support. Cases using forwarding modes 1 or 2 send nonzero subgroups and are explicitly excluded. Modes 0 and 3 are retained. The staged script preserves every selected case's arguments and deadline. A client that crashes after printing SUCCESS still fails. Selection tests prove @@ -19,8 +19,9 @@ these exclusions and failure sensitivity, and refuse a changed script anchor. The publisher advertises the `moq-test-00` prefix. The client subscribes to 16-field namespaces below it. Mock-clock integration tests in [`ietf_conformance.rs`](../../rs/moq-net/tests/integration/ietf_conformance.rs) -prove this routing path on both drafts and topologies before the live suite. -The live harness waits for that prefix at the subscriber relay before starting +prove this routing path on both drafts, through one relay and through two +relays joined with native lite-07. The live harness waits for that prefix at the +relay before starting cases and rejects upstream publisher reconnects, so a recovered session cannot hide a failure. @@ -52,19 +53,16 @@ Every other check is upstream's: forwarded object IDs, properties, payloads, END\_OF\_GROUP status objects and header bits, and FIRST\_OBJECT. Any other priority, a second END\_OF\_TRACK, or one anywhere else still fails. -Measured on this branch: +Measured over six consecutive runs: -| Draft | Direct origin | One relay | Two relays | -| --- | --- | --- | --- | -| 16 | 29 pass | 29 pass | 0 pass, 29 fail | -| 18 | 29 pass | 29 pass | 0 pass, 29 fail | +| Draft | Direct origin | One relay | +| --- | --- | --- | +| 16 | 29 pass | 29 pass | +| 18 | 29 pass | 29 pass | -The command still exits 1, so the nightly job stays red until the -[passthrough quest](../../quest/m1/ietf-extension/README.md)'s remaining work -lands. The two-relay path needs metadata from TRACK\_STATUS before subscribing, -and the pinned MoQTestPublisher inherits NOT\_SUPPORTED for that request. The -relay preserves that refusal, rather than widening query-only demand into -SUBSCRIBE. Separately, the direct draft-18 control has failed the "Large scale: -Many groups and objects" case with `Group Mismatch: Actual=1 Expected=0`; the -upstream verifier expects groups in order although each arrives on its own -stream. +A two-relay cell is not run yet. The edge needs metadata from TRACK\_STATUS +before subscribing, and the pinned MoQTestPublisher inherits NOT\_SUPPORTED for +that request, so every case fails before media. The relay preserves that +refusal, rather than widening query-only demand into SUBSCRIBE. The +[passthrough quest](../../quest/m1/ietf-extension/README.md) tracks adding the +cell back once a pinned publisher answers it. diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh index 6e346ef697c..517fae082a5 100755 --- a/test/ietf-conformance/run.sh +++ b/test/ietf-conformance/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Pinned upstream conformance, subgroup zero, single relay and lite-07 cluster. +# Pinned upstream conformance under the relay profile, subgroup zero, one relay. set -euo pipefail CONFORMANCE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -33,7 +33,7 @@ openssl req -config "$HARNESS_RUN/openssl.cnf" -x509 -newkey rsa:2048 -nodes \ >"$HARNESS_RUN/certificate.log" 2>&1 start_relay() { - local label="$1" version="$2" seed="${3:-}" + local label="$1" version="$2" harness_port "$label" local port="$HARNESS_PORT" config="$HARNESS_RUN/$label.toml" cat >"$config" <>"$config" <&2 exit 1 fi -echo "Moxygen conformance: all shared drafts and both topologies passed" +echo "Moxygen conformance: all shared drafts passed through the relay" diff --git a/test/justfile b/test/justfile index e3b85d5589a..cafbc825dc1 100644 --- a/test/justfile +++ b/test/justfile @@ -85,7 +85,7 @@ interop-report results out *args: wire-compat *args: ./interop/compat.sh "$@" -# Pinned Moxygen sections 1-7, subgroup zero, through one relay and a lite-07 cluster. +# Pinned Moxygen sections 1-7, subgroup zero, through one relay under the conformance profile. ietf-conformance: ./ietf-conformance/run.sh