diff --git a/.github/workflows/ietf-conformance.yml b/.github/workflows/ietf-conformance.yml new file mode 100644 index 00000000000..afe1f63e61b --- /dev/null +++ b/.github/workflows/ietf-conformance.yml @@ -0,0 +1,35 @@ +name: IETF conformance + +permissions: + contents: read + +on: + workflow_call: + +jobs: + moxygen: + runs-on: ubuntu-24.04 + environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + - uses: ./.github/actions/rust-cache + with: + access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + - name: Pinned Moxygen through the relay under the conformance profile + run: nix develop --command just test ietf-conformance + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/ietf-conformance + MOQ_TEST_KEEP: 1 + - name: Keep conformance reports and endpoint logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: ietf-conformance + path: ${{ runner.temp }}/ietf-conformance/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b4b77351474..aff832c04e5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -52,6 +52,10 @@ jobs: uses: ./.github/workflows/features.yml secrets: inherit + ietf-conformance: + uses: ./.github/workflows/ietf-conformance.yml + secrets: inherit + runner-vm: uses: ./.github/workflows/runner-vm.yml diff --git a/flake.nix b/flake.nix index e2e6a919763..923d0090805 100644 --- a/flake.nix +++ b/flake.nix @@ -577,6 +577,10 @@ legacyPackages = { inherit (pkgs) gst_all_1; + # Nightly conformance needs both pinned Moxygen endpoints. Keep this + # C++ dependency build out of the default shell and flake checks. + moxygen = pkgs.callPackage ./nix/moxygen.nix { }; + runner-userns-profile = import ./nix/runner/unshare.nix { inherit pkgs; }; # `nix develop .#obs`: the default shell plus obs-studio, which linking diff --git a/nix/moxygen-profile.patch b/nix/moxygen-profile.patch new file mode 100644 index 00000000000..a5efe587b17 --- /dev/null +++ b/nix/moxygen-profile.patch @@ -0,0 +1,116 @@ +--- a/moxygen/moqtest/MoQTestClient.cpp ++++ b/moxygen/moqtest/MoQTestClient.cpp +@@ -554,6 +554,10 @@ + XLOG(DBG1) << "MoQTest DEBUGGING: calling onObjectStatus"; + + ObjectHeader header = objHeader; ++ if (header.status == ObjectStatus::END_OF_TRACK) { ++ validateEndOfTrack(state, header); ++ return; ++ } + // Validate the received data + if (header.status != ObjectStatus::END_OF_GROUP) { + XLOG(ERR) +@@ -594,6 +598,35 @@ + } + } + ++void MoQTestClient::validateEndOfTrack( ++ const ReceiveState& state, ++ const ObjectHeader& header) { ++ // A subscription that runs to the track's end may learn where it ended from ++ // an END_OF_TRACK placed directly after the last object, either inline or ++ // opening the next group. It carries no payload and ticks off nothing. ++ const AbsoluteLocation location{header.group, header.id}; ++ const AbsoluteLocation inlineEnd{window_.last.group, window_.last.object + 1}; ++ const AbsoluteLocation nextGroup{window_.last.group + 1, 0}; ++ if (state.type != ReceivingType::SUBSCRIBE || !window_.endOfTrack || ++ endOfTrackReceived_ || ++ (location != inlineEnd && location != nextGroup)) { ++ recordSemanticsFailure( ++ folly::to( ++ "Unexpected End of Track at group=", ++ header.group, ++ " id=", ++ header.id, ++ " Expected group=", ++ nextGroup.group, ++ " id=0 or group=", ++ inlineEnd.group, ++ " id=", ++ inlineEnd.object)); ++ return; ++ } ++ endOfTrackReceived_ = true; ++} ++ + void MoQTestClient::onEndOfStream() { + XLOG(DBG1) << "MoQTest DEBUGGING: calling onEndOfStream"; + } +@@ -793,6 +826,16 @@ + publishDoneReceived_ = true; + } + ++namespace { ++// The origin alternates priority by group, but advertises the even-group value ++// as the track's publisher priority. A relay that ranks egress by track ++// priority forwards that value for every group; anything else still fails. ++bool priorityMatches(uint64_t groupID, uint8_t priority) { ++ return priority == publisherPriorityForGroup(groupID) || ++ priority == kMoQTestPublisherPriority; ++} ++} // namespace ++ + void MoQTestClient::validateSubgroupHeader( + const ReceiveState& state, + uint64_t groupID, +@@ -811,8 +854,11 @@ + return; + } + ++ // A relay may end the track on a stream of its own, past the last group. ++ // No payload group exists there to end; onObjectStatus checks its contents. ++ const bool terminal = groupID > window_.last.group; + auto expectEndOfGroup = subgroupCarriesLastObject(params_, subgroupID); +- if (options.containsLastInGroup != expectEndOfGroup) { ++ if (!terminal && options.containsLastInGroup != expectEndOfGroup) { + recordSemanticsFailure( + folly::to( + "End of Group Signal Mismatch for group=", +@@ -839,7 +885,7 @@ + // The publisher may elide the priority from the wire, but the value the + // subscriber ends up with must still be the one the publisher chose. + auto expectedPriority = publisherPriorityForGroup(groupID); +- if (priority != expectedPriority) { ++ if (!priorityMatches(groupID, priority)) { + recordSemanticsFailure( + folly::to( + "Subgroup Priority Mismatch for group=", +@@ -876,7 +922,7 @@ + } + + auto expectedPriority = publisherPriorityForGroup(header.group); +- if (header.priority != expectedPriority) { ++ if (!header.priority || !priorityMatches(header.group, *header.priority)) { + recordSemanticsFailure( + folly::to( + "Datagram Priority Mismatch for group=", +--- a/moxygen/moqtest/MoQTestClient.h ++++ b/moxygen/moqtest/MoQTestClient.h +@@ -254,6 +254,7 @@ + ReceiveState& state, + const std::optional& trackAlias, + const ObjectHeader& objHeader); ++ void validateEndOfTrack(const ReceiveState& state, const ObjectHeader& header); + void onEndOfStream(); + void onError(ReceiveState& state, ResetStreamErrorCode); + void onPublishDone(); +@@ -318,6 +319,8 @@ + // Last group the joining FETCH half covers; unset when there is no join. + std::optional fetchHalfLastGroup_; + bool publishDoneReceived_{false}; ++ // A track ends at most once. ++ bool endOfTrackReceived_{false}; + // A datagram that arrives after its deadline is as good as dropped. + uint64_t datagramDrops_{0}; + diff --git a/nix/moxygen-sources.nix b/nix/moxygen-sources.nix new file mode 100644 index 00000000000..4de3c2a895b --- /dev/null +++ b/nix/moxygen-sources.nix @@ -0,0 +1,44 @@ +{ + moxygen = { + owner = "facebookexperimental"; + repo = "moxygen"; + rev = "c27ae6358df62a1195da479b2e07c50dd8174279"; + hash = "sha256-ZbTb3kOyFYL0MfALDgoQC50NX+OR6geC4iUXc/phSik="; + }; + folly = { + owner = "facebook"; + repo = "folly"; + rev = "e89841cb61bf8f5283cda05b1bd35de99557a557"; + hash = "sha256-PMOouxfCqzzkdzIC3lO0iwSi82YjwQ5empXOUSV1j8E="; + }; + mvfst = { + owner = "facebook"; + repo = "mvfst"; + rev = "00fb3329dc43c6095668083d92a626031aa1eead"; + hash = "sha256-tTDiMMGZ82THJnrZc5V4xgScm+RaQiAgYUXbGLtY3qs="; + }; + proxygen = { + owner = "facebook"; + repo = "proxygen"; + rev = "05d1c685b865a2c90f6ab13aef6f01ccd53c7931"; + hash = "sha256-ogSop9Vc1Arfa9FOyhxo1F8n2QQ+ytBmlB7SK7Am0xE="; + }; + wangle = { + owner = "facebook"; + repo = "wangle"; + rev = "a39ad0756413158253824d08fa3d212dcfac4e42"; + hash = "sha256-OpmCdhEokiH0kTzsDGchHMtwHsvE/2ZjZ61ph3SeF0Y="; + }; + fizz = { + owner = "facebookincubator"; + repo = "fizz"; + rev = "327e167726bdde385a4fbb368ea148aeb2641acb"; + hash = "sha256-tQC/2apy2BS5+Ckb7U3DRwwsMs3Zbv5sUvW/g+wzywo="; + }; + fast_float = { + owner = "fastfloat"; + repo = "fast_float"; + rev = "v8.0.0"; + hash = "sha256-shP+me3iqTRrsPGYrvcbnJNRZouQbW62T24xfkEgGSE="; + }; +} diff --git a/nix/moxygen.nix b/nix/moxygen.nix new file mode 100644 index 00000000000..ef8879bc552 --- /dev/null +++ b/nix/moxygen.nix @@ -0,0 +1,104 @@ +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + ninja, + pkg-config, + gperf, + python3, + perl, + openssl, + glog, + gflags, + fmt, + double-conversion, + boost, + libevent, + libsodium, + zstd, + zlib, + c-ares, + libunwind, +}: +let + pins = import ./moxygen-sources.nix; + sources = lib.mapAttrs (_: pin: fetchFromGitHub pin) pins; +in +stdenv.mkDerivation { + pname = "moxygen-conformance"; + version = "unstable-2026-10-23"; + src = sources.moxygen; + cmakeDir = "../standalone"; + # The relay conformance profile: the client accepts track-level priority and + # an END_OF_TRACK directly after the last object. Every other check stays. + patches = [ ./moxygen-profile.patch ]; + nativeBuildInputs = [ + cmake + ninja + pkg-config + gperf + python3 + perl + ]; + buildInputs = [ + openssl + glog + gflags + fmt + double-conversion + boost + libevent + libsodium + zstd + zlib + c-ares + libunwind + ]; + # The standalone client uses blockingWait, which needs Folly's concrete executor. + postPatch = + lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: source: '' + cp -R ${source} .vendor-${name} + chmod -R u+w .vendor-${name} + patchShebangs .vendor-${name} + '') (builtins.removeAttrs sources [ "moxygen" ]) + ) + + '' + substituteInPlace moxygen/moqtest/CMakeLists.txt \ + --replace-fail 'moqtest_client PUBLIC' \ + 'moqtest_client PUBLIC Folly::folly_executors_manual_executor' + ''; + preConfigure = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: _: '' + cmakeFlagsArray+=("-DFETCHCONTENT_SOURCE_DIR_${lib.toUpper name}=$PWD/.vendor-${name}") + '') (builtins.removeAttrs sources [ "moxygen" ]) + ); + cmakeFlags = [ + "-DBUILD_TESTS=OFF" + "-DBUILD_MOQTEST=ON" + "-DBUILD_SAMPLES=OFF" + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" + ]; + ninjaFlags = [ + "moqtest_client" + "moqtest_server" + ]; + installPhase = '' + runHook preInstall + mkdir -p $out/bin $out/libexec/moxygen/moqtest $out/share/moxygen + for executable in moqtest_client moqtest_server; do + install -m755 moxygen/moqtest/$executable $out/libexec/moxygen/moqtest/$executable + ln -s $out/libexec/moxygen/moqtest/$executable $out/bin/$executable + done + install -m755 $src/moxygen/moqtest/conformance_test.sh $out/share/moxygen/conformance_test.sh + cp ${./moxygen-sources.nix} $out/share/moxygen/sources.nix + runHook postInstall + ''; + meta = { + description = "Pinned Moxygen client and origin for relay conformance"; + homepage = "https://github.com/facebookexperimental/moxygen"; + license = lib.licenses.asl20; + platforms = lib.platforms.linux; + }; +} diff --git a/quest/m1/README.md b/quest/m1/README.md index 0d3574d8ffe..47fd0782b09 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -86,7 +86,7 @@ tracks, so per-group stats stop costing cluster announces. - [Video preference in bindings](/quest/m1/ffi-video-preference.md) - every binding's video catalog record carries hang's rendition `preference` - [Binding encoder config](/quest/m1/binding-encoder-config.md) - binding encoders take an exact rational framerate, and the publication track name moves to options only the producers take - [FFI shutdown requests the drain synchronously](/quest/m1/ffi-shutdown-sync.md) - `shutdown()` becomes a sync drain request awaited with `closed()`, so a dropped future can no longer cancel -- [moq-transport passthrough](/quest/m1/ietf-extension/README.md) - object IDs, properties, end status, datagram objects, and unknown track properties survive moq-transport -> moq-relay (and a relay cluster) -> moq-transport; moxygen's moq-test sections 1-7 pass nightly, except cases that use a subgroup other than 0 +- [moq-transport passthrough](/quest/m1/ietf-extension/README.md) - object IDs, properties, end status, datagram objects, and unknown track properties survive moq-transport -> moq-relay (and a relay cluster) -> moq-transport; moxygen's moq-test sections 1-7 pass nightly under the relay conformance profile, except cases that use a subgroup other than 0 - [Announcement takeover](/quest/m1/announce-takeover.md) - a new independent route explicitly continues instances served through its predecessor, with shared Rust/JS semantics and lite-07 wire - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH; merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - a warm relay serves a reader with a wider max age the cached groups a cold relay would, with no wire change diff --git a/quest/m1/ietf-extension/README.md b/quest/m1/ietf-extension/README.md index 14a9094d7bd..0d2c380a11e 100644 --- a/quest/m1/ietf-extension/README.md +++ b/quest/m1/ietf-extension/README.md @@ -10,7 +10,10 @@ Tracks advertising DYNAMIC_GROUPS relay NEW_GROUP_REQUEST back to the original publisher, with redundant requests coalesced. Across the cluster, refresh demand carries them on native lite-07. moxygen's moq-test conformance (sections 1-7, except cases that use a -subgroup other than 0) passes through moq-relay in a nightly CI job. +subgroup other than 0) passes through moq-relay in a nightly CI job under the +relay conformance profile. The profile accepts the track's publisher priority +on every group and an END_OF_TRACK directly after the last object. It still +checks forwarded object IDs, properties, END_OF_GROUP and FIRST_OBJECT. Boundaries: @@ -76,12 +79,57 @@ Work owned by this README, after the children land: END_OF_GROUP status object, and a FIRST_OBJECT bit that differs. Every child must preserve these byte for byte. - Check routing first. Sections 1-7 send a SUBSCRIBE for a 16-field namespace - under the announced `moq-test-00` prefix. Whether `origin.dynamic` routes it - is unverified. If it does not, bring the routing decision to the maintainer - before working around it. + under the announced `moq-test-00` prefix. Mock-clock integration tests now + prove that path on drafts 16 and 18, through one relay and through two relays + joined by lite-07; the live harness reaches the publisher too. - FETCH cases are skipped upstream. Section 8 stays out while PUBLISH is declined. +### Conformance profile and remaining work + +Decided with the maintainer on 2026-10-23: keep the relay's track-level egress +priority and its END_OF_TRACK, and verify under a relay conformance profile +instead of the literal upstream suite. Changing per-group priority transport or +terminal representation to satisfy one verifier would cost scheduling and wire +complexity for no behavioral gain. + +The pinned harness is in +[`test/ietf-conformance`](../../../test/ietf-conformance/README.md), with Moxygen +`c27ae6358df62a1195da479b2e07c50dd8174279` and its matching dependency revisions. +It keeps all selected case arguments and deadlines, excludes only the agreed +nonzero-subgroup cases, and fails on client crashes or publisher reconnects. +`nix/moxygen-profile.patch` applies the profile to the client verifier: it also +accepts the track's advertised priority on every group, and one END_OF_TRACK +directly after the last object, whose stream ends no payload group. All other +upstream checks stay, including object IDs, properties, END_OF_GROUP and +FIRST_OBJECT. + +Measured on the profile over six runs: the direct-origin controls and the +single-relay cells pass 29/29 on drafts 16 and 18, and the nightly job gates on +them. Two-relay cells failed 29/29 before media, so the harness does not run +them yet. Remaining, in order: + +- **A publisher that answers metadata queries.** Cluster cells fail because the + edge's TRACK_INFO query becomes TRACK_STATUS upstream, and the pinned + MoQTestPublisher inherits NOT_SUPPORTED. Preserve that refusal: widening + query-only demand into a SUBSCRIBE is not an authorized fallback. + [Range widening](/quest/m1/compress/ietf-range-widen.md), claimed by + [moq-dev/moq#6465](https://github.com/moq-dev/moq/pull/6465), owns that + separate cold-metadata boundary. Needs a Moxygen revision whose test + publisher answers TRACK_STATUS; then bump the pins, rebase the profile, and + add the two-relay cell back to `test/ietf-conformance/run.sh` (it ran in + [d96126c65a](https://github.com/moq-dev/moq/commit/d96126c65a52027820884cee7cb9741e79fb674b)). +- **The direct draft-18 control.** One checkpoint run failed upstream's "Large + scale: Many groups and objects" with `Group Mismatch: Actual=1 Expected=0` + without any relay; it has not recurred in six runs since. The verifier + expects groups in order although each arrives on its own stream, which its + own comment calls a spurious failure. Resolve it at the verifier, not with a + retry. + +Then run both direct-origin controls and all four relay cells, fix any +independently justified forwarding failures at source, and keep the nightly +gate passing before deleting this README. + ## Related - [#4926](https://github.com/moq-dev/moq/issues/4926) - the declined request for an object-level API; this carries the fields for relays only diff --git a/rs/moq-net/tests/integration/ietf_conformance.rs b/rs/moq-net/tests/integration/ietf_conformance.rs new file mode 100644 index 00000000000..163ae649fe2 --- /dev/null +++ b/rs/moq-net/tests/integration/ietf_conformance.rs @@ -0,0 +1,86 @@ +#![cfg(feature = "async")] + +//! Moxygen announces one prefix, then serves its sixteen-field test namespaces. + +use crate::support::harness::{MockConnectOptions, connect_mock, spawn}; +use moq_net::{Hop, Version, broadcast, origin}; +use std::sync::Arc; + +// Moxygen c27ae6358df62a1195da479b2e07c50dd8174279, moqtest/Utils.cpp. +const PREFIX: &str = "moq-test-00"; +const NAMESPACE: &str = "moq-test-00/0/0/0/1/2/2/1024/100/50/1/1/1/-1/-1/0"; + +fn origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + spawn(driver); + producer +} + +async fn routing(version: &str, cluster: bool) { + let version: Version = version.parse().unwrap(); + let publisher = origin(1); + let relay = origin(2); + let reader = origin(4); + let prefix = Arc::new(publisher.dynamic_route(PREFIX, Default::default()).unwrap()); + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("test", moq_net::track::Info::default()).unwrap(); + let serving = prefix.clone(); + let answer = broadcast.consume(); + moq_clock::sim::spawn(async move { + let request = serving.requested_broadcast().await.unwrap(); + assert_eq!(request.path().as_str(), NAMESPACE); + request.accept(answer); + }); + + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(relay.clone()); + let mut links = vec![connect_mock(options).await]; + let downstream = if cluster { + let second = origin(3); + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse().unwrap()); + options.server_publish = Some(relay.consume()); + options.client_subscribe = Some(second.clone()); + options.client_extensions.ietf = true; + options.server_extensions.ietf = true; + links.push(connect_mock(options).await); + second + } else { + relay.clone() + }; + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(downstream.consume()); + options.client_subscribe = Some(reader.clone()); + links.push(connect_mock(options).await); + + reader.consume().routed(NAMESPACE).await.unwrap(); + let remote = reader.consume().request_broadcast(NAMESPACE, None).await.unwrap(); + let mut subscription = remote.track("test").unwrap().subscribe(None).await.unwrap(); + crate::support::harness::subscribed(&track).await; + let mut group = track.append_group().unwrap(); + group.write_frame(None, &b"conformance"[..]).unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + let mut received = subscription.recv_group().await.unwrap().unwrap(); + assert_eq!(received.sequence, 0); + assert_eq!( + received.read_frame().await.unwrap().unwrap().payload.as_ref(), + b"conformance" + ); + assert!(received.read_frame().await.unwrap().is_none()); + drop((links, prefix, broadcast, track)); +} + +#[moq_clock::test] +async fn conformance_namespaces_route_through_one_relay() { + for version in ["moq-transport-16", "moq-transport-18"] { + routing(version, false).await; + } +} + +#[moq_clock::test] +async fn conformance_namespaces_route_through_a_lite_cluster() { + for version in ["moq-transport-16", "moq-transport-18"] { + routing(version, true).await; + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 99e9941f634..7669be7f7b2 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -34,6 +34,7 @@ mod head_fetch; mod history_groups; mod idle_cancel_refetch; mod idle_session; +mod ietf_conformance; mod late_lower_group; mod lazy_discovery; mod mesh_withdraw; diff --git a/test/README.md b/test/README.md index f6e975981c6..c565adaffcb 100644 --- a/test/README.md +++ b/test/README.md @@ -10,6 +10,7 @@ questions to a harness. For builds and CI selection, see the | --- | --- | --- | | [interop](interop/README.md) | `just test interop --all` | clients built from this checkout interoperate; without `--all`, only Rust runs | | [wire compatibility](interop/README.md) | `just test wire-compat` | this checkout exchanges data with published versions | +| [IETF conformance](ietf-conformance/README.md) | `just test ietf-conformance` | pinned Moxygen sections 1-7, under the relay conformance profile, preserve subgroup-zero objects through one relay | | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | diff --git a/test/ietf-conformance/README.md b/test/ietf-conformance/README.md new file mode 100644 index 00000000000..a252809a15d --- /dev/null +++ b/test/ietf-conformance/README.md @@ -0,0 +1,68 @@ +# IETF relay conformance + +Run `nix develop --command just test ietf-conformance` on Linux. The harness +builds the Moxygen revision and matching Meta dependencies pinned in +[`nix/moxygen-sources.nix`](../../nix/moxygen-sources.nix), then runs its +`conformance_test.sh` through this checkout's relay. This is separate from the +published-package smoke matrix. + +Sections 1-7 run on both shared drafts, 16 and 18, through one relay. Each cell +runs 29 upstream SUBSCRIBE cases. A direct-origin control runs the same cases +once per draft, separating suite or pin failures from relay behavior. FETCH is +skipped using upstream's `SKIP_FETCH`; section 8 uses PUBLISH and is outside the +relay's support. Cases using forwarding modes 1 or 2 send +nonzero subgroups and are explicitly excluded. Modes 0 and 3 are retained. +The staged script preserves every selected case's arguments and deadline. +A client that crashes after printing SUCCESS still fails. Selection tests prove +these exclusions and failure sensitivity, and refuse a changed script anchor. + +The publisher advertises the `moq-test-00` prefix. The client subscribes to +16-field namespaces below it. Mock-clock integration tests in +[`ietf_conformance.rs`](../../rs/moq-net/tests/integration/ietf_conformance.rs) +prove this routing path on both drafts, through one relay and through two +relays joined with native lite-07. The live harness waits for that prefix at the +relay before starting +cases and rejects upstream publisher reconnects, so a recovered session cannot +hide a failure. + +The nightly workflow runs this recipe. `MOQ_TEST_KEEP=1` retains the upstream +reports, client and server logs, relay logs, configs, and endpoints. All children +and port reservations belong to the shared harness and are reaped on exit. +The pinned C++ build is opt-in under `legacyPackages..moxygen`; ordinary +checks and development shells do not build it. Updating the pins also requires +reviewing the selected cases, rebasing the profile patch, and rerunning the full +matrix. The build recipe adds the client's missing Folly ManualExecutor link +dependency and patches dependency script interpreters for Nix. + +## Conformance profile + +The client runs under a relay conformance profile, applied by +[`nix/moxygen-profile.patch`](../../nix/moxygen-profile.patch). The test origin +alternates priority by group and never ends the track in-band. This relay ranks +egress by the track's priority and keeps END\_OF\_TRACK, as decided in +[the runner decision](https://github.com/moq-dev/moq/blob/5f09861634b793056f840408a589f89a8c464fb0/quest/m1/interop-coverage/runner-cases.md). +The profile therefore accepts: + +- the track's advertised publisher priority (200) on any group, besides the + per-group value; +- one END\_OF\_TRACK on the subscription, directly after the last object, + either inline or opening the next group. That terminal stream skips the + END\_OF\_GROUP header expectation, since it ends no payload group. + +Every other check is upstream's: forwarded object IDs, properties, payloads, +END\_OF\_GROUP status objects and header bits, and FIRST\_OBJECT. Any other +priority, a second END\_OF\_TRACK, or one anywhere else still fails. + +Measured over six consecutive runs: + +| Draft | Direct origin | One relay | +| --- | --- | --- | +| 16 | 29 pass | 29 pass | +| 18 | 29 pass | 29 pass | + +A two-relay cell is not run yet. The edge needs metadata from TRACK\_STATUS +before subscribing, and the pinned MoQTestPublisher inherits NOT\_SUPPORTED for +that request, so every case fails before media. The relay preserves that +refusal, rather than widening query-only demand into SUBSCRIBE. The +[passthrough quest](../../quest/m1/ietf-extension/README.md) tracks adding the +cell back once a pinned publisher answers it. diff --git a/test/ietf-conformance/prepare.py b/test/ietf-conformance/prepare.py new file mode 100644 index 00000000000..34f2bb08c5e --- /dev/null +++ b/test/ietf-conformance/prepare.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Stage the pinned upstream suite with the relay's explicit subgroup scope.""" + +import sys +from pathlib import Path + + +def prepare(source: str) -> str: + selection = ' local args=("$@") # SC2124: assign as array\n\n section_enabled || return 0\n' + if source.count(selection) != 1: + raise ValueError("upstream run_test selection changed") + source = source.replace(selection, selection + ''' # These upstream cases send subgroups other than zero, outside relay support. + for argument in "${args[@]}"; do + case "$argument" in + --forwarding_preference=1|--forwarding_preference=2) + echo "EXCLUDED (nonzero subgroup): $test_name" + return 0 + ;; + --forwarding_preference=0|--forwarding_preference=3) ;; + --forwarding_preference=*) + echo "unknown upstream forwarding preference: $argument" >&2 + exit 1 + ;; + esac + done +''', 1) + status = ' local exit_code="$2"\n' + if source.count(status) != 1: + raise ValueError("upstream verdict status changed") + return source.replace(status, status + ''' + # A success line before a crash is still a failed client. + if [ "$exit_code" -ne 0 ]; then + echo "Client exited with status $exit_code" + return 0 + fi +''', 1) + + +if __name__ == "__main__": + Path(sys.argv[2]).write_text(prepare(Path(sys.argv[1]).read_text())) diff --git a/test/ietf-conformance/run.sh b/test/ietf-conformance/run.sh new file mode 100755 index 00000000000..517fae082a5 --- /dev/null +++ b/test/ietf-conformance/run.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Pinned upstream conformance under the relay profile, subgroup zero, one relay. +set -euo pipefail + +CONFORMANCE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$CONFORMANCE_DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$CONFORMANCE_DIR/../lib/harness.sh" + +system=$(nix eval --impure --raw --expr builtins.currentSystem) +package=$(nix build --cores 4 --no-link --print-out-paths "$WORKSPACE#legacyPackages.$system.moxygen") +python3 "$CONFORMANCE_DIR/selection.test.py" "$package/share/moxygen/conformance_test.sh" +cargo build --locked -p moq-relay +relay="$WORKSPACE/${CARGO_TARGET_DIR:-target}/debug/moq-relay" +# Cargo accepts an absolute target directory too. +[[ "${CARGO_TARGET_DIR:-}" != /* ]] || relay="$CARGO_TARGET_DIR/debug/moq-relay" + +harness_begin ietf-conformance "$(harness_env MOQ_TEST_RESULTS)just test ietf-conformance" +python3 "$CONFORMANCE_DIR/prepare.py" "$package/share/moxygen/conformance_test.sh" "$HARNESS_RUN/conformance.sh" +# LibreSSL in the dev shell has no system-wide openssl.cnf. +cat >"$HARNESS_RUN/openssl.cnf" <<'EOF' +[req] +prompt = no +distinguished_name = dn +x509_extensions = extensions +[dn] +CN = localhost +[extensions] +subjectAltName = DNS:localhost,IP:127.0.0.1 +EOF +openssl req -config "$HARNESS_RUN/openssl.cnf" -x509 -newkey rsa:2048 -nodes \ + -keyout "$HARNESS_RUN/key.pem" -out "$HARNESS_RUN/cert.pem" -days 1 \ + >"$HARNESS_RUN/certificate.log" 2>&1 + +start_relay() { + local label="$1" version="$2" + harness_port "$label" + local port="$HARNESS_PORT" config="$HARNESS_RUN/$label.toml" + cat >"$config" <&2 + return 1 +} + +conformance_client() { + cd "$HARNESS_RUN" + SECTIONS="1 2 3 4 5 6 7" SKIP_FETCH=1 MOXYGEN_DIR="$package/libexec" \ + bash ./conformance.sh "$@" Q +} + +run_suite() { + local name="$1" url="$2" status=0 + harness_endpoint "$name-subscriber" "$url" + echo "=== $name / draft $version / sections 1-7 / subgroup 0 ===" + # Upstream owns each case's deadline. No retry or replacement verdict. + harness_spawn "$name-client" "$HARNESS_RUN/$name-client.log" \ + conformance_client "$url" "$version" + harness_wait "$HARNESS_PID" || status=$? + cat "$HARNESS_RUN/$name-client.log" + if ((status != 0)) || ! publisher_healthy; then + harness_result fail suite=ietf-conformance case="$name" version="moq-transport-$version" transport=quic + failed=1 + else + harness_result pass suite=ietf-conformance case="$name" version="moq-transport-$version" transport=quic + fi +} + +# The pinned MoQVersions.h supports drafts 16 and 18; this relay supports both. +failed=0 +# A two-relay cell waits on a test publisher that answers TRACK_STATUS; see the +# passthrough quest. +for version in 16 18; do + label="single-$version" + start_relay "$label-origin" "$version" + relay_port="$HARNESS_PORT" + relay_pid="$HARNESS_PID" + harness_port "$label-publisher" + publisher_port="$HARNESS_PORT" + harness_spawn "$label-publisher" "$HARNESS_RUN/$label-publisher.log" \ + "$package/bin/moqtest_server" --port="$HARNESS_PORT" \ + --cert="$HARNESS_RUN/cert.pem" --key="$HARNESS_RUN/key.pem" \ + --relay_url="https://127.0.0.1:$relay_port/" --versions="$version" --transport=quic --logging=INFO + publisher_pid="$HARNESS_PID" + namespace_ready + # A direct-origin control distinguishes suite/pin failures from relay behavior. + run_suite "reference-$version" "https://127.0.0.1:$publisher_port/test" + run_suite "$label" "https://127.0.0.1:$relay_port/" + harness_reap "$publisher_pid" + harness_reap "$relay_pid" +done +if ((failed != 0)); then + echo "Moxygen conformance failed; see each cell's client and endpoint logs" >&2 + exit 1 +fi +echo "Moxygen conformance: all shared drafts passed through the relay" diff --git a/test/ietf-conformance/selection.test.py b/test/ietf-conformance/selection.test.py new file mode 100644 index 00000000000..746c3d2d933 --- /dev/null +++ b/test/ietf-conformance/selection.test.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Prove scoped upstream selection and failure sensitivity without a network.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from prepare import prepare + +UPSTREAM = Path(sys.argv.pop(1)).read_text() + + +class Selection(unittest.TestCase): + def run_suite(self, exit_code=0): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / "conformance.sh" + script.write_text(prepare(UPSTREAM)) + client = root / "moxygen/moqtest/moqtest_client" + client.parent.mkdir(parents=True) + client.write_text('''#!/usr/bin/env python3 +import json, os, sys +with open(os.environ["CONFORMANCE_CALLS"], "a") as log: + log.write(json.dumps(sys.argv[1:]) + "\\n") +print("MoQTest verification result: SUCCESS") +sys.exit(int(os.environ["CONFORMANCE_EXIT"])) +''') + client.chmod(0o755) + calls = root / "calls.jsonl" + env = dict(os.environ, SECTIONS="1 2 3 4 5 6 7", SKIP_FETCH="1", + MOXYGEN_DIR=str(root), CONFORMANCE_CALLS=str(calls), + CONFORMANCE_EXIT=str(exit_code)) + result = subprocess.run(["bash", str(script), "http://127.0.0.1:1", "16", "Q"], + cwd=root, env=env, capture_output=True, text=True) + return result, [json.loads(line) for line in calls.read_text().splitlines()] + + def test_every_selected_case_uses_subscribe_and_supported_forwarding(self): + result, calls = self.run_suite() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(calls), 29) + preferences = set() + for call in calls: + self.assertIn("--request=subscribe", call) + preference = next(arg for arg in call if arg.startswith("--forwarding_preference=")) + self.assertIn(preference, ["--forwarding_preference=0", "--forwarding_preference=3"]) + preferences.add(preference) + self.assertEqual(len(preferences), 2) + self.assertIn("EXCLUDED (nonzero subgroup)", result.stdout) + print(f"selected {len(calls)} upstream cases") + + def test_a_client_crash_after_success_still_fails_the_suite(self): + result, calls = self.run_suite(5) + self.assertTrue(calls) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Client exited with status 5", result.stdout) + + def test_an_upstream_selection_change_is_refused(self): + with self.assertRaises(ValueError): + prepare(UPSTREAM.replace("local args=", "local renamed=")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/justfile b/test/justfile index fc079055c94..cafbc825dc1 100644 --- a/test/justfile +++ b/test/justfile @@ -85,6 +85,10 @@ interop-report results out *args: wire-compat *args: ./interop/compat.sh "$@" +# Pinned Moxygen sections 1-7, subgroup zero, through one relay under the conformance profile. +ietf-conformance: + ./ietf-conformance/run.sh + # Installs each client from its public registry, inside test/smoke's own Nix # shell rather than this repo's toolchain. The first argument picks a smoke # recipe and the rest pass through, e.g. `just test smoke full` or `just test