From 1adab931570221289962e858948549537b3093f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:50:42 -0800 Subject: [PATCH 1/4] fix(uring): partition nightly socket measurements by axis --- .config/nextest.toml | 7 +++++ rs/moq-uring/README.md | 6 +++- .../tests/integration/socket_queues.rs | 30 +++++++++++++++---- sh/gh/nightly.test.py | 3 +- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 231558d9ae1..19eda5aad37 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -23,6 +23,12 @@ retries = 0 # executes them through cargo bench smokes. default-filter = "not kind(bench)" +# Split the socket benchmark's grid within the hang budget, but measure one +# axis at a time so its load does not compete with another axis. +[[profile.default.overrides]] +filter = "package(moq-uring) & test(benchmark_socket_queues)" +test-group = "socket-queues" + # Limit ring concurrency within one invocation. Unit-test workers use small # rings; integration workers keep production geometry. sh/rs/nextest.sh raises # hosted CI's locked-memory limit and uses the host's hard limit locally. @@ -63,6 +69,7 @@ slow-timeout = { period = "60s", terminate-after = 3 } [test-groups] io-uring = { max-threads = 4 } +socket-queues = { max-threads = 1 } v4l2 = { max-threads = 1 } # Nightly's ignored benchmark tests (`just gh nightly rust-test`) keep each test's diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index f76be803dae..f0fd139d763 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -131,7 +131,7 @@ just rs bench-udp --sample-size 20 --measurement-time 2 --warm-up-time 1 just rs bench-echo ``` -`benchmark_socket_queues` (an ignored integration test, run nightly) attributes +`benchmark_socket_queues` (ignored integration tests, run nightly) attributes UDP loss between a sender and a steered worker group, on this ring and on moq-sock's `recvmmsg` under Tokio. It sweeps the kernel queue, the receive pool depth, workers, flows and GSO one at a time across worker stalls, CPU @@ -141,6 +141,10 @@ at a socket (`sk_drops` via `SO_MEMINFO`), dropped on the path, or unattributed, beside `rx_enobufs`, peak queued bytes, delay and receive CPU. `rx_enobufs` is pool starvation, which leaves datagrams queued in the socket: with a 16-buffer pool it fired hundreds of times and dropped nothing. +Each sweep axis runs as a separate test within nextest's hang budget. The +`socket-queues` test group serializes them so measurements do not compete. +All cells still run three rounds by default (`SOCKET_QUEUES_RUNS` overrides +the count), interleaving cells and receive paths within each axis. ```bash just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs index 073e776002e..1b7c7ebb88c 100644 --- a/rs/moq-uring/tests/integration/socket_queues.rs +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -42,8 +42,10 @@ //! just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate //! ``` //! -//! `SOCKET_QUEUES_RUNS` sets the rounds (3 by default); each round runs every -//! cell once, so repeated cells interleave rather than run back to back. +//! Each sweep axis is a separate, serial test so the full grid does not exceed +//! nextest's per-test hang budget. `SOCKET_QUEUES_RUNS` sets the rounds (3 by +//! default); each round runs every cell of that axis on both paths, so repeated +//! cells interleave rather than run back to back. #![expect( clippy::disallowed_methods, @@ -232,9 +234,25 @@ fn cells() -> Vec { cells } -#[test] -#[ignore = "benchmark: run explicitly; prints a table"] -fn benchmark_socket_queues() { +mod benchmark_socket_queues { + macro_rules! axes { + ($($axis:ident),+ $(,)?) => { + $( + #[test] + #[ignore = "benchmark: run explicitly; prints a table"] + fn $axis() { + super::benchmark(stringify!($axis)); + } + )+ + }; + } + + axes!( + default, queue, stall, pool, workers, flows, gso, contend, overload, path + ); +} + +fn benchmark(axis: &str) { let runs = std::env::var("SOCKET_QUEUES_RUNS") .ok() .map(|runs| runs.parse::().expect("SOCKET_QUEUES_RUNS is a count")) @@ -250,7 +268,7 @@ fn benchmark_socket_queues() { socket_drops,path_drops,unattributed,snmp_rcvbuf_errors,backlog_drops,enobufs,peak_queue_kib,p50_ms,p99_ms,max_ms,\ goodput_mbps,rx_cpu_us_per_dgram" ); - let cells = cells(); + let cells: Vec<_> = cells().into_iter().filter(|cell| cell.axis == axis).collect(); for run in 0..runs { for path in &paths { for cell in &cells { diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 8ea23e81a51..f83aebe4475 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -377,7 +377,8 @@ def test_every_nightly_benchmark_test_exists(self): named |= set(re.findall(r"test\((\w+)\)", call)) self.assertTrue(named) sources = "".join(path.read_text() for path in (ROOT / "rs").glob("*/**/*.rs") if "/target/" not in str(path)) - missing = {name for name in named if not re.search(rf"\bfn {name}\b", sources)} + # A filter can name one test or a module containing several benchmark cases. + missing = {name for name in named if not re.search(rf"\b(?:fn|mod) {name}\b", sources)} self.assertEqual(missing, set(), "Nightly benchmark tests that no longer exist") def test_failed_release_build_does_not_hide_later_work(self): From 0868cefaf56eb57ea876f118b8252f56f7c7bd58 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 13:02:48 -0800 Subject: [PATCH 2/4] test(uring): reject socket cells without a benchmark axis --- rs/moq-uring/tests/integration/socket_queues.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs index 1b7c7ebb88c..424f15e428e 100644 --- a/rs/moq-uring/tests/integration/socket_queues.rs +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -237,6 +237,8 @@ fn cells() -> Vec { mod benchmark_socket_queues { macro_rules! axes { ($($axis:ident),+ $(,)?) => { + pub(super) const AXES: &[&str] = &[$(stringify!($axis)),+]; + $( #[test] #[ignore = "benchmark: run explicitly; prints a table"] @@ -268,7 +270,14 @@ fn benchmark(axis: &str) { socket_drops,path_drops,unattributed,snmp_rcvbuf_errors,backlog_drops,enobufs,peak_queue_kib,p50_ms,p99_ms,max_ms,\ goodput_mbps,rx_cpu_us_per_dgram" ); - let cells: Vec<_> = cells().into_iter().filter(|cell| cell.axis == axis).collect(); + let cells = cells(); + assert!( + cells + .iter() + .all(|cell| benchmark_socket_queues::AXES.contains(&cell.axis)), + "every cell's axis must have a benchmark test" + ); + let cells: Vec<_> = cells.into_iter().filter(|cell| cell.axis == axis).collect(); for run in 0..runs { for path in &paths { for cell in &cells { From 05c889eaa408478079fe8087d88ae7b442f67af9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 13:23:22 -0800 Subject: [PATCH 3/4] fix(bench): report unavailable socket axes as capability skips Co-Authored-By: GPT-6.1 Sol --- bench/README.md | 4 +- .../tests/integration/socket_queues.rs | 4 ++ sh/gh/bench-report.py | 3 ++ sh/gh/bench-report.test.py | 41 +++++++++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/bench/README.md b/bench/README.md index ee67fa96f5e..d65475ce393 100644 --- a/bench/README.md +++ b/bench/README.md @@ -840,7 +840,9 @@ The table includes these measurements: (`just gh nightly rust-test`) run in release builds, one at a time, under nextest's `bench` profile, which keeps each test's output in a JUnit report; their series are named by package and test path, such as - `rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. + `rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. A test with no runnable + path prints only `# benchmark skipped: REASON`; the report omits that capability + skip while rejecting an empty table or missing measurements. - Shipping sizes, in bytes: the release artifacts, minimal FFI build, and JS bundles the [size report](../sh/size/README.md) measures, recorded even on a night its growth check fails. Release build times stay out. diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs index 424f15e428e..446056b69e0 100644 --- a/rs/moq-uring/tests/integration/socket_queues.rs +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -263,6 +263,10 @@ fn benchmark(axis: &str) { true => vec![Path::Tokio, Path::Uring], false => vec![Path::Tokio], }; + if axis == "pool" && !paths.contains(&Path::Uring) { + println!("# benchmark skipped: io_uring is unavailable; Tokio has no receive pool"); + return; + } print_host(); println!( diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index bb3cb8aed8c..49596a7536f 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -210,6 +210,9 @@ def rust_tests(directory): test = case.get("name").replace("::", "/") output = case.findtext("system-out") or "" text = "\n".join(line for line in output.splitlines() if not LIBTEST.fullmatch(line)) + # A capability skip emits only this marker, never an empty CSV table. + if re.fullmatch(r"# benchmark skipped: [^\n]+", text.strip()): + continue entries += printed(text, f"rust-test/{package}/{test}", f"{path.name} {package} {test}") return checked(entries) diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index 7fe179b659e..b6ebe13535e 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -197,6 +197,47 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): with self.assertRaises(ValueError): report.rust_tests(self.root) + def test_rust_capability_skip_keeps_the_other_measurements(self): + measured = ( + "moq-uring::integration", + "socket_queues::benchmark_socket_queues::default", + "path,p99_ms\ntokio,2\n", + ) + self.junit( + "nextest.xml", + [ + measured, + ( + "moq-uring::integration", + "socket_queues::benchmark_socket_queues::pool", + "\nrunning 1 test\n# benchmark skipped: io_uring is unavailable; Tokio has no receive pool\n" + "test socket_queues::benchmark_socket_queues::pool ... ok\n", + ), + ], + ) + self.assertEqual( + report.rust_tests(self.root), + [ + { + "name": "rust-test/moq-uring/socket_queues/benchmark_socket_queues/default/path=tokio/p99_ms", + "unit": "ms", + "value": 2, + } + ], + ) + for stdout in ( + "", + "path,p99_ms\n", + "# benchmark skipped: \n", + "# benchmark skipped: unavailable\npath,p99_ms\n", + ): + self.junit( + "nextest.xml", + [measured, ("moq-uring::integration", "socket_queues::benchmark_socket_queues::pool", stdout)], + ) + with self.assertRaises(ValueError): + report.rust_tests(self.root) + def test_sizes_record_shipping_bytes(self): (self.root / "sizes.json").write_text( json.dumps( From fa1a85d64b75b10428c4ebe98d6a3bf7686961ee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 13:38:37 -0800 Subject: [PATCH 4/4] fix(uring): reserve the shared ring budget for socket axes Co-Authored-By: GPT-6.1 Sol --- .config/nextest.toml | 8 ++++---- rs/moq-uring/README.md | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 19eda5aad37..e0b32532186 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -23,11 +23,12 @@ retries = 0 # executes them through cargo bench smokes. default-filter = "not kind(bench)" -# Split the socket benchmark's grid within the hang budget, but measure one -# axis at a time so its load does not compete with another axis. +# Each socket axis occupies the whole ring budget, keeping it exclusive with +# other axes and ring tests while its offered load is measured. [[profile.default.overrides]] filter = "package(moq-uring) & test(benchmark_socket_queues)" -test-group = "socket-queues" +test-group = "io-uring" +threads-required = 4 # Limit ring concurrency within one invocation. Unit-test workers use small # rings; integration workers keep production geometry. sh/rs/nextest.sh raises @@ -69,7 +70,6 @@ slow-timeout = { period = "60s", terminate-after = 3 } [test-groups] io-uring = { max-threads = 4 } -socket-queues = { max-threads = 1 } v4l2 = { max-threads = 1 } # Nightly's ignored benchmark tests (`just gh nightly rust-test`) keep each test's diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index f0fd139d763..9356a12fd1e 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -141,8 +141,8 @@ at a socket (`sk_drops` via `SO_MEMINFO`), dropped on the path, or unattributed, beside `rx_enobufs`, peak queued bytes, delay and receive CPU. `rx_enobufs` is pool starvation, which leaves datagrams queued in the socket: with a 16-buffer pool it fired hundreds of times and dropped nothing. -Each sweep axis runs as a separate test within nextest's hang budget. The -`socket-queues` test group serializes them so measurements do not compete. +Each sweep axis runs as a separate test within nextest's hang budget. Each +reserves the `io-uring` group's four slots so measurements do not compete. All cells still run three rounds by default (`SOCKET_QUEUES_RUNS` overrides the count), interleaving cells and receive paths within each axis. @@ -155,7 +155,8 @@ just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ign Locked memory is charged per user across processes, not per nextest run. Unit tests use 128 SQ entries and 512 CQ entries instead of the production 256/2048 geometry. Integration tests still exercise production rings. -The nextest group allows four ring tests per invocation. +The nextest group allows four ring tests per invocation. Each socket-queue +benchmark axis occupies all four slots, excluding other ring tests while measuring. The Rust recipes that run nextest use `sh/rs/nextest.sh`: local Linux shells raise their soft locked-memory limit to the host's hard limit, while GitHub-hosted jobs