diff --git a/.config/nextest.toml b/.config/nextest.toml index 231558d9ae1..e0b32532186 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -23,6 +23,13 @@ retries = 0 # executes them through cargo bench smokes. default-filter = "not kind(bench)" +# Each socket axis occupies the whole ring budget, keeping it exclusive with +# other axes and ring tests while its offered load is measured. +[[profile.default.overrides]] +filter = "package(moq-uring) & test(benchmark_socket_queues)" +test-group = "io-uring" +threads-required = 4 + # Limit ring concurrency within one invocation. Unit-test workers use small # rings; integration workers keep production geometry. sh/rs/nextest.sh raises # hosted CI's locked-memory limit and uses the host's hard limit locally. diff --git a/bench/README.md b/bench/README.md index ee67fa96f5e..d65475ce393 100644 --- a/bench/README.md +++ b/bench/README.md @@ -840,7 +840,9 @@ The table includes these measurements: (`just gh nightly rust-test`) run in release builds, one at a time, under nextest's `bench` profile, which keeps each test's output in a JUnit report; their series are named by package and test path, such as - `rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. + `rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. A test with no runnable + path prints only `# benchmark skipped: REASON`; the report omits that capability + skip while rejecting an empty table or missing measurements. - Shipping sizes, in bytes: the release artifacts, minimal FFI build, and JS bundles the [size report](../sh/size/README.md) measures, recorded even on a night its growth check fails. Release build times stay out. diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index f76be803dae..9356a12fd1e 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -131,7 +131,7 @@ just rs bench-udp --sample-size 20 --measurement-time 2 --warm-up-time 1 just rs bench-echo ``` -`benchmark_socket_queues` (an ignored integration test, run nightly) attributes +`benchmark_socket_queues` (ignored integration tests, run nightly) attributes UDP loss between a sender and a steered worker group, on this ring and on moq-sock's `recvmmsg` under Tokio. It sweeps the kernel queue, the receive pool depth, workers, flows and GSO one at a time across worker stalls, CPU @@ -141,6 +141,10 @@ at a socket (`sk_drops` via `SO_MEMINFO`), dropped on the path, or unattributed, beside `rx_enobufs`, peak queued bytes, delay and receive CPU. `rx_enobufs` is pool starvation, which leaves datagrams queued in the socket: with a 16-buffer pool it fired hundreds of times and dropped nothing. +Each sweep axis runs as a separate test within nextest's hang budget. Each +reserves the `io-uring` group's four slots so measurements do not compete. +All cells still run three rounds by default (`SOCKET_QUEUES_RUNS` overrides +the count), interleaving cells and receive paths within each axis. ```bash just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate @@ -151,7 +155,8 @@ just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ign Locked memory is charged per user across processes, not per nextest run. Unit tests use 128 SQ entries and 512 CQ entries instead of the production 256/2048 geometry. Integration tests still exercise production rings. -The nextest group allows four ring tests per invocation. +The nextest group allows four ring tests per invocation. Each socket-queue +benchmark axis occupies all four slots, excluding other ring tests while measuring. The Rust recipes that run nextest use `sh/rs/nextest.sh`: local Linux shells raise their soft locked-memory limit to the host's hard limit, while GitHub-hosted jobs diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs index 073e776002e..446056b69e0 100644 --- a/rs/moq-uring/tests/integration/socket_queues.rs +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -42,8 +42,10 @@ //! just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate //! ``` //! -//! `SOCKET_QUEUES_RUNS` sets the rounds (3 by default); each round runs every -//! cell once, so repeated cells interleave rather than run back to back. +//! Each sweep axis is a separate, serial test so the full grid does not exceed +//! nextest's per-test hang budget. `SOCKET_QUEUES_RUNS` sets the rounds (3 by +//! default); each round runs every cell of that axis on both paths, so repeated +//! cells interleave rather than run back to back. #![expect( clippy::disallowed_methods, @@ -232,9 +234,27 @@ fn cells() -> Vec { cells } -#[test] -#[ignore = "benchmark: run explicitly; prints a table"] -fn benchmark_socket_queues() { +mod benchmark_socket_queues { + macro_rules! axes { + ($($axis:ident),+ $(,)?) => { + pub(super) const AXES: &[&str] = &[$(stringify!($axis)),+]; + + $( + #[test] + #[ignore = "benchmark: run explicitly; prints a table"] + fn $axis() { + super::benchmark(stringify!($axis)); + } + )+ + }; + } + + axes!( + default, queue, stall, pool, workers, flows, gso, contend, overload, path + ); +} + +fn benchmark(axis: &str) { let runs = std::env::var("SOCKET_QUEUES_RUNS") .ok() .map(|runs| runs.parse::().expect("SOCKET_QUEUES_RUNS is a count")) @@ -243,6 +263,10 @@ fn benchmark_socket_queues() { true => vec![Path::Tokio, Path::Uring], false => vec![Path::Tokio], }; + if axis == "pool" && !paths.contains(&Path::Uring) { + println!("# benchmark skipped: io_uring is unavailable; Tokio has no receive pool"); + return; + } print_host(); println!( @@ -251,6 +275,13 @@ fn benchmark_socket_queues() { goodput_mbps,rx_cpu_us_per_dgram" ); let cells = cells(); + assert!( + cells + .iter() + .all(|cell| benchmark_socket_queues::AXES.contains(&cell.axis)), + "every cell's axis must have a benchmark test" + ); + let cells: Vec<_> = cells.into_iter().filter(|cell| cell.axis == axis).collect(); for run in 0..runs { for path in &paths { for cell in &cells { diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index bb3cb8aed8c..49596a7536f 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -210,6 +210,9 @@ def rust_tests(directory): test = case.get("name").replace("::", "/") output = case.findtext("system-out") or "" text = "\n".join(line for line in output.splitlines() if not LIBTEST.fullmatch(line)) + # A capability skip emits only this marker, never an empty CSV table. + if re.fullmatch(r"# benchmark skipped: [^\n]+", text.strip()): + continue entries += printed(text, f"rust-test/{package}/{test}", f"{path.name} {package} {test}") return checked(entries) diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index 7fe179b659e..b6ebe13535e 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -197,6 +197,47 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): with self.assertRaises(ValueError): report.rust_tests(self.root) + def test_rust_capability_skip_keeps_the_other_measurements(self): + measured = ( + "moq-uring::integration", + "socket_queues::benchmark_socket_queues::default", + "path,p99_ms\ntokio,2\n", + ) + self.junit( + "nextest.xml", + [ + measured, + ( + "moq-uring::integration", + "socket_queues::benchmark_socket_queues::pool", + "\nrunning 1 test\n# benchmark skipped: io_uring is unavailable; Tokio has no receive pool\n" + "test socket_queues::benchmark_socket_queues::pool ... ok\n", + ), + ], + ) + self.assertEqual( + report.rust_tests(self.root), + [ + { + "name": "rust-test/moq-uring/socket_queues/benchmark_socket_queues/default/path=tokio/p99_ms", + "unit": "ms", + "value": 2, + } + ], + ) + for stdout in ( + "", + "path,p99_ms\n", + "# benchmark skipped: \n", + "# benchmark skipped: unavailable\npath,p99_ms\n", + ): + self.junit( + "nextest.xml", + [measured, ("moq-uring::integration", "socket_queues::benchmark_socket_queues::pool", stdout)], + ) + with self.assertRaises(ValueError): + report.rust_tests(self.root) + def test_sizes_record_shipping_bytes(self): (self.root / "sizes.json").write_text( json.dumps( diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 8ea23e81a51..f83aebe4475 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -377,7 +377,8 @@ def test_every_nightly_benchmark_test_exists(self): named |= set(re.findall(r"test\((\w+)\)", call)) self.assertTrue(named) sources = "".join(path.read_text() for path in (ROOT / "rs").glob("*/**/*.rs") if "/target/" not in str(path)) - missing = {name for name in named if not re.search(rf"\bfn {name}\b", sources)} + # A filter can name one test or a module containing several benchmark cases. + missing = {name for name in named if not re.search(rf"\b(?:fn|mod) {name}\b", sources)} self.assertEqual(missing, set(), "Nightly benchmark tests that no longer exist") def test_failed_release_build_does_not_hide_later_work(self):