Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ retries = 0
# executes them through cargo bench smokes.
default-filter = "not kind(bench)"

# Each socket axis occupies the whole ring budget, keeping it exclusive with
# other axes and ring tests while its offered load is measured.
[[profile.default.overrides]]
filter = "package(moq-uring) & test(benchmark_socket_queues)"
test-group = "io-uring"
threads-required = 4

# Limit ring concurrency within one invocation. Unit-test workers use small
# rings; integration workers keep production geometry. sh/rs/nextest.sh raises
# hosted CI's locked-memory limit and uses the host's hard limit locally.
Expand Down
4 changes: 3 additions & 1 deletion bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -840,7 +840,9 @@ The table includes these measurements:
(`just gh nightly rust-test`) run in release builds, one at a time, under
nextest's `bench` profile, which keeps each test's output in a JUnit report;
their series are named by package and test path, such as
`rust-test/moq-congestion/gcc/bench/benchmark_per_packet`.
`rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. A test with no runnable
path prints only `# benchmark skipped: REASON`; the report omits that capability
skip while rejecting an empty table or missing measurements.
- Shipping sizes, in bytes: the release artifacts, minimal FFI build, and JS
bundles the [size report](../sh/size/README.md) measures, recorded even on a
night its growth check fails. Release build times stay out.
Expand Down
9 changes: 7 additions & 2 deletions rs/moq-uring/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ just rs bench-udp --sample-size 20 --measurement-time 2 --warm-up-time 1
just rs bench-echo
```

`benchmark_socket_queues` (an ignored integration test, run nightly) attributes
`benchmark_socket_queues` (ignored integration tests, run nightly) attributes
UDP loss between a sender and a steered worker group, on this ring and on
moq-sock's `recvmmsg` under Tokio. It sweeps the kernel queue, the receive pool
depth, workers, flows and GSO one at a time across worker stalls, CPU
Expand All @@ -141,6 +141,10 @@ at a socket (`sk_drops` via `SO_MEMINFO`), dropped on the path, or
unattributed, beside `rx_enobufs`, peak queued bytes, delay and receive CPU.
`rx_enobufs` is pool starvation, which leaves datagrams queued in the socket:
with a 16-buffer pool it fired hundreds of times and dropped nothing.
Each sweep axis runs as a separate test within nextest's hang budget. Each
reserves the `io-uring` group's four slots so measurements do not compete.
All cells still run three rounds by default (`SOCKET_QUEUES_RUNS` overrides
the count), interleaving cells and receive paths within each axis.

```bash
just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate
Expand All @@ -151,7 +155,8 @@ just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ign
Locked memory is charged per user across processes, not per nextest run.
Unit tests use 128 SQ entries and 512 CQ entries instead of the production
256/2048 geometry. Integration tests still exercise production rings.
The nextest group allows four ring tests per invocation.
The nextest group allows four ring tests per invocation. Each socket-queue
benchmark axis occupies all four slots, excluding other ring tests while measuring.

The Rust recipes that run nextest use `sh/rs/nextest.sh`: local Linux shells raise their
soft locked-memory limit to the host's hard limit, while GitHub-hosted jobs
Expand Down
41 changes: 36 additions & 5 deletions rs/moq-uring/tests/integration/socket_queues.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@
//! just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate
//! ```
//!
//! `SOCKET_QUEUES_RUNS` sets the rounds (3 by default); each round runs every
//! cell once, so repeated cells interleave rather than run back to back.
//! Each sweep axis is a separate, serial test so the full grid does not exceed
//! nextest's per-test hang budget. `SOCKET_QUEUES_RUNS` sets the rounds (3 by
//! default); each round runs every cell of that axis on both paths, so repeated
//! cells interleave rather than run back to back.

#![expect(
clippy::disallowed_methods,
Expand Down Expand Up @@ -232,9 +234,27 @@ fn cells() -> Vec<Cell> {
cells
}

#[test]
#[ignore = "benchmark: run explicitly; prints a table"]
fn benchmark_socket_queues() {
mod benchmark_socket_queues {
macro_rules! axes {
($($axis:ident),+ $(,)?) => {
pub(super) const AXES: &[&str] = &[$(stringify!($axis)),+];

$(
#[test]
#[ignore = "benchmark: run explicitly; prints a table"]
fn $axis() {
super::benchmark(stringify!($axis));
}
)+
};
}

axes!(
default, queue, stall, pool, workers, flows, gso, contend, overload, path

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Preserve reporting when the standalone pool axis cannot run

On a Linux host where uring_supported() returns false and CI_STRICT is unset, this new pool test has only Path::Tokio, and the existing skip at lines 284–286 removes every cell. It therefore passes with a CSV header but no measurement rows. sh/gh/bench-report.py:206–213 feeds every JUnit testcase into printed(), which raises ValueError("benchmark produced no measurements") at lines 178–179 for this output. Consequently just gh nightly rust-test fails to generate its entire report, including otherwise valid Tokio measurements. Before the split, those measurements shared the same testcase and kept the report nonempty. Please make the unavailable pool case explicitly skippable by the reporting pipeline (without accepting arbitrary empty benchmarks), and add a JUnit/report regression fixture for the unsupported-io_uring path.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 05c889eaa4. An unavailable pool axis prints only an explicit capability-skip marker before any CSV header. The JUnit importer accepts that marker alone, keeps Tokio measurements, and still rejects accidental empty/header-only output. The regression fixture covers Tokio plus skipped pool and malformed skip outputs. All 16 report tests and just ci pass; the new head received a clean local Claude review.

(Written by GPT-6.1 Sol with xhigh effort)

);
}

fn benchmark(axis: &str) {
let runs = std::env::var("SOCKET_QUEUES_RUNS")
.ok()
.map(|runs| runs.parse::<usize>().expect("SOCKET_QUEUES_RUNS is a count"))
Expand All @@ -243,6 +263,10 @@ fn benchmark_socket_queues() {
true => vec![Path::Tokio, Path::Uring],
false => vec![Path::Tokio],
};
if axis == "pool" && !paths.contains(&Path::Uring) {
println!("# benchmark skipped: io_uring is unavailable; Tokio has no receive pool");
return;
}

print_host();
println!(
Expand All @@ -251,6 +275,13 @@ fn benchmark_socket_queues() {
goodput_mbps,rx_cpu_us_per_dgram"
);
let cells = cells();
assert!(
cells
.iter()
.all(|cell| benchmark_socket_queues::AXES.contains(&cell.axis)),
"every cell's axis must have a benchmark test"
);
let cells: Vec<_> = cells.into_iter().filter(|cell| cell.axis == axis).collect();
for run in 0..runs {
for path in &paths {
for cell in &cells {
Expand Down
3 changes: 3 additions & 0 deletions sh/gh/bench-report.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,9 @@ def rust_tests(directory):
test = case.get("name").replace("::", "/")
output = case.findtext("system-out") or ""
text = "\n".join(line for line in output.splitlines() if not LIBTEST.fullmatch(line))
# A capability skip emits only this marker, never an empty CSV table.
if re.fullmatch(r"# benchmark skipped: [^\n]+", text.strip()):
continue
entries += printed(text, f"rust-test/{package}/{test}", f"{path.name} {package} {test}")
return checked(entries)

Expand Down
41 changes: 41 additions & 0 deletions sh/gh/bench-report.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,47 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self):
with self.assertRaises(ValueError):
report.rust_tests(self.root)

def test_rust_capability_skip_keeps_the_other_measurements(self):
measured = (
"moq-uring::integration",
"socket_queues::benchmark_socket_queues::default",
"path,p99_ms\ntokio,2\n",
)
self.junit(
"nextest.xml",
[
measured,
(
"moq-uring::integration",
"socket_queues::benchmark_socket_queues::pool",
"\nrunning 1 test\n# benchmark skipped: io_uring is unavailable; Tokio has no receive pool\n"
"test socket_queues::benchmark_socket_queues::pool ... ok\n",
),
],
)
self.assertEqual(
report.rust_tests(self.root),
[
{
"name": "rust-test/moq-uring/socket_queues/benchmark_socket_queues/default/path=tokio/p99_ms",
"unit": "ms",
"value": 2,
}
],
)
for stdout in (
"",
"path,p99_ms\n",
"# benchmark skipped: \n",
"# benchmark skipped: unavailable\npath,p99_ms\n",
):
self.junit(
"nextest.xml",
[measured, ("moq-uring::integration", "socket_queues::benchmark_socket_queues::pool", stdout)],
)
with self.assertRaises(ValueError):
report.rust_tests(self.root)

def test_sizes_record_shipping_bytes(self):
(self.root / "sizes.json").write_text(
json.dumps(
Expand Down
3 changes: 2 additions & 1 deletion sh/gh/nightly.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,8 @@ def test_every_nightly_benchmark_test_exists(self):
named |= set(re.findall(r"test\((\w+)\)", call))
self.assertTrue(named)
sources = "".join(path.read_text() for path in (ROOT / "rs").glob("*/**/*.rs") if "/target/" not in str(path))
missing = {name for name in named if not re.search(rf"\bfn {name}\b", sources)}
# A filter can name one test or a module containing several benchmark cases.
missing = {name for name in named if not re.search(rf"\b(?:fn|mod) {name}\b", sources)}
self.assertEqual(missing, set(), "Nightly benchmark tests that no longer exist")

def test_failed_release_build_does_not_hide_later_work(self):
Expand Down
Loading