From 1a42601b0855715411f670a0a6e9d678792f134c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 07:59:01 +0100 Subject: [PATCH 1/8] osbench: the comparison measures the systems, not the setup The 2026-10-24 full run compared rows that were not alike: - persist ran on the boot stick's FAT32 over USB 2.0 under ToyOS and on ext4 on NVMe under Linux. It now runs on each system's persistent filesystem: ToyOS's DATA volume and Linux's home, both on the bench's NVMe. The files group's disk-backed cached operations join it as fs.disk.*. Every fs and persist line names what served it. The osbench metal row refuses a boot whose DATA was not a TOYOS-DATA partition the NVMe driver serves, so memory never stands in for the disk silently. - the settle took three probes over a tenth of a millisecond for steady, so the AMD bench's schedutil step at 1.36 GHz read as settled. A clock is now settled once every reading for 50 ms agrees within 1% (osbench_rules::settle, host-tested), at most 0.5 s. - names differed between systems, so rows never paired: fs.shm/fs.tmp are fs.mem; fair.flat/onejob are fair.shared and fair.scope/perjob fair.apart, one line per job by name. - time.yield compared sched_yield with a spin hint. ToyOS reports it unsupported. - mem.first_touch.2m faulted 32 4 KiB pages under ToyOS against 32 2 MiB pages under Linux. Both variants now fault every byte, per MiB. - osbench all ran every group in one Linux process where ToyOS runs one per group. It now starts each group as a child on both. - a long row's clock could fall into the AMD bench's package limit unmarked. A settled row now also settles after itself (mhz_after), and osbench-compare counts per system the runs whose clock fell under nine tenths of where it settled. The crunch workloads settle first. Linux records its clocksource, minimum frequency and platform profile. osbench-compare refuses a system's runs on one machine that disagree on their setup and states each system's setup above its table. tests/guest/osbench/README.md is the procedure: one revision, four or more interleaved runs each, Linux under ToyOS's frequency request and on the TSC, and both on the bench's NVMe. Co-Authored-By: Claude Opus 5.5 --- ...ompares-toyos-with-linux-on-one-machine.md | 8 ++ ...eaving-idle-and-not-their-settled-clock.md | 9 +- ...linux-reads-its-clock-through-a-syscall.md | 25 ++++ tests/guest/osbench/README.md | 107 +++++++++++++++ tests/guest/osbench/compare/src/lib.rs | 127 ++++++++++++++++-- tests/guest/osbench/rules/Cargo.toml | 2 +- tests/guest/osbench/rules/src/lib.rs | 82 ++++++++++- tests/guest/osbench/src/clock.rs | 25 ++-- tests/guest/osbench/src/crunch.rs | 3 + tests/guest/osbench/src/fair.rs | 62 ++++----- tests/guest/osbench/src/files.rs | 82 +++++------ tests/guest/osbench/src/main.rs | 61 +++++---- tests/guest/osbench/src/mem.rs | 31 +++-- tests/guest/osbench/src/procs.rs | 4 +- tests/guest/osbench/src/report.rs | 15 ++- tests/guest/osbench/src/storage.rs | 47 ++++--- tests/guest/osbench/src/sys/linux.rs | 14 +- tests/guest/osbench/src/sys/toyos.rs | 6 +- tests/guest/osbench/src/system.rs | 7 +- tests/guest/osbench/src/time.rs | 16 +-- tests/guest/osbench/src/work.rs | 4 +- tests/images/benchcase.toml | 10 +- tests/toyos.rs | 43 +++++- 23 files changed, 590 insertions(+), 200 deletions(-) create mode 100644 issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md create mode 100644 tests/guest/osbench/README.md diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index fb51fb9df22..2147b4f9abd 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -34,6 +34,14 @@ its groups: - a short CPU crunch, one thread then all, that stops at TjMax less 15 C where the package's temperature can be read: `crunch.*`. +`tests/guest/osbench/README.md` is how a comparison is run: both halves +from one revision, interleaved runs, Linux under ToyOS's frequency request +and on the TSC, and `persist` on the bench's NVMe under both. ToyOS reaches +that disk only once the bench's TOYOS-DATA partition is designated, which +neither bench's is: until then the `osbench` row refuses each boot. The AMD +bench's Linux reads no TSC +(`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`). + `internet.*`, HTTPS to Cloudflare's speed test, is exploratory evidence of a path neither system owns, run only when a pass names its row: no automated test depends on the public internet. diff --git a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md index 1bcde5520b8..4cf06a614c6 100644 --- a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md +++ b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md @@ -28,9 +28,12 @@ counter reading. Since round 2 every benchmark but the wake and fairness groups' (whose CPUs idle or are all busy by design) runs after `clock::settle`, which spins the -measuring CPU until three probes of a dependent `add` chain agree within 1%, -and every result line prints that clock as `mhz_settled` and the clock just -after as `mhz`; each group also prints `cpu.clock`, 200 probes from idle. +measuring CPU until every reading of a dependent `add` chain for 50 ms +agrees within 1% (`osbench_rules::settle`), or for 0.5 s, and every result +line prints that clock as `mhz_settled` and the clock just after as `mhz`; +each group also prints `cpu.clock`, 200 probes from idle. Three probes in a +row, the rule until 2026-10-27, took the AMD bench's `schedutil` step for +the settled clock: its Linux runs of 2026-10-24 settled at 1,357 MHz. **Exit**: a T14 run on each system shows `mhz_settled` near the all-CPU rate per CPU within the run-to-run spread, or names the benchmark whose CPU did diff --git a/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md b/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md new file mode 100644 index 00000000000..ad797819bec --- /dev/null +++ b/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-10-27 +--- + +# The AMD bench's Linux reads its clock through a syscall + +On 2026-10-24 `osbench`'s `time.now`, `Instant::now` a million times a +round, read 1,212,895 ps a call under the AMD bench's Ubuntu (Linux +6.8.0-147, both runs within 0.1%) against 14,302 ps under the T14's, the same +static binary. A read that slow is a clocksource the vDSO cannot read, so +each one is a syscall: an inference from the cost, since that session did +not record the clocksource. ToyOS reads the TSC on both benches, so every +per-operation timing of the AMD bench's Linux carries some 1.2 µs that its +ToyOS runs do not. + +osbench now records `clocksource` with Linux's conditions, and +`tests/guest/osbench/README.md` runs a comparison with `clocksource=tsc`. + +**Exit**: the AMD bench's Linux reads `clocksource=tsc` in an `osbench` run, +and its `time.now` is within twice the T14's. + +**Owner**: the orchestrator, which runs the benches. Changing the bench's +Linux boot configuration is the owner's decision. diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md new file mode 100644 index 00000000000..82a69749578 --- /dev/null +++ b/tests/guest/osbench/README.md @@ -0,0 +1,107 @@ +# osbench: ToyOS and Linux on one bench + +`osbench` is one program built for both systems (`src/main.rs`). A comparison +is one bench's runs of each, read together by `osbench-compare`. What follows +is what makes those runs comparable. A run taken any other way measures the +setup, not the system. + +## Build both halves from one revision + +The ToyOS half is the `osbench` metal row's image (`tests/images/benchcase.toml`): + +```text +cargo test --test toyos-build -- --metal --metal-readback osbench +``` + +The Linux half is static (`x86_64-unknown-linux-musl`, linked by the +toolchain's own LLD per `.cargo/config.toml`), so a bench runs it without a +toolchain: + +```text +rustup target add --toolchain nightly-2026-07-22 x86_64-unknown-linux-musl +cargo +nightly-2026-07-22 build -p osbench --target x86_64-unknown-linux-musl --profile toyos +``` + +That nightly is the build's stage-0 compiler (`build/sysroot.rs`'s +`STAGE0_CARGO`), the upstream one nearest ToyOS's own. With the `toyos` +profile osbench's own code is generated alike for both; std's platform layer +and the C library beneath it are each system's own. + +## Run + +- **At least four runs of each system, interleaved**: Linux, ToyOS, Linux, + ToyOS. A ToyOS run is one boot and one readback directory. A Linux run is + `osbench all osbench`, + with `--metal-host` for a bench other than the T14. +- **Compare** every run of the bench in one call: + `cargo run -p osbench-compare -- `. It + refuses runs of one system that disagree on their setup (`SETUP`) and + states each system's. Each row's verdict says whether the runs resolve a + difference. +- **A limit the machine reaches is marked, not hidden.** A settled row + carries the clock settled before it (`mhz_settled`) and after it + (`mhz_after`). Where the second is under nine tenths of the first, the + measurement ran into a power or heat limit. The AMD bench's package + limit held a long single-thread row at 1.4 to 1.9 GHz after a 4.29 GHz + start. The verdict counts those runs per system. No program reads the + package's limit state or temperature on ToyOS, so the clock is what both + systems report. + +## Linux makes the requests ToyOS makes + +**The CPU's frequency.** Linux runs under the frequency request ToyOS's +kernel makes of the same CPU, so the clock is the hardware's answer to one +request on both systems: + +- **The T14** (i5-1135G7, `intel_pstate` with HWP). ToyOS asks for HWP's + lowest to highest performance at EPP 0x80 (`kernel/pure/cpu.rs`). Ubuntu's + default, `powersave` at `balance_performance` (EPP 0x80), is that request. + Leave it, and check `governor=powersave*8`, `epp=balance_performance*8`, + `intel_no_turbo=0`. +- **The AMD bench** (Ryzen 7 4800U, `acpi-cpufreq`). ToyOS writes no P-state + or boost register, so the firmware's P0 and Core Performance Boost stand. + Linux's `performance` governor asks for that. Set it before the Linux runs + and give back `schedutil` after them: + + ```text + echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor + echo schedutil | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor + ``` + + Check `governor=performance*16` and `boost=1`. + +When ToyOS's request changes, this section changes with it. + +**The clock.** Every timestamp is read from the TSC on ToyOS, so Linux runs +with `clocksource=tsc`. A clocksource the vDSO cannot read costs every +`Instant::now` a syscall: on 2026-10-24 the AMD bench's Linux took 1.21 µs a +read (`time.now`) against the T14's 14 ns, and every per-operation timing +there carried it. + +## Both write the bench's NVMe + +`persist` and `fs.disk` run on each system's persistent filesystem on the +bench's NVMe. On Linux that is the runner's home, ext4 on the bench's root +partition. On ToyOS it is DATA: the bench's TOYOS-DATA partition, its fourth, +named `ToyOS data`. It is served through the DATA file server and the NVMe +driver. The partition holds a ToyOS volume only once its first block carries +the designation stamp. Until then the file server writes nothing to it and +keeps DATA in memory, and the row refuses the boot by name. The first boot +after the stamp formats the volume, so it is not a measured run. Every row's +`served_by` names the filesystem and the disk it measured. + +## What is not a comparison + +The table also holds rows only one system can measure. They show the +system's figure beside the other's reason: `unsupported` and `unmeasured` +rows, Linux's `.slack1ns` variants and `fair.session`, and ToyOS's +`tail.diag`. They are records, not verdicts. diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs index 58c197eebe3..83421f2cd96 100644 --- a/tests/guest/osbench/compare/src/lib.rs +++ b/tests/guest/osbench/compare/src/lib.rs @@ -11,6 +11,16 @@ //! uncorrected for the table's other rows. Better or worse is judged only //! for a unit whose better direction is known ([`better`]); for any other //! the verdict says which way ToyOS differs. +//! +//! A row whose clock fell across a run's measurement, the clock settled +//! after it (`mhz_after`) under nine tenths of the one settled before it +//! (`mhz_settled`) ([`clock_fell`]), ran into a limit of the machine's, power +//! or heat, which no program reads on both systems: the verdict says in how +//! many of each system's runs it did. +//! +//! A system's runs on one machine are one configuration ([`SETUP`]): runs that +//! disagree on it are refused, since their median would mix two, and each +//! machine's table states what each system's runs were taken under. use std::collections::BTreeMap; use std::fmt::Write as _; @@ -26,9 +36,19 @@ pub enum Os { Toyos, } +impl Os { + pub fn name(self) -> &'static str { + match self { + Os::Linux => "Linux", + Os::Toyos => "ToyOS", + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum Outcome { - Measured { unit: String, p50: u64 }, + /// `clock_fell` as [`clock_fell`] judges the line's two clocks. + Measured { unit: String, p50: u64, clock_fell: bool }, /// Unmeasured, unsupported or refused, with the run's reason. Not { kind: String, why: String }, } @@ -49,7 +69,7 @@ impl Run { /// One run's lines. A benchmark a run reports in more than one group, as /// every group's `cpu.clock`, is named `@`, and the `k`th of -/// one name in a group, as `fair.flat`'s jobs, `#k`. A run with no +/// one name in a group `#k`. A run with no /// osbench line, two systems' lines or one group started twice is refused. pub fn parse(text: &str) -> Result { let mut os = None; @@ -89,7 +109,8 @@ pub fn parse(text: &str) -> Result { let fields: BTreeMap<&str, &str> = words.filter_map(|w| w.split_once('=')).collect(); let outcome = if let Some(unit) = fields.get("unit") { let p50 = fields.get("p50").and_then(|v| v.parse().ok()).ok_or_else(|| format!("{bench}: no p50"))?; - Outcome::Measured { unit: unit.to_string(), p50 } + let mhz = |key: &str| fields.get(key).and_then(|v| v.parse().ok()); + Outcome::Measured { unit: unit.to_string(), p50, clock_fell: clock_fell(mhz("mhz_settled"), mhz("mhz_after")) } } else if let Some((kind, why)) = ["unmeasured", "unsupported", "refused"].iter().find_map(|k| fields.get(k).map(|why| (*k, *why))) { @@ -114,6 +135,13 @@ pub fn parse(text: &str) -> Result { Ok(Run { os: os.ok_or("no osbench line")?, conds, benches }) } +/// Whether the clock fell across a measurement: the clock settled after it +/// under nine tenths of the clock settled before it. A line that settled no +/// clock says nothing either way. +pub fn clock_fell(settled: Option, after: Option) -> bool { + matches!((settled, after), (Some(settled), Some(after)) if after * 10 < settled * 9) +} + pub fn median(values: &[u64]) -> f64 { let mut v = values.to_vec(); v.sort_unstable(); @@ -212,6 +240,44 @@ pub fn better(unit: &str) -> Option { /// varies by the run. const PATHS: [(&str, &str, usize); 2] = [("lan.", "lan.peer_link", 2), ("internet.", "internet.link", 1)]; +/// The conditions every run of one system on one machine must share: the +/// kernel, the CPU's frequency request and the clock every timestamp is read +/// from, and what serves each filesystem the `fs` and `persist` rows use. +pub const SETUP: [&str; 13] = [ + "kernel", + "cpufreq_driver", + "governor", + "epp", + "min_khz", + "max_khz", + "boost", + "intel_no_turbo", + "platform_profile", + "clocksource", + "smt_active", + "fs.mem.served_by", + "fs.disk.served_by", +]; + +/// `os`'s setup on these runs, as `key=value` in [`SETUP`]'s order, or the +/// first key its runs disagree on. +fn setup(runs: &[&Run], os: Os) -> Result, String> { + let runs: Vec<&&Run> = runs.iter().filter(|r| r.os == os).collect(); + let mut said = Vec::new(); + for key in SETUP { + let values: std::collections::BTreeSet> = runs.iter().map(|r| r.conds.get(key)).collect(); + match values.iter().collect::>()[..] { + [] | [None] => {} + [Some(value)] => said.push(format!("{key}={value}")), + _ => { + let seen: Vec<&str> = values.iter().map(|v| v.map_or("absent", String::as_str)).collect(); + return Err(format!("the {} runs disagree on {key}: {}", os.name(), seen.join(" / "))); + } + } + } + Ok(said) +} + fn path(said: &str, words: usize) -> String { said.split(['_', ' ']).take(words).collect::>().join(" ") } @@ -239,6 +305,12 @@ pub fn compare(runs: &[Run]) -> Result { write!(out, " {cond}: {seen}.").unwrap(); } writeln!(out, "\n").unwrap(); + for os in [Os::Linux, Os::Toyos] { + let said = setup(&runs, os).map_err(|why| format!("{machine}: {why}"))?; + if !said.is_empty() { + writeln!(out, "{} setup: {}.\n", os.name(), said.join(", ")).unwrap(); + } + } writeln!(out, "| bench | unit | Linux p50s | Linux median | Linux range | ToyOS p50s | ToyOS median | ToyOS range | ToyOS/Linux | p | verdict |").unwrap(); writeln!(out, "|---|---|---|---|---|---|---|---|---|---|---|").unwrap(); let names: std::collections::BTreeSet<&String> = runs.iter().flat_map(|r| r.benches.keys()).collect(); @@ -262,14 +334,16 @@ pub fn compare(runs: &[Run]) -> Result { fn compare_one(runs: &[&Run], name: &str) -> String { let mut unit = None; let mut values: BTreeMap> = BTreeMap::new(); + let mut fell: BTreeMap = BTreeMap::new(); let mut not: BTreeMap> = BTreeMap::new(); for run in runs { match run.benches.get(name) { - Some(Outcome::Measured { unit: u, p50 }) => { + Some(Outcome::Measured { unit: u, p50, clock_fell }) => { if unit.get_or_insert(u.as_str()) != u { return "| | | | | | | | | units differ between runs".into(); } values.entry(run.os).or_default().push(*p50); + *fell.entry(run.os).or_default() += usize::from(*clock_fell); } Some(Outcome::Not { kind, why }) => not.entry(run.os).or_default().push(format!("{kind}: {why}")), None => not.entry(run.os).or_default().push("absent".into()), @@ -309,13 +383,17 @@ fn compare_one(runs: &[&Run], name: &str) -> String { _ => ("-".into(), "-".into(), String::new()), }; let mut verdict = verdict; + for (os, fell) in fell.iter().filter(|(_, n)| **n > 0) { + let sep = if verdict.is_empty() { "" } else { "; " }; + let of = values.get(os).map_or(0, Vec::len); + verdict = format!("{verdict}{sep}the clock fell in {fell} of {of} {} run(s)", os.name()); + } for (os, reasons) in ¬ { let mut distinct = reasons.clone(); distinct.sort(); distinct.dedup(); - let who = if *os == Os::Linux { "Linux" } else { "ToyOS" }; let sep = if verdict.is_empty() { "" } else { "; " }; - verdict = format!("{verdict}{sep}{who} {} run(s) {}", reasons.len(), distinct.join(" / ")); + verdict = format!("{verdict}{sep}{} {} run(s) {}", os.name(), reasons.len(), distinct.join(" / ")); } format!("{} | {lp} | {lm} | {lr} | {tp} | {tm} | {tr} | {ratio} | {p} | {verdict}", unit.unwrap_or_default()) } @@ -345,7 +423,7 @@ mod tests { ); assert_eq!(r.os, Os::Toyos); assert_eq!(r.machine(), Some("Bench_CPU")); - assert_eq!(r.benches["lan.wired.tcp.rtt.64"], Outcome::Measured { unit: "ns".into(), p50: 812 }); + assert_eq!(r.benches["lan.wired.tcp.rtt.64"], Outcome::Measured { unit: "ns".into(), p50: 812, clock_fell: false }); assert_eq!( r.benches["lan.wifi.tcp.upload"], Outcome::Not { kind: "unsupported".into(), why: "ToyOS_has_no_Wi-Fi_driver".into() } @@ -360,8 +438,8 @@ mod tests { let groups = ["sched", "fair"].map(|g| format!("osbench v=2 os=linux group={g} start\nosbench v=2 os=linux bench=cpu.clock unit=MHz n=1 min=1 p50=4000 max=1 mean=1\n")).concat(); let run = parse(&groups).unwrap(); assert_eq!(run.benches.keys().collect::>(), ["cpu.clock@fair", "cpu.clock@sched"]); - let jobs = "osbench v=2 os=linux bench=fair.flat unit=permille n=5 min=1 p50=63 max=1 mean=1 job=build\n".repeat(2); - assert_eq!(parse(&jobs).unwrap().benches.keys().collect::>(), ["fair.flat#1", "fair.flat#2"]); + let jobs = "osbench v=2 os=linux bench=wake.storm unit=ns n=5 min=1 p50=63 max=1 mean=1\n".repeat(2); + assert_eq!(parse(&jobs).unwrap().benches.keys().collect::>(), ["wake.storm#1", "wake.storm#2"]); let mixed = "osbench v=2 os=linux cond cpus=8\nosbench v=2 os=toyos cond cpus=8\n"; assert!(parse(mixed).unwrap_err().contains("two systems")); assert!(parse("nothing here\n").is_err()); @@ -486,6 +564,37 @@ mod tests { assert!(compare(&runs).unwrap().contains("| the runs disagree on lan.peer_link |")); } + #[test] + fn a_row_whose_clock_fell_in_a_run_says_so_per_system() { + assert!(clock_fell(Some(4289), Some(1884))); + assert!(!clock_fell(Some(4289), Some(3861)), "a tenth down is not a fall"); + assert!(clock_fell(Some(4289), Some(3860))); + assert!(!clock_fell(None, Some(1396)), "no settled clock, no judgement"); + let deflate = |os: &str, p50: u64, after: u64| { + run(os, &[&format!("bench=crunch.single.deflate unit=ns n=63 min=1 p50={p50} max=1 mean=1 mhz_settled=4289 mhz_after={after} mhz=1300")]) + }; + let mut runs: Vec = [(65, 4200), (66, 4210), (64, 4190), (65, 4200)].iter().map(|&(p, m)| deflate("linux", p, m)).collect(); + runs.extend([(63, 1884), (66, 1396), (63, 4200), (64, 4280)].iter().map(|&(p, m)| deflate("toyos", p, m))); + let table = compare(&runs).unwrap(); + let row = table.lines().find(|l| l.starts_with("| crunch.single.deflate")).unwrap(); + assert!(row.ends_with("not resolved; the clock fell in 2 of 4 ToyOS run(s) |"), "{row}"); + } + + #[test] + fn a_systems_runs_on_two_setups_are_refused_and_one_setup_is_stated() { + let under = |os: &str, governor: &str| { + run(os, &[&format!("cond governor={governor}"), "cond clocksource=tsc", "bench=time.now unit=ps n=15 min=1 p50=14 max=1 mean=1"]) + }; + let mixed = [under("linux", "performance*16"), under("linux", "schedutil*16"), under("toyos", "absent")]; + let why = compare(&mixed).unwrap_err(); + assert!(why.contains("the Linux runs disagree on governor: performance*16 / schedutil*16"), "{why}"); + let one = [under("linux", "performance*16"), under("linux", "performance*16")]; + let table = compare(&one).unwrap(); + assert!(table.contains("Linux setup: governor=performance*16, clocksource=tsc."), "{table}"); + let absent = [under("linux", "performance*16"), run("linux", &["bench=time.now unit=ps n=15 min=1 p50=14 max=1 mean=1"])]; + assert!(compare(&absent).unwrap_err().contains("disagree on governor: absent / performance*16")); + } + #[test] fn internet_runs_that_left_by_different_kinds_of_link_are_not_compared() { let left = |os: &str, said: &str| { diff --git a/tests/guest/osbench/rules/Cargo.toml b/tests/guest/osbench/rules/Cargo.toml index 6f8008706fe..92cdce32e6b 100644 --- a/tests/guest/osbench/rules/Cargo.toml +++ b/tests/guest/osbench/rules/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "osbench-rules" -description = "osbench's pure rules, host-tested: when a munmap pair's x2APIC readings admit its figure, where the CPU crunch stops for heat, and an energy counter's span across its wrap." +description = "osbench's pure rules, host-tested: when a munmap pair's x2APIC readings admit its figure, where the CPU crunch stops for heat, an energy counter's span across its wrap, and when a CPU's clock has settled." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/tests/guest/osbench/rules/src/lib.rs b/tests/guest/osbench/rules/src/lib.rs index cb55320caaf..663c7d36557 100644 --- a/tests/guest/osbench/rules/src/lib.rs +++ b/tests/guest/osbench/rules/src/lib.rs @@ -1,7 +1,8 @@ //! The judgements osbench makes over what it read, apart from the reading, so //! a host tests them: whether a munmap pair's figure stands ([`pair`]), where -//! the CPU crunch stops for heat ([`heat`]), and how far an energy counter -//! advanced across its wrap ([`energy`]). +//! the CPU crunch stops for heat ([`heat`]), how far an energy counter +//! advanced across its wrap ([`energy`]), and when a CPU's clock has settled +//! ([`settle`]). pub mod pair { //! A munmap pair is an initiator that unmaps and a sibling that touched @@ -243,3 +244,80 @@ pub mod energy { } } } + +pub mod settle { + //! A clock has settled once every reading over [`STEADY_NS`] agrees + //! within 1%. A frequency governor can hold a CPU that just turned busy + //! at a lower clock for tens of milliseconds before it steps up, so a + //! span much shorter than that takes the step for the settled clock. + + /// How long the readings must agree. + pub const STEADY_NS: u64 = 50_000_000; + + /// The readings since the last one that disagreed. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct Plateau { + since_ns: u64, + lo: u64, + hi: u64, + } + + impl Plateau { + pub fn new(at_ns: u64, mhz: u64) -> Self { + Plateau { since_ns: at_ns, lo: mhz, hi: mhz } + } + + /// The reading `mhz` taken at `at_ns`: whether every reading for + /// [`STEADY_NS`] up to it agrees within 1%. A reading that does not + /// agree with those before it starts the span again from itself. + pub fn settled_with(&mut self, at_ns: u64, mhz: u64) -> bool { + let (lo, hi) = (self.lo.min(mhz), self.hi.max(mhz)); + if (hi - lo) * 100 > hi { + *self = Plateau::new(at_ns, mhz); + } else { + (self.lo, self.hi) = (lo, hi); + } + at_ns - self.since_ns >= STEADY_NS + } + } + + #[cfg(test)] + mod tests { + use super::*; + + const MS: u64 = 1_000_000; + + /// The first time a probe every quarter millisecond from `at` settles. + fn settles(readings: impl Fn(u64) -> u64) -> Option<(u64, u64)> { + let mut plateau = Plateau::new(0, readings(0)); + (1..4000).map(|i| i * MS / 4).find_map(|at| plateau.settled_with(at, readings(at)).then(|| (at, readings(at)))) + } + + #[test] + fn a_governors_30_ms_step_is_not_the_settled_clock() { + let ramp = |at: u64| if at < 30 * MS { 1362 } else { 4033 }; + assert_eq!(settles(ramp), Some((80 * MS, 4033))); + } + + #[test] + fn readings_within_one_percent_hold_the_span() { + let wobble = |at: u64| if (at / MS).is_multiple_of(2) { 4000 } else { 3970 }; + assert_eq!(settles(wobble), Some((50 * MS, wobble(50 * MS)))); + } + + #[test] + fn a_reading_that_disagrees_starts_the_span_again() { + let mut plateau = Plateau::new(0, 4000); + assert!(!plateau.settled_with(40 * MS, 4000)); + assert!(!plateau.settled_with(41 * MS, 3900), "2.5% below"); + assert!(!plateau.settled_with(90 * MS, 3900), "49 ms since the disagreement"); + assert!(plateau.settled_with(91 * MS, 3900)); + } + + #[test] + fn a_slow_drift_settles_only_once_it_stops() { + let drift = |at: u64| 3000 + (at / MS).min(200) * 5; + assert_eq!(settles(drift).map(|(_, mhz)| mhz), Some(4000)); + } + } +} diff --git a/tests/guest/osbench/src/clock.rs b/tests/guest/osbench/src/clock.rs index e6a7fe2eeae..6ae0728a30c 100644 --- a/tests/guest/osbench/src/clock.rs +++ b/tests/guest/osbench/src/clock.rs @@ -5,6 +5,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; +use osbench_rules::settle::Plateau; + use crate::report::result; use crate::ns_since; @@ -28,21 +30,22 @@ pub fn mhz() -> u64 { ADDS * TURNS * 1000 / ns } +/// The longest a settle spins before it takes the clock it has. +const SETTLE_LIMIT: Duration = Duration::from_millis(500); + /// A steady-state warm-up of the calling CPU, for a benchmark to run after: -/// probes back to back for at least 10 ms and until three in a row agree -/// within 1%, or for 200 ms. The clock it reached is printed beside the next -/// result as `mhz_settled`, and `mhz` beside it is the clock just after. +/// [`mhz`] back to back until `osbench_rules::settle` calls the clock +/// settled, or for [`SETTLE_LIMIT`]. The clock it reached is printed beside +/// the next result as `mhz_settled`, and `mhz` beside it is the clock just +/// after. pub fn settle() { let start = Instant::now(); - let mut last = [0u64; 3]; + let at = || u64::try_from(start.elapsed().as_nanos()).expect("a settle under 584 years"); + let mut plateau = Plateau::new(at(), mhz()); loop { - last.rotate_left(1); - last[2] = ADDS * TURNS * 1000 / burst(); - let (lo, hi) = (last.iter().min().copied().unwrap_or(0), last.iter().max().copied().unwrap_or(0)); - let steady = lo > 0 && (hi - lo) * 100 <= hi; - let spent = start.elapsed(); - if (steady && spent >= Duration::from_millis(10)) || spent >= Duration::from_millis(200) { - SETTLED.store(last[2], Ordering::Relaxed); + let reading = mhz(); + if plateau.settled_with(at(), reading) || start.elapsed() >= SETTLE_LIMIT { + SETTLED.store(reading, Ordering::Relaxed); return; } } diff --git a/tests/guest/osbench/src/crunch.rs b/tests/guest/osbench/src/crunch.rs index 2d3115fdf9a..aef7f3bf746 100644 --- a/tests/guest/osbench/src/crunch.rs +++ b/tests/guest/osbench/src/crunch.rs @@ -218,6 +218,9 @@ fn run_phase(phase: &str, threads: usize, inputs: &Arc, heat: Option<&He continue; } } + // The clock before the slice, beside the one after it: a slice + // long enough to meet the package's power limit says so. + crate::clock::settle(); let (mut ns, elapsed) = slice(workload, threads, inputs, &stopped); let units = ns.len() as u64; let mut fields = vec![ diff --git a/tests/guest/osbench/src/fair.rs b/tests/guest/osbench/src/fair.rs index cf3af89326f..a2f37cd3bd9 100644 --- a/tests/guest/osbench/src/fair.rs +++ b/tests/guest/osbench/src/fair.rs @@ -14,17 +14,17 @@ //! machine's CPUs. //! //! Each system runs it once with all three workloads in this program's own -//! group, and once with a group per workload. On ToyOS the group is the Job -//! object its threads share CPU time under: `onejob` leaves the three in this -//! program's, and `perjob` starts each workload in a Job of its own -//! (`SysCap::new_job`, which needs `Rights::JOBS`). On Linux `flat` leaves -//! them in this process's session and cgroup, where the fair unit is the -//! thread; `session` gives each a session of its own, whose autogroup is the -//! unit where autogroup applies; and `scope` puts each in a transient -//! `systemd-run --user --scope` with equal `CPUWeight`, whose cgroup is the -//! unit where the user's manager has the `cpu` controller. A `units` line says -//! which unit applied: `3xcgroup` is a cgroup per workload, `1xcgroup` one -//! cgroup for all three. +//! group, `shared`, and once with a group per workload, `apart`; a line per +//! job, `fair..`. On ToyOS the group is the Job object its +//! threads share CPU time under, and `apart` starts each workload in a Job of +//! its own (`SysCap::new_job`, which needs `Rights::JOBS`). On Linux `shared` +//! leaves them in this process's session and cgroup, where the fair unit is +//! the thread, and `apart` puts each in a transient `systemd-run --user +//! --scope` with equal `CPUWeight`, whose cgroup is the unit where the user's +//! manager has the `cpu` controller; Linux also runs `session`, a session +//! each, whose autogroup is the unit where autogroup applies. A `units` line +//! says which unit applied: `3xcgroup` is a cgroup per workload, `1xcgroup` +//! one cgroup for all three. use std::process::{Command, Stdio}; use std::thread; @@ -196,8 +196,8 @@ fn units(_: Grouping, keys: &[String]) -> String { #[cfg(target_os = "toyos")] fn units(grouping: Grouping, _: &[String]) -> String { match grouping { - Grouping::Plain => "1xjob", - Grouping::Job => "3xjob", + Grouping::Shared => "1xjob", + Grouping::Apart => "3xjob", } .into() } @@ -213,38 +213,30 @@ fn stop(children: impl IntoIterator) /// How the coordinator starts each job. #[derive(Clone, Copy)] pub enum Grouping { - /// Every job in this program's group: ToyOS's `onejob`, Linux's `flat`. - Plain, - /// Each workload in a ToyOS Job of its own. - #[cfg(target_os = "toyos")] - Job, + /// Every job in this program's group. + Shared, + /// Each workload in a group of its own: a ToyOS Job, a Linux scope's cgroup. + Apart, + /// Each workload in a Linux session, whose autogroup is the unit. #[cfg(target_os = "linux")] Session, - #[cfg(target_os = "linux")] - Scope, } impl Grouping { fn name(self) -> &'static str { match self { - #[cfg(target_os = "toyos")] - Grouping::Plain => "onejob", - #[cfg(target_os = "toyos")] - Grouping::Job => "perjob", - #[cfg(target_os = "linux")] - Grouping::Plain => "flat", + Grouping::Shared => "shared", + Grouping::Apart => "apart", #[cfg(target_os = "linux")] Grouping::Session => "session", - #[cfg(target_os = "linux")] - Grouping::Scope => "scope", } } fn command(self, exe: &std::path::Path) -> Result { match self { - Grouping::Plain => Ok(Command::new(exe)), + Grouping::Shared => Ok(Command::new(exe)), #[cfg(target_os = "toyos")] - Grouping::Job => { + Grouping::Apart => { use std::os::toyos::process::CommandExt; use toyos::syscap::SysCap; static CAP: std::sync::OnceLock> = std::sync::OnceLock::new(); @@ -272,7 +264,7 @@ impl Grouping { Ok(command) } #[cfg(target_os = "linux")] - Grouping::Scope => { + Grouping::Apart => { let mut command = Command::new("systemd-run"); command.args(["--user", "--scope", "--quiet", "--collect", "-p", "CPUWeight=100", "--"]).arg(exe); Ok(command) @@ -353,11 +345,11 @@ pub fn contend(grouping: Grouping) { let mut permille: Vec = counts.iter().zip(&totals).map(|(c, t)| c * 1000 / t).collect(); permille.sort_unstable(); let cpus_milli = permille[permille.len() / 2] * cpus; - let mut extra = vec![("job", kind.to_string()), ("cpus_milli_p50", cpus_milli.to_string())]; + let mut extra = vec![("cpus_milli_p50", cpus_milli.to_string())]; if *kind == "helpers" { extra.push(("helpers_spawned", helpers.clone())); } - result(&bench, "permille", &mut permille, &extra); + result(&format!("{bench}.{kind}"), "permille", &mut permille, &extra); } let mut rate: Vec = totals.iter().map(|t| t * 1_000_000_000 / windows.width).collect(); result(&format!("{bench}.total"), "chunks/s", &mut rate, &[]); @@ -365,10 +357,10 @@ pub fn contend(grouping: Grouping) { #[cfg(target_os = "linux")] pub fn groupings() -> [Grouping; 3] { - [Grouping::Plain, Grouping::Session, Grouping::Scope] + [Grouping::Shared, Grouping::Apart, Grouping::Session] } #[cfg(target_os = "toyos")] pub fn groupings() -> [Grouping; 2] { - [Grouping::Plain, Grouping::Job] + [Grouping::Shared, Grouping::Apart] } diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index a820e7712c5..6af0630fc2c 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -1,12 +1,12 @@ -//! Files, through std's `fs` on both systems, under a directory each one -//! keeps in memory: the same operations on Linux's VFS over tmpfs and on -//! ToyOS's two kinds of file service, the kernel's own tmpfs (syscalls) and a -//! file server process (IPC per request). Persistent storage is `storage`'s. +//! Files, through std's `fs` on both systems: the same operations under a +//! filesystem each keeps in memory (`mem`) and under the one on the bench's +//! disk (`disk`, which `storage` runs beside its durable writes). Nothing here +//! asks for durability, so on the disk these are the cached path. //! -//! The `` in each name says where it ran: `shm` is Linux's -//! `/dev/shm`; `tmp` is ToyOS's `/tmp`, which the kernel serves; `served` is -//! ToyOS's `/home`, which the DATA file server serves, and which a boot with -//! no DATA partition keeps in memory (the file server says so in the log). +//! `mem` is Linux's `/dev/shm` through the VFS, and ToyOS's `/tmp`, which the +//! kernel serves through syscalls. `disk` is each system's +//! [`crate::sys::persistent_dir`]: Linux's VFS and page cache over the +//! runner's home, and ToyOS's DATA file server, through IPC per request. use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; @@ -20,19 +20,22 @@ const MIB: usize = 1 << 20; /// A directory to measure under, and what serves it. pub struct Target { pub name: &'static str, - pub dir: &'static str, - pub served_by: &'static str, + pub dir: String, + pub served_by: String, } -pub fn memory_targets() -> Vec { - if cfg!(target_os = "toyos") { - vec![ - Target { name: "tmp", dir: "/tmp", served_by: "the kernel's tmpfs, through syscalls" }, - Target { name: "served", dir: "/home", served_by: "the DATA file server, through IPC" }, - ] +pub fn memory() -> Target { + let (dir, served_by) = if cfg!(target_os = "toyos") { + ("/tmp", "the kernel's tmpfs, through syscalls") } else { - vec![Target { name: "shm", dir: "/dev/shm", served_by: "tmpfs, through the VFS" }] - } + ("/dev/shm", "tmpfs, through the VFS") + }; + Target { name: "mem", dir: dir.into(), served_by: served_by.into() } +} + +/// The persistent filesystem's directory, or `None` where the system has none. +pub fn disk() -> Option { + crate::sys::persistent_dir().map(|(dir, served_by)| Target { name: "disk", dir, served_by }) } /// A scratch directory of this process's under `dir`, removed whole on drop. @@ -77,32 +80,34 @@ fn page_at(i: usize, pages: usize) -> u64 { (x % pages as u64) * 4096 } -/// Every benchmark under `target`. Each one's comment is its warm-up and -/// repetitions. +/// Every benchmark under `target`, each line naming what served it. Each +/// one's comment is its warm-up and repetitions. pub fn suite(target: &Target) { - condition(&format!("fs.{}.dir", target.name), target.dir); - condition(&format!("fs.{}.served_by", target.name), target.served_by); + condition(&format!("fs.{}.served_by", target.name), &target.served_by); #[cfg(target_os = "linux")] - condition(&format!("fs.{}.mount", target.name), crate::sys::mount_options(target.dir)); - let scratch = match Scratch::new(target.dir) { + condition(&format!("fs.{}.mount", target.name), crate::sys::mount_options(&target.dir)); + let scratch = match Scratch::new(&target.dir) { Ok(s) => s, Err(e) => return refuse(&format!("fs.{}", target.name), format!("no scratch directory under {}: {e}", target.dir)), }; let name = |op: &str| format!("fs.{}.{op}", target.name); + let with = |fields: &[(&'static str, String)]| -> Vec<(&'static str, String)> { + fields.iter().cloned().chain([("served_by", target.served_by.clone())]).collect() + }; // Opening an existing 4 KiB file for reading and closing it: 2,000 after 100. let small = scratch.at("small"); if let Err(e) = fill(&small, 4096) { return refuse(&name("setup"), e); } - each(&name("open_close"), 100, 2000, &[], |_| io("open", File::open(&small)).map(drop)); + each(&name("open_close"), 100, 2000, &with(&[]), |_| io("open", File::open(&small)).map(drop)); // `metadata` of the same file: 2,000 after 100. - each(&name("stat"), 100, 2000, &[], |_| io("stat", fs::metadata(&small)).map(drop)); + each(&name("stat"), 100, 2000, &with(&[]), |_| io("stat", fs::metadata(&small)).map(drop)); // A new file, 100 bytes written to it, closed and removed: 1,000 after 50. let fresh = scratch.at("fresh"); - each(&name("create_write_unlink"), 50, 1000, &[("bytes", "100".into())], |_| { + each(&name("create_write_unlink"), 50, 1000, &with(&[("bytes", "100".into())]), |_| { let mut f = io("create", File::create(&fresh))?; io("write", f.write_all(&[7u8; 100]))?; drop(f); @@ -120,11 +125,11 @@ pub fn suite(target: &Target) { match OpenOptions::new().read(true).write(true).open(&big) { Ok(mut f) => { let mut buf = vec![0u8; 4096]; - each(&name("read_4k_at"), 100, 2000, &[("file_bytes", (16 * MIB).to_string())], |i| { + each(&name("read_4k_at"), 100, 2000, &with(&[("file_bytes", (16 * MIB).to_string())]), |i| { io("seek", f.seek(SeekFrom::Start(page_at(i, pages))))?; io("read", f.read_exact(&mut buf)) }); - each(&name("write_4k_at"), 100, 2000, &[("file_bytes", (16 * MIB).to_string())], |i| { + each(&name("write_4k_at"), 100, 2000, &with(&[("file_bytes", (16 * MIB).to_string())]), |i| { io("seek", f.seek(SeekFrom::Start(page_at(i, pages))))?; io("write", f.write_all(&buf)) }); @@ -143,7 +148,7 @@ pub fn suite(target: &Target) { Ok(()) })(); match made { - Ok(()) => each(&name("readdir.1000"), 5, 50, &[("entries", "1000".into())], |_| { + Ok(()) => each(&name("readdir.1000"), 5, 50, &with(&[("entries", "1000".into())]), |_| { let mut n = 0; for entry in io("read_dir", fs::read_dir(&listed))? { std::hint::black_box(io("entry", entry)?.file_name()); @@ -158,7 +163,7 @@ pub fn suite(target: &Target) { // A file renamed within its directory, back and forth: 2,000 after 100. let (a, b) = (scratch.at("a"), scratch.at("b")); match File::create(&a) { - Ok(_) => each(&name("rename"), 100, 2000, &[], |i| { + Ok(_) => each(&name("rename"), 100, 2000, &with(&[]), |i| { let (from, to) = if i % 2 == 0 { (&a, &b) } else { (&b, &a) }; io("rename", fs::rename(from, to)) }), @@ -167,7 +172,7 @@ pub fn suite(target: &Target) { // An empty directory made and removed: 1,000 after 50. let sub = scratch.at("sub"); - each(&name("mkdir_rmdir"), 50, 1000, &[], |_| { + each(&name("mkdir_rmdir"), 50, 1000, &with(&[]), |_| { io("mkdir", fs::create_dir(&sub))?; io("rmdir", fs::remove_dir(&sub)) }); @@ -175,7 +180,7 @@ pub fn suite(target: &Target) { // A directory of 100 files of 64 bytes made, then every file and the // directory removed: ns per whole tree, 20 after 2. let tree = scratch.at("tree"); - each(&name("tree.100"), 2, 20, &[("files", "100".into())], |_| { + each(&name("tree.100"), 2, 20, &with(&[("files", "100".into())]), |_| { io("mkdir", fs::create_dir(&tree))?; for i in 0..100 { io("write", fs::write(tree.join(format!("f{i:03}")), [3u8; 64]))?; @@ -189,17 +194,18 @@ pub fn suite(target: &Target) { // 64 MiB written to a new file and read back whole, in 4 KiB and 1 MiB // calls: MB/s, five rounds after one. Last, as the largest: a server that // cannot hold it takes no other benchmark's number with it. - sequential(&name("seq"), &scratch.at("seq"), 64 * MIB, 5); + sequential(&name("seq"), &scratch.at("seq"), 64 * MIB, 5, &with(&[])); } /// `len` bytes written to `path` and read back, in 4 KiB and in 1 MiB calls: /// `_write.` and `_read.`, MB/s, `rounds` after -/// one. A write round creates the file anew; the read rounds read the last -/// one written. -pub fn sequential(prefix: &str, path: &Path, len: usize, rounds: usize) { +/// one, each line with `fields` beside its own. A write round creates the +/// file anew; the read rounds read the last one written. +fn sequential(prefix: &str, path: &Path, len: usize, rounds: usize, fields: &[(&'static str, String)]) { let block: Vec = (0..MIB).map(|i| (i % 253) as u8).collect(); for call in [4096usize, MIB] { - let extra = [("call_bytes", call.to_string()), ("file_bytes", len.to_string())]; + let extra: Vec<_> = + [("call_bytes", call.to_string()), ("file_bytes", len.to_string())].into_iter().chain(fields.iter().cloned()).collect(); throughput(&format!("{prefix}_write.{call}"), rounds, len as u64, &extra, || { let mut f = io("create", File::create(path))?; for chunk in 0..len / call { diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index d9e64686c28..b89a4cbe56a 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -1,11 +1,12 @@ //! `osbench`: the same system workloads, built from one source for ToyOS //! and for Linux, so one machine's two systems can be compared. //! -//! `osbench ` runs a group, or `all` of them in [`GROUPS`]' order; the -//! groups in [`BY_NAME`] need another party and run only by name. A ToyOS job -//! list passes no arguments, so there the group is the name the program was -//! started under, `osbench_`. Every other command is one -//! this program gives its own children. +//! `osbench ` runs a group, or `all` of them in [`GROUPS`]' order, each +//! in a process of its own as a ToyOS job list runs them; the groups in +//! [`BY_NAME`] need another party and run only by name. A ToyOS job list +//! passes no arguments, so there the group is the name the program was started +//! under, `osbench_`. Every other command is one this program gives its +//! own children. //! //! Every result is one line of `key=value` words (`report`): a distribution //! with its sample count, minimum, p50, p90, p99, p99.9, maximum and mean, @@ -16,25 +17,12 @@ //! (`unsupported`), not a failure. //! //! Where a facility differs the code does too, in `sys` and said at the -//! benchmark: native IPC, persistent storage's medium, the tail group's -//! diagnostics. Everything else is the same source on both. +//! benchmark: native IPC, the directories each filesystem is reached at, the +//! tail group's diagnostics. Everything else is the same source on both. //! -//! The Linux build is static (`x86_64-unknown-linux-musl`, linked by the -//! toolchain's own LLD per `.cargo/config.toml`), so a bench runs it without a -//! toolchain: -//! -//! ```text -//! rustup target add --toolchain nightly-2026-07-22 x86_64-unknown-linux-musl -//! cargo +nightly-2026-07-22 build -p osbench --target x86_64-unknown-linux-musl --profile toyos -//! ``` -//! -//! That nightly is the build's stage-0 compiler (`build/sysroot.rs`'s -//! `STAGE0_CARGO`, which this follows), the upstream one nearest ToyOS's own. //! On Linux `nvme` and `energy` read what only root may (the raw namespace, -//! the package's energy counter) and say so unprivileged. -//! With the `toyos` profile every guest binary is built with, osbench's own -//! code is generated alike for both; std's platform layer and the C library -//! beneath it (musl's on Linux) are each system's own. +//! the package's energy counter) and say so unprivileged. `README.md` is how +//! the Linux half is built and how a comparison is run. mod arch; mod boot; @@ -137,16 +125,14 @@ fn fair() { } fn files() { - for target in files::memory_targets() { - files::suite(&target); - } + files::suite(&files::memory()); } /// Every group the machine alone runs, in the order `all` and the ToyOS job /// list run them: `boot` first, since it reads how long the boot took to /// reach it; `energy` while the machine is still quiet; `crunch`, which heats -/// it, after every other measurement; and `persist` last, since it writes the -/// volume a ToyOS boot's log is on. +/// it, after every other measurement; and `persist` last, so what the disk +/// writes back after it disturbs no other measurement. const GROUPS: [(&str, fn()); 17] = [ ("boot", boot::suite), ("identity", identity::suite), @@ -182,6 +168,25 @@ fn group(name: &str, run: fn()) { report::group(name, &format!("done secs={}", t.elapsed().as_secs())); } +/// Every group in [`GROUPS`], each a child of its own, so no group runs in a +/// process an earlier one shaped: exits 1 if any did not measure. +fn all() { + let exe = std::env::current_exe().expect("a path to this program"); + let mut failed = Vec::new(); + for (name, _) in GROUPS { + let status = std::process::Command::new(&exe).arg(name).status(); + match status { + Ok(status) if status.success() => {} + Ok(status) => failed.push(format!("{name} ({status})")), + Err(e) => failed.push(format!("{name} (would not start: {e})")), + } + } + if !failed.is_empty() { + eprintln!("osbench: groups that did not measure: {}", failed.join(", ")); + std::process::exit(1); + } +} + fn main() { let args: Vec = std::env::args().collect(); let invoked = args.first().and_then(|a| a.rsplit('/').next()).unwrap_or_default(); @@ -191,7 +196,7 @@ fn main() { }; let rest = args.get(2..).unwrap_or_default(); match command { - "all" => GROUPS.iter().for_each(|(name, run)| group(name, *run)), + "all" => return all(), "nop" => return, "spin" => return wake::spin(), "fair-job" => return fair::job(rest), diff --git a/tests/guest/osbench/src/mem.rs b/tests/guest/osbench/src/mem.rs index 3fc9bd177f1..865fbf85428 100644 --- a/tests/guest/osbench/src/mem.rs +++ b/tests/guest/osbench/src/mem.rs @@ -5,8 +5,8 @@ //! //! Both systems fault each anonymous page in on its first touch: ToyOS in //! 4 KiB pages always, Linux in 4 KiB pages where huge pages are refused and -//! in 2 MiB pages where they are asked for and given. `map_touch_unmap` is the -//! whole cost. The allocator is each system's own: ToyOS std's dlmalloc +//! in 2 MiB pages where they are asked for and given. `first_touch` faults in +//! the same bytes either way, and `map_touch_unmap` is the whole cost. The allocator is each system's own: ToyOS std's dlmalloc //! under one lock, Linux's static musl malloc. use std::alloc::{alloc, dealloc, realloc, Layout}; @@ -38,37 +38,40 @@ pub fn suite() { realloc_grow(); } -/// A fresh 64 MiB mapping written once per page: ns per page touched, each -/// round's mean, and the whole map-touch-unmap per page; 4 KiB pages with -/// huge pages refused, and 2 MiB pages with them asked for. Ten rounds after one. +/// A fresh 64 MiB mapping written once per 4 KiB, so every byte of it is +/// faulted in whatever page size backs it: ns per MiB touched, each round's, +/// and the whole map-touch-unmap per MiB; with huge pages refused (`4k`) and +/// asked for (`2m`). Ten rounds after one. fn first_touch() { const LEN: usize = 64 * MIB; - for (name, step, map) in [("4k", PAGE, sys::map_anon_4k as fn(usize) -> *mut u8), ("2m", HUGE, sys::map_anon_huge)] { - let pages = (LEN / step) as u64; + const MIBS: u64 = (LEN / MIB) as u64; + for (name, map) in [("4k", sys::map_anon_4k as fn(usize) -> *mut u8), ("2m", sys::map_anon_huge)] { let round = || { let t = Instant::now(); let ptr = map(LEN); let mapped = ns_since(t); let before = sys::counts(); let t = Instant::now(); - for page in 0..LEN / step { + for page in 0..LEN / PAGE { // SAFETY: inside the mapping just made. - unsafe { ptr.add(page * step).write_volatile(1) }; + unsafe { ptr.add(page * PAGE).write_volatile(1) }; } let touched = ns_since(t); let faults = sys::counts().faults.zip(before.faults).map(|(b, a)| b - a); + let huge = sys::huge_backing(); let t = Instant::now(); // SAFETY: the whole mapping, unused from here. unsafe { sys::unmap(ptr, LEN) }; - (touched / pages, (mapped + touched + ns_since(t)) / pages, faults) + (touched / MIBS, (mapped + touched + ns_since(t)) / MIBS, faults, huge) }; round(); crate::clock::settle(); - let rounds: Vec<(u64, u64, Option)> = (0..10).map(|_| round()).collect(); + let rounds: Vec<(u64, u64, Option, String)> = (0..10).map(|_| round()).collect(); let faults = rounds.iter().filter_map(|r| r.2).max().map_or("unread".to_string(), |f| f.to_string()); - let extra = [("page_bytes", step.to_string()), ("faults_per_round_max", faults), ("huge_pages", sys::huge_backing())]; - result(&format!("mem.first_touch.{name}"), "ns/page", &mut rounds.iter().map(|r| r.0).collect::>(), &extra); - result(&format!("mem.map_touch_unmap.{name}"), "ns/page", &mut rounds.iter().map(|r| r.1).collect::>(), &extra); + let huge = rounds.last().map(|r| r.3.clone()).unwrap_or_default(); + let extra = [("mapping_bytes", LEN.to_string()), ("faults_per_round_max", faults), ("huge_pages_last_round", huge)]; + result(&format!("mem.first_touch.{name}"), "ns/MiB", &mut rounds.iter().map(|r| r.0).collect::>(), &extra); + result(&format!("mem.map_touch_unmap.{name}"), "ns/MiB", &mut rounds.iter().map(|r| r.1).collect::>(), &extra); } } diff --git a/tests/guest/osbench/src/procs.rs b/tests/guest/osbench/src/procs.rs index 29d870202d4..7fd728918a6 100644 --- a/tests/guest/osbench/src/procs.rs +++ b/tests/guest/osbench/src/procs.rs @@ -145,8 +145,8 @@ fn c_spawn_wait(path: &CString, argv: &[&CString]) -> Result<(), sys::Refused> { } pub fn suite() { - let memory = crate::files::memory_targets().remove(0).dir; - let dir = match Scratch::new(memory) { + let memory = crate::files::memory().dir; + let dir = match Scratch::new(&memory) { Ok(d) => d, Err(e) => return refuse("proc", format!("no scratch directory under {memory}: {e}")), }; diff --git a/tests/guest/osbench/src/report.rs b/tests/guest/osbench/src/report.rs index 3d45bba87df..109f484e962 100644 --- a/tests/guest/osbench/src/report.rs +++ b/tests/guest/osbench/src/report.rs @@ -60,9 +60,10 @@ pub fn any_refused() -> bool { /// One measured distribution, by nearest rank. A percentile the sample count /// cannot resolve prints `-`: p99 needs 100 samples and p99.9 a thousand. /// `mhz` is the reporting CPU's clock read just after the measurement -/// (`clock::mhz`), and `mhz_settled` the clock the benchmark's warm-up -/// reached before it (`clock::settle`), so every number says the clock it -/// was taken at. +/// (`clock::mhz`). Where the benchmark's warm-up settled the clock first, +/// `mhz_settled` is the clock it reached, and `mhz_after` the clock a settle +/// just after the measurement reaches: a CPU that only idled meanwhile is +/// back at its clock, and one a power or heat limit holds is not. pub fn result(bench: &str, unit: &str, samples: &mut [u64], extra: &[(&str, String)]) { if samples.is_empty() { refuse(bench, "no samples"); @@ -85,11 +86,15 @@ pub fn result(bench: &str, unit: &str, samples: &mut [u64], extra: &[(&str, Stri for (key, value) in extra { line.push_str(&format!(" {key}={}", word(value))); } + let after = crate::clock::mhz(); match crate::clock::take_settled() { 0 => {} - settled => line.push_str(&format!(" mhz_settled={settled}")), + settled => { + crate::clock::settle(); + line.push_str(&format!(" mhz_settled={settled} mhz_after={}", crate::clock::take_settled())); + } } - line.push_str(&format!(" mhz={}", crate::clock::mhz())); + line.push_str(&format!(" mhz={after}")); emit(&line); } diff --git a/tests/guest/osbench/src/storage.rs b/tests/guest/osbench/src/storage.rs index c44b91133ae..62342c27a51 100644 --- a/tests/guest/osbench/src/storage.rs +++ b/tests/guest/osbench/src/storage.rs @@ -1,34 +1,39 @@ -//! Persistent storage, each system on the medium it has, so **not like for -//! like**: what these say is what a durable write costs on the storage a -//! measured boot has, never which kernel is faster. +//! The bench's disk, through each system's persistent filesystem +//! ([`crate::files::disk`]): durable writes first, each made durable by +//! `sync_all` or `sync_data` before the next, then `files`' cached operations +//! under the same directory as `fs.disk.*`. Every line's `served_by` names +//! the filesystem and the disk. //! -//! On a ToyOS benchcase boot that is the boot USB stick's `TOYOS-LOG` -//! partition, FAT32, through the LOG file server and usbd: the boot has no -//! DATA partition, so `/home` and `/state` are memory (`files`'s `served`), -//! and the stick's other volumes are read-only. On Linux it is a directory in -//! the runner's home, on whatever disk and filesystem hold it, which the -//! conditions name. Every file here is removed at the end; the most alive at -//! once is under 10 MiB, inside what the log volume leaves free. +//! On Linux that is the runner's home, on whatever filesystem and disk hold +//! it. On ToyOS it is `/home`, on the DATA volume: the bcachefs crate's +//! format, through the DATA file server, the block service and the disk's +//! driver. A ToyOS boot whose DATA is in memory measures memory here, which +//! the `osbench` metal row refuses (`tests/toyos.rs`). Every file here is +//! removed at the end. use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; -use crate::files::Scratch; +use crate::files::{self, Scratch}; use crate::measure::{each, io, throughput}; -use crate::report::{condition, refuse}; -use crate::sys; +use crate::report::refuse; const MIB: usize = 1 << 20; pub fn suite() { - let Some((dir, medium)) = sys::persistent_dir() else { + let Some(disk) = files::disk() else { return refuse("persist", "no persistent directory to write in"); }; - condition("persist.medium", medium); - let scratch = match Scratch::new(&dir) { + durable(&disk); + files::suite(&disk); +} + +fn durable(disk: &files::Target) { + let scratch = match Scratch::new(&disk.dir) { Ok(s) => s, Err(e) => return refuse("persist", format!("no scratch directory: {e}")), }; + let served = [("served_by", disk.served_by.clone())]; let block = [0x6bu8; 4096]; // 4 KiB written over the start of a 1 MiB file, then `sync_all`: 100 after 5. @@ -39,20 +44,20 @@ pub fn suite() { })(); match made { Ok(mut f) => { - each("persist.write_4k_fsync", 5, 100, &[], |_| { + each("persist.write_4k_fsync", 5, 100, &served, |_| { io("seek", f.seek(SeekFrom::Start(0)))?; io("write", f.write_all(&block))?; io("fsync", f.sync_all()) }); // `sync_all` of the same file with nothing new in it: 200 after 10. - each("persist.fsync_clean", 10, 200, &[], |_| io("fsync", f.sync_all())); + each("persist.fsync_clean", 10, 200, &served, |_| io("fsync", f.sync_all())); } Err(e) => refuse("persist.write_4k_fsync", e), } // 4 KiB appended, then `sync_data`: 100 after 5, so the file grows by 420 KiB. match OpenOptions::new().create(true).append(true).open(scratch.at("log")) { - Ok(mut f) => each("persist.append_4k_fsync", 5, 100, &[], |_| { + Ok(mut f) => each("persist.append_4k_fsync", 5, 100, &served, |_| { io("write", f.write_all(&block))?; io("fsync", f.sync_data()) }), @@ -62,7 +67,7 @@ pub fn suite() { // A new file, 4 KiB written and `sync_all`ed, closed and removed: 50 // after 5. The directory is not synced. let fresh = scratch.at("fresh"); - each("persist.create_fsync_unlink", 5, 50, &[], |_| { + each("persist.create_fsync_unlink", 5, 50, &served, |_| { let mut f = io("create", File::create(&fresh))?; io("write", f.write_all(&block))?; io("fsync", f.sync_all())?; @@ -75,7 +80,7 @@ pub fn suite() { // calls, which both systems answer from their caches. let seq = scratch.at("seq"); let chunk = vec![0x2du8; MIB]; - let extra = [("file_bytes", (8 * MIB).to_string()), ("call_bytes", MIB.to_string())]; + let extra = [("file_bytes", (8 * MIB).to_string()), ("call_bytes", MIB.to_string()), served[0].clone()]; throughput("persist.seq_write_fsync", 3, 8 * MIB as u64, &extra, || { let mut f = io("create", File::create(&seq))?; for _ in 0..8 { diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index 0e8372a662e..c7f36918ac9 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -225,7 +225,8 @@ fn busy_permille(span: Duration) -> Option { (total > 0).then(|| (total - (idle1 - idle0)) * 1000 / total) } -/// What a Linux number depends on, read and never changed. +/// What a Linux number depends on, read and never changed. `README.md` says +/// which of them a comparison sets. pub fn conditions() { condition("kernel", read("/proc/sys/kernel/osrelease").unwrap_or_default()); let distro = fs::read_to_string("/etc/os-release").ok().and_then(|text| { @@ -238,6 +239,7 @@ pub fn conditions() { condition("cpufreq_driver", per_cpu("cpufreq/scaling_driver")); condition("governor", per_cpu("cpufreq/scaling_governor")); condition("epp", per_cpu("cpufreq/energy_performance_preference")); + condition("min_khz", per_cpu("cpufreq/scaling_min_freq")); condition("max_khz", per_cpu("cpufreq/scaling_max_freq")); condition( "intel_no_turbo", @@ -245,6 +247,11 @@ pub fn conditions() { ); condition("boost", read("/sys/devices/system/cpu/cpufreq/boost").unwrap_or_else(|| "absent".into())); condition("amd_pstate", read("/sys/devices/system/cpu/amd_pstate/status").unwrap_or_else(|| "absent".into())); + condition("platform_profile", read("/sys/firmware/acpi/platform_profile").unwrap_or_else(|| "absent".into())); + // What every timestamp here is read from: one the vDSO cannot read costs + // each `Instant::now` a syscall. + let clocksource = read("/sys/devices/system/clocksource/clocksource0/current_clocksource"); + condition("clocksource", clocksource.unwrap_or_else(|| "absent".into())); for supply in fs::read_dir("/sys/class/power_supply").into_iter().flatten().flatten() { let at = supply.path(); let get = |f: &str| read(&at.join(f).to_string_lossy()); @@ -514,8 +521,9 @@ pub fn mount_options(dir: &str) -> String { mount_of(dir).map_or_else(|| "unknown".into(), |(fstype, _, options)| format!("{fstype} {options}")) } -/// The runner's home, and the filesystem and disk model holding it; the -/// path itself names the user and is never printed. +/// The runner's home, and the filesystem and disk model holding it, as +/// `ext4, NVMe disk, model `; the path itself names the user and is +/// never printed. pub fn persistent_dir() -> Option<(String, String)> { let home = std::env::var("HOME").ok()?; let (fstype, source, _) = mount_of(&home)?; diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index d4a50d33247..eb867e37e3b 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -358,9 +358,11 @@ pub fn map_anon_4k(len: usize) -> *mut u8 { map_anon(len) } -/// The one writable persistent volume a benchcase boot has, and what it is. +/// DATA's `/home`, and what serves it. A program cannot ask which disk that +/// is, or whether DATA is in memory instead; the boot's log says both. pub fn persistent_dir() -> Option<(String, String)> { - Some(("/log".into(), "boot USB stick TOYOS-LOG partition, FAT32, LOG file server over usbd mass storage".into())) + let served_by = "DATA volume, bcachefs crate's format, DATA file server over the block service"; + Some(("/home".into(), served_by.into())) } /// What carries a connection on ToyOS: netstack drives one card, Ethernet. diff --git a/tests/guest/osbench/src/system.rs b/tests/guest/osbench/src/system.rs index cbbffa3fb5f..79d35246766 100644 --- a/tests/guest/osbench/src/system.rs +++ b/tests/guest/osbench/src/system.rs @@ -12,10 +12,9 @@ use crate::{chunk, ns_since}; const MIB: usize = 1 << 20; -/// What one chunk of the fairness work costs one thread, 200 after one: the -/// same instructions on both systems, so a difference between them is the -/// clock that thread ran at. It is a 2 ms single-threaded burst, so it reads -/// the clock a CPU leaving idle has, not the clock it settles at. +/// What one chunk of the fairness work costs one thread, 200 after a settle: +/// the same instructions on both systems, so a difference between them is the +/// clock that thread ran at. pub fn chunk_rate() { chunk(); crate::clock::settle(); diff --git a/tests/guest/osbench/src/time.rs b/tests/guest/osbench/src/time.rs index 99e1d7d426d..42be598d1af 100644 --- a/tests/guest/osbench/src/time.rs +++ b/tests/guest/osbench/src/time.rs @@ -7,11 +7,6 @@ use std::time::{Duration, Instant}; use crate::report::result; use crate::ns_since; -/// What std's `yield_now` is on each system: Linux's is `sched_yield`; -/// ToyOS's enters no kernel (`sdk/std/sys/thread.rs`: there is no yield -/// syscall, and it is a spin-loop hint). -const YIELD: &str = if cfg!(target_os = "toyos") { "spin_hint_no_syscall" } else { "sched_yield" }; - pub fn suite() { clock_read(); sleeps(""); @@ -72,9 +67,14 @@ fn sleeps(suffix: &str) { } } -/// `thread::yield_now` with nothing else to run: 15 rounds of 100,000 calls -/// after one, picoseconds per call. +/// `thread::yield_now` with nothing else to run, `sched_yield` on Linux: 15 +/// rounds of 100,000 calls after one, picoseconds per call. ToyOS has no +/// yield to measure: its std's `yield_now` is a spin-loop hint +/// (`sdk/std/sys/thread.rs`). fn yields() { + if cfg!(target_os = "toyos") { + return crate::report::unsupported("time.yield", "no yield syscall: std's yield_now is a spin-loop hint"); + } const CALLS: u64 = 100_000; let round = || { let t = Instant::now(); @@ -86,5 +86,5 @@ fn yields() { round(); crate::clock::settle(); let mut ps: Vec = (0..15).map(|_| round()).collect(); - result("time.yield", "ps", &mut ps, &[("yield_is", YIELD.into()), ("calls_per_sample", CALLS.to_string())]); + result("time.yield", "ps", &mut ps, &[("calls_per_sample", CALLS.to_string())]); } diff --git a/tests/guest/osbench/src/work.rs b/tests/guest/osbench/src/work.rs index 87d71f290d5..783aeb1429d 100644 --- a/tests/guest/osbench/src/work.rs +++ b/tests/guest/osbench/src/work.rs @@ -52,8 +52,8 @@ pub fn cc(args: &[String]) { } pub fn suite() { - let memory = crate::files::memory_targets().remove(0).dir; - let dir = match Scratch::new(memory) { + let memory = crate::files::memory().dir; + let dir = match Scratch::new(&memory) { Ok(d) => d, Err(e) => return refuse("work", format!("no scratch directory under {memory}: {e}")), }; diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index f810120f758..58cc9ea5e5e 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -4,9 +4,9 @@ # Only what a job list needs to run and to reach the log on the boot stick, the # ACPI server, and netstack for `osbench lan` and `osbench internet`, each its # own metal row (`tests/toyos.rs`): no compositor or sound, so little but the -# measurement asks for a CPU. DATA is memory here, the stick having no DATA -# partition, and the log volume is the one persistent volume a job can write -# (`osbench persist`). +# measurement asks for a CPU. The stick has no DATA partition, so DATA is the +# bench's: the TOYOS-DATA partition on its NVMe, which `osbench persist` +# writes, and which the `osbench` row refuses a boot to be without. # # The runner's capability carries `jobs`, for the fairness run that starts each # workload as a job of its own; `counters`, `trace` and `roster`, so `osbench @@ -40,13 +40,13 @@ serves = ["netstack"] devices = ["pci:02:00:00/8086:15fc"] # The order `osbench all` runs the groups in. The bound is the metal row's, its -# 370 s on the 60 s every list has (`tests/toyos.rs`), restated for a boot of +# 465 s on the 60 s every list has (`tests/toyos.rs`), restated for a boot of # this config built without the row; a derived metal config states its own. [programs.test-runner] receives = ["power", "netstack"] syscap = ["jobs", "counters", "trace", "roster", "inventory", "dup"] args = [ - "--bound-ms=430000", + "--bound-ms=525000", "osbench_boot", "osbench_identity", "osbench_energy", diff --git a/tests/toyos.rs b/tests/toyos.rs index 0b7a11526cf..031b27dd707 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1108,10 +1108,17 @@ const METAL: &[(&str, metal::Metal)] = &[ ( // The numbers compared with the same program on the bench's own Linux. // Evidence and not a verdict: they are read off the log, and the row - // holds only that every group ran to its end. The T14 and the AMD bench. - // One boot runs the whole suite, so three readbacks are three runs. + // holds only that every group ran to its end on the bench's NVMe. + // The T14 and the AMD bench. One boot runs the whole suite, so three + // readbacks are three runs. "osbench", - metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0], &OSBENCH_GROUPS) }, + metal::Metal { + arms: BENCHCASE, + judge: |b| { + data_on_the_nvme(b[0])?; + osbench_ran(b[0], &OSBENCH_GROUPS) + }, + }, ), ( // `osbench lan` against the `osbench-peer` the other bench's Linux runs, @@ -1619,7 +1626,7 @@ const TESTCASES: &[metal::Arm] = &[metal::once( /// `osbench`'s groups, each a job named by its symlink, in `osbench all`'s /// order: `boot` first, which reads how long the boot took to reach it, and -/// `persist` last, which writes the log volume. +/// `persist` last, which writes the disk. const OSBENCH_GROUPS: [&str; 17] = [ "osbench_boot", "osbench_identity", @@ -1646,10 +1653,12 @@ const OSBENCH_GROUPS: [&str; 17] = [ /// served 64 MiB write stalled and `files` took 80 s against 13. The groups /// since add at most 50 s by their own spans: `crunch`'s one-thread 16 s /// (ToyOS reads no temperature, so its all-core phase does not run), and -/// under 30 s of memcpy sizes, pipe sizes and 1 ms timers. So a list bound of -/// 430 s leaves that stall 136 s. +/// under 30 s of memcpy sizes, pipe sizes and 1 ms timers. Each of the some +/// ninety benchmarks that settle the clock settles it before and after +/// itself, 50 ms where the clock holds and 0.5 s at most (`clock::settle`): +/// under 95 s. So a list bound of 525 s leaves that stall 136 s. const BENCHCASE: &[metal::Arm] = &[metal::Arm { - adds_ms: 370_000, + adds_ms: 465_000, ..metal::once("benchcase", "tests/images/benchcase.toml", &[], &OSBENCH_GROUPS) }]; @@ -1688,6 +1697,26 @@ fn osbench_ran(boot: &metal::Readback, jobs: &[&str]) -> Result<(), String> { Ok(()) } +/// DATA, which `osbench persist` writes, was a volume on a TOYOS-DATA +/// partition the NVMe driver serves: mounted, or formatted off its +/// designation, and not memory standing in for a partition that is not ours. +/// The block service refuses DATA where two disks carry one, so a mount +/// alongside the NVMe's partition is that partition's. +fn data_on_the_nvme(boot: &metal::Readback) -> Result<(), String> { + let log = boot.log(); + let text = log.text(); + let served = format!(" of type {} at block ", toyos_gpt::Guid::TOYOS_DATA); + if !text.lines().any(|l| l.contains("nvme] nvme ") && l.contains(&served)) { + return Err("the NVMe driver served no TOYOS-DATA partition, so persist measured no disk of the bench's".into()); + } + let mounted = ["fileserver: mounted the DATA volume", "fileserver: block 0 designates this partition for ToyOS; formatting it"]; + if !mounted.iter().any(|said| text.contains(said)) { + let why = text.lines().find(|l| l.contains("fileserver: the DATA")).unwrap_or("the file server said nothing of DATA"); + return Err(format!("DATA was not the NVMe's partition, so persist measured no disk: {why}")); + } + Ok(()) +} + /// A timing boot was in ACPI mode before its first job and stayed there, as a /// shipped image and the bench's Linux are. In legacy mode the T14's firmware /// stops every CPU every 2.2 s, and its time would be measured as ToyOS's From 8d3ea1a89d91bfda9b0a0287afcfccd3c43a29a5 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 08:28:21 +0100 Subject: [PATCH 2/8] toyos-metal: --designate-data stamps the bench's ToyOS DATA, and the bench rows judge it designated The owner chose (2026-10-28) to give both benches' fourth partition, TOYOS-DATA named "ToyOS data", a ToyOS volume, so osbench persist measures the bench's NVMe through ToyOS's real storage stack. --designate-data finds the partition by type and name across every disk the machine lists, never by node: exactly one, on a fixed disk, unmounted, nothing stacked on it. It refuses one that carries a filesystem signature or a ToyOS volume, and reports one that already carries its stamp without writing it. It writes the build's own stamp (toyos_build::image::designation, now public) through the partition's by-partuuid node, then reads the block back and requires the exact bytes. --dry-run makes every read and prints the write. nvme_leaves_the_benchs_disk_alone now judges DATA designated: mounted, or formatted off its stamp, and never memory in its place. Its QEMU twin, a_host_disk_beside_the_stick_is_left_alone, keeps the foreign DATA it boots beside. The osbench row refuses the boot that formats the volume as a run and states how long the format took. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + build/image.rs | 5 +- lib/bcachefs/src/lib.rs | 2 +- tests/common/hostdisk.rs | 48 +++++++--- tests/guest/osbench/README.md | 18 +++- tests/metal/Cargo.toml | 2 + tests/metal/src/internal.rs | 2 +- tests/metal/src/lib.rs | 88 +++++++++++++++++- tests/metal/src/stamp.rs | 166 ++++++++++++++++++++++++++++++++++ tests/toyos.rs | 39 +++++--- 10 files changed, 336 insertions(+), 35 deletions(-) create mode 100644 tests/metal/src/stamp.rs diff --git a/Cargo.lock b/Cargo.lock index 82931a647c4..461c34194d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3802,6 +3802,7 @@ dependencies = [ name = "toyos-metal" version = "0.1.0" dependencies = [ + "bcachefs", "serde", "toml", "toyos-build", diff --git a/build/image.rs b/build/image.rs index 409fc64efed..6ea247ab7d9 100644 --- a/build/image.rs +++ b/build/image.rs @@ -733,8 +733,9 @@ fn name_of(part: &toyos_gpt::Partition) -> String { String::from_utf16(part.name()).unwrap_or_else(|e| panic!("partition {}'s name is no UTF-16: {e}", part.unique_guid())) } -/// Block 0 of a volume: the magic and its block count. -fn designation(blocks: u64) -> [u8; SECTOR] { +/// Block 0 of a volume of `blocks` 4 KiB blocks, designated for ToyOS: the +/// magic and its block count. +pub fn designation(blocks: u64) -> [u8; SECTOR] { let mut block = [0u8; SECTOR]; block[..bcachefs::DESIGNATION_MAGIC.len()].copy_from_slice(&bcachefs::DESIGNATION_MAGIC); let at = bcachefs::DESIGNATION_BLOCKS_OFFSET; diff --git a/lib/bcachefs/src/lib.rs b/lib/bcachefs/src/lib.rs index 2d01bf93eb1..48685b07aa6 100644 --- a/lib/bcachefs/src/lib.rs +++ b/lib/bcachefs/src/lib.rs @@ -27,7 +27,7 @@ pub use block_io::{BlockIO, BlockBuf, BlockNum, DeviceError, TransferError}; #[cfg(feature = "std")] pub use block_io::VecBlockIO; pub use fs::{file_entry_fits, Formatted, Mounted, ReadOnly, ReadWrite, Recorded, FsError, Extent}; -pub use superblock::{DESIGNATION_BLOCKS_OFFSET, DESIGNATION_MAGIC, FsUuid, Superblock}; +pub use superblock::{DESIGNATION_BLOCKS_OFFSET, DESIGNATION_MAGIC, FsUuid, MAGIC, Superblock}; /// Records the largest single allocation each test thread makes, so a test can /// assert what parsing a crafted block asks the allocator for. diff --git a/tests/common/hostdisk.rs b/tests/common/hostdisk.rs index 5523b9fb8d3..6b772f9d192 100644 --- a/tests/common/hostdisk.rs +++ b/tests/common/hostdisk.rs @@ -1,5 +1,6 @@ //! Another system's disk, as a bench's internal disk is before ToyOS is -//! installed on it, and the judge of what ToyOS did to it. +//! installed on it, and the judge of what ToyOS did to it, there and on a +//! bench whose DATA is designated. //! //! The layout is the benches': the host's ESP, its `/boot` and its root, and //! the owner's ToyOS DATA, unformatted. Every block of every partition carries @@ -54,12 +55,23 @@ fn said_by_nvme(line: &str) -> Option<&str> { (at.len() == 7 && at[2] == b':' && at[5] == b'.').then_some(said) } -/// A boot off a stick drove an NVMe disk that is another system's and left -/// it alone: a driver brought a controller up and offered its partitions, it -/// opened at least one session and each on a ToyOS DATA partition, and the file server -/// found that DATA neither ours nor designated, and so neither mounted nor -/// formatted it. Answers how many partitions the disks' tables named. -pub fn left_alone(log: &Serial) -> Result { +/// What the bench's DATA partition is to the file server: another system's, +/// as this module's disk is, or designated for ToyOS, as a bench's is once +/// `toyos-metal --designate-data` stamped it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Data { + Foreign, + Designated, +} + +/// A boot off a stick drove an NVMe disk that is another system's but for +/// its DATA and left it alone: a driver brought a controller up and offered +/// its partitions, it opened at least one session and each on a ToyOS DATA +/// partition, and the file server found that DATA as `data` says: neither +/// ours nor designated, so neither mounted nor formatted, or designated, so +/// mounted or formatted off its stamp and never memory in its place. +/// Answers how many partitions the disks' tables named. +pub fn left_alone(log: &Serial, data: Data) -> Result { log.must_say("diskserver: every disk driver has settled")?; let said: Vec<&str> = log.text().lines().filter_map(said_by_nvme).collect(); if !said.iter().any(|s| s.starts_with("up: ")) { @@ -74,7 +86,7 @@ pub fn left_alone(log: &Serial) -> Result { if kinds.is_empty() { return Err(format!("no nvme driver named a partition of its disk\n{}", log.text())); } - let data = Guid::TOYOS_DATA.to_string(); + let data_type = Guid::TOYOS_DATA.to_string(); let opened: Vec<&str> = said .iter() .filter_map(|s| s.strip_prefix("session ")) @@ -86,12 +98,24 @@ pub fn left_alone(log: &Serial) -> Result { } for guid in opened { match kinds.iter().find(|(g, _)| *g == guid) { - Some((_, kind)) if *kind == data => {} + Some((_, kind)) if *kind == data_type => {} other => return Err(format!("an nvme driver opened a session on {guid}, of type {other:?}, which is no ToyOS DATA")), } } - log.must_say("fileserver: no volume of ours and no designation stamp; nothing will be written to this partition")?; - log.must_not_say("fileserver: mounted the DATA volume")?; - log.must_not_say("designates this partition for ToyOS; formatting it")?; + const MOUNTED: &str = "fileserver: mounted the DATA volume"; + const FORMATTED: &str = "fileserver: block 0 designates this partition for ToyOS; formatting it"; + match data { + Data::Foreign => { + log.must_say("fileserver: no volume of ours and no designation stamp; nothing will be written to this partition")?; + log.must_not_say(MOUNTED)?; + log.must_not_say(FORMATTED)?; + } + Data::Designated => { + if !log.text().contains(MOUNTED) && !log.text().contains(FORMATTED) { + return Err(format!("the designated DATA was neither mounted nor formatted\n{}", log.text())); + } + log.must_not_say("are in memory and will not survive a reboot")?; + } + } Ok(kinds.len()) } diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md index 82a69749578..809d88f609f 100644 --- a/tests/guest/osbench/README.md +++ b/tests/guest/osbench/README.md @@ -93,11 +93,19 @@ there carried it. bench's NVMe. On Linux that is the runner's home, ext4 on the bench's root partition. On ToyOS it is DATA: the bench's TOYOS-DATA partition, its fourth, named `ToyOS data`. It is served through the DATA file server and the NVMe -driver. The partition holds a ToyOS volume only once its first block carries -the designation stamp. Until then the file server writes nothing to it and -keeps DATA in memory, and the row refuses the boot by name. The first boot -after the stamp formats the volume, so it is not a measured run. Every row's -`served_by` names the filesystem and the disk it measured. +driver. The `osbench` row refuses a boot whose DATA was anything else. Every +row's `served_by` names the filesystem and the disk it measured. + +The partition holds a ToyOS volume once, ever, after its first block +carries the designation stamp: + +```text +cargo run -p toyos-metal -- --designate-data --host --dry-run +cargo run -p toyos-metal -- --designate-data --host +``` + +The first ToyOS boot after the stamp formats the volume. The `osbench` row +refuses that boot as a run and says how long the format took. ## What is not a comparison diff --git a/tests/metal/Cargo.toml b/tests/metal/Cargo.toml index f76030bffef..5d758f6e43e 100644 --- a/tests/metal/Cargo.toml +++ b/tests/metal/Cargo.toml @@ -14,6 +14,8 @@ toyos-build = { path = "../.." } # The outside judge of the log partition the boot left on the stick. toyos-fat32-check = { path = "../../lib/fat32/check" } toyos-gpt = { path = "../../lib/gpt" } +# A ToyOS volume's magic, which the DATA designation never writes over. +bcachefs = { path = "../../lib/bcachefs" } toyos-quiesce = { path = "../../lib/quiesce" } toyos-tco = { path = "../../lib/tco" } diff --git a/tests/metal/src/internal.rs b/tests/metal/src/internal.rs index d701148e5a2..a4302fa098a 100644 --- a/tests/metal/src/internal.rs +++ b/tests/metal/src/internal.rs @@ -29,7 +29,7 @@ const MIB: u64 = 1 << 20; const BENCH: u32 = 4; /// The fourth partition's name, which the owner gave the bench's DATA. -const DATA_LABEL: &str = "ToyOS data"; +pub(crate) const DATA_LABEL: &str = "ToyOS data"; /// The partitions an internal install owns, in table order from partition 5, /// laid out as the image lays out its own. diff --git a/tests/metal/src/lib.rs b/tests/metal/src/lib.rs index cd1235724ad..c5134caa012 100644 --- a/tests/metal/src/lib.rs +++ b/tests/metal/src/lib.rs @@ -1,7 +1,8 @@ //! `toyos-metal` — the loop that flashes ToyOS to a metal bench's stick, boots //! it once, and answers with [`toyos_build::bootlog`]'s verdict on what that boot //! wrote to the log partition; and beside it, the internal install -//! ([`internal`]) that writes an image to the bench's own NVMe for a demo boot. +//! ([`internal`]) that writes an image to the bench's own NVMe for a demo boot, +//! and the designation of the bench's ToyOS DATA ([`stamp`]). //! //! It runs on the development host and reaches the machine over `ssh`: the //! T14 bench unless `--host` names another, whose account's home holds the @@ -40,6 +41,7 @@ use crate::timings::{Machine, Record, Stick}; pub mod image; pub mod internal; +pub mod stamp; pub mod testargs; pub mod timings; @@ -94,6 +96,7 @@ pub const STICK_SECS: u64 = 30; const DD: &str = "/usr/bin/dd"; const EFIBOOTMGR: &str = "/usr/bin/efibootmgr"; const MOUNT: &str = "/usr/bin/mount"; +const OD: &str = "/usr/bin/od"; const PARTX: &str = "/usr/bin/partx"; const SGDISK: &str = "/usr/sbin/sgdisk"; const UMOUNT: &str = "/usr/bin/umount"; @@ -145,6 +148,8 @@ pub enum Refusal { Ambiguous { carrying: String, nodes: Vec }, /// The disk named to be adopted as scratch is not one this loop erases. Adopt(String), + /// The bench's DATA partition is not one this loop designates. + Designate(String), File { path: String, why: String }, Table(String), /// The table does not hold exactly one partition of a type the loop needs. @@ -255,6 +260,7 @@ impl fmt::Display for Refusal { } ), Self::Adopt(why) => write!(f, "nothing was erased: {why}"), + Self::Designate(why) => write!(f, "no partition was designated: {why}"), Self::Ambiguous { carrying, nodes } => write!( f, "{} are each a recorded disk carrying {carrying}, so nothing was written: unplug \ @@ -661,6 +667,12 @@ enum Job { CreateInternal, Write(Owned), Scratch(Sd), + /// What signatures the partition at the node carries, writing nothing. + Signatures(String), + /// The partition's first block, as `od` writes bytes. + FirstBlock(String), + /// The stamp on stdin written over the partition's first block. + Stamp(String), } /// One word as the machine's login shell must receive it: `ssh` hands the whole @@ -792,6 +804,9 @@ impl Target { &sd.node().whole(), ]), Job::CreateInternal => create(&internal, Owned::Esp.index(), INTERNAL_LABEL), + Job::Signatures(node) => words(&[WIPEFS, "--no-act", "--noheadings", "--output", "TYPE", node]), + Job::FirstBlock(node) => words(&[OD, "-An", "-v", "-tx1", &format!("-N{}", stamp::BLOCK), node]), + Job::Stamp(node) => words(&[DD, &format!("of={node}"), &format!("bs={}", stamp::BLOCK), "count=1", "conv=fsync,nocreat"]), Job::Write(o) => words(&[DD, &format!("of={}", o.node(&self.internal)), "bs=4M", "conv=fsync,nocreat"]), } } @@ -1649,6 +1664,7 @@ declare_flags!(METAL = { KEY = "--key", Next; DEVICE = "--device", Next; ADOPT_SCRATCH = "--adopt-scratch", Next; + DESIGNATE_DATA = "--designate-data", None; RECORD = "--record", Next; READBACK = "--readback", Next; FAT32_CHECK = "--fat32-check", None; @@ -1694,6 +1710,9 @@ pub struct Args { /// The disk [`adopt_scratch`] erases into a scratch disk. An action of its /// own, beside nothing but the flags that say which machine and `--dry-run`. adopt_scratch: Option, + /// Stamp the bench's ToyOS DATA partition ([`designate_data`]). An action + /// of its own, beside nothing but the machine's flags and `--dry-run`. + designate_data: bool, } impl Args { @@ -1719,6 +1738,7 @@ impl Args { install_internal: value(&INSTALL_INTERNAL).map(PathBuf::from), boot_internal: METAL.present(args, &BOOT_INTERNAL), adopt_scratch: value(&ADOPT_SCRATCH).map(Sd::parse).transpose()?, + designate_data: METAL.present(args, &DESIGNATE_DATA), }; if let Some(host) = value(&HOST) { let (user, machine) = host_of(host)?; @@ -1735,8 +1755,11 @@ impl Args { // flag beside it that describes anything else is refused rather than // left for the action to fall through into — which is how a capture // once flashed whatever `--image` defaulted to. - let actions: [(&Flag, &[&Flag]); 2] = - [(&RECORD, &[&HOST, &KEY]), (&ADOPT_SCRATCH, &[&HOST, &KEY, &DRY_RUN])]; + let actions: [(&Flag, &[&Flag]); 3] = [ + (&RECORD, &[&HOST, &KEY]), + (&ADOPT_SCRATCH, &[&HOST, &KEY, &DRY_RUN]), + (&DESIGNATE_DATA, &[&HOST, &KEY, &DRY_RUN]), + ]; for (action, alone) in actions { if !METAL.present(args, action) { continue; @@ -1851,6 +1874,55 @@ fn adopt_scratch(target: &Target, dry_run: bool, sd: Sd) -> Result<(), Refusal> Ok(()) } +/// Designate a recorded bench's ToyOS DATA partition ([`stamp`]): found by +/// type and name, refused where it carries a signature or a ToyOS volume, +/// stamped through its GUID's node, and read back. The partition's contents +/// are lost; its next ToyOS boot formats it. A dry run reads everything and +/// writes nothing. +fn designate_data(target: &Target, dry_run: bool) -> Result<(), Refusal> { + let driver = Driver::reach(target.clone(), dry_run)?; + driver.require_sudo()?; + driver.recorded_machine()?; + let data = stamp::find(&driver.ssh("listing the disks", stamp::QUERY)?).map_err(Refusal::Designate)?; + let node = data.by_guid(); + let root = |job: Job| driver.target.remote(&job, &driver.home); + let signatures = driver.ssh("reading the partition's signatures", &root(Job::Signatures(node.clone())))?; + if !signatures.trim().is_empty() { + let found = signatures.split_whitespace().collect::>().join(", "); + return Err(Refusal::Designate(format!("{} carries {found}", data.node))); + } + let first = |what: &str| -> Result { + let bytes = stamp::od_bytes(&driver.ssh(what, &root(Job::FirstBlock(node.clone())))?).map_err(Refusal::Designate)?; + Ok(stamp::classify(&data, &bytes)) + }; + let said = format!("{} ({}, {} blocks of {})", data.node, internal::DATA_LABEL, data.blocks, stamp::BLOCK); + match first("reading the partition's first block")? { + stamp::Block0::Stamped => { + println!("{said} already carries its designation; nothing was written"); + return Ok(()); + } + stamp::Block0::Volume => { + return Err(Refusal::Designate(format!("{said} holds a ToyOS volume, which a designation would destroy"))); + } + stamp::Block0::Other => {} + } + let file = std::env::temp_dir().join(format!("toyos-metal-designation-{}", std::process::id())); + std::fs::write(&file, data.stamp()).map_err(|e| Refusal::File { path: file.display().to_string(), why: e.to_string() })?; + let feed = Feed { path: &file, at: 0, len: stamp::BLOCK as u64 }; + let wrote = driver.as_root("writing the designation", Job::Stamp(node.clone()), Some(feed)); + let _ = std::fs::remove_file(&file); + let Some(out) = wrote? else { + println!("dry run: {said} was read and nothing was written"); + return Ok(()); + }; + driver.fed("writing the designation", &out, stamp::BLOCK as u64)?; + if first("reading the designation back")? != stamp::Block0::Stamped { + return Err(Refusal::Designate(format!("{said} was written and does not read back as its designation"))); + } + println!("{said} is designated: the next ToyOS boot formats it"); + Ok(()) +} + /// Bytes of a local file one root command reads on its stdin. #[derive(Debug, Clone, Copy)] struct Feed<'a> { @@ -1914,6 +1986,10 @@ pub fn run(args: &Args) -> Result, Refusal> { adopt_scratch(&args.target, args.dry_run, sd)?; return Ok(None); } + if args.designate_data { + designate_data(&args.target, args.dry_run)?; + return Ok(None); + } let began = std::time::Instant::now(); let Some(asked) = &args.image else { @@ -3212,6 +3288,12 @@ mod tests { } let both = Args::parse(&words(&["--record", "/dev/sda", "--adopt-scratch", "/dev/sdb"])).unwrap_err(); assert!(matches!(both, Refusal::Usage(_))); + let designate = Args::parse(&words(&["--designate-data", "--dry-run", "--host", "toyos@bench-amd"])).unwrap(); + assert!(designate.designate_data && designate.dry_run); + for beside in boot.iter().copied().chain([&["--adopt-scratch", "/dev/sdb"][..]]) { + let line = [&["--designate-data"][..], beside].concat(); + assert!(matches!(Args::parse(&words(&line)), Err(Refusal::Usage(ref why)) if why.contains(beside[0])), "{line:?}"); + } } /// **The mount point is under the home the machine names**, and a home diff --git a/tests/metal/src/stamp.rs b/tests/metal/src/stamp.rs new file mode 100644 index 00000000000..b8629457369 --- /dev/null +++ b/tests/metal/src/stamp.rs @@ -0,0 +1,166 @@ +//! Designating a bench's ToyOS DATA partition: the stamp the DATA file server +//! formats a volume off (`bcachefs::DESIGNATION_MAGIC`), written into the +//! partition's first 4 KiB block. +//! +//! **The partition is found by what it is, never by its node**: the one +//! partition of every disk the machine lists of type TOYOS-DATA and named +//! [`internal::DATA_LABEL`], on a fixed disk, unmounted, with nothing stacked +//! on it, and written through its GUID's own node. Its first block is read +//! before the write and after it. A partition that already carries this +//! stamp, or a ToyOS volume, is never written. + +use toyos_gpt::Guid; + +use crate::internal; + +/// The block the stamp fills, the DATA volume's. +pub const BLOCK: usize = 4096; + +/// Every block device, for [`find`]. +pub const QUERY: &str = "lsblk --bytes --noheadings --paths --pairs --output \ + NAME,TYPE,RM,PKNAME,SIZE,PARTTYPE,PARTUUID,PARTLABEL,MOUNTPOINTS"; + +/// The DATA partition [`find`] found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Data { + pub node: String, + pub unique: Guid, + pub blocks: u64, +} + +impl Data { + /// The node udev names the partition by its GUID, which no renumbering of + /// the machine's disks moves. + pub fn by_guid(&self) -> String { + format!("/dev/disk/by-partuuid/{}", self.unique.to_string().to_ascii_lowercase()) + } + + /// The stamp for this partition: it names the partition's own block count. + pub fn stamp(&self) -> [u8; BLOCK] { + toyos_build::image::designation(self.blocks) + } +} + +/// The one DATA partition [`QUERY`] answered, or why none can be written. +pub fn find(listing: &str) -> Result { + let rows: Vec<_> = + listing.lines().filter(|l| !l.trim().is_empty()).map(internal::pairs).collect::>()?; + let field = |row: &std::collections::BTreeMap, key: &str| row.get(key).cloned().unwrap_or_default(); + let data: Vec<_> = rows + .iter() + .filter(|r| field(r, "TYPE") == "part" && internal::guid(&field(r, "PARTTYPE")) == Some(Guid::TOYOS_DATA)) + .collect(); + let [part] = data[..] else { + return Err(format!("the machine lists {} TOYOS-DATA partitions, and only one is designated", data.len())); + }; + let node = field(part, "NAME"); + if field(part, "PARTLABEL") != internal::DATA_LABEL { + return Err(format!("{node} is named {:?}, where a bench's DATA is {:?}", field(part, "PARTLABEL"), internal::DATA_LABEL)); + } + let disk = rows.iter().find(|r| field(r, "NAME") == field(part, "PKNAME") && field(r, "TYPE") == "disk"); + if disk.is_none_or(|d| field(d, "RM") != "0") { + return Err(format!("{node} is on no fixed disk the machine lists, and a bench's DATA is on its own")); + } + if !field(part, "MOUNTPOINTS").trim().is_empty() { + return Err(format!("{node} is mounted")); + } + if let Some(above) = rows.iter().find(|r| field(r, "PKNAME") == node) { + return Err(format!("{} is stacked on {node}", field(above, "NAME"))); + } + let unique = internal::guid(&field(part, "PARTUUID")).ok_or_else(|| format!("{node} states no GUID"))?; + let bytes: u64 = field(part, "SIZE").parse().map_err(|_| format!("{node} reads SIZE={:?}", field(part, "SIZE")))?; + if bytes == 0 || !bytes.is_multiple_of(BLOCK as u64) { + return Err(format!("{node} is {bytes} bytes, no whole number of {BLOCK}-byte blocks")); + } + Ok(Data { node, unique, blocks: bytes / BLOCK as u64 }) +} + +/// What a DATA partition's first block holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Block0 { + /// This partition's stamp, byte for byte. + Stamped, + /// A ToyOS volume's superblock: never written over. + Volume, + /// Anything else, a stamp for another size among it, which the stamp + /// replaces. + Other, +} + +pub fn classify(data: &Data, block0: &[u8]) -> Block0 { + if block0 == data.stamp() { + Block0::Stamped + } else if block0.starts_with(&bcachefs::MAGIC) { + Block0::Volume + } else { + Block0::Other + } +} + +/// `od -An -v -tx1`'s bytes. +pub fn od_bytes(text: &str) -> Result, String> { + text.split_whitespace().map(|b| u8::from_str_radix(b, 16).map_err(|_| format!("{b:?} is no byte od writes"))).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + const UNIQUE: &str = "11111111-2222-3333-4444-555555555555"; + + fn listing(label: &str, rm: &str, mounted: &str, extra: &str) -> String { + format!( + "NAME=\"/dev/nvme0n1\" TYPE=\"disk\" RM=\"{rm}\" PKNAME=\"\" SIZE=\"1024209543168\" PARTTYPE=\"\" PARTUUID=\"\" PARTLABEL=\"\" MOUNTPOINTS=\"\"\n\ + NAME=\"/dev/nvme0n1p3\" TYPE=\"part\" RM=\"0\" PKNAME=\"/dev/nvme0n1\" SIZE=\"34359738368\" PARTTYPE=\"0fc63daf-8483-4772-8e79-3d69d8477de4\" PARTUUID=\"aaaaaaaa-2222-3333-4444-555555555555\" PARTLABEL=\"\" MOUNTPOINTS=\"/\"\n\ + NAME=\"/dev/nvme0n1p4\" TYPE=\"part\" RM=\"0\" PKNAME=\"/dev/nvme0n1\" SIZE=\"644245094400\" PARTTYPE=\"064e3777-5076-4c71-8e07-90ad24cfe8d6\" PARTUUID=\"{UNIQUE}\" PARTLABEL=\"{label}\" MOUNTPOINTS=\"{mounted}\"\n{extra}" + ) + } + + fn bench() -> String { + listing("ToyOS\\x20data", "0", "", "") + } + + #[test] + fn the_benchs_data_is_found_by_type_and_name_and_written_by_its_guid() { + let data = find(&bench()).unwrap(); + assert_eq!((data.node.as_str(), data.blocks), ("/dev/nvme0n1p4", 157_286_400)); + assert_eq!(data.by_guid(), format!("/dev/disk/by-partuuid/{UNIQUE}")); + let stamp = data.stamp(); + assert_eq!(&stamp[..16], b"TOYOS-FORMAT-ME\0"); + assert_eq!(u64::from_le_bytes(stamp[16..24].try_into().unwrap()), 157_286_400); + assert!(stamp[24..].iter().all(|&b| b == 0)); + } + + #[test] + fn anything_but_one_fixed_free_data_partition_named_toyos_data_is_refused() { + let refused = |text: String, said: &str| assert!(find(&text).unwrap_err().contains(said), "{said}"); + refused(listing("data", "0", "", ""), "is named \"data\""); + refused(listing("ToyOS\\x20data", "1", "", ""), "no fixed disk"); + refused(listing("ToyOS\\x20data", "0", "/mnt", ""), "is mounted"); + let above = "NAME=\"/dev/mapper/x\" TYPE=\"crypt\" RM=\"0\" PKNAME=\"/dev/nvme0n1p4\" SIZE=\"1\" PARTTYPE=\"\" PARTUUID=\"\" PARTLABEL=\"\" MOUNTPOINTS=\"\"\n"; + refused(listing("ToyOS\\x20data", "0", "", above), "/dev/mapper/x is stacked on /dev/nvme0n1p4"); + let second = bench().replace("nvme0n1p4", "nvme0n1p5").replace(UNIQUE, "bbbbbbbb-2222-3333-4444-555555555555"); + refused(format!("{}{second}", bench()), "lists 2 TOYOS-DATA partitions"); + refused(bench().replace("064e3777", "0fc63daf"), "lists 0 TOYOS-DATA partitions"); + refused(bench().replace("644245094400", "644245094401"), "no whole number"); + } + + #[test] + fn a_first_block_is_this_stamp_a_volume_or_anything_else() { + let data = find(&bench()).unwrap(); + assert_eq!(classify(&data, &data.stamp()), Block0::Stamped); + let mut volume = [0u8; BLOCK]; + volume[..4].copy_from_slice(&bcachefs::MAGIC); + assert_eq!(classify(&data, &volume), Block0::Volume); + assert_eq!(classify(&data, &[0u8; BLOCK]), Block0::Other); + let other_size = toyos_build::image::designation(data.blocks - 1); + assert_eq!(classify(&data, &other_size), Block0::Other, "a stamp for another size designates nothing"); + assert_eq!(classify(&data, &data.stamp()[..BLOCK - 1]), Block0::Other, "a short read is no stamp"); + } + + #[test] + fn od_bytes_read_back_as_written() { + assert_eq!(od_bytes(" 54 4f 59\n 00 ff\n").unwrap(), vec![0x54, 0x4f, 0x59, 0, 0xff]); + assert!(od_bytes("zz").is_err()); + } +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 031b27dd707..1df0975095b 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1408,12 +1408,16 @@ const METAL: &[(&str, metal::Metal)] = &[ metal::Metal { arms: TESTCASES, judge: |b| usbd_serves_the_boot_stick(&b[0].kernel(), &b[0].log()) }, ), ( - // The bench's own NVMe disk, its host's partitions and the owner's - // unformatted ToyOS DATA, driven beside the stick by the row's class - // and left alone (`hostdisk::left_alone`). The T14 and the AMD bench; + // The bench's own NVMe disk, driven beside the stick by the row's + // class: its host's partitions left alone, and the owner's ToyOS DATA, + // designated (`toyos-metal --designate-data`), the one partition a + // session opened (`hostdisk::left_alone`). The T14 and the AMD bench; // the metal loop's next boot of the host off that disk is the other half. "nvme_leaves_the_benchs_disk_alone", - metal::Metal { arms: TESTCASES, judge: |b| common::hostdisk::left_alone(&b[0].log()).map(|_| ()) }, + metal::Metal { + arms: TESTCASES, + judge: |b| common::hostdisk::left_alone(&b[0].log(), common::hostdisk::Data::Designated).map(|_| ()), + }, ), ( // The stick written through usbd while the boot deadline resets the @@ -1698,25 +1702,38 @@ fn osbench_ran(boot: &metal::Readback, jobs: &[&str]) -> Result<(), String> { } /// DATA, which `osbench persist` writes, was a volume on a TOYOS-DATA -/// partition the NVMe driver serves: mounted, or formatted off its -/// designation, and not memory standing in for a partition that is not ours. -/// The block service refuses DATA where two disks carry one, so a mount -/// alongside the NVMe's partition is that partition's. +/// partition the NVMe driver serves, mounted, and not memory standing in for +/// a partition that is not ours. The block service refuses DATA where two +/// disks carry one, so a mount alongside the NVMe's partition is that +/// partition's. The boot that formats a freshly designated partition is no +/// measured run, and says how long the format took. fn data_on_the_nvme(boot: &metal::Readback) -> Result<(), String> { + const FORMATTING: &str = "fileserver: block 0 designates this partition for ToyOS; formatting it"; + const SERVING: &str = "fileserver: Data serving "; let log = boot.log(); let text = log.text(); let served = format!(" of type {} at block ", toyos_gpt::Guid::TOYOS_DATA); if !text.lines().any(|l| l.contains("nvme] nvme ") && l.contains(&served)) { return Err("the NVMe driver served no TOYOS-DATA partition, so persist measured no disk of the bench's".into()); } - let mounted = ["fileserver: mounted the DATA volume", "fileserver: block 0 designates this partition for ToyOS; formatting it"]; - if !mounted.iter().any(|said| text.contains(said)) { + if let Some(began) = text.lines().find(|l| l.contains(FORMATTING)) { + let ended = text.lines().find(|l| l.contains(SERVING)); + let secs = ended.and_then(since_boot).zip(since_boot(began)).map(|(end, start)| end - start); + let took = secs.map_or("its end unlogged".to_string(), |s| format!("{s:.3} s")); + return Err(format!("this boot formatted DATA off its designation ({took}), and the format's boot is no measured run")); + } + if !text.contains("fileserver: mounted the DATA volume") { let why = text.lines().find(|l| l.contains("fileserver: the DATA")).unwrap_or("the file server said nothing of DATA"); return Err(format!("DATA was not the NVMe's partition, so persist measured no disk: {why}")); } Ok(()) } +/// A log line's seconds since boot: `[2026-10-24 14:17:17 12.664 fileserver]`. +fn since_boot(line: &str) -> Option { + line.strip_prefix('[')?.split_whitespace().nth(2)?.parse().ok() +} + /// A timing boot was in ACPI mode before its first job and stayed there, as a /// shipped image and the bench's Linux are. In legacy mode the T14's firmware /// stops every CPU every 2.2 s, and its time would be measured as ToyOS's @@ -4686,7 +4703,7 @@ fn a_host_disk_beside_the_stick_is_left_alone(test_config: &Path) -> Result<(), console.push_str(&qemu.await_exit(by)?); let log = serial::Serial::named("the boot", console); log.must_say("vendor=8086 device=5845 prog_if=02")?; - match common::hostdisk::left_alone(&log)? { + match common::hostdisk::left_alone(&log, common::hostdisk::Data::Foreign)? { 4 => {} named => return Err(format!("the driver named {named} partitions of a disk that has 4")), } From ba76f730a40ef425cf65a7b3fec4d0d2c8b0cec2 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 08:39:21 +0100 Subject: [PATCH 3/8] toyos-metal: the designation waits in a TempDir, as every host scratch does Co-Authored-By: Claude Opus 5.5 --- tests/metal/Cargo.toml | 3 ++- tests/metal/src/lib.rs | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/metal/Cargo.toml b/tests/metal/Cargo.toml index 5d758f6e43e..5d619007c9f 100644 --- a/tests/metal/Cargo.toml +++ b/tests/metal/Cargo.toml @@ -16,10 +16,11 @@ toyos-fat32-check = { path = "../../lib/fat32/check" } toyos-gpt = { path = "../../lib/gpt" } # A ToyOS volume's magic, which the DATA designation never writes over. bcachefs = { path = "../../lib/bcachefs" } +# Where the designation's one block waits for the root command that writes it. +toyos-tmpdir = { path = "../../lib/tmpdir" } toyos-quiesce = { path = "../../lib/quiesce" } toyos-tco = { path = "../../lib/tco" } [dev-dependencies] -toyos-tmpdir = { path = "../../lib/tmpdir" } # The slot table written into the test images `internal::admit` reads. toyos-update = { path = "../../lib/update" } diff --git a/tests/metal/src/lib.rs b/tests/metal/src/lib.rs index c5134caa012..2dda81e3bae 100644 --- a/tests/metal/src/lib.rs +++ b/tests/metal/src/lib.rs @@ -1906,12 +1906,11 @@ fn designate_data(target: &Target, dry_run: bool) -> Result<(), Refusal> { } stamp::Block0::Other => {} } - let file = std::env::temp_dir().join(format!("toyos-metal-designation-{}", std::process::id())); + let scratch = toyos_tmpdir::TempDir::new("designation"); + let file = scratch.path().join("stamp"); std::fs::write(&file, data.stamp()).map_err(|e| Refusal::File { path: file.display().to_string(), why: e.to_string() })?; let feed = Feed { path: &file, at: 0, len: stamp::BLOCK as u64 }; - let wrote = driver.as_root("writing the designation", Job::Stamp(node.clone()), Some(feed)); - let _ = std::fs::remove_file(&file); - let Some(out) = wrote? else { + let Some(out) = driver.as_root("writing the designation", Job::Stamp(node.clone()), Some(feed))? else { println!("dry run: {said} was read and nothing was written"); return Ok(()); }; From 7fea95d5921a877d0858512d7b2ef5eef77ab574 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 09:06:02 +0100 Subject: [PATCH 4/8] review r1 and the owner's ruling: DATA waits for ToyOS to designate it, and the rows say what their clocks and media were The owner ruled (2026-10-28) that ToyOS manages its partitions itself, so toyos-metal --designate-data is a workaround. It goes, with its tests, the public designation and the bcachefs magic it exported. What depends on DATA being the bench's partition waits for ToyOS-native designation: - persist on ToyOS's DATA, and fs.disk; - the osbench row's requirement that DATA be the NVMe's partition; - nvme_leaves_the_benchs_disk_alone's designated premise. Until then ToyOS's persist stays on the boot stick's LOG volume and files keeps ToyOS's served /home. What lands now: - every fs and persist line names its medium: memory, or the disk's kind as Linux's sysfs or ToyOS's boot gives it (nvme, usb, sata). The compare refuses a verdict on a row whose media differ, so persist reads "the media differ: Linux nvme, ToyOS usb" and not ToyOS worse; - the osbench row refuses the boot that formats a newly designated DATA as a run and says how long it took, since the AMD bench's partition is stamped, which reddens no ordinary boot; - persist runs before crunch, which ToyOS cannot cool down after (r1 B1); - a settle that ran out its time marks the line unsettled, the compare counts those runs and judges no fall from them, and it says where the systems' median settled clocks lie more than a tenth apart, the case a fall cannot see (r1 B5 and suggestions); - the README defines a resolved row as complete separation at four runs each, corrects "three of one" (3 against 5 reaches 2/56), states the ~2.9% chance rate of an uncorrected table, names the clock-fall cause an inference and its blind spot, writes EPP 128 by construction, sets the AMD governor before each Linux run, and checks fair.apart.units (r1 B4, B5, B6 and suggestions); - issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 - build/image.rs | 5 +- ...ompares-toyos-with-linux-on-one-machine.md | 9 +- ...ime-on-a-tsc-it-never-asks-is-invariant.md | 34 ++++ lib/bcachefs/src/lib.rs | 2 +- tests/common/hostdisk.rs | 48 ++--- tests/guest/osbench/README.md | 83 +++++---- tests/guest/osbench/compare/src/lib.rs | 125 +++++++++++-- tests/guest/osbench/src/clock.rs | 26 ++- tests/guest/osbench/src/files.rs | 45 +++-- tests/guest/osbench/src/main.rs | 12 +- tests/guest/osbench/src/report.rs | 15 +- tests/guest/osbench/src/storage.rs | 34 ++-- tests/guest/osbench/src/sys/linux.rs | 22 ++- tests/guest/osbench/src/sys/toyos.rs | 10 +- tests/images/benchcase.toml | 6 +- tests/metal/Cargo.toml | 5 +- tests/metal/src/internal.rs | 2 +- tests/metal/src/lib.rs | 87 +-------- tests/metal/src/stamp.rs | 166 ------------------ tests/toyos.rs | 59 +++---- 21 files changed, 344 insertions(+), 452 deletions(-) create mode 100644 issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md delete mode 100644 tests/metal/src/stamp.rs diff --git a/Cargo.lock b/Cargo.lock index 461c34194d7..82931a647c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3802,7 +3802,6 @@ dependencies = [ name = "toyos-metal" version = "0.1.0" dependencies = [ - "bcachefs", "serde", "toml", "toyos-build", diff --git a/build/image.rs b/build/image.rs index 6ea247ab7d9..409fc64efed 100644 --- a/build/image.rs +++ b/build/image.rs @@ -733,9 +733,8 @@ fn name_of(part: &toyos_gpt::Partition) -> String { String::from_utf16(part.name()).unwrap_or_else(|e| panic!("partition {}'s name is no UTF-16: {e}", part.unique_guid())) } -/// Block 0 of a volume of `blocks` 4 KiB blocks, designated for ToyOS: the -/// magic and its block count. -pub fn designation(blocks: u64) -> [u8; SECTOR] { +/// Block 0 of a volume: the magic and its block count. +fn designation(blocks: u64) -> [u8; SECTOR] { let mut block = [0u8; SECTOR]; block[..bcachefs::DESIGNATION_MAGIC.len()].copy_from_slice(&bcachefs::DESIGNATION_MAGIC); let at = bcachefs::DESIGNATION_BLOCKS_OFFSET; diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index 2147b4f9abd..f93e23ed6b1 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -35,11 +35,10 @@ its groups: where the package's temperature can be read: `crunch.*`. `tests/guest/osbench/README.md` is how a comparison is run: both halves -from one revision, interleaved runs, Linux under ToyOS's frequency request -and on the TSC, and `persist` on the bench's NVMe under both. ToyOS reaches -that disk only once the bench's TOYOS-DATA partition is designated, which -neither bench's is: until then the `osbench` row refuses each boot. The AMD -bench's Linux reads no TSC +from one revision, interleaved runs, and Linux under ToyOS's frequency +request and on the TSC. `persist` is not yet one disk: ToyOS writes the boot +stick's LOG volume, and moves to the bench's NVMe once ToyOS designates its +DATA partition itself. The AMD bench's Linux reads no TSC (`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`). `internet.*`, HTTPS to Cloudflare's speed test, is exploratory evidence of a diff --git a/issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md b/issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md new file mode 100644 index 00000000000..1c3f7899bc4 --- /dev/null +++ b/issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md @@ -0,0 +1,34 @@ +--- +status: open +kind: defect +opened: 2026-10-28 +--- + +# The kernel keeps time on a TSC it never asks is invariant + +x86-64's clock is the TSC, measured once against the HPET over 50 ms at boot +(`kernel/src/arch/x86_64/hpet.rs`), and every timestamp after that converts +TSC ticks at the rate that one window found. Nothing reads CPUID +80000007H:EDX[8], which Intel (the SDM's "Invariant TSC") and +AMD (its PPRs' `CPUID_Fn80000007_EDX[TscInvariant]`) define as the TSC +counting at one rate across P-, C- and T-states. A CPU without it keeps a +wrong clock silently: every timeout, deadline and measurement on it is off +by however its rate moved, and no line says so. + +The AMD bench (Ryzen 7 4800U) is why this was looked at. Its Linux's +clocksource watchdog measured the TSC at 1802.69 MHz against the HPET over +one 502 ms interval 2.6 s into boot, where ToyOS's calibration against the +same HPET reads 1796.52 MHz, about 3400 ppm apart; Linux marked the TSC +unstable and fell back to the HPET. Whether that CPU's TSC drifts is being +measured (`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`), +and the bit is missing here whatever the measurement shows. The bit's +document citations above are the fix's to read at the site; this record +names them and has not quoted them. + +**Exit**: the kernel reads the bit on every CPU before it keeps time on the +TSC, and a CPU that does not state it fails the boot by name or keeps time +on a counter that is invariant. Each answer covers a guest whose CPU model +leaves the bit clear. QEMU's TCG `qemu64` is one, an inference not yet +checked against QEMU's model. + +**Owner**: the kernel's x86-64 clock (`kernel/src/arch/x86_64/hpet.rs`). diff --git a/lib/bcachefs/src/lib.rs b/lib/bcachefs/src/lib.rs index 48685b07aa6..2d01bf93eb1 100644 --- a/lib/bcachefs/src/lib.rs +++ b/lib/bcachefs/src/lib.rs @@ -27,7 +27,7 @@ pub use block_io::{BlockIO, BlockBuf, BlockNum, DeviceError, TransferError}; #[cfg(feature = "std")] pub use block_io::VecBlockIO; pub use fs::{file_entry_fits, Formatted, Mounted, ReadOnly, ReadWrite, Recorded, FsError, Extent}; -pub use superblock::{DESIGNATION_BLOCKS_OFFSET, DESIGNATION_MAGIC, FsUuid, MAGIC, Superblock}; +pub use superblock::{DESIGNATION_BLOCKS_OFFSET, DESIGNATION_MAGIC, FsUuid, Superblock}; /// Records the largest single allocation each test thread makes, so a test can /// assert what parsing a crafted block asks the allocator for. diff --git a/tests/common/hostdisk.rs b/tests/common/hostdisk.rs index 6b772f9d192..5523b9fb8d3 100644 --- a/tests/common/hostdisk.rs +++ b/tests/common/hostdisk.rs @@ -1,6 +1,5 @@ //! Another system's disk, as a bench's internal disk is before ToyOS is -//! installed on it, and the judge of what ToyOS did to it, there and on a -//! bench whose DATA is designated. +//! installed on it, and the judge of what ToyOS did to it. //! //! The layout is the benches': the host's ESP, its `/boot` and its root, and //! the owner's ToyOS DATA, unformatted. Every block of every partition carries @@ -55,23 +54,12 @@ fn said_by_nvme(line: &str) -> Option<&str> { (at.len() == 7 && at[2] == b':' && at[5] == b'.').then_some(said) } -/// What the bench's DATA partition is to the file server: another system's, -/// as this module's disk is, or designated for ToyOS, as a bench's is once -/// `toyos-metal --designate-data` stamped it. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Data { - Foreign, - Designated, -} - -/// A boot off a stick drove an NVMe disk that is another system's but for -/// its DATA and left it alone: a driver brought a controller up and offered -/// its partitions, it opened at least one session and each on a ToyOS DATA -/// partition, and the file server found that DATA as `data` says: neither -/// ours nor designated, so neither mounted nor formatted, or designated, so -/// mounted or formatted off its stamp and never memory in its place. -/// Answers how many partitions the disks' tables named. -pub fn left_alone(log: &Serial, data: Data) -> Result { +/// A boot off a stick drove an NVMe disk that is another system's and left +/// it alone: a driver brought a controller up and offered its partitions, it +/// opened at least one session and each on a ToyOS DATA partition, and the file server +/// found that DATA neither ours nor designated, and so neither mounted nor +/// formatted it. Answers how many partitions the disks' tables named. +pub fn left_alone(log: &Serial) -> Result { log.must_say("diskserver: every disk driver has settled")?; let said: Vec<&str> = log.text().lines().filter_map(said_by_nvme).collect(); if !said.iter().any(|s| s.starts_with("up: ")) { @@ -86,7 +74,7 @@ pub fn left_alone(log: &Serial, data: Data) -> Result { if kinds.is_empty() { return Err(format!("no nvme driver named a partition of its disk\n{}", log.text())); } - let data_type = Guid::TOYOS_DATA.to_string(); + let data = Guid::TOYOS_DATA.to_string(); let opened: Vec<&str> = said .iter() .filter_map(|s| s.strip_prefix("session ")) @@ -98,24 +86,12 @@ pub fn left_alone(log: &Serial, data: Data) -> Result { } for guid in opened { match kinds.iter().find(|(g, _)| *g == guid) { - Some((_, kind)) if *kind == data_type => {} + Some((_, kind)) if *kind == data => {} other => return Err(format!("an nvme driver opened a session on {guid}, of type {other:?}, which is no ToyOS DATA")), } } - const MOUNTED: &str = "fileserver: mounted the DATA volume"; - const FORMATTED: &str = "fileserver: block 0 designates this partition for ToyOS; formatting it"; - match data { - Data::Foreign => { - log.must_say("fileserver: no volume of ours and no designation stamp; nothing will be written to this partition")?; - log.must_not_say(MOUNTED)?; - log.must_not_say(FORMATTED)?; - } - Data::Designated => { - if !log.text().contains(MOUNTED) && !log.text().contains(FORMATTED) { - return Err(format!("the designated DATA was neither mounted nor formatted\n{}", log.text())); - } - log.must_not_say("are in memory and will not survive a reboot")?; - } - } + log.must_say("fileserver: no volume of ours and no designation stamp; nothing will be written to this partition")?; + log.must_not_say("fileserver: mounted the DATA volume")?; + log.must_not_say("designates this partition for ToyOS; formatting it")?; Ok(kinds.len()) } diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md index 809d88f609f..04fe7fc96cd 100644 --- a/tests/guest/osbench/README.md +++ b/tests/guest/osbench/README.md @@ -32,9 +32,11 @@ and the C library beneath it are each system's own. - **At least four runs of each system, interleaved**: Linux, ToyOS, Linux, ToyOS. A ToyOS run is one boot and one readback directory. A Linux run is `osbench all `. It refuses runs of one system that disagree on their setup (`SETUP`) and states each system's. Each row's verdict says whether the runs resolve a - difference. -- **A limit the machine reaches is marked, not hidden.** A settled row - carries the clock settled before it (`mhz_settled`) and after it - (`mhz_after`). Where the second is under nine tenths of the first, the - measurement ran into a power or heat limit. The AMD bench's package - limit held a long single-thread row at 1.4 to 1.9 GHz after a 4.29 GHz - start. The verdict counts those runs per system. No program reads the + difference. Each p is its row's alone: with no difference anywhere, about + 2.9% of rows resolve by chance, a few in a table this size. +- **The clocks are told beside each verdict.** No program reads the package's limit state or temperature on ToyOS, so the clock is what both - systems report. + systems report. A settled row carries the clock settled before it + (`mhz_settled`) and after it (`mhz_after`). The verdict counts per system + the runs where the second is under nine tenths of the first. A power or + heat limit reached inside the row is the likely cause, an inference: the + AMD bench's package limit held a long single-thread row at 1.4 to 1.9 GHz + after a 4.29 GHz start. A row that began already limited is not marked + that way, since both its settles read the limited clock. So the verdict + also says where the systems' median settled clocks lie more than a tenth + apart. A settle that ran out its 0.5 s says so (`unsettled`), and is + counted rather than judged. ## Linux makes the requests ToyOS makes @@ -64,13 +71,24 @@ request on both systems: - **The T14** (i5-1135G7, `intel_pstate` with HWP). ToyOS asks for HWP's lowest to highest performance at EPP 0x80 (`kernel/pure/cpu.rs`). Ubuntu's - default, `powersave` at `balance_performance` (EPP 0x80), is that request. - Leave it, and check `governor=powersave*8`, `epp=balance_performance*8`, - `intel_no_turbo=0`. + default `powersave` governor makes HWP's request with the EPP its + `energy_performance_preference` holds, so write 0x80 there before each + Linux run, which `intel_pstate` takes as a number: + + ```text + echo 128 | sudo tee /sys/devices/system/cpu/cpufreq/policy*/energy_performance_preference + ``` + + Check `governor=powersave*8` and `intel_no_turbo=0`; `epp` reads 128 by + the name the driver gives it, `balance_performance` where that is 0x80 on + this CPU. Linux's lowest request is `cpuinfo_min_freq`, which need not be + HWP's lowest, ToyOS's: that reaches a CPU waking from idle, not a settled + row. - **The AMD bench** (Ryzen 7 4800U, `acpi-cpufreq`). ToyOS writes no P-state or boost register, so the firmware's P0 and Core Performance Boost stand. - Linux's `performance` governor asks for that. Set it before the Linux runs - and give back `schedutil` after them: + Linux's `performance` governor asks for that. Set it before each Linux + run, since every ToyOS boot between them boots Linux back to `schedutil`, + and give `schedutil` back after the last: ```text echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor @@ -81,31 +99,28 @@ request on both systems: When ToyOS's request changes, this section changes with it. +**The groups.** `fair.apart` sets each workload in a group of its own: a +ToyOS Job, and on Linux a scope's cgroup, which is a group only where the +user's manager delegates the `cpu` controller. Check +`fair.apart.units=3xcgroup`. With `1xcgroup` Linux's `apart` is +thread-fair, and the pair compares nothing. + **The clock.** Every timestamp is read from the TSC on ToyOS, so Linux runs with `clocksource=tsc`. A clocksource the vDSO cannot read costs every `Instant::now` a syscall: on 2026-10-24 the AMD bench's Linux took 1.21 µs a read (`time.now`) against the T14's 14 ns, and every per-operation timing there carried it. -## Both write the bench's NVMe - -`persist` and `fs.disk` run on each system's persistent filesystem on the -bench's NVMe. On Linux that is the runner's home, ext4 on the bench's root -partition. On ToyOS it is DATA: the bench's TOYOS-DATA partition, its fourth, -named `ToyOS data`. It is served through the DATA file server and the NVMe -driver. The `osbench` row refuses a boot whose DATA was anything else. Every -row's `served_by` names the filesystem and the disk it measured. - -The partition holds a ToyOS volume once, ever, after its first block -carries the designation stamp: - -```text -cargo run -p toyos-metal -- --designate-data --host --dry-run -cargo run -p toyos-metal -- --designate-data --host -``` +## The disk is not yet one disk -The first ToyOS boot after the stamp formats the volume. The `osbench` row -refuses that boot as a run and says how long the format took. +`persist` writes each system's persistent filesystem. On Linux that is the +runner's home, ext4 on the bench's NVMe. On ToyOS it is still the boot +stick's LOG volume, FAT32 over USB 2.0, because ToyOS's DATA is the bench's +partition only once ToyOS has designated it, and ToyOS cannot yet do that +itself. So `persist` compares two media. Its lines name theirs, and +`osbench-compare` refuses a verdict on any row whose media differ. The +`osbench` row refuses the boot that formats a newly designated DATA as a +run, and says how long the format took. ## What is not a comparison diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs index 83421f2cd96..4f5af738b46 100644 --- a/tests/guest/osbench/compare/src/lib.rs +++ b/tests/guest/osbench/compare/src/lib.rs @@ -12,11 +12,22 @@ //! for a unit whose better direction is known ([`better`]); for any other //! the verdict says which way ToyOS differs. //! -//! A row whose clock fell across a run's measurement, the clock settled -//! after it (`mhz_after`) under nine tenths of the one settled before it -//! (`mhz_settled`) ([`clock_fell`]), ran into a limit of the machine's, power -//! or heat, which no program reads on both systems: the verdict says in how -//! many of each system's runs it did. +//! Each row's clocks are told beside its verdict, since no program reads a +//! power or heat limit on both systems: +//! +//! - in how many runs the clock settled after the row was under nine tenths +//! of the one settled before it ([`clock_fell`]), which a limit reached +//! inside the row most likely explains; +//! - where the two systems' median clocks settled before the row lie more +//! than a tenth apart ([`clocks_apart`]), which also catches a row that +//! started already limited, as a fall cannot; +//! - in how many runs a settle ran out its time (`unsettled`), whose clocks +//! say nothing either way. +//! +//! A row whose lines name their medium (`medium=`, the memory or the kind of +//! disk under a filesystem) is compared only where every run's medium is one: +//! a row written to an NVMe disk under one system and a USB stick under the +//! other has a verdict of what the two media are, not of the systems. //! //! A system's runs on one machine are one configuration ([`SETUP`]): runs that //! disagree on it are refused, since their median would mix two, and each @@ -47,8 +58,17 @@ impl Os { #[derive(Clone, Debug, PartialEq, Eq)] pub enum Outcome { - /// `clock_fell` as [`clock_fell`] judges the line's two clocks. - Measured { unit: String, p50: u64, clock_fell: bool }, + /// `clock_fell` as [`clock_fell`] judges the line's two clocks, unless + /// either is `unsettled`; `settled` the clock settled before it; `medium` + /// as the line names it. + Measured { + unit: String, + p50: u64, + clock_fell: bool, + settled: Option, + unsettled: bool, + medium: Option, + }, /// Unmeasured, unsupported or refused, with the run's reason. Not { kind: String, why: String }, } @@ -110,7 +130,15 @@ pub fn parse(text: &str) -> Result { let outcome = if let Some(unit) = fields.get("unit") { let p50 = fields.get("p50").and_then(|v| v.parse().ok()).ok_or_else(|| format!("{bench}: no p50"))?; let mhz = |key: &str| fields.get(key).and_then(|v| v.parse().ok()); - Outcome::Measured { unit: unit.to_string(), p50, clock_fell: clock_fell(mhz("mhz_settled"), mhz("mhz_after")) } + let unsettled = fields.contains_key("unsettled"); + Outcome::Measured { + unit: unit.to_string(), + p50, + clock_fell: !unsettled && clock_fell(mhz("mhz_settled"), mhz("mhz_after")), + settled: mhz("mhz_settled"), + unsettled, + medium: fields.get("medium").map(|m| m.to_string()), + } } else if let Some((kind, why)) = ["unmeasured", "unsupported", "refused"].iter().find_map(|k| fields.get(k).map(|why| (*k, *why))) { @@ -142,6 +170,12 @@ pub fn clock_fell(settled: Option, after: Option) -> bool { matches!((settled, after), (Some(settled), Some(after)) if after * 10 < settled * 9) } +/// Whether two systems' median settled clocks lie more than a tenth of the +/// higher apart. +pub fn clocks_apart(a: f64, b: f64) -> bool { + (a - b).abs() * 10.0 > a.max(b) +} + pub fn median(values: &[u64]) -> f64 { let mut v = values.to_vec(); v.sort_unstable(); @@ -243,7 +277,7 @@ const PATHS: [(&str, &str, usize); 2] = [("lan.", "lan.peer_link", 2), ("interne /// The conditions every run of one system on one machine must share: the /// kernel, the CPU's frequency request and the clock every timestamp is read /// from, and what serves each filesystem the `fs` and `persist` rows use. -pub const SETUP: [&str; 13] = [ +pub const SETUP: [&str; 14] = [ "kernel", "cpufreq_driver", "governor", @@ -256,7 +290,8 @@ pub const SETUP: [&str; 13] = [ "clocksource", "smt_active", "fs.mem.served_by", - "fs.disk.served_by", + "fs.served.served_by", + "persist.served_by", ]; /// `os`'s setup on these runs, as `key=value` in [`SETUP`]'s order, or the @@ -335,15 +370,21 @@ fn compare_one(runs: &[&Run], name: &str) -> String { let mut unit = None; let mut values: BTreeMap> = BTreeMap::new(); let mut fell: BTreeMap = BTreeMap::new(); + let mut unsettled_runs: BTreeMap = BTreeMap::new(); + let mut settled: BTreeMap> = BTreeMap::new(); + let mut media: BTreeMap> = BTreeMap::new(); let mut not: BTreeMap> = BTreeMap::new(); for run in runs { match run.benches.get(name) { - Some(Outcome::Measured { unit: u, p50, clock_fell }) => { + Some(Outcome::Measured { unit: u, p50, clock_fell, settled: mhz, unsettled, medium }) => { if unit.get_or_insert(u.as_str()) != u { return "| | | | | | | | | units differ between runs".into(); } values.entry(run.os).or_default().push(*p50); *fell.entry(run.os).or_default() += usize::from(*clock_fell); + *unsettled_runs.entry(run.os).or_default() += usize::from(*unsettled); + settled.entry(run.os).or_default().extend(mhz.filter(|_| !*unsettled)); + media.entry(run.os).or_default().extend(medium.clone()); } Some(Outcome::Not { kind, why }) => not.entry(run.os).or_default().push(format!("{kind}: {why}")), None => not.entry(run.os).or_default().push("absent".into()), @@ -361,7 +402,12 @@ fn compare_one(runs: &[&Run], name: &str) -> String { }; let [lp, lm, lr] = side(Os::Linux); let [tp, tm, tr] = side(Os::Toyos); + let all_media: std::collections::BTreeSet<&String> = media.values().flatten().collect(); let (ratio, p, verdict) = match (values.get(&Os::Linux), values.get(&Os::Toyos)) { + (Some(_), Some(_)) if all_media.len() > 1 => { + let named = |os: Os| media.get(&os).map_or("unnamed".to_string(), |m| m.iter().cloned().collect::>().join("+")); + ("-".into(), "-".into(), format!("the media differ: Linux {}, ToyOS {}", named(Os::Linux), named(Os::Toyos))) + } (Some(l), Some(t)) => { let ratio = median(t) / median(l); let exact = mann_whitney(t, l); @@ -383,10 +429,21 @@ fn compare_one(runs: &[&Run], name: &str) -> String { _ => ("-".into(), "-".into(), String::new()), }; let mut verdict = verdict; - for (os, fell) in fell.iter().filter(|(_, n)| **n > 0) { + let mut note = |said: String| { let sep = if verdict.is_empty() { "" } else { "; " }; - let of = values.get(os).map_or(0, Vec::len); - verdict = format!("{verdict}{sep}the clock fell in {fell} of {of} {} run(s)", os.name()); + verdict = format!("{verdict}{sep}{said}"); + }; + if let (Some(l), Some(t)) = (settled.get(&Os::Linux).filter(|v| !v.is_empty()), settled.get(&Os::Toyos).filter(|v| !v.is_empty())) { + let (l, t) = (median(l), median(t)); + if clocks_apart(l, t) { + note(format!("settled clocks apart: Linux {l:.0} MHz, ToyOS {t:.0} MHz")); + } + } + for (counts, said) in [(&fell, "the clock fell"), (&unsettled_runs, "the clock did not settle")] { + for (os, n) in counts.iter().filter(|(_, n)| **n > 0) { + let of = values.get(os).map_or(0, Vec::len); + note(format!("{said} in {n} of {of} {} run(s)", os.name())); + } } for (os, reasons) in ¬ { let mut distinct = reasons.clone(); @@ -423,7 +480,10 @@ mod tests { ); assert_eq!(r.os, Os::Toyos); assert_eq!(r.machine(), Some("Bench_CPU")); - assert_eq!(r.benches["lan.wired.tcp.rtt.64"], Outcome::Measured { unit: "ns".into(), p50: 812, clock_fell: false }); + assert_eq!( + r.benches["lan.wired.tcp.rtt.64"], + Outcome::Measured { unit: "ns".into(), p50: 812, clock_fell: false, settled: None, unsettled: false, medium: None } + ); assert_eq!( r.benches["lan.wifi.tcp.upload"], Outcome::Not { kind: "unsupported".into(), why: "ToyOS_has_no_Wi-Fi_driver".into() } @@ -580,6 +640,41 @@ mod tests { assert!(row.ends_with("not resolved; the clock fell in 2 of 4 ToyOS run(s) |"), "{row}"); } + #[test] + fn clocks_settled_apart_and_settles_cut_short_are_told() { + assert!(clocks_apart(4289.0, 1884.0)); + assert!(!clocks_apart(4186.0, 3788.0), "9.5% apart is within a tenth"); + assert!(clocks_apart(4186.0, 3767.0)); + let at = |os: &str, v: u64, settled: u64, unsettled: &str| { + run(os, &[&format!("bench=fs.mem.stat unit=ns n=2000 min=1 p50={v} max=1 mean=1 mhz_settled={settled} mhz_after=1000{unsettled} mhz=1")]) + }; + let mut runs: Vec = [10, 11, 12, 13].iter().map(|v| at("linux", *v, 4200, "")).collect(); + runs.extend([20, 21, 22].iter().map(|v| at("toyos", *v, 1884, ""))); + runs.push(at("toyos", 23, 4200, " unsettled=after")); + let row = compare(&runs).unwrap().lines().find(|l| l.starts_with("| fs.mem.stat")).unwrap().to_string(); + assert!(row.contains("settled clocks apart: Linux 4200 MHz, ToyOS 1884 MHz"), "{row}"); + assert!(row.contains("the clock fell in 4 of 4 Linux run(s)"), "{row}"); + assert!(row.contains("the clock fell in 3 of 4 ToyOS run(s); the clock did not settle in 1 of 4 ToyOS run(s) |"), "{row}"); + } + + #[test] + fn a_row_on_two_media_is_no_comparison_of_the_systems() { + let fsync = |os: &str, v: u64, medium: &str| { + run(os, &[&format!("bench=persist.fsync_clean unit=ns n=200 min=1 p50={v} max=1 mean=1 medium={medium}")]) + }; + let mut runs: Vec = [67, 68, 66, 67].iter().map(|v| fsync("linux", *v, "nvme")).collect(); + runs.extend([1687, 1690, 1700, 1695].iter().map(|v| fsync("toyos", *v, "usb"))); + let row = compare(&runs).unwrap().lines().find(|l| l.starts_with("| persist.fsync_clean")).unwrap().to_string(); + assert!(row.ends_with("| - | - | the media differ: Linux nvme, ToyOS usb |"), "{row}"); + let mut runs: Vec = [67, 68, 66, 67].iter().map(|v| fsync("linux", *v, "nvme")).collect(); + runs.extend([70, 71, 72, 73].iter().map(|v| fsync("toyos", *v, "nvme"))); + assert!(compare(&runs).unwrap().contains("| ToyOS worse |"), "one medium is a comparison"); + let mut runs: Vec = [67, 68, 66].iter().map(|v| fsync("linux", *v, "nvme")).collect(); + runs.push(fsync("linux", 69, "sata")); + runs.extend([70, 71, 72, 73].iter().map(|v| fsync("toyos", *v, "nvme"))); + assert!(compare(&runs).unwrap().contains("the media differ: Linux nvme+sata, ToyOS nvme"), "nor one system's runs on two"); + } + #[test] fn a_systems_runs_on_two_setups_are_refused_and_one_setup_is_stated() { let under = |os: &str, governor: &str| { diff --git a/tests/guest/osbench/src/clock.rs b/tests/guest/osbench/src/clock.rs index 6ae0728a30c..0df954f0c1c 100644 --- a/tests/guest/osbench/src/clock.rs +++ b/tests/guest/osbench/src/clock.rs @@ -2,7 +2,7 @@ //! `arch::add_chain`'s adds wait a cycle each, so cycles are the number of //! adds. -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; use std::time::{Duration, Instant}; use osbench_rules::settle::Plateau; @@ -10,8 +10,16 @@ use osbench_rules::settle::Plateau; use crate::report::result; use crate::ns_since; -/// What the last [`settle`] settled at, for the next result line to print. -static SETTLED: AtomicU64 = AtomicU64::new(0); +/// What the last [`settle`] reached, for the next result line to print. +static SETTLED: Mutex> = Mutex::new(None); + +/// The clock a settle reached, and whether it held: `capped` is a settle +/// that ran out its [`SETTLE_LIMIT`] and took the reading it had. +#[derive(Clone, Copy)] +pub struct Settled { + pub mhz: u64, + pub capped: bool, +} /// Turns per probe: 100,000 cycles, 25 to 50 µs on either bench. const TURNS: u64 = 1_000; @@ -44,16 +52,18 @@ pub fn settle() { let mut plateau = Plateau::new(at(), mhz()); loop { let reading = mhz(); - if plateau.settled_with(at(), reading) || start.elapsed() >= SETTLE_LIMIT { - SETTLED.store(reading, Ordering::Relaxed); + let held = plateau.settled_with(at(), reading); + if held || start.elapsed() >= SETTLE_LIMIT { + *SETTLED.lock().expect("no settle panics holding it") = Some(Settled { mhz: reading, capped: !held }); return; } } } -/// The clock the last [`settle`] reached, once: zero when none has run since. -pub fn take_settled() -> u64 { - SETTLED.swap(0, Ordering::Relaxed) +/// The clock the last [`settle`] reached, once: `None` when none has run +/// since. +pub fn take_settled() -> Option { + SETTLED.lock().expect("no settle panics holding it").take() } /// `cpu.clock`: 200 probes back to back after one, MHz. The first probes of a diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index 6af0630fc2c..9cc69c5009a 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -1,12 +1,12 @@ //! Files, through std's `fs` on both systems: the same operations under a -//! filesystem each keeps in memory (`mem`) and under the one on the bench's -//! disk (`disk`, which `storage` runs beside its durable writes). Nothing here -//! asks for durability, so on the disk these are the cached path. +//! filesystem each keeps in memory (`mem`), and on ToyOS under `/home` +//! (`served`). Persistent storage is `storage`'s. //! //! `mem` is Linux's `/dev/shm` through the VFS, and ToyOS's `/tmp`, which the -//! kernel serves through syscalls. `disk` is each system's -//! [`crate::sys::persistent_dir`]: Linux's VFS and page cache over the -//! runner's home, and ToyOS's DATA file server, through IPC per request. +//! kernel serves through syscalls. `served` is ToyOS's DATA file server, +//! through IPC per request: on the volume the bench's DATA partition holds +//! where it holds one of ours, and in memory where it does not (the file +//! server says which in the log). use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; @@ -17,11 +17,22 @@ use crate::report::{condition, refuse}; const MIB: usize = 1 << 20; -/// A directory to measure under, and what serves it. +/// A directory to measure under, what serves it, and the medium beneath, +/// where the program can tell: `memory`, or the disk's kind. pub struct Target { pub name: &'static str, pub dir: String, pub served_by: String, + pub medium: Option, +} + +impl Target { + /// What every line measured under the target carries. + pub fn fields(&self) -> Vec<(&'static str, String)> { + let mut fields = vec![("served_by", self.served_by.clone())]; + fields.extend(self.medium.clone().map(|m| ("medium", m))); + fields + } } pub fn memory() -> Target { @@ -30,12 +41,23 @@ pub fn memory() -> Target { } else { ("/dev/shm", "tmpfs, through the VFS") }; - Target { name: "mem", dir: dir.into(), served_by: served_by.into() } + Target { name: "mem", dir: dir.into(), served_by: served_by.into(), medium: Some("memory".into()) } +} + +/// Every directory `files` measures under. +pub fn targets() -> Vec { + let mut targets = vec![memory()]; + if cfg!(target_os = "toyos") { + // Memory or the DATA partition's volume: the program cannot tell which. + let served_by = "the DATA file server, through IPC".into(); + targets.push(Target { name: "served", dir: "/home".into(), served_by, medium: None }); + } + targets } /// The persistent filesystem's directory, or `None` where the system has none. pub fn disk() -> Option { - crate::sys::persistent_dir().map(|(dir, served_by)| Target { name: "disk", dir, served_by }) + crate::sys::persistent_dir().map(|(dir, served_by, medium)| Target { name: "disk", dir, served_by, medium: Some(medium) }) } /// A scratch directory of this process's under `dir`, removed whole on drop. @@ -80,7 +102,8 @@ fn page_at(i: usize, pages: usize) -> u64 { (x % pages as u64) * 4096 } -/// Every benchmark under `target`, each line naming what served it. Each +/// Every benchmark under `target`, each line naming what served it and its +/// medium. Each /// one's comment is its warm-up and repetitions. pub fn suite(target: &Target) { condition(&format!("fs.{}.served_by", target.name), &target.served_by); @@ -92,7 +115,7 @@ pub fn suite(target: &Target) { }; let name = |op: &str| format!("fs.{}.{op}", target.name); let with = |fields: &[(&'static str, String)]| -> Vec<(&'static str, String)> { - fields.iter().cloned().chain([("served_by", target.served_by.clone())]).collect() + fields.iter().cloned().chain(target.fields()).collect() }; // Opening an existing 4 KiB file for reading and closing it: 2,000 after 100. diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index b89a4cbe56a..52264cd265f 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -125,14 +125,16 @@ fn fair() { } fn files() { - files::suite(&files::memory()); + for target in files::targets() { + files::suite(&target); + } } /// Every group the machine alone runs, in the order `all` and the ToyOS job /// list run them: `boot` first, since it reads how long the boot took to -/// reach it; `energy` while the machine is still quiet; `crunch`, which heats -/// it, after every other measurement; and `persist` last, so what the disk -/// writes back after it disturbs no other measurement. +/// reach it; `energy` while the machine is still quiet; and `crunch`, which +/// heats it, last: ToyOS reads no temperature to cool down by, so a group +/// after it would start at a clock its Linux run does not. const GROUPS: [(&str, fn()); 17] = [ ("boot", boot::suite), ("identity", identity::suite), @@ -149,8 +151,8 @@ const GROUPS: [(&str, fn()); 17] = [ ("work", work::suite), ("tail", tail::suite), ("nvme", nvme::suite), - ("crunch", crunch::suite), ("persist", storage::suite), + ("crunch", crunch::suite), ]; /// The groups that need another party: `lan` the `osbench-peer` on the other diff --git a/tests/guest/osbench/src/report.rs b/tests/guest/osbench/src/report.rs index 109f484e962..bd9bf2ee20a 100644 --- a/tests/guest/osbench/src/report.rs +++ b/tests/guest/osbench/src/report.rs @@ -64,6 +64,8 @@ pub fn any_refused() -> bool { /// `mhz_settled` is the clock it reached, and `mhz_after` the clock a settle /// just after the measurement reaches: a CPU that only idled meanwhile is /// back at its clock, and one a power or heat limit holds is not. +/// `unsettled` names a settle that ran out its limit and took the clock it +/// had. pub fn result(bench: &str, unit: &str, samples: &mut [u64], extra: &[(&str, String)]) { if samples.is_empty() { refuse(bench, "no samples"); @@ -87,11 +89,14 @@ pub fn result(bench: &str, unit: &str, samples: &mut [u64], extra: &[(&str, Stri line.push_str(&format!(" {key}={}", word(value))); } let after = crate::clock::mhz(); - match crate::clock::take_settled() { - 0 => {} - settled => { - crate::clock::settle(); - line.push_str(&format!(" mhz_settled={settled} mhz_after={}", crate::clock::take_settled())); + if let Some(before) = crate::clock::take_settled() { + crate::clock::settle(); + let settled_after = crate::clock::take_settled().expect("the settle just run"); + line.push_str(&format!(" mhz_settled={} mhz_after={}", before.mhz, settled_after.mhz)); + let unsettled: Vec<&str> = + [("before", before.capped), ("after", settled_after.capped)].iter().filter(|(_, c)| *c).map(|(w, _)| *w).collect(); + if !unsettled.is_empty() { + line.push_str(&format!(" unsettled={}", unsettled.join("+"))); } } line.push_str(&format!(" mhz={after}")); diff --git a/tests/guest/osbench/src/storage.rs b/tests/guest/osbench/src/storage.rs index 62342c27a51..af1d628a39e 100644 --- a/tests/guest/osbench/src/storage.rs +++ b/tests/guest/osbench/src/storage.rs @@ -1,22 +1,21 @@ -//! The bench's disk, through each system's persistent filesystem -//! ([`crate::files::disk`]): durable writes first, each made durable by -//! `sync_all` or `sync_data` before the next, then `files`' cached operations -//! under the same directory as `fs.disk.*`. Every line's `served_by` names -//! the filesystem and the disk. +//! Persistent storage, each system on the medium a measured boot has, so +//! **not like for like** until ToyOS's DATA is the bench's partition: what +//! these say is what a durable write costs there, each line naming it. //! -//! On Linux that is the runner's home, on whatever filesystem and disk hold -//! it. On ToyOS it is `/home`, on the DATA volume: the bcachefs crate's -//! format, through the DATA file server, the block service and the disk's -//! driver. A ToyOS boot whose DATA is in memory measures memory here, which -//! the `osbench` metal row refuses (`tests/toyos.rs`). Every file here is -//! removed at the end. +//! On a ToyOS benchcase boot that is the boot USB stick's `TOYOS-LOG` +//! partition, FAT32, through the LOG file server and usbd: the stick's other +//! volumes are read-only, and DATA is the bench's partition only once ToyOS +//! has designated it. On Linux it is a directory in the runner's home, on +//! whatever disk and filesystem hold it. Every file here is removed at the +//! end; the most alive at once is under 10 MiB, inside what the log volume +//! leaves free. use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use crate::files::{self, Scratch}; use crate::measure::{each, io, throughput}; -use crate::report::refuse; +use crate::report::{condition, refuse}; const MIB: usize = 1 << 20; @@ -24,16 +23,12 @@ pub fn suite() { let Some(disk) = files::disk() else { return refuse("persist", "no persistent directory to write in"); }; - durable(&disk); - files::suite(&disk); -} - -fn durable(disk: &files::Target) { let scratch = match Scratch::new(&disk.dir) { Ok(s) => s, Err(e) => return refuse("persist", format!("no scratch directory: {e}")), }; - let served = [("served_by", disk.served_by.clone())]; + condition("persist.served_by", &disk.served_by); + let served = disk.fields(); let block = [0x6bu8; 4096]; // 4 KiB written over the start of a 1 MiB file, then `sync_all`: 100 after 5. @@ -80,7 +75,8 @@ fn durable(disk: &files::Target) { // calls, which both systems answer from their caches. let seq = scratch.at("seq"); let chunk = vec![0x2du8; MIB]; - let extra = [("file_bytes", (8 * MIB).to_string()), ("call_bytes", MIB.to_string()), served[0].clone()]; + let extra: Vec<_> = + [("file_bytes", (8 * MIB).to_string()), ("call_bytes", MIB.to_string())].into_iter().chain(served.iter().cloned()).collect(); throughput("persist.seq_write_fsync", 3, 8 * MIB as u64, &extra, || { let mut f = io("create", File::create(&seq))?; for _ in 0..8 { diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index c7f36918ac9..f8670f14d07 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -521,10 +521,11 @@ pub fn mount_options(dir: &str) -> String { mount_of(dir).map_or_else(|| "unknown".into(), |(fstype, _, options)| format!("{fstype} {options}")) } -/// The runner's home, and the filesystem and disk model holding it, as -/// `ext4, NVMe disk, model `; the path itself names the user and is -/// never printed. -pub fn persistent_dir() -> Option<(String, String)> { +/// The runner's home, the filesystem and disk model holding it, as `ext4, +/// NVMe disk, model `, and the disk's kind as a medium: `nvme`, `usb`, +/// `sata`, `mmc`, `virtio`, `other`, or `none` for no block device. The path +/// itself names the user and is never printed. +pub fn persistent_dir() -> Option<(String, String, String)> { let home = std::env::var("HOME").ok()?; let (fstype, source, _) = mount_of(&home)?; // btrfs reports an anonymous `0:NN` that names no block device; its @@ -537,19 +538,22 @@ pub fn persistent_dir() -> Option<(String, String)> { sysfs(format!("{}:{}", libc::major(rdev), libc::minor(rdev))) }); let Some(device) = device else { - return Some((home, format!("{fstype}, on no block device"))); + return Some((home, format!("{fstype}, on no block device"), "none".into())); }; let (disk, mapped) = backing_disk(device); let model = read(&disk.join("device/model").to_string_lossy()).unwrap_or_else(|| "unknown".into()); let name = disk.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); - let kind = [("nvme", "NVMe"), ("sd", "SCSI, SATA or USB"), ("mmcblk", "MMC"), ("vd", "virtio")] - .iter() + // A SCSI disk is a USB one where sysfs puts it under a USB device. + let usb = disk.to_string_lossy().contains("/usb"); + let sd = if usb { ("USB", "usb") } else { ("SCSI or SATA", "sata") }; + let (kind, medium) = [("nvme", ("NVMe", "nvme")), ("sd", sd), ("mmcblk", ("MMC", "mmc")), ("vd", ("virtio", "virtio"))] + .into_iter() .find(|(prefix, _)| name.starts_with(prefix)) - .map_or("other", |(_, kind)| kind); + .map_or(("other", "other"), |(_, kind)| kind); let through = if mapped { " through device-mapper" } else { "" }; // The device's own name is never printed: an installer or a user chose // a mapper's (`luks-`, a volume group named for the host). - Some((home, format!("{fstype}{through}, {kind} disk, model {model}"))) + Some((home, format!("{fstype}{through}, {kind} disk, model {model}"), medium.into())) } /// The `major:minor` of the filesystem holding `path`, off mountinfo's third diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index eb867e37e3b..1927e12ef6b 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -358,11 +358,11 @@ pub fn map_anon_4k(len: usize) -> *mut u8 { map_anon(len) } -/// DATA's `/home`, and what serves it. A program cannot ask which disk that -/// is, or whether DATA is in memory instead; the boot's log says both. -pub fn persistent_dir() -> Option<(String, String)> { - let served_by = "DATA volume, bcachefs crate's format, DATA file server over the block service"; - Some(("/home".into(), served_by.into())) +/// The one writable volume a benchcase boot certainly has on a disk, what +/// serves it, and its medium. +pub fn persistent_dir() -> Option<(String, String, String)> { + let served_by = "boot USB stick TOYOS-LOG partition, FAT32, LOG file server over usbd mass storage"; + Some(("/log".into(), served_by.into(), "usb".into())) } /// What carries a connection on ToyOS: netstack drives one card, Ethernet. diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 58cc9ea5e5e..580062e5459 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -5,8 +5,8 @@ # ACPI server, and netstack for `osbench lan` and `osbench internet`, each its # own metal row (`tests/toyos.rs`): no compositor or sound, so little but the # measurement asks for a CPU. The stick has no DATA partition, so DATA is the -# bench's: the TOYOS-DATA partition on its NVMe, which `osbench persist` -# writes, and which the `osbench` row refuses a boot to be without. +# bench's TOYOS-DATA partition where ToyOS has designated it, and memory where +# it has not; the log volume is what `osbench persist` writes. # # The runner's capability carries `jobs`, for the fairness run that starts each # workload as a job of its own; `counters`, `trace` and `roster`, so `osbench @@ -62,8 +62,8 @@ args = [ "osbench_work", "osbench_tail", "osbench_nvme", - "osbench_crunch", "osbench_persist", + "osbench_crunch", "reboot", ] diff --git a/tests/metal/Cargo.toml b/tests/metal/Cargo.toml index 5d619007c9f..f76030bffef 100644 --- a/tests/metal/Cargo.toml +++ b/tests/metal/Cargo.toml @@ -14,13 +14,10 @@ toyos-build = { path = "../.." } # The outside judge of the log partition the boot left on the stick. toyos-fat32-check = { path = "../../lib/fat32/check" } toyos-gpt = { path = "../../lib/gpt" } -# A ToyOS volume's magic, which the DATA designation never writes over. -bcachefs = { path = "../../lib/bcachefs" } -# Where the designation's one block waits for the root command that writes it. -toyos-tmpdir = { path = "../../lib/tmpdir" } toyos-quiesce = { path = "../../lib/quiesce" } toyos-tco = { path = "../../lib/tco" } [dev-dependencies] +toyos-tmpdir = { path = "../../lib/tmpdir" } # The slot table written into the test images `internal::admit` reads. toyos-update = { path = "../../lib/update" } diff --git a/tests/metal/src/internal.rs b/tests/metal/src/internal.rs index a4302fa098a..d701148e5a2 100644 --- a/tests/metal/src/internal.rs +++ b/tests/metal/src/internal.rs @@ -29,7 +29,7 @@ const MIB: u64 = 1 << 20; const BENCH: u32 = 4; /// The fourth partition's name, which the owner gave the bench's DATA. -pub(crate) const DATA_LABEL: &str = "ToyOS data"; +const DATA_LABEL: &str = "ToyOS data"; /// The partitions an internal install owns, in table order from partition 5, /// laid out as the image lays out its own. diff --git a/tests/metal/src/lib.rs b/tests/metal/src/lib.rs index 2dda81e3bae..cd1235724ad 100644 --- a/tests/metal/src/lib.rs +++ b/tests/metal/src/lib.rs @@ -1,8 +1,7 @@ //! `toyos-metal` — the loop that flashes ToyOS to a metal bench's stick, boots //! it once, and answers with [`toyos_build::bootlog`]'s verdict on what that boot //! wrote to the log partition; and beside it, the internal install -//! ([`internal`]) that writes an image to the bench's own NVMe for a demo boot, -//! and the designation of the bench's ToyOS DATA ([`stamp`]). +//! ([`internal`]) that writes an image to the bench's own NVMe for a demo boot. //! //! It runs on the development host and reaches the machine over `ssh`: the //! T14 bench unless `--host` names another, whose account's home holds the @@ -41,7 +40,6 @@ use crate::timings::{Machine, Record, Stick}; pub mod image; pub mod internal; -pub mod stamp; pub mod testargs; pub mod timings; @@ -96,7 +94,6 @@ pub const STICK_SECS: u64 = 30; const DD: &str = "/usr/bin/dd"; const EFIBOOTMGR: &str = "/usr/bin/efibootmgr"; const MOUNT: &str = "/usr/bin/mount"; -const OD: &str = "/usr/bin/od"; const PARTX: &str = "/usr/bin/partx"; const SGDISK: &str = "/usr/sbin/sgdisk"; const UMOUNT: &str = "/usr/bin/umount"; @@ -148,8 +145,6 @@ pub enum Refusal { Ambiguous { carrying: String, nodes: Vec }, /// The disk named to be adopted as scratch is not one this loop erases. Adopt(String), - /// The bench's DATA partition is not one this loop designates. - Designate(String), File { path: String, why: String }, Table(String), /// The table does not hold exactly one partition of a type the loop needs. @@ -260,7 +255,6 @@ impl fmt::Display for Refusal { } ), Self::Adopt(why) => write!(f, "nothing was erased: {why}"), - Self::Designate(why) => write!(f, "no partition was designated: {why}"), Self::Ambiguous { carrying, nodes } => write!( f, "{} are each a recorded disk carrying {carrying}, so nothing was written: unplug \ @@ -667,12 +661,6 @@ enum Job { CreateInternal, Write(Owned), Scratch(Sd), - /// What signatures the partition at the node carries, writing nothing. - Signatures(String), - /// The partition's first block, as `od` writes bytes. - FirstBlock(String), - /// The stamp on stdin written over the partition's first block. - Stamp(String), } /// One word as the machine's login shell must receive it: `ssh` hands the whole @@ -804,9 +792,6 @@ impl Target { &sd.node().whole(), ]), Job::CreateInternal => create(&internal, Owned::Esp.index(), INTERNAL_LABEL), - Job::Signatures(node) => words(&[WIPEFS, "--no-act", "--noheadings", "--output", "TYPE", node]), - Job::FirstBlock(node) => words(&[OD, "-An", "-v", "-tx1", &format!("-N{}", stamp::BLOCK), node]), - Job::Stamp(node) => words(&[DD, &format!("of={node}"), &format!("bs={}", stamp::BLOCK), "count=1", "conv=fsync,nocreat"]), Job::Write(o) => words(&[DD, &format!("of={}", o.node(&self.internal)), "bs=4M", "conv=fsync,nocreat"]), } } @@ -1664,7 +1649,6 @@ declare_flags!(METAL = { KEY = "--key", Next; DEVICE = "--device", Next; ADOPT_SCRATCH = "--adopt-scratch", Next; - DESIGNATE_DATA = "--designate-data", None; RECORD = "--record", Next; READBACK = "--readback", Next; FAT32_CHECK = "--fat32-check", None; @@ -1710,9 +1694,6 @@ pub struct Args { /// The disk [`adopt_scratch`] erases into a scratch disk. An action of its /// own, beside nothing but the flags that say which machine and `--dry-run`. adopt_scratch: Option, - /// Stamp the bench's ToyOS DATA partition ([`designate_data`]). An action - /// of its own, beside nothing but the machine's flags and `--dry-run`. - designate_data: bool, } impl Args { @@ -1738,7 +1719,6 @@ impl Args { install_internal: value(&INSTALL_INTERNAL).map(PathBuf::from), boot_internal: METAL.present(args, &BOOT_INTERNAL), adopt_scratch: value(&ADOPT_SCRATCH).map(Sd::parse).transpose()?, - designate_data: METAL.present(args, &DESIGNATE_DATA), }; if let Some(host) = value(&HOST) { let (user, machine) = host_of(host)?; @@ -1755,11 +1735,8 @@ impl Args { // flag beside it that describes anything else is refused rather than // left for the action to fall through into — which is how a capture // once flashed whatever `--image` defaulted to. - let actions: [(&Flag, &[&Flag]); 3] = [ - (&RECORD, &[&HOST, &KEY]), - (&ADOPT_SCRATCH, &[&HOST, &KEY, &DRY_RUN]), - (&DESIGNATE_DATA, &[&HOST, &KEY, &DRY_RUN]), - ]; + let actions: [(&Flag, &[&Flag]); 2] = + [(&RECORD, &[&HOST, &KEY]), (&ADOPT_SCRATCH, &[&HOST, &KEY, &DRY_RUN])]; for (action, alone) in actions { if !METAL.present(args, action) { continue; @@ -1874,54 +1851,6 @@ fn adopt_scratch(target: &Target, dry_run: bool, sd: Sd) -> Result<(), Refusal> Ok(()) } -/// Designate a recorded bench's ToyOS DATA partition ([`stamp`]): found by -/// type and name, refused where it carries a signature or a ToyOS volume, -/// stamped through its GUID's node, and read back. The partition's contents -/// are lost; its next ToyOS boot formats it. A dry run reads everything and -/// writes nothing. -fn designate_data(target: &Target, dry_run: bool) -> Result<(), Refusal> { - let driver = Driver::reach(target.clone(), dry_run)?; - driver.require_sudo()?; - driver.recorded_machine()?; - let data = stamp::find(&driver.ssh("listing the disks", stamp::QUERY)?).map_err(Refusal::Designate)?; - let node = data.by_guid(); - let root = |job: Job| driver.target.remote(&job, &driver.home); - let signatures = driver.ssh("reading the partition's signatures", &root(Job::Signatures(node.clone())))?; - if !signatures.trim().is_empty() { - let found = signatures.split_whitespace().collect::>().join(", "); - return Err(Refusal::Designate(format!("{} carries {found}", data.node))); - } - let first = |what: &str| -> Result { - let bytes = stamp::od_bytes(&driver.ssh(what, &root(Job::FirstBlock(node.clone())))?).map_err(Refusal::Designate)?; - Ok(stamp::classify(&data, &bytes)) - }; - let said = format!("{} ({}, {} blocks of {})", data.node, internal::DATA_LABEL, data.blocks, stamp::BLOCK); - match first("reading the partition's first block")? { - stamp::Block0::Stamped => { - println!("{said} already carries its designation; nothing was written"); - return Ok(()); - } - stamp::Block0::Volume => { - return Err(Refusal::Designate(format!("{said} holds a ToyOS volume, which a designation would destroy"))); - } - stamp::Block0::Other => {} - } - let scratch = toyos_tmpdir::TempDir::new("designation"); - let file = scratch.path().join("stamp"); - std::fs::write(&file, data.stamp()).map_err(|e| Refusal::File { path: file.display().to_string(), why: e.to_string() })?; - let feed = Feed { path: &file, at: 0, len: stamp::BLOCK as u64 }; - let Some(out) = driver.as_root("writing the designation", Job::Stamp(node.clone()), Some(feed))? else { - println!("dry run: {said} was read and nothing was written"); - return Ok(()); - }; - driver.fed("writing the designation", &out, stamp::BLOCK as u64)?; - if first("reading the designation back")? != stamp::Block0::Stamped { - return Err(Refusal::Designate(format!("{said} was written and does not read back as its designation"))); - } - println!("{said} is designated: the next ToyOS boot formats it"); - Ok(()) -} - /// Bytes of a local file one root command reads on its stdin. #[derive(Debug, Clone, Copy)] struct Feed<'a> { @@ -1985,10 +1914,6 @@ pub fn run(args: &Args) -> Result, Refusal> { adopt_scratch(&args.target, args.dry_run, sd)?; return Ok(None); } - if args.designate_data { - designate_data(&args.target, args.dry_run)?; - return Ok(None); - } let began = std::time::Instant::now(); let Some(asked) = &args.image else { @@ -3287,12 +3212,6 @@ mod tests { } let both = Args::parse(&words(&["--record", "/dev/sda", "--adopt-scratch", "/dev/sdb"])).unwrap_err(); assert!(matches!(both, Refusal::Usage(_))); - let designate = Args::parse(&words(&["--designate-data", "--dry-run", "--host", "toyos@bench-amd"])).unwrap(); - assert!(designate.designate_data && designate.dry_run); - for beside in boot.iter().copied().chain([&["--adopt-scratch", "/dev/sdb"][..]]) { - let line = [&["--designate-data"][..], beside].concat(); - assert!(matches!(Args::parse(&words(&line)), Err(Refusal::Usage(ref why)) if why.contains(beside[0])), "{line:?}"); - } } /// **The mount point is under the home the machine names**, and a home diff --git a/tests/metal/src/stamp.rs b/tests/metal/src/stamp.rs deleted file mode 100644 index b8629457369..00000000000 --- a/tests/metal/src/stamp.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Designating a bench's ToyOS DATA partition: the stamp the DATA file server -//! formats a volume off (`bcachefs::DESIGNATION_MAGIC`), written into the -//! partition's first 4 KiB block. -//! -//! **The partition is found by what it is, never by its node**: the one -//! partition of every disk the machine lists of type TOYOS-DATA and named -//! [`internal::DATA_LABEL`], on a fixed disk, unmounted, with nothing stacked -//! on it, and written through its GUID's own node. Its first block is read -//! before the write and after it. A partition that already carries this -//! stamp, or a ToyOS volume, is never written. - -use toyos_gpt::Guid; - -use crate::internal; - -/// The block the stamp fills, the DATA volume's. -pub const BLOCK: usize = 4096; - -/// Every block device, for [`find`]. -pub const QUERY: &str = "lsblk --bytes --noheadings --paths --pairs --output \ - NAME,TYPE,RM,PKNAME,SIZE,PARTTYPE,PARTUUID,PARTLABEL,MOUNTPOINTS"; - -/// The DATA partition [`find`] found. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Data { - pub node: String, - pub unique: Guid, - pub blocks: u64, -} - -impl Data { - /// The node udev names the partition by its GUID, which no renumbering of - /// the machine's disks moves. - pub fn by_guid(&self) -> String { - format!("/dev/disk/by-partuuid/{}", self.unique.to_string().to_ascii_lowercase()) - } - - /// The stamp for this partition: it names the partition's own block count. - pub fn stamp(&self) -> [u8; BLOCK] { - toyos_build::image::designation(self.blocks) - } -} - -/// The one DATA partition [`QUERY`] answered, or why none can be written. -pub fn find(listing: &str) -> Result { - let rows: Vec<_> = - listing.lines().filter(|l| !l.trim().is_empty()).map(internal::pairs).collect::>()?; - let field = |row: &std::collections::BTreeMap, key: &str| row.get(key).cloned().unwrap_or_default(); - let data: Vec<_> = rows - .iter() - .filter(|r| field(r, "TYPE") == "part" && internal::guid(&field(r, "PARTTYPE")) == Some(Guid::TOYOS_DATA)) - .collect(); - let [part] = data[..] else { - return Err(format!("the machine lists {} TOYOS-DATA partitions, and only one is designated", data.len())); - }; - let node = field(part, "NAME"); - if field(part, "PARTLABEL") != internal::DATA_LABEL { - return Err(format!("{node} is named {:?}, where a bench's DATA is {:?}", field(part, "PARTLABEL"), internal::DATA_LABEL)); - } - let disk = rows.iter().find(|r| field(r, "NAME") == field(part, "PKNAME") && field(r, "TYPE") == "disk"); - if disk.is_none_or(|d| field(d, "RM") != "0") { - return Err(format!("{node} is on no fixed disk the machine lists, and a bench's DATA is on its own")); - } - if !field(part, "MOUNTPOINTS").trim().is_empty() { - return Err(format!("{node} is mounted")); - } - if let Some(above) = rows.iter().find(|r| field(r, "PKNAME") == node) { - return Err(format!("{} is stacked on {node}", field(above, "NAME"))); - } - let unique = internal::guid(&field(part, "PARTUUID")).ok_or_else(|| format!("{node} states no GUID"))?; - let bytes: u64 = field(part, "SIZE").parse().map_err(|_| format!("{node} reads SIZE={:?}", field(part, "SIZE")))?; - if bytes == 0 || !bytes.is_multiple_of(BLOCK as u64) { - return Err(format!("{node} is {bytes} bytes, no whole number of {BLOCK}-byte blocks")); - } - Ok(Data { node, unique, blocks: bytes / BLOCK as u64 }) -} - -/// What a DATA partition's first block holds. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Block0 { - /// This partition's stamp, byte for byte. - Stamped, - /// A ToyOS volume's superblock: never written over. - Volume, - /// Anything else, a stamp for another size among it, which the stamp - /// replaces. - Other, -} - -pub fn classify(data: &Data, block0: &[u8]) -> Block0 { - if block0 == data.stamp() { - Block0::Stamped - } else if block0.starts_with(&bcachefs::MAGIC) { - Block0::Volume - } else { - Block0::Other - } -} - -/// `od -An -v -tx1`'s bytes. -pub fn od_bytes(text: &str) -> Result, String> { - text.split_whitespace().map(|b| u8::from_str_radix(b, 16).map_err(|_| format!("{b:?} is no byte od writes"))).collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - const UNIQUE: &str = "11111111-2222-3333-4444-555555555555"; - - fn listing(label: &str, rm: &str, mounted: &str, extra: &str) -> String { - format!( - "NAME=\"/dev/nvme0n1\" TYPE=\"disk\" RM=\"{rm}\" PKNAME=\"\" SIZE=\"1024209543168\" PARTTYPE=\"\" PARTUUID=\"\" PARTLABEL=\"\" MOUNTPOINTS=\"\"\n\ - NAME=\"/dev/nvme0n1p3\" TYPE=\"part\" RM=\"0\" PKNAME=\"/dev/nvme0n1\" SIZE=\"34359738368\" PARTTYPE=\"0fc63daf-8483-4772-8e79-3d69d8477de4\" PARTUUID=\"aaaaaaaa-2222-3333-4444-555555555555\" PARTLABEL=\"\" MOUNTPOINTS=\"/\"\n\ - NAME=\"/dev/nvme0n1p4\" TYPE=\"part\" RM=\"0\" PKNAME=\"/dev/nvme0n1\" SIZE=\"644245094400\" PARTTYPE=\"064e3777-5076-4c71-8e07-90ad24cfe8d6\" PARTUUID=\"{UNIQUE}\" PARTLABEL=\"{label}\" MOUNTPOINTS=\"{mounted}\"\n{extra}" - ) - } - - fn bench() -> String { - listing("ToyOS\\x20data", "0", "", "") - } - - #[test] - fn the_benchs_data_is_found_by_type_and_name_and_written_by_its_guid() { - let data = find(&bench()).unwrap(); - assert_eq!((data.node.as_str(), data.blocks), ("/dev/nvme0n1p4", 157_286_400)); - assert_eq!(data.by_guid(), format!("/dev/disk/by-partuuid/{UNIQUE}")); - let stamp = data.stamp(); - assert_eq!(&stamp[..16], b"TOYOS-FORMAT-ME\0"); - assert_eq!(u64::from_le_bytes(stamp[16..24].try_into().unwrap()), 157_286_400); - assert!(stamp[24..].iter().all(|&b| b == 0)); - } - - #[test] - fn anything_but_one_fixed_free_data_partition_named_toyos_data_is_refused() { - let refused = |text: String, said: &str| assert!(find(&text).unwrap_err().contains(said), "{said}"); - refused(listing("data", "0", "", ""), "is named \"data\""); - refused(listing("ToyOS\\x20data", "1", "", ""), "no fixed disk"); - refused(listing("ToyOS\\x20data", "0", "/mnt", ""), "is mounted"); - let above = "NAME=\"/dev/mapper/x\" TYPE=\"crypt\" RM=\"0\" PKNAME=\"/dev/nvme0n1p4\" SIZE=\"1\" PARTTYPE=\"\" PARTUUID=\"\" PARTLABEL=\"\" MOUNTPOINTS=\"\"\n"; - refused(listing("ToyOS\\x20data", "0", "", above), "/dev/mapper/x is stacked on /dev/nvme0n1p4"); - let second = bench().replace("nvme0n1p4", "nvme0n1p5").replace(UNIQUE, "bbbbbbbb-2222-3333-4444-555555555555"); - refused(format!("{}{second}", bench()), "lists 2 TOYOS-DATA partitions"); - refused(bench().replace("064e3777", "0fc63daf"), "lists 0 TOYOS-DATA partitions"); - refused(bench().replace("644245094400", "644245094401"), "no whole number"); - } - - #[test] - fn a_first_block_is_this_stamp_a_volume_or_anything_else() { - let data = find(&bench()).unwrap(); - assert_eq!(classify(&data, &data.stamp()), Block0::Stamped); - let mut volume = [0u8; BLOCK]; - volume[..4].copy_from_slice(&bcachefs::MAGIC); - assert_eq!(classify(&data, &volume), Block0::Volume); - assert_eq!(classify(&data, &[0u8; BLOCK]), Block0::Other); - let other_size = toyos_build::image::designation(data.blocks - 1); - assert_eq!(classify(&data, &other_size), Block0::Other, "a stamp for another size designates nothing"); - assert_eq!(classify(&data, &data.stamp()[..BLOCK - 1]), Block0::Other, "a short read is no stamp"); - } - - #[test] - fn od_bytes_read_back_as_written() { - assert_eq!(od_bytes(" 54 4f 59\n 00 ff\n").unwrap(), vec![0x54, 0x4f, 0x59, 0, 0xff]); - assert!(od_bytes("zz").is_err()); - } -} diff --git a/tests/toyos.rs b/tests/toyos.rs index 1df0975095b..c25413339fe 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1108,14 +1108,14 @@ const METAL: &[(&str, metal::Metal)] = &[ ( // The numbers compared with the same program on the bench's own Linux. // Evidence and not a verdict: they are read off the log, and the row - // holds only that every group ran to its end on the bench's NVMe. - // The T14 and the AMD bench. One boot runs the whole suite, so three - // readbacks are three runs. + // holds only that every group ran to its end, on a boot that did not + // spend itself formatting DATA. The T14 and the AMD bench. One boot + // runs the whole suite, so three readbacks are three runs. "osbench", metal::Metal { arms: BENCHCASE, judge: |b| { - data_on_the_nvme(b[0])?; + not_the_format_boot(b[0])?; osbench_ran(b[0], &OSBENCH_GROUPS) }, }, @@ -1408,16 +1408,12 @@ const METAL: &[(&str, metal::Metal)] = &[ metal::Metal { arms: TESTCASES, judge: |b| usbd_serves_the_boot_stick(&b[0].kernel(), &b[0].log()) }, ), ( - // The bench's own NVMe disk, driven beside the stick by the row's - // class: its host's partitions left alone, and the owner's ToyOS DATA, - // designated (`toyos-metal --designate-data`), the one partition a - // session opened (`hostdisk::left_alone`). The T14 and the AMD bench; + // The bench's own NVMe disk, its host's partitions and the owner's + // unformatted ToyOS DATA, driven beside the stick by the row's class + // and left alone (`hostdisk::left_alone`). The T14 and the AMD bench; // the metal loop's next boot of the host off that disk is the other half. "nvme_leaves_the_benchs_disk_alone", - metal::Metal { - arms: TESTCASES, - judge: |b| common::hostdisk::left_alone(&b[0].log(), common::hostdisk::Data::Designated).map(|_| ()), - }, + metal::Metal { arms: TESTCASES, judge: |b| common::hostdisk::left_alone(&b[0].log()).map(|_| ()) }, ), ( // The stick written through usbd while the boot deadline resets the @@ -1630,7 +1626,7 @@ const TESTCASES: &[metal::Arm] = &[metal::once( /// `osbench`'s groups, each a job named by its symlink, in `osbench all`'s /// order: `boot` first, which reads how long the boot took to reach it, and -/// `persist` last, which writes the disk. +/// `crunch` last, which heats the machine. const OSBENCH_GROUPS: [&str; 17] = [ "osbench_boot", "osbench_identity", @@ -1647,8 +1643,8 @@ const OSBENCH_GROUPS: [&str; 17] = [ "osbench_work", "osbench_tail", "osbench_nvme", - "osbench_crunch", "osbench_persist", + "osbench_crunch", ]; /// The suite runs past every boot's bound on purpose: `tail` alone waits out @@ -1701,32 +1697,21 @@ fn osbench_ran(boot: &metal::Readback, jobs: &[&str]) -> Result<(), String> { Ok(()) } -/// DATA, which `osbench persist` writes, was a volume on a TOYOS-DATA -/// partition the NVMe driver serves, mounted, and not memory standing in for -/// a partition that is not ours. The block service refuses DATA where two -/// disks carry one, so a mount alongside the NVMe's partition is that -/// partition's. The boot that formats a freshly designated partition is no -/// measured run, and says how long the format took. -fn data_on_the_nvme(boot: &metal::Readback) -> Result<(), String> { +/// The boot that formats a freshly designated DATA partition is no measured +/// run: its start went to the format, and the volume it leaves is new. It +/// says how long the format took. +fn not_the_format_boot(boot: &metal::Readback) -> Result<(), String> { const FORMATTING: &str = "fileserver: block 0 designates this partition for ToyOS; formatting it"; const SERVING: &str = "fileserver: Data serving "; let log = boot.log(); let text = log.text(); - let served = format!(" of type {} at block ", toyos_gpt::Guid::TOYOS_DATA); - if !text.lines().any(|l| l.contains("nvme] nvme ") && l.contains(&served)) { - return Err("the NVMe driver served no TOYOS-DATA partition, so persist measured no disk of the bench's".into()); - } - if let Some(began) = text.lines().find(|l| l.contains(FORMATTING)) { - let ended = text.lines().find(|l| l.contains(SERVING)); - let secs = ended.and_then(since_boot).zip(since_boot(began)).map(|(end, start)| end - start); - let took = secs.map_or("its end unlogged".to_string(), |s| format!("{s:.3} s")); - return Err(format!("this boot formatted DATA off its designation ({took}), and the format's boot is no measured run")); - } - if !text.contains("fileserver: mounted the DATA volume") { - let why = text.lines().find(|l| l.contains("fileserver: the DATA")).unwrap_or("the file server said nothing of DATA"); - return Err(format!("DATA was not the NVMe's partition, so persist measured no disk: {why}")); - } - Ok(()) + let Some(began) = text.lines().find(|l| l.contains(FORMATTING)) else { + return Ok(()); + }; + let ended = text.lines().find(|l| l.contains(SERVING)); + let secs = ended.and_then(since_boot).zip(since_boot(began)).map(|(end, start)| end - start); + let took = secs.map_or("its end unlogged".to_string(), |s| format!("{s:.3} s")); + Err(format!("this boot formatted DATA off its designation ({took}), and the format's boot is no measured run")) } /// A log line's seconds since boot: `[2026-10-24 14:17:17 12.664 fileserver]`. @@ -4703,7 +4688,7 @@ fn a_host_disk_beside_the_stick_is_left_alone(test_config: &Path) -> Result<(), console.push_str(&qemu.await_exit(by)?); let log = serial::Serial::named("the boot", console); log.must_say("vendor=8086 device=5845 prog_if=02")?; - match common::hostdisk::left_alone(&log, common::hostdisk::Data::Foreign)? { + match common::hostdisk::left_alone(&log)? { 4 => {} named => return Err(format!("the driver named {named} partitions of a disk that has 4")), } From 2eb28ee65f55b01022220706b1c5817d0beb889b Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 09:35:23 +0100 Subject: [PATCH 5/8] osbench: the baseline is Linux as installed, and a matched request is a labelled experiment The planning architect's guidance (2026-10-28) is that a trustworthy default Linux is the baseline, and a hardware-policy setting is a controlled experiment beside it, never its replacement. The README now says so. The frequency request ToyOS makes (EPP 0x80 on the T14, performance on the AMD) is the experiment. The AMD's Linux on the HPET is its baseline. A Linux forced onto the TSC has its stability checks switched off, so it is no baseline. Four runs a side is where a row can first resolve, not a precision guarantee. The AMD clocksource record keeps the bench's default. Ten minutes of a steady TSC rate (1796.6248 MHz against the HPET and the ACPI PM timer) does not explain the boot-time watchdog's 1.7 ms over 502 ms, so that cause is the record's exit. The invariant-TSC finding leaves this PR for #1082, filed accurately: deep idle already reads the bit, and the gap is the system clock's. Co-Authored-By: Claude Opus 5.5 --- ...ompares-toyos-with-linux-on-one-machine.md | 11 ++--- ...linux-reads-its-clock-through-a-syscall.md | 42 ++++++++++++------ ...ime-on-a-tsc-it-never-asks-is-invariant.md | 34 -------------- tests/guest/osbench/README.md | 44 ++++++++++++------- 4 files changed, 62 insertions(+), 69 deletions(-) delete mode 100644 issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index f307e33991e..6677c3fbe46 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -34,11 +34,12 @@ its groups: - a short CPU crunch, one thread then all, that stops at TjMax less 15 C where the package's temperature can be read: `crunch.*`. -`tests/guest/osbench/README.md` is how a comparison is run: both halves -from one revision, interleaved runs, and Linux under ToyOS's frequency -request and on the TSC. `persist` is not yet one disk: ToyOS writes the boot -stick's LOG volume, and moves to the bench's NVMe once ToyOS designates its -DATA partition itself. The AMD bench's Linux reads no TSC +`tests/guest/osbench/README.md` is how a comparison is run: both halves from +one revision, interleaved runs, Linux as installed for the baseline, and +Linux under ToyOS's frequency request as a labelled experiment. `persist` is +not yet one disk: ToyOS writes the boot stick's LOG volume, and moves to the +bench's NVMe once ToyOS designates its DATA partition itself. The AMD +bench's Linux reads its clock through a syscall (`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`). `internet.*`, HTTPS to Cloudflare's speed test, is exploratory evidence of a diff --git a/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md b/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md index ad797819bec..2161cee0747 100644 --- a/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md +++ b/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md @@ -6,20 +6,34 @@ opened: 2026-10-27 # The AMD bench's Linux reads its clock through a syscall -On 2026-10-24 `osbench`'s `time.now`, `Instant::now` a million times a -round, read 1,212,895 ps a call under the AMD bench's Ubuntu (Linux -6.8.0-147, both runs within 0.1%) against 14,302 ps under the T14's, the same -static binary. A read that slow is a clocksource the vDSO cannot read, so -each one is a syscall: an inference from the cost, since that session did -not record the clocksource. ToyOS reads the TSC on both benches, so every -per-operation timing of the AMD bench's Linux carries some 1.2 µs that its -ToyOS runs do not. +On 2026-10-24, osbench's `time.now` (`Instant::now` a million times a round) +read 1,212,895 ps a call under the AMD bench's Ubuntu (Linux 6.8.0-147). +Under the T14's it read 14,302 ps, with the same static binary. The AMD's +Linux keeps time on the HPET: 2.6 s into the one boot whose log was read, +its clocksource watchdog found the TSC 1,698,571 ns ahead of the HPET over +502,290,025 ns, marked it unstable and switched. At that cost each timestamp +is a syscall, an inference from the cost, and every per-operation timing of +that Linux carries about 1.2 µs that ToyOS's, which reads the TSC, does not. +That is the baseline's own behaviour: osbench records `clocksource`, and the +README treats a Linux forced onto the TSC as an experiment, never the +baseline. -osbench now records `clocksource` with Linux's conditions, and -`tests/guest/osbench/README.md` runs a comparison with `clocksource=tsc`. +What the watchdog's interval was is unknown. Its own figures put the TSC at +1802.69 MHz across it. Over 10 minutes later in the boot the TSC counts +1796.6248 MHz against the HPET and against the ACPI PM timer, in every idle +and busy phase, worst residual 23 µs, and the CPU states an invariant TSC +(2026-10-28, `tsc-drift`, PR #1079's evidence). A steady average can still +hide a step. The candidates are: +- Linux's calibration at that point; +- a reference read delayed inside the interval; +- an offset between the CPUs the two reads ran on; +- the TSC's or the HPET's rate or state changing around that time in boot. -**Exit**: the AMD bench's Linux reads `clocksource=tsc` in an `osbench` run, -and its `time.now` is within twice the T14's. +**Exit**: the interval's cause is known, from per-CPU offsets and steps +measured across idle and load and from the watchdog's figures in more than +one boot. Then either the bench's Linux keeps its default and this records +why, or the cause is a ToyOS-relevant TSC defect, filed against ToyOS's +clock. -**Owner**: the orchestrator, which runs the benches. Changing the bench's -Linux boot configuration is the owner's decision. +**Owner**: the orchestrator, which runs the benches. A change to the bench's +Linux boot configuration is the owner's. diff --git a/issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md b/issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md deleted file mode 100644 index 1c3f7899bc4..00000000000 --- a/issues/the-kernel-keeps-time-on-a-tsc-it-never-asks-is-invariant.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-28 ---- - -# The kernel keeps time on a TSC it never asks is invariant - -x86-64's clock is the TSC, measured once against the HPET over 50 ms at boot -(`kernel/src/arch/x86_64/hpet.rs`), and every timestamp after that converts -TSC ticks at the rate that one window found. Nothing reads CPUID -80000007H:EDX[8], which Intel (the SDM's "Invariant TSC") and -AMD (its PPRs' `CPUID_Fn80000007_EDX[TscInvariant]`) define as the TSC -counting at one rate across P-, C- and T-states. A CPU without it keeps a -wrong clock silently: every timeout, deadline and measurement on it is off -by however its rate moved, and no line says so. - -The AMD bench (Ryzen 7 4800U) is why this was looked at. Its Linux's -clocksource watchdog measured the TSC at 1802.69 MHz against the HPET over -one 502 ms interval 2.6 s into boot, where ToyOS's calibration against the -same HPET reads 1796.52 MHz, about 3400 ppm apart; Linux marked the TSC -unstable and fell back to the HPET. Whether that CPU's TSC drifts is being -measured (`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`), -and the bit is missing here whatever the measurement shows. The bit's -document citations above are the fix's to read at the site; this record -names them and has not quoted them. - -**Exit**: the kernel reads the bit on every CPU before it keeps time on the -TSC, and a CPU that does not state it fails the boot by name or keeps time -on a counter that is invariant. Each answer covers a guest whose CPU model -leaves the bit clear. QEMU's TCG `qemu64` is one, an inference not yet -checked against QEMU's model. - -**Owner**: the kernel's x86-64 clock (`kernel/src/arch/x86_64/hpet.rs`). diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md index 04fe7fc96cd..ba690bc99f6 100644 --- a/tests/guest/osbench/README.md +++ b/tests/guest/osbench/README.md @@ -37,6 +37,8 @@ and the C library beneath it are each system's own. 0.05 is 2/70, so a resolved row is one whose every run of one system beat every run of the other. Fewer runs of either system than four cannot reach that, except against more of the other: three against five can (2/56). + Four a side is where a row can first resolve, a starting sample and no + promise of precision: a row whose runs overlap is unresolved, not equal. Boots of one image differ: the AMD bench's `crunch.single.deflate` read 63.0 to 63.6 ms on some boots and 66.6 to 66.8 ms on others (2026-10-27), so no single boot stands for its system. @@ -63,17 +65,23 @@ and the C library beneath it are each system's own. apart. A settle that ran out its 0.5 s says so (`unsettled`), and is counted rather than judged. -## Linux makes the requests ToyOS makes +## Linux as installed, and Linux under ToyOS's requests -**The CPU's frequency.** Linux runs under the frequency request ToyOS's -kernel makes of the same CPU, so the clock is the hardware's answer to one -request on both systems: +**The baseline is Linux as the bench boots it**: its own governor, its own +clocksource, its own checks. What it does on that machine is ordinary OS +behaviour and part of what is compared. A comparison names its +configuration in its setup lines, and `osbench-compare` refuses to mix two. + +**A matched request is a controlled experiment, labelled as one.** It asks +the CPU's frequency control for what ToyOS's kernel asks of the same CPU, so +the clock is the hardware's answer to one request on both systems. It +answers a different question from the baseline, and never replaces it: - **The T14** (i5-1135G7, `intel_pstate` with HWP). ToyOS asks for HWP's lowest to highest performance at EPP 0x80 (`kernel/pure/cpu.rs`). Ubuntu's - default `powersave` governor makes HWP's request with the EPP its - `energy_performance_preference` holds, so write 0x80 there before each - Linux run, which `intel_pstate` takes as a number: + `powersave` governor makes HWP's request with the EPP its + `energy_performance_preference` holds, so the experiment writes 0x80 there + before each Linux run, which `intel_pstate` takes as a number: ```text echo 128 | sudo tee /sys/devices/system/cpu/cpufreq/policy*/energy_performance_preference @@ -86,9 +94,9 @@ request on both systems: row. - **The AMD bench** (Ryzen 7 4800U, `acpi-cpufreq`). ToyOS writes no P-state or boost register, so the firmware's P0 and Core Performance Boost stand. - Linux's `performance` governor asks for that. Set it before each Linux - run, since every ToyOS boot between them boots Linux back to `schedutil`, - and give `schedutil` back after the last: + Linux's `performance` governor asks for that. The experiment sets it + before each Linux run, since every ToyOS boot between them boots Linux + back to `schedutil`, and gives `schedutil` back after the last: ```text echo performance | sudo tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor @@ -97,7 +105,7 @@ request on both systems: Check `governor=performance*16` and `boost=1`. -When ToyOS's request changes, this section changes with it. +When ToyOS's request changes, the experiment changes with it. **The groups.** `fair.apart` sets each workload in a group of its own: a ToyOS Job, and on Linux a scope's cgroup, which is a group only where the @@ -105,11 +113,15 @@ user's manager delegates the `cpu` controller. Check `fair.apart.units=3xcgroup`. With `1xcgroup` Linux's `apart` is thread-fair, and the pair compares nothing. -**The clock.** Every timestamp is read from the TSC on ToyOS, so Linux runs -with `clocksource=tsc`. A clocksource the vDSO cannot read costs every -`Instant::now` a syscall: on 2026-10-24 the AMD bench's Linux took 1.21 µs a -read (`time.now`) against the T14's 14 ns, and every per-operation timing -there carried it. +**The clock.** Every timestamp is read from the TSC on ToyOS. The AMD +bench's Linux keeps time on the HPET, having found its TSC unstable at boot +(`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`), so +each of its `Instant::now` is a syscall: 1.21 µs on 2026-10-24 against the +T14's 14 ns, carried by every per-operation timing there. That is its +baseline, stated by the `clocksource` setup line. A Linux forced onto the +TSC (`tsc=reliable`) has its stability checks switched off, so it is no +baseline. It is an experiment, and only once the instability's cause is +known. ## The disk is not yet one disk From a6a4c60d840d473e0f824bd8e8255d0cb93c22f5 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 10:32:45 +0100 Subject: [PATCH 6/8] osbench: what the validation run showed, folded in The 2026-10-28 validation run at 2eb28ee65, one ToyOS boot and two baseline Linux runs per bench, passed its row on both. It showed: - The AMD's baseline Linux boosts at 4,022 to 4,126 MHz, dithering by some 2% within any 50 ms, so the 1% plateau never formed and 84 rows read unsettled, at 0.5 s twice each. The governor steps the settle exists to catch are a quarter of the clock and more, so it now holds within 5%. - The format-boot refusal moves to the DATA follow-up. Neither bench's p4 is designated (the AMD's stamp was cleared before any ToyOS boot read it), so nothing reaches it until ToyOS designates DATA. - Why the AMD's Linux keeps time on the HPET. Over 30 boots' kernel logs its watchdog fired in 18. In 13, Linux's refined TSC calibration read 1799.54-1799.61 MHz, 1,630 ppm high. In 5, with a right calibration, a 1.7 ms step early in boot was seen, cause unknown. At runtime (tsc-probe, 15 min, 16 CPUs, idle gaps to 1 s) no CPU read behind another and no step passed 1.2 us. The README records it, and issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md closes: the bench keeps its default, and the unexplained step is ToyOS's clock's concern (#1082). - The T14's default Linux requests 0x80002a04 (IA32_HWP_REQUEST, every CPU): EPP 0x80 and highest 0x2a, as ToyOS asks, and lowest 4 against ToyOS's 1. The README says so. Co-Authored-By: Claude Opus 5.5 --- ...ompares-toyos-with-linux-on-one-machine.md | 5 +-- ...eaving-idle-and-not-their-settled-clock.md | 13 ++++--- ...linux-reads-its-clock-through-a-syscall.md | 39 ------------------- tests/guest/osbench/README.md | 31 +++++++++------ tests/guest/osbench/rules/src/lib.rs | 22 ++++++----- tests/toyos.rs | 35 ++--------------- 6 files changed, 46 insertions(+), 99 deletions(-) delete mode 100644 issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index 6677c3fbe46..4ce5bfc1134 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -38,9 +38,8 @@ its groups: one revision, interleaved runs, Linux as installed for the baseline, and Linux under ToyOS's frequency request as a labelled experiment. `persist` is not yet one disk: ToyOS writes the boot stick's LOG volume, and moves to the -bench's NVMe once ToyOS designates its DATA partition itself. The AMD -bench's Linux reads its clock through a syscall -(`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`). +bench's NVMe once ToyOS designates its DATA partition itself. The README +says why the AMD bench's Linux keeps time on the HPET. `internet.*`, HTTPS to Cloudflare's speed test, is exploratory evidence of a path neither system owns, run only when a pass names its row: no automated diff --git a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md index 4cf06a614c6..1576c8eaeeb 100644 --- a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md +++ b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md @@ -29,11 +29,14 @@ counter reading. Since round 2 every benchmark but the wake and fairness groups' (whose CPUs idle or are all busy by design) runs after `clock::settle`, which spins the measuring CPU until every reading of a dependent `add` chain for 50 ms -agrees within 1% (`osbench_rules::settle`), or for 0.5 s, and every result -line prints that clock as `mhz_settled` and the clock just after as `mhz`; -each group also prints `cpu.clock`, 200 probes from idle. Three probes in a -row, the rule until 2026-10-27, took the AMD bench's `schedutil` step for -the settled clock: its Linux runs of 2026-10-24 settled at 1,357 MHz. +agrees within 5% (`osbench_rules::settle`), or for 0.5 s, and every result +line prints that clock as `mhz_settled`, the clock a second settle reaches +after the row as `mhz_after`, and the clock just after as `mhz`; each group +also prints `cpu.clock`, 200 probes from idle. Three probes in a row, the +rule until 2026-10-27, took the AMD bench's `schedutil` step for the +settled clock: its Linux runs of 2026-10-24 settled at 1,357 MHz. A 1% +span never formed there on 2026-10-28, where Linux's boosting clock +dithered between 4,022 and 4,126 MHz, so 84 of its rows read `unsettled`. **Exit**: a T14 run on each system shows `mhz_settled` near the all-CPU rate per CPU within the run-to-run spread, or names the benchmark whose CPU did diff --git a/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md b/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md deleted file mode 100644 index 2161cee0747..00000000000 --- a/issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-27 ---- - -# The AMD bench's Linux reads its clock through a syscall - -On 2026-10-24, osbench's `time.now` (`Instant::now` a million times a round) -read 1,212,895 ps a call under the AMD bench's Ubuntu (Linux 6.8.0-147). -Under the T14's it read 14,302 ps, with the same static binary. The AMD's -Linux keeps time on the HPET: 2.6 s into the one boot whose log was read, -its clocksource watchdog found the TSC 1,698,571 ns ahead of the HPET over -502,290,025 ns, marked it unstable and switched. At that cost each timestamp -is a syscall, an inference from the cost, and every per-operation timing of -that Linux carries about 1.2 µs that ToyOS's, which reads the TSC, does not. -That is the baseline's own behaviour: osbench records `clocksource`, and the -README treats a Linux forced onto the TSC as an experiment, never the -baseline. - -What the watchdog's interval was is unknown. Its own figures put the TSC at -1802.69 MHz across it. Over 10 minutes later in the boot the TSC counts -1796.6248 MHz against the HPET and against the ACPI PM timer, in every idle -and busy phase, worst residual 23 µs, and the CPU states an invariant TSC -(2026-10-28, `tsc-drift`, PR #1079's evidence). A steady average can still -hide a step. The candidates are: -- Linux's calibration at that point; -- a reference read delayed inside the interval; -- an offset between the CPUs the two reads ran on; -- the TSC's or the HPET's rate or state changing around that time in boot. - -**Exit**: the interval's cause is known, from per-CPU offsets and steps -measured across idle and load and from the watchdog's figures in more than -one boot. Then either the bench's Linux keeps its default and this records -why, or the cause is a ToyOS-relevant TSC defect, filed against ToyOS's -clock. - -**Owner**: the orchestrator, which runs the benches. A change to the bench's -Linux boot configuration is the owner's. diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md index ba690bc99f6..9a867578bf4 100644 --- a/tests/guest/osbench/README.md +++ b/tests/guest/osbench/README.md @@ -114,14 +114,25 @@ user's manager delegates the `cpu` controller. Check thread-fair, and the pair compares nothing. **The clock.** Every timestamp is read from the TSC on ToyOS. The AMD -bench's Linux keeps time on the HPET, having found its TSC unstable at boot -(`issues/the-amd-benchs-linux-reads-its-clock-through-a-syscall.md`), so -each of its `Instant::now` is a syscall: 1.21 µs on 2026-10-24 against the -T14's 14 ns, carried by every per-operation timing there. That is its -baseline, stated by the `clocksource` setup line. A Linux forced onto the -TSC (`tsc=reliable`) has its stability checks switched off, so it is no -baseline. It is an experiment, and only once the instability's cause is -known. +bench's Linux keeps time on the HPET in most boots: its clocksource watchdog +distrusts the TSC early in boot. Each `Instant::now` there is then a +syscall's cost, 1.21 µs against the T14's 14 ns, carried by every +per-operation timing. That is its baseline, stated by the `clocksource` +setup line. In 30 boots' kernel logs (2026-10-28) the watchdog fired in 18: +- in 13, Linux's refined calibration read 1799.54 to 1799.61 MHz, some + 1,630 ppm above the TSC's 1796.62 MHz; +- in 5, with a right calibration, the TSC ran some 1.7 ms ahead of the HPET + over half a second, a step whose cause is unknown. + +At runtime neither shows: across 16 CPUs and idle gaps up to 1 s, no CPU's +TSC read behind another's and no step exceeded the 1.2 µs of a read. A +Linux forced onto the TSC (`tsc=reliable`) has those checks switched off, +so it is no baseline. + +**On the T14 the baseline already makes ToyOS's request but for its +floor.** Its Linux's `IA32_HWP_REQUEST` read `0x80002a04` on every CPU +(2026-10-28): EPP 0x80 and highest 0x2a as ToyOS asks, and lowest 4 where +ToyOS asks 1. ## The disk is not yet one disk @@ -130,9 +141,7 @@ runner's home, ext4 on the bench's NVMe. On ToyOS it is still the boot stick's LOG volume, FAT32 over USB 2.0, because ToyOS's DATA is the bench's partition only once ToyOS has designated it, and ToyOS cannot yet do that itself. So `persist` compares two media. Its lines name theirs, and -`osbench-compare` refuses a verdict on any row whose media differ. The -`osbench` row refuses the boot that formats a newly designated DATA as a -run, and says how long the format took. +`osbench-compare` refuses a verdict on any row whose media differ. ## What is not a comparison diff --git a/tests/guest/osbench/rules/src/lib.rs b/tests/guest/osbench/rules/src/lib.rs index e2d35d37bee..257e03e0176 100644 --- a/tests/guest/osbench/rules/src/lib.rs +++ b/tests/guest/osbench/rules/src/lib.rs @@ -398,9 +398,12 @@ pub mod window { pub mod settle { //! A clock has settled once every reading over [`STEADY_NS`] agrees - //! within 1%. A frequency governor can hold a CPU that just turned busy + //! within 5%. A frequency governor can hold a CPU that just turned busy //! at a lower clock for tens of milliseconds before it steps up, so a - //! span much shorter than that takes the step for the settled clock. + //! span much shorter than that takes the step for the settled clock. The + //! steps are a quarter of the clock and more (the AMD bench's 1.36, 1.8 + //! and 4 GHz), while a boosting CPU's clock dithers by some 2% from one + //! reading to the next, so 5% tells the two apart. /// How long the readings must agree. pub const STEADY_NS: u64 = 50_000_000; @@ -419,11 +422,11 @@ pub mod settle { } /// The reading `mhz` taken at `at_ns`: whether every reading for - /// [`STEADY_NS`] up to it agrees within 1%. A reading that does not + /// [`STEADY_NS`] up to it agrees within 5%. A reading that does not /// agree with those before it starts the span again from itself. pub fn settled_with(&mut self, at_ns: u64, mhz: u64) -> bool { let (lo, hi) = (self.lo.min(mhz), self.hi.max(mhz)); - if (hi - lo) * 100 > hi { + if (hi - lo) * 20 > hi { *self = Plateau::new(at_ns, mhz); } else { (self.lo, self.hi) = (lo, hi); @@ -451,8 +454,9 @@ pub mod settle { } #[test] - fn readings_within_one_percent_hold_the_span() { - let wobble = |at: u64| if (at / MS).is_multiple_of(2) { 4000 } else { 3970 }; + fn a_boosting_clocks_dither_holds_the_span() { + // The AMD bench's Linux readings on 2026-10-28: 4022 to 4126 MHz. + let wobble = |at: u64| if (at / MS).is_multiple_of(2) { 4126 } else { 4022 }; assert_eq!(settles(wobble), Some((50 * MS, wobble(50 * MS)))); } @@ -460,9 +464,9 @@ pub mod settle { fn a_reading_that_disagrees_starts_the_span_again() { let mut plateau = Plateau::new(0, 4000); assert!(!plateau.settled_with(40 * MS, 4000)); - assert!(!plateau.settled_with(41 * MS, 3900), "2.5% below"); - assert!(!plateau.settled_with(90 * MS, 3900), "49 ms since the disagreement"); - assert!(plateau.settled_with(91 * MS, 3900)); + assert!(!plateau.settled_with(41 * MS, 3700), "7.5% below"); + assert!(!plateau.settled_with(90 * MS, 3700), "49 ms since the disagreement"); + assert!(plateau.settled_with(91 * MS, 3700)); } #[test] diff --git a/tests/toyos.rs b/tests/toyos.rs index 50276fd2bcf..416a2e487b2 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1113,17 +1113,10 @@ const METAL: &[(&str, metal::Metal)] = &[ ( // The numbers compared with the same program on the bench's own Linux. // Evidence and not a verdict: they are read off the log, and the row - // holds only that every group ran to its end, on a boot that did not - // spend itself formatting DATA. The T14 and the AMD bench. One boot - // runs the whole suite, so three readbacks are three runs. + // holds only that every group ran to its end. The T14 and the AMD bench. + // One boot runs the whole suite, so three readbacks are three runs. "osbench", - metal::Metal { - arms: BENCHCASE, - judge: |b| { - not_the_format_boot(b[0])?; - osbench_ran(b[0], &OSBENCH_GROUPS) - }, - }, + metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0], &OSBENCH_GROUPS) }, ), ( // `osbench lan` against the `osbench-peer` the other bench's Linux runs, @@ -1702,28 +1695,6 @@ fn osbench_ran(boot: &metal::Readback, jobs: &[&str]) -> Result<(), String> { Ok(()) } -/// The boot that formats a freshly designated DATA partition is no measured -/// run: its start went to the format, and the volume it leaves is new. It -/// says how long the format took. -fn not_the_format_boot(boot: &metal::Readback) -> Result<(), String> { - const FORMATTING: &str = "fileserver: block 0 designates this partition for ToyOS; formatting it"; - const SERVING: &str = "fileserver: Data serving "; - let log = boot.log(); - let text = log.text(); - let Some(began) = text.lines().find(|l| l.contains(FORMATTING)) else { - return Ok(()); - }; - let ended = text.lines().find(|l| l.contains(SERVING)); - let secs = ended.and_then(since_boot).zip(since_boot(began)).map(|(end, start)| end - start); - let took = secs.map_or("its end unlogged".to_string(), |s| format!("{s:.3} s")); - Err(format!("this boot formatted DATA off its designation ({took}), and the format's boot is no measured run")) -} - -/// A log line's seconds since boot: `[2026-10-24 14:17:17 12.664 fileserver]`. -fn since_boot(line: &str) -> Option { - line.strip_prefix('[')?.split_whitespace().nth(2)?.parse().ok() -} - /// A timing boot was in ACPI mode before its first job and stayed there, as a /// shipped image and the bench's Linux are. In legacy mode the T14's firmware /// stops every CPU every 2.2 s, and its time would be measured as ToyOS's From fa7258221d484c31602dd05f8be1a13101cd82fb Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 11:12:04 +0100 Subject: [PATCH 7/8] review r2: the clock reading leaves the time read out, Linux's clocksource is per run, and the counts are right MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - r2 1: the AMD's Linux spread was its HPET read's cost (1.21 µs) on a 25 µs probe, not the CPU's dither. ToyOS on the same CPU settled within 1% every time (0 of 92 capped). A clock reading is now the difference of a 100,000- and a 400,000-cycle chain, each the fastest of five, so the read's cost cancels. The settle holds within 3%, and is pinned by host tests at the smallest steps measured, the AMD's 1.65 to 1.85 GHz (11%) and the T14's 3.79 to 4.19 GHz (9.5%). The module doc, the issue and the README name the real cause. - r2 2: the AMD's watchdog fired in 17 of 30 boots: 12 with Linux's calibration high and 5 with the step. A 13th boot kept a TSC clock 1,630 ppm fast, and its clocksource line reads as a right one's. - r2 3: the clocksource is part of Linux as booted. The compare states it per run (PER_RUN) and never refuses a mix, which would have refused most interleaved AMD comparisons. - r2 4: the T14 has no matched-request experiment. Its baseline already requests 0x80002a04, ToyOS's request but for the floor. - Suggestions: an unnamed medium beside a named one is a medium of its own; two rewraps; the hot-paths issue notes that no row measures a lone fault now. Co-Authored-By: Claude Opus 5.5 --- ...eaving-idle-and-not-their-settled-clock.md | 16 +++-- ...y-hot-paths-are-slower-than-2-mib-pages.md | 2 + tests/guest/osbench/README.md | 62 +++++++++--------- tests/guest/osbench/compare/src/lib.rs | 65 +++++++++++++++---- tests/guest/osbench/rules/src/lib.rs | 25 ++++--- tests/guest/osbench/src/clock.rs | 42 ++++++++---- tests/guest/osbench/src/files.rs | 3 +- tests/guest/osbench/src/mem.rs | 5 +- 8 files changed, 148 insertions(+), 72 deletions(-) diff --git a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md index 1576c8eaeeb..ff18a6dd5d1 100644 --- a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md +++ b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md @@ -29,14 +29,18 @@ counter reading. Since round 2 every benchmark but the wake and fairness groups' (whose CPUs idle or are all busy by design) runs after `clock::settle`, which spins the measuring CPU until every reading of a dependent `add` chain for 50 ms -agrees within 5% (`osbench_rules::settle`), or for 0.5 s, and every result +agrees within 3% (`osbench_rules::settle`), or for 0.5 s, and every result line prints that clock as `mhz_settled`, the clock a second settle reaches after the row as `mhz_after`, and the clock just after as `mhz`; each group -also prints `cpu.clock`, 200 probes from idle. Three probes in a row, the -rule until 2026-10-27, took the AMD bench's `schedutil` step for the -settled clock: its Linux runs of 2026-10-24 settled at 1,357 MHz. A 1% -span never formed there on 2026-10-28, where Linux's boosting clock -dithered between 4,022 and 4,126 MHz, so 84 of its rows read `unsettled`. +also prints `cpu.clock`, 200 probes from idle. A reading is the difference +of a short add chain's time and a long one's, so the cost of reading the +time is no part of it. Three probes in a row, the rule until 2026-10-27, +took the AMD bench's `schedutil` step for the settled clock: its Linux runs +of 2026-10-24 settled at 1,357 MHz. On 2026-10-28 every settle of the AMD's +Linux ran out its time under a 1% rule (84 of 84) where ToyOS's on the same +CPU never did (0 of 92). That Linux timed through the HPET, whose 1.2 µs a +read sat inside a 25 µs probe, reading the clock some 5% low and moving it +by the read's jitter. **Exit**: a T14 run on each system shows `mhz_settled` near the all-CPU rate per CPU within the run-to-run spread, or names the benchmark whose CPU did diff --git a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md index c363a28f498..497c7d6c12d 100644 --- a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md +++ b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md @@ -38,6 +38,8 @@ did; the T14's numbers are below: (`UserTables::leaf_table`); cause not found. Accepted as a measured tradeoff by the coordinator on PR #1055, under the owner's delegation of implementation tradeoffs (2026-10-23), against about 380 ns saved per page touched in order. + That row now faults every 4 KiB (PR #1079, so both systems fault the + same bytes), so no osbench row measures a lone fault any more. - **Spawn.** `proc.spawn_wait_loop.100` rose 2.3% at `a75abd3f6` and is within 1.5% at `6d826e313`. - **Frees.** A clear walks the 64 futex buckets once per table of leaves that diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md index 9a867578bf4..a08e43896d3 100644 --- a/tests/guest/osbench/README.md +++ b/tests/guest/osbench/README.md @@ -49,12 +49,16 @@ and the C library beneath it are each system's own. - **Compare** every run of the bench in one call: `cargo run -p osbench-compare -- `. It refuses runs of one system that disagree on their setup (`SETUP`) and - states each system's. Each row's verdict says whether the runs resolve a + states each system's, and what each run's boot chose (`PER_RUN`, the + clocksource) run by run. Each row's verdict says whether the runs resolve a difference. Each p is its row's alone: with no difference anywhere, about 2.9% of rows resolve by chance, a few in a table this size. - **The clocks are told beside each verdict.** No program reads the package's limit state or temperature on ToyOS, so the clock is what both - systems report. A settled row carries the clock settled before it + systems report. A clock reading is the difference between a short add + chain's time and a long one's, so what reading the time costs is no part + of it; a settle holds once every reading for 50 ms agrees within 3%. A + settled row carries the clock settled before it (`mhz_settled`) and after it (`mhz_after`). The verdict counts per system the runs where the second is under nine tenths of the first. A power or heat limit reached inside the row is the likely cause, an inference: the @@ -71,27 +75,22 @@ and the C library beneath it are each system's own. clocksource, its own checks. What it does on that machine is ordinary OS behaviour and part of what is compared. A comparison names its configuration in its setup lines, and `osbench-compare` refuses to mix two. +What a boot chooses for itself, the clocksource, belongs to Linux as booted: +each run states its own, and runs that differ are compared together. **A matched request is a controlled experiment, labelled as one.** It asks the CPU's frequency control for what ToyOS's kernel asks of the same CPU, so the clock is the hardware's answer to one request on both systems. It answers a different question from the baseline, and never replaces it: -- **The T14** (i5-1135G7, `intel_pstate` with HWP). ToyOS asks for HWP's - lowest to highest performance at EPP 0x80 (`kernel/pure/cpu.rs`). Ubuntu's - `powersave` governor makes HWP's request with the EPP its - `energy_performance_preference` holds, so the experiment writes 0x80 there - before each Linux run, which `intel_pstate` takes as a number: - - ```text - echo 128 | sudo tee /sys/devices/system/cpu/cpufreq/policy*/energy_performance_preference - ``` - - Check `governor=powersave*8` and `intel_no_turbo=0`; `epp` reads 128 by - the name the driver gives it, `balance_performance` where that is 0x80 on - this CPU. Linux's lowest request is `cpuinfo_min_freq`, which need not be - HWP's lowest, ToyOS's: that reaches a CPU waking from idle, not a settled - row. +- **The T14** (i5-1135G7, `intel_pstate` with HWP) has no experiment: its + baseline already makes ToyOS's request but for the floor. ToyOS asks for + HWP's lowest to highest performance at EPP 0x80 (`kernel/pure/cpu.rs`). + The baseline's `IA32_HWP_REQUEST` read `0x80002a04` on every CPU + (2026-10-28): EPP 0x80 (`epp=balance_performance*8`), highest 0x2a, and + lowest 4 where ToyOS asks 1. The floor reaches a CPU waking from idle, not + a settled row. Check `governor=powersave*8`, `epp=balance_performance*8` + and `intel_no_turbo=0`. - **The AMD bench** (Ryzen 7 4800U, `acpi-cpufreq`). ToyOS writes no P-state or boost register, so the firmware's P0 and Core Performance Boost stand. Linux's `performance` governor asks for that. The experiment sets it @@ -114,25 +113,26 @@ user's manager delegates the `cpu` controller. Check thread-fair, and the pair compares nothing. **The clock.** Every timestamp is read from the TSC on ToyOS. The AMD -bench's Linux keeps time on the HPET in most boots: its clocksource watchdog -distrusts the TSC early in boot. Each `Instant::now` there is then a -syscall's cost, 1.21 µs against the T14's 14 ns, carried by every -per-operation timing. That is its baseline, stated by the `clocksource` -setup line. In 30 boots' kernel logs (2026-10-28) the watchdog fired in 18: -- in 13, Linux's refined calibration read 1799.54 to 1799.61 MHz, some +bench's Linux chooses its clocksource at each boot. Its watchdog moves it +off the TSC when, early in boot, the TSC and the HPET disagree. On the HPET +each `Instant::now` is a syscall's cost, 1.21 µs against the T14's 14 ns, +carried by every per-operation timing of that run. In 30 boots' kernel logs +(2026-10-28) the watchdog fired in 17: +- in 12, Linux's refined calibration read 1799.54 to 1799.61 MHz, some 1,630 ppm above the TSC's 1796.62 MHz; - in 5, with a right calibration, the TSC ran some 1.7 ms ahead of the HPET over half a second, a step whose cause is unknown. -At runtime neither shows: across 16 CPUs and idle gaps up to 1 s, no CPU's -TSC read behind another's and no step exceeded the 1.2 µs of a read. A -Linux forced onto the TSC (`tsc=reliable`) has those checks switched off, -so it is no baseline. +In a 13th boot the calibration read 1799.565 MHz and the watchdog let it +stand, so that Linux kept a TSC clock some 1,630 ppm fast for the whole +boot. Its `clocksource=tsc` reads the same as a right one's. The 12 others +kept the TSC on a right calibration. Each run's clocksource is part of the +Linux it measured, so the compare states it per run. -**On the T14 the baseline already makes ToyOS's request but for its -floor.** Its Linux's `IA32_HWP_REQUEST` read `0x80002a04` on every CPU -(2026-10-28): EPP 0x80 and highest 0x2a as ToyOS asks, and lowest 4 where -ToyOS asks 1. +At runtime neither fault shows: across 16 CPUs and idle gaps up to 1 s, no +CPU's TSC read behind another's and no step exceeded the 1.2 µs of a read. +A Linux forced onto the TSC (`tsc=reliable`) has those checks switched off, +so it is no baseline. ## The disk is not yet one disk diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs index 4f5af738b46..feea0194dac 100644 --- a/tests/guest/osbench/compare/src/lib.rs +++ b/tests/guest/osbench/compare/src/lib.rs @@ -31,7 +31,9 @@ //! //! A system's runs on one machine are one configuration ([`SETUP`]): runs that //! disagree on it are refused, since their median would mix two, and each -//! machine's table states what each system's runs were taken under. +//! machine's table states what each system's runs were taken under. What a +//! system chooses afresh at each boot ([`PER_RUN`]) is part of that system as +//! booted: it is stated run by run and never refused. use std::collections::BTreeMap; use std::fmt::Write as _; @@ -275,9 +277,9 @@ pub fn better(unit: &str) -> Option { const PATHS: [(&str, &str, usize); 2] = [("lan.", "lan.peer_link", 2), ("internet.", "internet.link", 1)]; /// The conditions every run of one system on one machine must share: the -/// kernel, the CPU's frequency request and the clock every timestamp is read -/// from, and what serves each filesystem the `fs` and `persist` rows use. -pub const SETUP: [&str; 14] = [ +/// kernel, the CPU's frequency request, and what serves each filesystem the +/// `fs` and `persist` rows use. +pub const SETUP: [&str; 13] = [ "kernel", "cpufreq_driver", "governor", @@ -287,13 +289,31 @@ pub const SETUP: [&str; 14] = [ "boost", "intel_no_turbo", "platform_profile", - "clocksource", "smt_active", "fs.mem.served_by", "fs.served.served_by", "persist.served_by", ]; +/// The conditions a system chooses at each boot, stated per run: the clock +/// every timestamp is read from, which Linux's watchdog can move off the TSC +/// in one boot and not the next. +pub const PER_RUN: [&str; 1] = ["clocksource"]; + +/// `os`'s [`PER_RUN`] conditions, each as `key=,...` in the +/// runs' order, `absent` for a run that names none. +fn per_run(runs: &[&Run], os: Os) -> Vec { + let runs: Vec<&&Run> = runs.iter().filter(|r| r.os == os).collect(); + PER_RUN + .iter() + .filter(|key| runs.iter().any(|r| r.conds.contains_key(**key))) + .map(|key| { + let said: Vec<&str> = runs.iter().map(|r| r.conds.get(*key).map_or("absent", String::as_str)).collect(); + format!("{key}={}", said.join(",")) + }) + .collect() +} + /// `os`'s setup on these runs, as `key=value` in [`SETUP`]'s order, or the /// first key its runs disagree on. fn setup(runs: &[&Run], os: Os) -> Result, String> { @@ -345,6 +365,10 @@ pub fn compare(runs: &[Run]) -> Result { if !said.is_empty() { writeln!(out, "{} setup: {}.\n", os.name(), said.join(", ")).unwrap(); } + let each = per_run(&runs, os); + if !each.is_empty() { + writeln!(out, "{} per run: {}.\n", os.name(), each.join(", ")).unwrap(); + } } writeln!(out, "| bench | unit | Linux p50s | Linux median | Linux range | ToyOS p50s | ToyOS median | ToyOS range | ToyOS/Linux | p | verdict |").unwrap(); writeln!(out, "|---|---|---|---|---|---|---|---|---|---|---|").unwrap(); @@ -372,7 +396,7 @@ fn compare_one(runs: &[&Run], name: &str) -> String { let mut fell: BTreeMap = BTreeMap::new(); let mut unsettled_runs: BTreeMap = BTreeMap::new(); let mut settled: BTreeMap> = BTreeMap::new(); - let mut media: BTreeMap> = BTreeMap::new(); + let mut media: BTreeMap>> = BTreeMap::new(); let mut not: BTreeMap> = BTreeMap::new(); for run in runs { match run.benches.get(name) { @@ -384,7 +408,7 @@ fn compare_one(runs: &[&Run], name: &str) -> String { *fell.entry(run.os).or_default() += usize::from(*clock_fell); *unsettled_runs.entry(run.os).or_default() += usize::from(*unsettled); settled.entry(run.os).or_default().extend(mhz.filter(|_| !*unsettled)); - media.entry(run.os).or_default().extend(medium.clone()); + media.entry(run.os).or_default().insert(medium.clone()); } Some(Outcome::Not { kind, why }) => not.entry(run.os).or_default().push(format!("{kind}: {why}")), None => not.entry(run.os).or_default().push("absent".into()), @@ -402,10 +426,15 @@ fn compare_one(runs: &[&Run], name: &str) -> String { }; let [lp, lm, lr] = side(Os::Linux); let [tp, tm, tr] = side(Os::Toyos); - let all_media: std::collections::BTreeSet<&String> = media.values().flatten().collect(); + // A side that names no medium is a medium of its own beside one that does. + let all_media: std::collections::BTreeSet<&Option> = media.values().flatten().collect(); + let differ = all_media.len() > 1 && all_media.iter().any(|m| m.is_some()); let (ratio, p, verdict) = match (values.get(&Os::Linux), values.get(&Os::Toyos)) { - (Some(_), Some(_)) if all_media.len() > 1 => { - let named = |os: Os| media.get(&os).map_or("unnamed".to_string(), |m| m.iter().cloned().collect::>().join("+")); + (Some(_), Some(_)) if differ => { + let named = |os: Os| { + let said = media.get(&os).into_iter().flatten().map(|m| m.as_deref().unwrap_or("unnamed")); + said.collect::>().join("+") + }; ("-".into(), "-".into(), format!("the media differ: Linux {}, ToyOS {}", named(Os::Linux), named(Os::Toyos))) } (Some(l), Some(t)) => { @@ -673,6 +702,9 @@ mod tests { runs.push(fsync("linux", 69, "sata")); runs.extend([70, 71, 72, 73].iter().map(|v| fsync("toyos", *v, "nvme"))); assert!(compare(&runs).unwrap().contains("the media differ: Linux nvme+sata, ToyOS nvme"), "nor one system's runs on two"); + let mut runs: Vec = [67, 68, 66, 67].iter().map(|v| fsync("linux", *v, "nvme")).collect(); + runs.extend([70, 71, 72, 73].iter().map(|v| run("toyos", &[&format!("bench=persist.fsync_clean unit=ns n=200 min=1 p50={v} max=1 mean=1")]))); + assert!(compare(&runs).unwrap().contains("the media differ: Linux nvme, ToyOS unnamed"), "nor a named medium and none"); } #[test] @@ -685,11 +717,22 @@ mod tests { assert!(why.contains("the Linux runs disagree on governor: performance*16 / schedutil*16"), "{why}"); let one = [under("linux", "performance*16"), under("linux", "performance*16")]; let table = compare(&one).unwrap(); - assert!(table.contains("Linux setup: governor=performance*16, clocksource=tsc."), "{table}"); + assert!(table.contains("Linux setup: governor=performance*16."), "{table}"); let absent = [under("linux", "performance*16"), run("linux", &["bench=time.now unit=ps n=15 min=1 p50=14 max=1 mean=1"])]; assert!(compare(&absent).unwrap_err().contains("disagree on governor: absent / performance*16")); } + #[test] + fn a_clocksource_each_boot_chose_is_stated_per_run_and_never_refused() { + let booted = |clocksource: &str| { + run("linux", &[&format!("cond clocksource={clocksource}"), "bench=time.now unit=ps n=15 min=1 p50=14 max=1 mean=1"]) + }; + let runs = [booted("hpet"), booted("tsc"), booted("hpet"), run("toyos", &["bench=time.now unit=ps n=15 min=1 p50=9 max=1 mean=1"])]; + let table = compare(&runs).unwrap(); + assert!(table.contains("Linux per run: clocksource=hpet,tsc,hpet."), "{table}"); + assert!(!table.contains("ToyOS per run"), "{table}"); + } + #[test] fn internet_runs_that_left_by_different_kinds_of_link_are_not_compared() { let left = |os: &str, said: &str| { diff --git a/tests/guest/osbench/rules/src/lib.rs b/tests/guest/osbench/rules/src/lib.rs index 257e03e0176..b6698f11078 100644 --- a/tests/guest/osbench/rules/src/lib.rs +++ b/tests/guest/osbench/rules/src/lib.rs @@ -398,12 +398,12 @@ pub mod window { pub mod settle { //! A clock has settled once every reading over [`STEADY_NS`] agrees - //! within 5%. A frequency governor can hold a CPU that just turned busy + //! within 3%. A frequency governor can hold a CPU that just turned busy //! at a lower clock for tens of milliseconds before it steps up, so a //! span much shorter than that takes the step for the settled clock. The - //! steps are a quarter of the clock and more (the AMD bench's 1.36, 1.8 - //! and 4 GHz), while a boosting CPU's clock dithers by some 2% from one - //! reading to the next, so 5% tells the two apart. + //! smallest steps measured are the AMD bench's 1.65 to 1.85 GHz and the + //! T14's 3.79 to 4.19 GHz, 11% and 9.5%; a boosting CPU's readings + //! agreed within 1% wherever reading the time was cheap. /// How long the readings must agree. pub const STEADY_NS: u64 = 50_000_000; @@ -422,11 +422,11 @@ pub mod settle { } /// The reading `mhz` taken at `at_ns`: whether every reading for - /// [`STEADY_NS`] up to it agrees within 5%. A reading that does not + /// [`STEADY_NS`] up to it agrees within 3%. A reading that does not /// agree with those before it starts the span again from itself. pub fn settled_with(&mut self, at_ns: u64, mhz: u64) -> bool { let (lo, hi) = (self.lo.min(mhz), self.hi.max(mhz)); - if (hi - lo) * 20 > hi { + if (hi - lo) * 100 > hi * 3 { *self = Plateau::new(at_ns, mhz); } else { (self.lo, self.hi) = (lo, hi); @@ -454,9 +454,16 @@ pub mod settle { } #[test] - fn a_boosting_clocks_dither_holds_the_span() { - // The AMD bench's Linux readings on 2026-10-28: 4022 to 4126 MHz. - let wobble = |at: u64| if (at / MS).is_multiple_of(2) { 4126 } else { 4022 }; + fn the_smallest_measured_steps_are_not_the_settled_clock() { + for (from, to) in [(1650, 1850), (3789, 4189)] { + let step = |at: u64| if at < 30 * MS { from } else { to }; + assert_eq!(settles(step), Some((80 * MS, to)), "{from} to {to} MHz"); + } + } + + #[test] + fn readings_within_a_percent_or_two_hold_the_span() { + let wobble = |at: u64| if (at / MS).is_multiple_of(2) { 4241 } else { 4170 }; assert_eq!(settles(wobble), Some((50 * MS, wobble(50 * MS)))); } diff --git a/tests/guest/osbench/src/clock.rs b/tests/guest/osbench/src/clock.rs index 0df954f0c1c..e6b600eccb0 100644 --- a/tests/guest/osbench/src/clock.rs +++ b/tests/guest/osbench/src/clock.rs @@ -1,6 +1,9 @@ //! The clock a CPU is running at, read the same way on both systems: //! `arch::add_chain`'s adds wait a cycle each, so cycles are the number of -//! adds. +//! adds. A probe times a short chain and a long one and takes the +//! difference, so the cost of reading the time, which both timings carry, is +//! no part of the clock it reads: 1.2 µs a read on a Linux timing through the +//! HPET, a twentieth of a 100,000-cycle chain. use std::sync::Mutex; use std::time::{Duration, Instant}; @@ -21,21 +24,38 @@ pub struct Settled { pub capped: bool, } -/// Turns per probe: 100,000 cycles, 25 to 50 µs on either bench. +/// Turns in a probe's short chain, 100,000 cycles; its long one is +/// [`LONG`] times that, so the difference is 300,000 cycles, 70 to 150 µs on +/// either bench. const TURNS: u64 = 1_000; +const LONG: u64 = 4; const ADDS: u64 = crate::arch::ADDS_PER_TURN; +/// The cycles a probe's difference spans. +const CYCLES: u64 = ADDS * TURNS * (LONG - 1); -fn burst() -> u64 { +fn chain(turns: u64) -> u64 { let t = Instant::now(); - crate::arch::add_chain(TURNS); - ns_since(t).max(1) + crate::arch::add_chain(turns); + ns_since(t) } -/// The clock the calling CPU runs at now, in MHz: the fastest of five probes, -/// so an interrupt inside one does not read as a slow clock. +/// One probe for `cpu.clock`'s distribution: the long chain's time less the +/// short one's, in ns. +fn burst() -> u64 { + let short = chain(TURNS); + chain(TURNS * LONG).saturating_sub(short).max(1) +} + +/// The clock the calling CPU runs at now, in MHz: the fastest of five of +/// each chain, taken in turn, so an interrupt inside one reads as neither a +/// slow clock nor, in the short chain, a fast one. pub fn mhz() -> u64 { - let ns = (0..5).map(|_| burst()).min().expect("five probes"); - ADDS * TURNS * 1000 / ns + let (mut short, mut long) = (u64::MAX, u64::MAX); + for _ in 0..5 { + short = short.min(chain(TURNS)); + long = long.min(chain(TURNS * LONG)); + } + CYCLES * 1000 / long.saturating_sub(short).max(1) } /// The longest a settle spins before it takes the clock it has. @@ -70,6 +90,6 @@ pub fn take_settled() -> Option { /// CPU leaving idle show its ramp; the rest, the clock it settles at. pub fn distribution() { burst(); - let mut mhz: Vec = (0..200).map(|_| ADDS * TURNS * 1000 / burst()).collect(); - result("cpu.clock", "MHz", &mut mhz, &[("adds_per_probe", (ADDS * TURNS).to_string())]); + let mut mhz: Vec = (0..200).map(|_| CYCLES * 1000 / burst()).collect(); + result("cpu.clock", "MHz", &mut mhz, &[("cycles_per_probe", CYCLES.to_string())]); } diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index 9cc69c5009a..401a9c57066 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -103,8 +103,7 @@ fn page_at(i: usize, pages: usize) -> u64 { } /// Every benchmark under `target`, each line naming what served it and its -/// medium. Each -/// one's comment is its warm-up and repetitions. +/// medium. Each one's comment is its warm-up and repetitions. pub fn suite(target: &Target) { condition(&format!("fs.{}.served_by", target.name), &target.served_by); #[cfg(target_os = "linux")] diff --git a/tests/guest/osbench/src/mem.rs b/tests/guest/osbench/src/mem.rs index b94db96ae82..74bb0a57266 100644 --- a/tests/guest/osbench/src/mem.rs +++ b/tests/guest/osbench/src/mem.rs @@ -6,8 +6,9 @@ //! Both systems fault each anonymous page in on its first touch: ToyOS in //! 4 KiB pages always, Linux in 4 KiB pages where huge pages are refused and //! in 2 MiB pages where they are asked for and given. `first_touch` faults in -//! the same bytes either way, and `map_touch_unmap` is the whole cost. The allocator is each system's own: ToyOS std's dlmalloc -//! under one lock, Linux's static musl malloc. +//! the same bytes either way, and `map_touch_unmap` is the whole cost. The +//! allocator is each system's own: ToyOS std's dlmalloc under one lock, +//! Linux's static musl malloc. use std::alloc::{alloc, dealloc, realloc, Layout}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; From 6be1ab62b38e7f90109554e2848ee51c66c08a97 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 11:29:50 +0100 Subject: [PATCH 8/8] osbench: a probe whose long chain took no longer reads no clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit r3's suggestion: cpu.clock's single probes took the long chain's time less the short one's, saturating at 1 ns. A short chain preempted past the long one then read as about 3x10^11 MHz, which corrupts that row's max and mean. osbench_rules::settle::probe_mhz answers None there. cpu.clock skips such a probe, and mhz, whose fastest-of-five chains cannot meet it unless the clock rose that far between them, reads again. A host test fails without the minimum (control m8, exit 101). The settle, mhz_settled, mhz_after and every p50 read as at fa7258221. clock.rs no longer states a false 70 to 150 µs: 300,000 cycles are 70 µs at 4.24 GHz and 230 µs at 1.3 GHz. Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/rules/src/lib.rs | 18 +++++++++++++++ tests/guest/osbench/src/clock.rs | 34 ++++++++++++++++------------ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tests/guest/osbench/rules/src/lib.rs b/tests/guest/osbench/rules/src/lib.rs index b6698f11078..eaa1737141b 100644 --- a/tests/guest/osbench/rules/src/lib.rs +++ b/tests/guest/osbench/rules/src/lib.rs @@ -408,6 +408,14 @@ pub mod settle { /// How long the readings must agree. pub const STEADY_NS: u64 = 50_000_000; + /// The clock in MHz that `cycles` more adds in the long chain than in + /// the short one say, timed at `long_ns` and `short_ns`: `None` where the + /// long chain took no longer, as when the short one was preempted, which + /// would otherwise read as a clock of many THz. + pub fn probe_mhz(cycles: u64, short_ns: u64, long_ns: u64) -> Option { + long_ns.checked_sub(short_ns).filter(|&ns| ns > 0).map(|ns| cycles * 1000 / ns) + } + /// The readings since the last one that disagreed. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Plateau { @@ -476,6 +484,16 @@ pub mod settle { assert!(plateau.settled_with(91 * MS, 3700)); } + #[test] + fn a_probe_reads_the_difference_and_a_preempted_short_chain_reads_nothing() { + // 300,000 cycles over 70.7 µs, ToyOS's 4,241 MHz on the AMD bench. + assert_eq!(probe_mhz(300_000, 23_600, 94_340), Some(4240)); + // The time read's cost, in both chains alike, cancels. + assert_eq!(probe_mhz(300_000, 23_600 + 1_210, 94_340 + 1_210), Some(4240)); + assert_eq!(probe_mhz(300_000, 500_000, 94_340), None, "short chain preempted past the long"); + assert_eq!(probe_mhz(300_000, 94_340, 94_340), None); + } + #[test] fn a_slow_drift_settles_only_once_it_stops() { let drift = |at: u64| 3000 + (at / MS).min(200) * 5; diff --git a/tests/guest/osbench/src/clock.rs b/tests/guest/osbench/src/clock.rs index e6b600eccb0..c317f321ee5 100644 --- a/tests/guest/osbench/src/clock.rs +++ b/tests/guest/osbench/src/clock.rs @@ -8,7 +8,7 @@ use std::sync::Mutex; use std::time::{Duration, Instant}; -use osbench_rules::settle::Plateau; +use osbench_rules::settle::{probe_mhz, Plateau}; use crate::report::result; use crate::ns_since; @@ -25,8 +25,8 @@ pub struct Settled { } /// Turns in a probe's short chain, 100,000 cycles; its long one is -/// [`LONG`] times that, so the difference is 300,000 cycles, 70 to 150 µs on -/// either bench. +/// [`LONG`] times that, so the difference is 300,000 cycles: 70 µs at the +/// AMD bench's 4.24 GHz, 230 µs at 1.3 GHz. const TURNS: u64 = 1_000; const LONG: u64 = 4; const ADDS: u64 = crate::arch::ADDS_PER_TURN; @@ -39,23 +39,29 @@ fn chain(turns: u64) -> u64 { ns_since(t) } -/// One probe for `cpu.clock`'s distribution: the long chain's time less the -/// short one's, in ns. -fn burst() -> u64 { +/// One probe for `cpu.clock`'s distribution, in MHz: `None` where the short +/// chain was preempted past the long one. +fn burst() -> Option { let short = chain(TURNS); - chain(TURNS * LONG).saturating_sub(short).max(1) + probe_mhz(CYCLES, short, chain(TURNS * LONG)) } /// The clock the calling CPU runs at now, in MHz: the fastest of five of /// each chain, taken in turn, so an interrupt inside one reads as neither a -/// slow clock nor, in the short chain, a fast one. +/// slow clock nor, in the short chain, a fast one. A clock that rose so far +/// between the two that the fastest long chain took no longer than the +/// fastest short one is read again. pub fn mhz() -> u64 { - let (mut short, mut long) = (u64::MAX, u64::MAX); - for _ in 0..5 { - short = short.min(chain(TURNS)); - long = long.min(chain(TURNS * LONG)); + loop { + let (mut short, mut long) = (u64::MAX, u64::MAX); + for _ in 0..5 { + short = short.min(chain(TURNS)); + long = long.min(chain(TURNS * LONG)); + } + if let Some(mhz) = probe_mhz(CYCLES, short, long) { + return mhz; + } } - CYCLES * 1000 / long.saturating_sub(short).max(1) } /// The longest a settle spins before it takes the clock it has. @@ -90,6 +96,6 @@ pub fn take_settled() -> Option { /// CPU leaving idle show its ramp; the rest, the clock it settles at. pub fn distribution() { burst(); - let mut mhz: Vec = (0..200).map(|_| CYCLES * 1000 / burst()).collect(); + let mut mhz: Vec = (0..200).filter_map(|_| burst()).collect(); result("cpu.clock", "MHz", &mut mhz, &[("cycles_per_probe", CYCLES.to_string())]); }