diff --git a/issues/the-nightly-is-frozen-so-nothing-runs-the-guest-matrix.md b/issues/the-nightly-is-frozen-so-nothing-runs-the-guest-matrix.md index 430386e8c2..b3a15fc037 100644 --- a/issues/the-nightly-is-frozen-so-nothing-runs-the-guest-matrix.md +++ b/issues/the-nightly-is-frozen-so-nothing-runs-the-guest-matrix.md @@ -15,6 +15,12 @@ writer, main's toolchain release, and the fresh-machine and macOS builds. Guest rows run only where an agent dispatches them on the guest runner (`guest-rows.yml`), so a regression in a row no PR selected goes unseen. +The unfiltered suite now carries the shared boots' guest rows, +`testcases_members_exit_0` and `shared_debug_members_exit_0`, and with them an +in-suite build of every guest binary and corpus case. Their cost on a TCG lane +is unmeasured, and resumption measures it rather than meeting it at a lane's +timeout. + `build/ci/qemu-version` moved to 11.1.3, the guest runner's and the dev host's QEMU, while `guest.yml`'s container still installs 11.1.1 from its snapshot archive, so its instrument step reds until the container moves to an image diff --git a/issues/the-t14-reboots-through-ubuntu-for-every-test.md b/issues/the-t14-reboots-through-ubuntu-for-every-test.md index 4f72d3bd86..cdf8dbee37 100644 --- a/issues/the-t14-reboots-through-ubuntu-for-every-test.md +++ b/issues/the-t14-reboots-through-ubuntu-for-every-test.md @@ -8,8 +8,10 @@ opened: 2026-09-29 The owner's direction: the tests run one after another in a ToyOS that stays booted, driven by the host over ssh, with no reboot between them and no Ubuntu. -The T14 is the integration tier: the shared boot runs only as `shared_metal` -and `c_corpus_metal`, and +The T14 is the integration tier for hardware: the shared boots run there as +`shared_metal` and `c_corpus_metal` stage them, and on QEMU as the guest rows +`testcases_members_exit_0` and `shared_debug_members_exit_0`, which stage +the same members in the same order without the bench rows' own jobs; `issues/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md` adds metal rows. diff --git a/tests/checks.rs b/tests/checks.rs index ee7b2a05f1..0f63f13c5b 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -1321,6 +1321,81 @@ mod checks { metal_checks::a_failing_shared_member_fails_itself_alone(); } + /// A shared boot's guest row judges each member by the runner's own end + /// for it and nothing else: another member's end, a longer name's that + /// opens with its own, a job's line, another program's or a kernel + /// record's does not answer for it. + #[test] + fn a_shared_member_is_judged_by_its_own_end() { + let jobs: Vec = ["test_rs_passes", "test_rs_fails", "40_stdio", "test_rs_stuck", "test_rs_stuck_longer", "test_rs_never"] + .iter() + .map(ToString::to_string) + .collect(); + let verdicts = member_verdicts(SHARED_CONSOLE, &jobs); + let said = |job: &str| verdicts.iter().find(|(name, _)| *name == job).map(|(_, member)| member).expect(job); + assert_eq!(said("test_rs_passes"), &Member::Passed); + assert_eq!(said("test_rs_stuck_longer"), &Member::Passed); + let failed = |job: &str| match said(job) { + Member::Failed(why) => why.clone(), + other => panic!("{job} is {other:?}, not failed"), + }; + let fails = failed("test_rs_fails"); + assert!(fails.starts_with("exited 101\n") && fails.contains("the census moved"), "{fails}"); + assert!(failed("40_stdio").starts_with("did not run: entity not found")); + let stuck = failed("test_rs_stuck"); + assert!(stuck.starts_with("started and never ended\n") && stuck.contains("still going"), "{stuck}"); + assert_eq!(said("test_rs_never"), &Member::Unreached); + assert_eq!(verdicts.iter().map(|(job, _)| *job).collect::>(), jobs.iter().map(String::as_str).collect::>()); + } + + /// A list's runner lines, each member's, and the jobs' and others' lines + /// that must not answer for any of them. + const SHARED_CONSOLE: &str = "\ +[ 1.200 cpu0 kernel] Boot: complete (1200ms) +[ 1.300 test-runner] ===READY=== +[ 1.301 test-runner] ===TEST_START test_rs_passes=== +[ 1.302 test-runner] ===TEST_END test_rs_passes exit=0=== +[ 1.303 test-runner] ===TEST_START test_rs_fails=== +[ 1.304 test-runner pid=7] assertion failed: the census moved +[ 1.305 test-runner] ===TEST_END test_rs_fails exit=101=== +[ 1.306 test-runner] ===TEST_START 40_stdio=== +[ 1.307 test-runner] ===TEST_END 40_stdio error=entity not found=== +[ 1.308 test-runner] ===TEST_START test_rs_stuck=== +[ 1.309 test-runner] ===TEST_START test_rs_stuck_longer=== +[ 1.310 test-runner] ===TEST_END test_rs_stuck_longer exit=0=== +[ 1.311 cpu1 kernel] ===TEST_END test_rs_never exit=0=== +[ 1.311 fileserver] ===TEST_END test_rs_never exit=0=== +[ 1.312 test-runner pid=11] ===TEST_END test_rs_stuck exit=0=== +[ 1.313 test-runner pid=11] still going +"; + + /// A shared boot ends green only by its list's reboot: the kernel's own + /// `Rebooting.` and QEMU's `guest-reset`, as well as every member's exit. + /// A silent reset after the last member, any other stop, a `Rebooting.` + /// a program says, and a wait that never saw QEMU exit are red. + #[test] + fn a_shared_boot_ends_by_its_lists_reboot() { + let jobs = vec!["test_rs_passes".to_string()]; + let ran = "\ +[ 1.300 test-runner] ===READY=== +[ 1.301 test-runner] ===TEST_START test_rs_passes=== +[ 1.302 test-runner] ===TEST_END test_rs_passes exit=0=== +[ 1.303 test-runner] ===TEST_START reboot=== +"; + let rebooted = format!("{ran}[ 1.400 cpu0 kernel] Rebooting.\n"); + let reset = || Ok(Some("guest-reset".to_string())); + assert_eq!(judge_shared("fixture", &rebooted, &jobs, reset()), Ok(1)); + let red = |console: &str, ended: Result, String>, why: &str| { + let said = judge_shared("fixture", console, &jobs, ended).expect_err(why); + assert!(said.contains("did not end by its list's reboot"), "{why}: {said}"); + }; + red(ran, reset(), "a silent reset after the last member"); + red(&format!("{ran}[ 1.400 toybox] Rebooting.\n"), reset(), "a program's Rebooting."); + red(&rebooted, Ok(Some("guest-shutdown".to_string())), "a power-off"); + red(&rebooted, Ok(None), "a stop QMP never named"); + red(&rebooted, Err("STALLED: the guest went quiet".to_string()), "a wait that never saw QEMU exit"); + } + #[test] fn metal_boots_last_job_is_behind_every_other() { metal_checks::a_boots_last_job_is_behind_every_other(); diff --git a/tests/common/compile.rs b/tests/common/compile.rs index b4ebb123e6..decd7ee6e4 100644 --- a/tests/common/compile.rs +++ b/tests/common/compile.rs @@ -50,8 +50,8 @@ fn first_error(stderr: &[u8]) -> String { } /// Compile `source` in `dir` to an object with the toolchain's clang, from -/// that directory and by its bare name. -fn compile_one(c: &CSysroot, dir: &Path, flags: &[&str], source: &str, object: &Path) { +/// that directory and by its bare name, or the first error it reported. +fn compile_one(c: &CSysroot, dir: &Path, flags: &[&str], source: &str, object: &Path) -> Result<(), String> { let output = Command::new(&c.clang) .args(c.args()) .args(flags) @@ -62,14 +62,17 @@ fn compile_one(c: &CSysroot, dir: &Path, flags: &[&str], source: &str, object: & .current_dir(dir) .output() .unwrap_or_else(|e| panic!("run {}: {e}", c.clang.display())); - assert!(output.status.success(), "{}", first_error(&output.stderr)); + if !output.status.success() { + return Err(first_error(&output.stderr)); + } + Ok(()) } /// Compile a corpus case, and its companion if it has one (`104+_inline.c` /// for `104_inline`), and return the objects. Each is compiled as TinyCC's /// runner does, in the corpus's directory by its bare name: `__FILE__` is /// part of what some cases print. -pub fn compile_c(name: &str) -> Vec { +pub fn compile_c(name: &str) -> Result, String> { let c = c_sysroot(); let mut sources = vec![format!("{name}.c")]; if let Some((prefix, suffix)) = name.split_once('_') { @@ -83,18 +86,18 @@ pub fn compile_c(name: &str) -> Vec { .enumerate() .map(|(i, source)| { let object = scratch(name, &format!("-{i}.o")); - compile_one(&c, &testcases_dir(), CORPUS_FLAGS, source, &object); - object + compile_one(&c, &testcases_dir(), CORPUS_FLAGS, source, &object)?; + Ok(object) }) .collect() } /// Compile `.c` in `dir`, one of ToyOS's own C programs, to an object, /// with warnings refused. -pub fn compile_own_c(dir: &Path, name: &str) -> PathBuf { +pub fn compile_own_c(dir: &Path, name: &str) -> Result { let object = scratch(name, ".o"); - compile_one(&c_sysroot(), dir, &["-Wall", "-Werror"], &format!("{name}.c"), &object); - object + compile_one(&c_sysroot(), dir, &["-Wall", "-Werror"], &format!("{name}.c"), &object)?; + Ok(object) } /// Link flags a case needs beyond the corpus's, by case name. @@ -109,13 +112,13 @@ const LINK_FLAGS: &[(&str, &[&str])] = &[ /// Link `objects` into a ToyOS executable through the clang driver — which /// names `ld.lld` and the sysroot's `libtoyos_c.a`, and makes a PIE — and -/// return its bytes. +/// return its bytes, or why it did not link. /// /// **Without DWARF, and with `.symtab`**, as `[profile.toyos]` links every Rust /// guest binary: the archive brings its Rust libraries' DWARF into every case /// and nothing reads it, while `symbolize` and the loader's dlopen binding read /// `.symtab`. -pub fn link_toyos(objects: &[PathBuf], name: &str) -> Vec { +pub fn link_toyos(objects: &[PathBuf], name: &str) -> Result, String> { let c = c_sysroot(); let out = scratch(name, ".elf"); let output = Command::new(&c.clang) @@ -130,14 +133,15 @@ pub fn link_toyos(objects: &[PathBuf], name: &str) -> Vec { for object in objects { let _ = fs::remove_file(object); } - // One line: the harness reads a failed link for the symbol a declared case - // stops on, and LLD reports each undefined symbol on a line of its own. - assert!( - output.status.success(), - "clang could not link {name}: {}", - String::from_utf8_lossy(&output.stderr).lines().collect::>().join(" | "), - ); + // One line, as a failed compile's is: each case's refusal is one line of + // the corpus compile's list and of `NOT_RUN`'s report. + if !output.status.success() { + return Err(format!( + "clang could not link {name}: {}", + String::from_utf8_lossy(&output.stderr).lines().collect::>().join(" | ") + )); + } let linked = fs::read(&out).unwrap_or_else(|e| panic!("read {}: {e}", out.display())); let _ = fs::remove_file(&out); - linked + Ok(linked) } diff --git a/tests/common/libcchildren.rs b/tests/common/libcchildren.rs index 495f1adbee..bd0f353a1a 100644 --- a/tests/common/libcchildren.rs +++ b/tests/common/libcchildren.rs @@ -40,8 +40,8 @@ pub fn libc_children() -> Result<(), String> { names.sort(); let bins: Vec<(String, Vec)> = names .iter() - .map(|name| (name.clone(), compile::link_toyos(&[compile::compile_own_c(&dir, name)], name))) - .collect(); + .map(|name| Ok((name.clone(), compile::link_toyos(&[compile::compile_own_c(&dir, name)?], name)?))) + .collect::>()?; let case = compile::repo_root().join("tests/images/libccase.toml"); let rust_tests = compile::repo_root().join("tests/guest/rust-tests"); let probes: Vec<(String, Vec)> = diff --git a/tests/common/metal.rs b/tests/common/metal.rs index 1f4cb2aed9..cf126c363e 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -692,23 +692,29 @@ pub fn reached( out } -/// Build one batch's image, and answer where it landed. -fn build( +/// What one batch's image is built from: its config with the job list derived +/// onto it, and the files it puts on ROOT beside the config's own. +pub struct Staged { + pub config: PathBuf, + pub files: Vec<(String, Vec)>, +} + +/// Stage `batch` in `home`, its runner given `bound_ms` for the whole list: +/// the bench's image and a guest row's boot of the same batch both start here. +pub fn stage( root: &Path, - dir: &Path, - label: &str, + home: &Path, batch: &Batch, + bound_ms: u64, rust_bins: &[(String, Vec)], - quiet: bool, -) -> Result { - let home = at(dir, label); - std::fs::create_dir_all(&home).map_err(|e| format!("{}: {e}", home.display()))?; +) -> Result { + std::fs::create_dir_all(home).map_err(|e| format!("{}: {e}", home.display()))?; let committed = root.join(batch.config); let text = std::fs::read_to_string(&committed) .map_err(|e| format!("{}: {e}", committed.display()))?; let jobs: Vec<&str> = batch.jobs.iter().map(String::as_str).collect(); - let derived = image::derive(&text, batch.bound_ms(), &jobs, &batch.links) + let derived = image::derive(&text, bound_ms, &jobs, &batch.links) .map_err(|why| format!("{}: {why}", committed.display()))?; // **The job list is in the file name, not only in the file.** // `build_test_image` memoizes ROOT on the config's *path*, so two runs whose @@ -762,6 +768,20 @@ fn build( } extra.extend(reached(&reachable, &batch.jobs, rust_bins)); } + Ok(Staged { config, files: extra }) +} + +/// Build one batch's image, and answer where it landed. +fn build( + root: &Path, + dir: &Path, + label: &str, + batch: &Batch, + rust_bins: &[(String, Vec)], + quiet: bool, +) -> Result { + let home = at(dir, label); + let staged = stage(root, &home, batch, batch.bound_ms(), rust_bins)?; // The build a boot asked for, or — where it asked for none — the one its // arms imply: an actuator outside `kernel/src/params.rs` needs the kernel @@ -796,9 +816,9 @@ fn build( let deadline = format!("{}{}", toyos_tco::DEADLINE_PARAM, batch.deadline_ms()); let mut params: Vec<&str> = batch.params.clone(); params.push(&deadline); - let mut plan = toyos_build::build::Plan::new(toyos_build::arch::Arch::X86_64, &config, features, ¶ms); + let mut plan = toyos_build::build::Plan::new(toyos_build::arch::Arch::X86_64, &staged.config, features, ¶ms); plan.machines = toyos_build::microcode::Machines::Benches; - let bytes = toyos_build::build::build_test_image(root, &plan, quiet, &extra); + let bytes = toyos_build::build::build_test_image(root, &plan, quiet, &staged.files); let image = home.join("image.img"); std::fs::write(&image, &bytes).map_err(|e| format!("{}: {e}", image.display()))?; Ok(image) diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 13bc821be0..3f7c6fcf71 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -177,6 +177,10 @@ static FASTEST_BOOT_MS: AtomicU32 = AtomicU32::new(u32::MAX); /// host pays them at 1×. const REFERENCE_BOOT_MS: u32 = 1424; +/// The most [`host_scale`] pays a ceiling for a slow host, so one anomalous +/// boot cannot disable every guard in the suite at once. +pub const HOST_SCALE_CAP: u32 = 8; + fn record_boot(took: Duration) { let ms = took.as_millis().min(u32::MAX as u128) as u32; FASTEST_BOOT_MS.fetch_min(ms, Ordering::SeqCst); @@ -186,8 +190,7 @@ fn record_boot(took: Duration) { /// a fraction so that a 1.4× host is not rounded to 1. /// /// **Only ever upward**, since a ceiling that shrank would report wedges that -/// are not there; and at most 8×, so one anomalous boot cannot disable every -/// guard in the suite at once. +/// are not there; and at most [`HOST_SCALE_CAP`]. fn host_scale() -> (u32, u32) { let fastest = FASTEST_BOOT_MS.load(Ordering::SeqCst); // Before the first boot there is no measurement, and the sentinel must not @@ -195,7 +198,7 @@ fn host_scale() -> (u32, u32) { if fastest == u32::MAX || fastest <= REFERENCE_BOOT_MS { return (1, 1); } - (fastest.min(REFERENCE_BOOT_MS * 8), REFERENCE_BOOT_MS) + (fastest.min(REFERENCE_BOOT_MS * HOST_SCALE_CAP), REFERENCE_BOOT_MS) } /// The fastest boot seen, the reference, and the scale it produced — for a run's @@ -407,7 +410,7 @@ pub const NEVER_ANNOUNCED: &str = const WINDOW_LINES: usize = 40; /// The last [`WINDOW_LINES`] lines of `said`, under the count of what was cut. -fn window(said: &str) -> String { +pub fn window(said: &str) -> String { let lines: Vec<&str> = said.lines().collect(); let kept = lines.len().min(WINDOW_LINES); let head = if lines.len() > kept { @@ -1410,14 +1413,14 @@ impl Drop for HeldCpus { /// Which of [`DECLARED_KERNEL_BUILDS`] this boot wants. /// /// **A parameter never decides a build.** Every actuator lives in the one test -/// kernel, so asking for one selects that kernel and nothing more; the third -/// build is asked for by name and by one test. +/// kernel, so asking for one selects that kernel, which a boot may also name, +/// and nothing more; the third build is asked for by name and by one test. fn kernel_of(options: &BootOptions) -> Vec<&'static str> { if options.kernel_params.is_empty() { return options.kernel_features.to_vec(); } assert!( - options.kernel_features.is_empty(), + options.kernel_features.is_empty() || options.kernel_features == toyos_build::build::TEST_KERNEL, "a boot asking to arm {:?} also asks for the kernel build {:?}; an actuator is a \ parameter and the test kernel carries all of them", options.kernel_params, @@ -1836,6 +1839,52 @@ impl QemuInstance { Ok(said) } + /// Everything the guest says until QEMU exits, onto `log`, and `Ok` once + /// it has exited 0, which under `-no-reboot` a reset, a power-off and a + /// triple fault all do: what stopped the guest is the caller's to ask. Or + /// why it did not exit, as [`ceiling_verdict`] reads `ceiling` paid out + /// for this guest. + pub fn await_end(&mut self, log: &mut String, ceiling: Duration) -> Result<(), String> { + let ceiling = self.budget(ceiling); + let from = log.len(); + let start = Instant::now(); + let (mut last_line, mut said_waiting) = (start, start); + let mut lines = 0usize; + let mut dying: Option = None; + loop { + match self.rx.recv_timeout(VERDICT_POLL) { + Ok(line) => { + last_line = Instant::now(); + lines += 1; + if dying.is_none() && super::serial::died(&line) == Some(super::serial::Died::Kernel) { + dying = Some(line.clone()); + } + log.push_str(&line); + log.push('\n'); + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + if said_waiting.elapsed() >= WAIT_SAID { + said_waiting = Instant::now(); + eprintln!( + " [qemu] the job list has run {}s of its {}s guard, the guest last heard {}s ago", + start.elapsed().as_secs(), + ceiling.as_secs(), + last_line.elapsed().as_secs() + ); + } + if let Some(why) = ceiling_verdict(dying.as_deref(), start.elapsed(), ceiling, last_line.elapsed(), lines) { + return Err(WaitVerdict::new(why, &[&log[from..]]).to_string()); + } + } + let status = self.child.wait().map_err(|e| format!("QEMU could not be waited for: {e}"))?; + if !status.success() { + return Err(format!("QEMU exited {status}")); + } + Ok(()) + } + /// [`budget_smp`] for a host-side wait on this guest's own vCPU count. pub fn budget(&self, ceiling: Duration) -> Duration { budget_smp(ceiling, self.smp) diff --git a/tests/guest/rust-tests/src/bin/readdir_bound.rs b/tests/guest/rust-tests/src/bin/readdir_bound.rs index e55353f431..3f985d8f29 100644 --- a/tests/guest/rust-tests/src/bin/readdir_bound.rs +++ b/tests/guest/rust-tests/src/bin/readdir_bound.rs @@ -4,13 +4,30 @@ use std::collections::BTreeSet; use std::fs; const DIR: &str = "/tmp/readdir_bound"; -const ENTRIES: usize = 6_000; +/// Long names, so few files make a listing of several replies: the names +/// alone are 128,000 bytes, nearly twice a 64 KiB reply. +const ENTRIES: usize = 640; +const NAME_BYTES: usize = 200; +const _: () = assert!(ENTRIES * NAME_BYTES > 64 * 1024); + +fn name(i: usize) -> String { + let mut name = format!("entry-{i:06}-"); + name.extend(std::iter::repeat_n('x', NAME_BYTES - name.len())); + name +} fn main() { fs::create_dir(DIR).expect("create listing directory"); - // The names alone exceed a 64 KiB reply, independently of wire overhead. for i in 0..ENTRIES { - fs::write(format!("{DIR}/entry-{i:06}"), b"").expect("create file"); + if let Err(e) = fs::write(format!("{DIR}/{}", name(i)), b"") { + // What it made goes first: a `/tmp` left full fails every later + // job of the boot that writes it. + for made in 0..i { + let _ = fs::remove_file(format!("{DIR}/{}", name(made))); + } + let _ = fs::remove_dir(DIR); + panic!("create file {i} of {ENTRIES}: {e:?}"); + } } let mut names = BTreeSet::new(); for entry in fs::read_dir(DIR).expect("open listing") { @@ -21,7 +38,7 @@ fn main() { } assert_eq!(names.len(), ENTRIES, "listing omitted entries"); for i in 0..ENTRIES { - let name = format!("entry-{i:06}"); + let name = name(i); assert!(names.contains(&name), "listing omitted {name}"); fs::remove_file(format!("{DIR}/{name}")).expect("remove file"); } diff --git a/tests/toyos.rs b/tests/toyos.rs index 75133d5c45..7d266e9deb 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -383,9 +383,9 @@ const DRIVEN_AND_SHARED: &[&str] = &[ // Its shared run is each bench's verdict on its own CPU's fault path; // `a_user_copy_faults_in_mid_copy_or_ends_as_bad_address` runs it under QEMU. "user_copy_faults", - // Its shared run is each bench's verdict; - // `tmp_holds_empty_files_by_what_each_costs` runs it under QEMU, and - // `virt_tmp_holds_empty_files_by_what_each_costs` on AArch64's. + // Its shared run is the x86-64 verdict; + // `virt_tmp_holds_empty_files_by_what_each_costs` builds it for AArch64 + // and runs it on that architecture's case. "tmp_empty_files", // Its shared run is ring's known answers in a guest; the crypto boot runs it // first on each bench, and `crypto_instrument_prints_its_table` before the @@ -453,8 +453,8 @@ const SCREEN_TESTS: &[ScreenTest] = &[ // the unprivileged copies and their EL1 fixup (`arch::uaccess`). ("virt_user_copy_faults", qemu::Profile::Virt, |profile, _| virt_job(profile, &format!("test_rs_{VIRT_USER_COPY}"))), ("virt_vm_map_shares", qemu::Profile::Virt, |profile, _| virt_job(profile, &format!("test_rs_{VIRT_VM_MAP}"))), - // `tmp_holds_empty_files_by_what_each_costs` on AArch64: the paged - // objects' table and its notices are shared memory-management code. + // `tmp_empty_files` on AArch64: the paged objects' table and its notices + // are shared memory-management code. ("virt_tmp_holds_empty_files_by_what_each_costs", qemu::Profile::Virt, |profile, _| virt_tmp_holds_empty_files_by_what_each_costs(profile)), ("virt_ring0_timer_in_syscall", qemu::Profile::Virt, |profile, _| virt_job(profile, &format!("test_rs_{VIRT_RING0_TIMER}"))), // Emulated, with `virt_el1_smp`, `virt_off_names_the_cpus_left_on` and @@ -526,6 +526,16 @@ type MachineTest = (&'static str, fn(&Path) -> Result<(), String>); /// The tests whose machine shape *is* the test, each on a boot of its own. const MACHINE_TESTS: &[MachineTest] = &[ + // The metal profile's two shared boots, each run whole: its members are + // guest binaries and corpus cases whose claims are a booted kernel's and + // its programs' together, which no host build has, and whose verdicts are + // order, completion, contents and counts, which QEMU establishes. Each + // boot is staged as the T14's is ([`shared_members`]) and its members are + // judged by their own exits, on QEMU x86-64's q35 (`Profile::Headless`, + // `qemu::SUITE_ARCH`): KVM on the guest runner, TCG where the host has + // none. AArch64 builds no member. + ("testcases_members_exit_0", |_| shared_members("testcases")), + ("shared_debug_members_exit_0", |_| shared_members("shared-debug")), // A `drives` row's one process per function, on QEMU x86-64's q35 with // two `edu` (`Profile::HeadlessTwoEdu`): which functions get one is // `toyos-supervisor`'s, host-tested, and this is the supervisor @@ -721,12 +731,6 @@ const MACHINE_TESTS: &[MachineTest] = &[ ("a_user_copy_highest_first_faults_or_ends_as_bad_address", |test_config| { a_user_copy_faults_in_mid_copy_or_ends_as_bad_address(test_config, &["copies-go-down"]) }), - // `/tmp` holds more empty files than its guest has MiB of memory, and - // each one's last holder going reaches its server: the memory file - // server's objects as the kernel makes, charges and reports them, which - // no host build has. One boot of QEMU's x86-64 machine; the shared boots - // run it on each bench. - ("tmp_holds_empty_files_by_what_each_costs", |test_config| tmp_holds_empty_files_by_what_each_costs(test_config)), // The boot's log written through usbd onto the stick it booted from, and // still there once the machine has stopped: the path every metal boot's // `/log` takes, which no other QEMU row reads, because their console is @@ -2349,12 +2353,13 @@ const FIRMWARE_LEFT: &[metal::Arm] = &[metal::Arm { ..metal::once("iommu-firmware-left", "tests/images/testcases.toml", &["iommu-firmware-left"], &[]) }]; -/// The two boots every discovered member rides on the T14, each behind its -/// rows' jobs: `testcases`, the shipping kernel's, carries every discovered -/// Rust binary, then the C corpus, then [`LAST_MEMBERS`]; `shared-debug` -/// carries [`ACTUATOR_TESTS`] on the kernel that has `SYS_DEBUG`, armed with -/// [`SELFTEST_PARAMS`]. A second boot rather than one image for the whole -/// set: what those need is a syscall number the rest must not have. +/// The two boots every discovered member rides, on the T14 behind its rows' +/// jobs and on QEMU by [`shared_members`]: `testcases`, the shipping kernel's, +/// carries every discovered Rust binary, then the C corpus, then +/// [`LAST_MEMBERS`]; `shared-debug` carries [`ACTUATOR_TESTS`] on the kernel +/// that has `SYS_DEBUG`, armed with [`SELFTEST_PARAMS`]. A second boot rather +/// than one image for the whole set: what those need is a syscall number the +/// rest must not have. fn shared_metal(c_bins: &[(String, Vec)]) -> Vec { let rust = |names: Vec| { names.into_iter().map(|n| metal::Member { job: format!("test_rs_{n}"), adds_ms: toyos_tco::RUST_MEMBER_MS }) @@ -2378,6 +2383,147 @@ fn shared_metal(c_bins: &[(String, Vec)]) -> Vec { ] } +/// The shared boot `label` of [`shared_metal`] on QEMU: staged as its T14 +/// boot is, its members' job list run to the list's `reboot`, and each +/// member judged by its own exit. +fn shared_members(label: &str) -> Result<(), String> { + let (c_bins, rust_bins) = build_shared_bins(); + let boot = shared_metal(c_bins) + .into_iter() + .find(|boot| boot.boot == label) + .unwrap_or_else(|| panic!("the metal profile has no shared boot {label:?}")); + let batches = metal::batches(&[], std::slice::from_ref(&boot))?; + let batch = &batches[label]; + // The runner's bound, widened as far as this harness pays any host: + // under QEMU the host's ceiling, paid for the host this boot measures, + // ends a list that does not end itself. + let bound_ms = batch.bound_ms() * u64::from(qemu::HOST_SCALE_CAP); + let home = common::lane::dir().join(label); + let staged = metal::stage(&compile::repo_root(), &home, batch, bound_ms, rust_bins)?; + let mut qemu = QemuInstance::boot_with_options( + &staged.config, + &[], + &[], + BootOptions { + qmp: true, + kernel_params: boot.params, + kernel_features: boot.features, + extra_root_files: staged.files, + ..Default::default() + }, + ); + // Opened at once: QMP delivers no event emitted before its client + // connected, and the list's reboot is that event. + let mut stop = qemu::QmpShutdown::open(qemu.qmp_socket(), qemu.budget(qemu::GUEST_QUIET)); + let mut console = format!("{}\n", qemu.boot_log()); + let ended = qemu.await_end(&mut console, Duration::from_millis(batch.bound_ms())).map(|()| stop.reason()); + let passed = judge_shared(label, &console, &batch.jobs, ended)?; + eprintln!(" [shared] {label}: {passed} member(s) exited 0, and the list's reboot ended the boot"); + Ok(()) +} + +/// A shared boot's verdict on what it left: `console`, and how QEMU ended, +/// with the reason its `SHUTDOWN` event named. Each of `jobs` by its own +/// exit ([`member_verdicts`]), and the boot by its list's `reboot`: the +/// kernel's `Rebooting.` and a `guest-reset`, since a reset, a power-off and +/// a triple fault all end a `-no-reboot` QEMU with status 0. A kernel death +/// anywhere reds it too. Answers how many members passed. +fn judge_shared(label: &str, console: &str, jobs: &[String], ended: Result, String>) -> Result { + let rebooted = match ended { + Err(why) => Err(why.lines().next().unwrap_or_default().to_string()), + Ok(reason) if reason.as_deref() != Some("guest-reset") => Err(format!("QEMU stopped it for {reason:?}, not guest-reset")), + Ok(_) if !console.lines().filter_map(toyos_logstream::parse).any(|said| { + said.source == toyos_logstream::Source::Kernel && said.text == bootlog::REBOOTING + }) => Err(format!("the kernel never said {:?}", bootlog::REBOOTING)), + Ok(_) => Ok(()), + }; + let verdicts = member_verdicts(console, jobs); + let death = serial::kernel_death(console); + let red: Vec<(&str, &Member)> = verdicts.iter().filter(|(_, member)| *member != Member::Passed).map(|(job, member)| (*job, member)).collect(); + if red.is_empty() && rebooted.is_ok() && death.is_none() { + return Ok(verdicts.len()); + } + let mut said = if red.is_empty() { + format!("each of {label}'s {} member(s) exited 0, and its boot did not end cleanly", verdicts.len()) + } else { + let names: Vec<&str> = red.iter().map(|(job, _)| *job).take(12).collect(); + let more = red.len().saturating_sub(names.len()); + let more = if more > 0 { format!(" and {more} more") } else { String::new() }; + format!("{} of {label}'s {} member(s) did not exit 0: {}{more}", red.len(), verdicts.len(), names.join(", ")) + }; + if let Err(why) = &rebooted { + said.push_str(&format!("\nthe boot did not end by its list's reboot: {why}")); + } + if let Some(line) = death { + said.push_str(&format!("\nthe kernel died: {}", line.trim())); + } + let mut unreached: Vec<&str> = Vec::new(); + for (job, member) in red { + match member { + Member::Failed(why) => said.push_str(&format!("\n--- {job}: {why}")), + Member::Unreached => unreached.push(job), + Member::Passed => unreachable!("only reds are listed"), + } + } + if !unreached.is_empty() { + said.push_str(&format!("\n--- the list never reached: {}", unreached.join(", "))); + } + Err(qemu::WaitVerdict::new(said, &[console]).to_string()) +} + +/// What a job list's console says of one of its jobs. +#[derive(Debug, PartialEq)] +enum Member { + Passed, + /// Why not, over its own lines. + Failed(String), + /// The list never got to it. + Unreached, +} + +/// What the runner said of each of `jobs` on a job list's `console`, in their +/// order. A marker is the runner's only under the head the supervisor +/// registered for it, with no `pid=`, which a job's line in the runner's ring +/// carries. +fn member_verdicts<'a>(console: &str, jobs: &'a [String]) -> Vec<(&'a str, Member)> { + use toyos_logstream::{parse, Source}; + let lines: Vec<&str> = console.lines().collect(); + let mut started: BTreeMap<&str, usize> = BTreeMap::new(); + let mut ended: BTreeMap<&str, (usize, &str)> = BTreeMap::new(); + for (at, line) in lines.iter().enumerate() { + let Some(said) = parse(line) else { continue }; + if said.source != Source::Program("test-runner") || said.rest.contains("pid=") { + continue; + } + let text = said.text; + if let Some(job) = text.strip_prefix("===TEST_START ").and_then(|rest| rest.strip_suffix("===")) { + started.entry(job).or_insert(at); + } else if let Some(rest) = text.strip_prefix("===TEST_END ").and_then(|rest| rest.strip_suffix("===")) { + let (job, said) = rest.split_once(' ').unwrap_or((rest, "")); + ended.entry(job).or_insert((at, said)); + } + } + let own = |from: usize, to: usize| qemu::window(&lines[from..to].join("\n")); + jobs.iter() + .map(|job| { + let member = match (started.get(job.as_str()), ended.get(job.as_str())) { + (_, Some((_, "exit=0"))) => Member::Passed, + (from, Some(&(to, said))) => { + let said = match (said.strip_prefix("exit="), said.strip_prefix("error=")) { + (Some(code), _) => format!("exited {code}"), + (_, Some(why)) => format!("did not run: {why}"), + _ => format!("ended saying {said:?}"), + }; + Member::Failed(format!("{said}\n{}", own(from.copied().unwrap_or(to), to + 1))) + } + (Some(&from), None) => Member::Failed(format!("started and never ended\n{}", own(from, lines.len()))), + (None, None) => Member::Unreached, + }; + (job.as_str(), member) + }) + .collect() +} + /// **One boot for both measurements this suite takes rather than reads.** /// `tests/images/latencycase.toml` is the only config that endows the real-time band, and /// the TLB bench is a boot parameter rather than a job — so arming this image @@ -2665,12 +2811,9 @@ fn discover_rust_tests() -> Vec { } fn compile_c_tests(names: &[String]) -> Vec<(String, Vec)> { - // Made before the hook below goes in: a C sysroot that cannot be made is - // no case's failure, and says so in its own words. + // Made before any case: a C sysroot that cannot be made is no case's + // failure, and says so in its own words. compile::c_sysroot(); - // Suppress panic messages during compilation — we handle failures via catch_unwind. - let prev_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(|_| {})); // A clang and a link per case, so the cases are spread over threads; the // result is in `names`' order whatever order they finish in. @@ -2682,21 +2825,14 @@ fn compile_c_tests(names: &[String]) -> Vec<(String, Vec)> { scope.spawn(move || { chunk .iter() - .map(|name| { - let built = std::panic::catch_unwind(|| { - compile::link_toyos(&compile::compile_c(name), name) - }); - (name, built.map_err(|e| panic_message(&e))) - }) + .map(|name| (name, compile::compile_c(name).and_then(|objects| compile::link_toyos(&objects, name)))) .collect::>() }) }) .collect(); - workers.into_iter().flat_map(|w| w.join().expect("a compile thread")).collect() + workers.into_iter().flat_map(|w| w.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect() }); - std::panic::set_hook(prev_hook); - let mut bins = Vec::new(); let mut broken: Vec<(&str, String)> = Vec::new(); for (name, built) in results { @@ -2730,10 +2866,6 @@ fn check_not_run() { let mut wrong: Vec = Vec::new(); let mut seen: BTreeSet<&str> = BTreeSet::new(); - compile::c_sysroot(); - let prev_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(|_| {})); - for entry in NOT_RUN { let case = entry.case; let before = wrong.len(); @@ -2745,10 +2877,8 @@ fn check_not_run() { wrong.push(format!("{case}: no such file in the corpus — a rename left this behind")); continue; } - let compiled = std::panic::catch_unwind(|| compile::compile_c(case)); - match (&entry.stage, compiled) { - (Stage::Refused(says), Err(e)) => { - let said = panic_message(&e); + match (&entry.stage, compile::compile_c(case)) { + (Stage::Refused(says), Err(said)) => { if !said.contains(says) { wrong.push(format!( "{case}: refused, but not for the declared reason.\n \ @@ -2760,15 +2890,12 @@ fn check_not_run() { "{case}: compiles now — it was declared to stop with {says:?}. \ The fix arrived; delete the entry and let the case run." )), - (_, Err(e)) => wrong.push(format!( - "{case}: no longer compiles, and it was declared to get further: {}", - panic_message(&e) + (_, Err(said)) => wrong.push(format!( + "{case}: no longer compiles, and it was declared to get further: {said}" )), (stage, Ok(objects)) => { - let linked = std::panic::catch_unwind(|| compile::link_toyos(&objects, case)); - match (stage, linked) { - (Stage::NoLink(symbol), Err(e)) => { - let said = panic_message(&e); + match (stage, compile::link_toyos(&objects, case)) { + (Stage::NoLink(symbol), Err(said)) => { if !said.contains(symbol) { wrong.push(format!( "{case}: the link fails on something else.\n \ @@ -2780,9 +2907,8 @@ fn check_not_run() { "{case}: links now — it was declared to fail on {symbol:?}. \ The fix arrived; delete the entry and let the case run." )), - (Stage::Built, Err(e)) => wrong.push(format!( - "{case}: no longer links, and it was declared to build: {}", - panic_message(&e) + (Stage::Built, Err(said)) => wrong.push(format!( + "{case}: no longer links, and it was declared to build: {said}" )), (Stage::Built, Ok(_)) => {} (Stage::Refused(_), _) => unreachable!("handled above"), @@ -2794,8 +2920,6 @@ fn check_not_run() { } } - std::panic::set_hook(prev_hook); - assert!( wrong.is_empty(), "NOT_RUN no longer describes the corpus. Every entry is attempted to its declared \ @@ -2804,17 +2928,6 @@ fn check_not_run() { ); } -/// What a caught panic said, first line, whole. A refusal quoted in `NOT_RUN` -/// is compared against this, so nothing here may shorten it. -fn panic_message(e: &Box) -> String { - let full = e - .downcast_ref::() - .cloned() - .or_else(|| e.downcast_ref::<&str>().map(|s| s.to_string())) - .unwrap_or_else(|| "".to_string()); - full.lines().next().unwrap_or_default().to_string() -} - /// A case's committed expectation as both comparators read it, the host's and /// the guest's `ccheck`, or `None` where none is committed. TinyCC's runner /// captured its warnings about the case with its output, and neither compares @@ -5934,29 +6047,8 @@ fn ring3_debug_traps_end_their_process(test_config: &Path) -> Result<(), String> /// the machine's memory and 1,024 more, was opened to read and removed. const TMP_FILES_SAID: &str = "empty files, one per MiB of memory and 1024 more, made, read-opened and removed"; -/// `tmp_empty_files` alone: more empty files in `/tmp` than the guest has -/// MiB of memory, each opened to read and closed, and removed. -fn tmp_holds_empty_files_by_what_each_costs(test_config: &Path) -> Result<(), String> { - const JOB: &str = "tmp_empty_files"; - let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); - let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], BootOptions::default()); - serial::Serial::boot(&qemu).must_be_clean()?; - let result = qemu.run_test("test_rs_tmp_empty_files", Duration::from_secs(300)); - if let Some(why) = &result.error { - return Err(format!("{why}\nthe job said:\n{}", result.stdout)); - } - if result.exit_code != Some(0) { - return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); - } - let Some(said) = result.stdout.lines().find(|l| l.contains(TMP_FILES_SAID)) else { - return Err(format!("the job never said {TMP_FILES_SAID:?}:\n{}", result.stdout)); - }; - eprintln!(" [tmp] {}", said.trim()); - Ok(()) -} - -/// The same job on AArch64: `tests/images/virttmpcase.toml`, whose one job it -/// is, beside the memory file server alone. +/// `tmp_empty_files` on AArch64: `tests/images/virttmpcase.toml`, whose one +/// job it is, beside the memory file server alone. fn virt_tmp_holds_empty_files_by_what_each_costs(profile: qemu::Profile) -> Result<(), String> { let config = compile::repo_root().join("tests/images/virttmpcase.toml"); let options = BootOptions { @@ -8644,21 +8736,24 @@ fn run_debug_mode(c_tests: &[(String, Vec)], rust_bins: &[(String, Vec)] /// Built test binaries, each by the name `run` takes. type Binaries = Vec<(String, Vec)>; -/// The binaries the shared boots carry, C and Rust: every Rust test binary, -/// and the C corpus, compiled once its declared cases have been attempted to -/// their stages. -fn build_shared_bins() -> (Binaries, Binaries) { - let rust_tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/guest/rust-tests"); - let mut rust_bins = qemu::build_toyos_bins(&rust_tests_dir); - rust_bins.push(crypto_parity_unasserted()); - let c_names = discover_c_tests(); - eprintln!( - "[toyos] Compiling {} C tests, and attempting {} declared ones...", - c_names.len(), - NOT_RUN.len() - ); - check_not_run(); - (compile_c_tests(&c_names), rust_bins) +/// The binaries the shared boots carry, C and Rust, built once a run: every +/// Rust test binary, and the C corpus, compiled once its declared cases have +/// been attempted to their stages. Once, because both shared rows carry them. +fn build_shared_bins() -> &'static (Binaries, Binaries) { + static BUILT: std::sync::OnceLock<(Binaries, Binaries)> = std::sync::OnceLock::new(); + BUILT.get_or_init(|| { + let rust_tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/guest/rust-tests"); + let mut rust_bins = qemu::build_toyos_bins(&rust_tests_dir); + rust_bins.push(crypto_parity_unasserted()); + let c_names = discover_c_tests(); + eprintln!( + "[toyos] Compiling {} C tests, and attempting {} declared ones...", + c_names.len(), + NOT_RUN.len() + ); + check_not_run(); + (compile_c_tests(&c_names), rust_bins) + }) } /// `crypto_parity` with debug assertions off, by the name its job runs. @@ -9253,7 +9348,7 @@ fn main() { } }; let (c_bins, rust_bins) = build_shared_bins(); - let boots = shared_metal(&c_bins); + let boots = shared_metal(c_bins); let taken = match metal::select(filters, &parsed.boots, bench, METAL, &boots) { Ok(taken) => taken, Err(refusal) => { @@ -9266,7 +9361,7 @@ fn main() { // and "measured nothing" — a run that staged images and never reached // the machine has no claim to make. run.exit( - match metal::run(mode, bench, &host, &taken, &rust_bins, !nocapture && !debug_mode) { + match metal::run(mode, bench, &host, &taken, rust_bins, !nocapture && !debug_mode) { metal::Verdict::Green => 0, metal::Verdict::Red => 1, metal::Verdict::Staged => 2, @@ -9283,7 +9378,7 @@ fn main() { if debug_mode { let (c_bins, rust_bins) = build_shared_bins(); - run_debug_mode(&c_bins, &rust_bins); + run_debug_mode(c_bins, rust_bins); return; } @@ -9292,7 +9387,11 @@ fn main() { // A filter that takes nothing would be dropped in silence beside one that // takes something. if let Some(dead) = filters.iter().find(|f| !declared().any(|name| name.contains(**f))) { - eprintln!("No test matches filter {dead:?}"); + if shared.iter().any(|member| member.contains(*dead)) { + eprintln!("No test matches filter {dead:?}, which names a shared member: a shared boot runs whole, as `members_exit_0`"); + } else { + eprintln!("No test matches filter {dead:?}"); + } run.exit(1); }