The shared boots run as guest rows, each member judged by its own exit

issues/the-nightly-is-frozen-so-nothing-runs-the-guest-matrix.md

@@ -15,6 +15,12 @@ writer, main's toolchain release, and the fresh-machine and macOS builds.
 Guest rows run only where an agent dispatches them on the guest runner
 (`guest-rows.yml`), so a regression in a row no PR selected goes unseen.
 
+The unfiltered suite now carries the shared boots' guest rows,
+`testcases_members_exit_0` and `shared_debug_members_exit_0`, and with them an
+in-suite build of every guest binary and corpus case. Their cost on a TCG lane
+is unmeasured, and resumption measures it rather than meeting it at a lane's
+timeout.
+
 `build/ci/qemu-version` moved to 11.1.3, the guest runner's and the dev host's
 QEMU, while `guest.yml`'s container still installs 11.1.1 from its snapshot
 archive, so its instrument step reds until the container moves to an image

issues/the-t14-reboots-through-ubuntu-for-every-test.md

@@ -8,8 +8,10 @@ opened: 2026-09-29
 
 The owner's direction: the tests run one after another in a ToyOS that stays
 booted, driven by the host over ssh, with no reboot between them and no Ubuntu.
-The T14 is the integration tier: the shared boot runs only as `shared_metal`
-and `c_corpus_metal`, and
+The T14 is the integration tier for hardware: the shared boots run there as
+`shared_metal` and `c_corpus_metal` stage them, and on QEMU as the guest rows
+`testcases_members_exit_0` and `shared_debug_members_exit_0`, which stage
+the same members in the same order without the bench rows' own jobs;
 `issues/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md` adds
 metal rows.
 

tests/checks.rs

@@ -1321,6 +1321,81 @@ mod checks {
         metal_checks::a_failing_shared_member_fails_itself_alone();
     }
 
+    /// A shared boot's guest row judges each member by the runner's own end
+    /// for it and nothing else: another member's end, a longer name's that
+    /// opens with its own, a job's line, another program's or a kernel
+    /// record's does not answer for it.
+    #[test]
+    fn a_shared_member_is_judged_by_its_own_end() {
+        let jobs: Vec<String> = ["test_rs_passes", "test_rs_fails", "40_stdio", "test_rs_stuck", "test_rs_stuck_longer", "test_rs_never"]
+            .iter()
+            .map(ToString::to_string)
+            .collect();
+        let verdicts = member_verdicts(SHARED_CONSOLE, &jobs);
+        let said = |job: &str| verdicts.iter().find(|(name, _)| *name == job).map(|(_, member)| member).expect(job);
+        assert_eq!(said("test_rs_passes"), &Member::Passed);
+        assert_eq!(said("test_rs_stuck_longer"), &Member::Passed);
+        let failed = |job: &str| match said(job) {
+            Member::Failed(why) => why.clone(),
+            other => panic!("{job} is {other:?}, not failed"),
+        };
+        let fails = failed("test_rs_fails");
+        assert!(fails.starts_with("exited 101\n") && fails.contains("the census moved"), "{fails}");
+        assert!(failed("40_stdio").starts_with("did not run: entity not found"));
+        let stuck = failed("test_rs_stuck");
+        assert!(stuck.starts_with("started and never ended\n") && stuck.contains("still going"), "{stuck}");
+        assert_eq!(said("test_rs_never"), &Member::Unreached);
+        assert_eq!(verdicts.iter().map(|(job, _)| *job).collect::<Vec<_>>(), jobs.iter().map(String::as_str).collect::<Vec<_>>());
+    }
+
+    /// A list's runner lines, each member's, and the jobs' and others' lines
+    /// that must not answer for any of them.
+    const SHARED_CONSOLE: &str = "\
+[ 1.200 cpu0 kernel] Boot: complete (1200ms)
+[ 1.300 test-runner] ===READY===
+[ 1.301 test-runner] ===TEST_START test_rs_passes===
+[ 1.302 test-runner] ===TEST_END test_rs_passes exit=0===
+[ 1.303 test-runner] ===TEST_START test_rs_fails===
+[ 1.304 test-runner pid=7] assertion failed: the census moved
+[ 1.305 test-runner] ===TEST_END test_rs_fails exit=101===
+[ 1.306 test-runner] ===TEST_START 40_stdio===
+[ 1.307 test-runner] ===TEST_END 40_stdio error=entity not found===
+[ 1.308 test-runner] ===TEST_START test_rs_stuck===
+[ 1.309 test-runner] ===TEST_START test_rs_stuck_longer===
+[ 1.310 test-runner] ===TEST_END test_rs_stuck_longer exit=0===
+[ 1.311 cpu1 kernel] ===TEST_END test_rs_never exit=0===
+[ 1.311 fileserver] ===TEST_END test_rs_never exit=0===
+[ 1.312 test-runner pid=11] ===TEST_END test_rs_stuck exit=0===
+[ 1.313 test-runner pid=11] still going
+";
+
+    /// A shared boot ends green only by its list's reboot: the kernel's own
+    /// `Rebooting.` and QEMU's `guest-reset`, as well as every member's exit.
+    /// A silent reset after the last member, any other stop, a `Rebooting.`
+    /// a program says, and a wait that never saw QEMU exit are red.
+    #[test]
+    fn a_shared_boot_ends_by_its_lists_reboot() {
+        let jobs = vec!["test_rs_passes".to_string()];
+        let ran = "\
+[ 1.300 test-runner] ===READY===
+[ 1.301 test-runner] ===TEST_START test_rs_passes===
+[ 1.302 test-runner] ===TEST_END test_rs_passes exit=0===
+[ 1.303 test-runner] ===TEST_START reboot===
+";
+        let rebooted = format!("{ran}[ 1.400 cpu0 kernel] Rebooting.\n");
+        let reset = || Ok(Some("guest-reset".to_string()));
+        assert_eq!(judge_shared("fixture", &rebooted, &jobs, reset()), Ok(1));
+        let red = |console: &str, ended: Result<Option<String>, String>, why: &str| {
+            let said = judge_shared("fixture", console, &jobs, ended).expect_err(why);
+            assert!(said.contains("did not end by its list's reboot"), "{why}: {said}");
+        };
+        red(ran, reset(), "a silent reset after the last member");
+        red(&format!("{ran}[ 1.400 toybox] Rebooting.\n"), reset(), "a program's Rebooting.");
+        red(&rebooted, Ok(Some("guest-shutdown".to_string())), "a power-off");
+        red(&rebooted, Ok(None), "a stop QMP never named");
+        red(&rebooted, Err("STALLED: the guest went quiet".to_string()), "a wait that never saw QEMU exit");
+    }
+
     #[test]
     fn metal_boots_last_job_is_behind_every_other() {
         metal_checks::a_boots_last_job_is_behind_every_other();

tests/common/compile.rs

@@ -50,8 +50,8 @@ fn first_error(stderr: &[u8]) -> String {
 }
 
 /// Compile `source` in `dir` to an object with the toolchain's clang, from
-/// that directory and by its bare name.
-fn compile_one(c: &CSysroot, dir: &Path, flags: &[&str], source: &str, object: &Path) {
+/// that directory and by its bare name, or the first error it reported.
+fn compile_one(c: &CSysroot, dir: &Path, flags: &[&str], source: &str, object: &Path) -> Result<(), String> {
     let output = Command::new(&c.clang)
         .args(c.args())
         .args(flags)
@@ -62,14 +62,17 @@ fn compile_one(c: &CSysroot, dir: &Path, flags: &[&str], source: &str, object: &
         .current_dir(dir)
         .output()
         .unwrap_or_else(|e| panic!("run {}: {e}", c.clang.display()));
-    assert!(output.status.success(), "{}", first_error(&output.stderr));
+    if !output.status.success() {
+        return Err(first_error(&output.stderr));
+    }
+    Ok(())
 }
 
 /// Compile a corpus case, and its companion if it has one (`104+_inline.c`
 /// for `104_inline`), and return the objects. Each is compiled as TinyCC's
 /// runner does, in the corpus's directory by its bare name: `__FILE__` is
 /// part of what some cases print.
-pub fn compile_c(name: &str) -> Vec<PathBuf> {
+pub fn compile_c(name: &str) -> Result<Vec<PathBuf>, String> {
     let c = c_sysroot();
     let mut sources = vec![format!("{name}.c")];
     if let Some((prefix, suffix)) = name.split_once('_') {
@@ -83,18 +86,18 @@ pub fn compile_c(name: &str) -> Vec<PathBuf> {
         .enumerate()
         .map(|(i, source)| {
             let object = scratch(name, &format!("-{i}.o"));
-            compile_one(&c, &testcases_dir(), CORPUS_FLAGS, source, &object);
-            object
+            compile_one(&c, &testcases_dir(), CORPUS_FLAGS, source, &object)?;
+            Ok(object)
         })
         .collect()
 }
 
 /// Compile `<name>.c` in `dir`, one of ToyOS's own C programs, to an object,
 /// with warnings refused.
-pub fn compile_own_c(dir: &Path, name: &str) -> PathBuf {
+pub fn compile_own_c(dir: &Path, name: &str) -> Result<PathBuf, String> {
     let object = scratch(name, ".o");
-    compile_one(&c_sysroot(), dir, &["-Wall", "-Werror"], &format!("{name}.c"), &object);
-    object
+    compile_one(&c_sysroot(), dir, &["-Wall", "-Werror"], &format!("{name}.c"), &object)?;
+    Ok(object)
 }
 
 /// Link flags a case needs beyond the corpus's, by case name.
@@ -109,13 +112,13 @@ const LINK_FLAGS: &[(&str, &[&str])] = &[
 
 /// Link `objects` into a ToyOS executable through the clang driver — which
 /// names `ld.lld` and the sysroot's `libtoyos_c.a`, and makes a PIE — and
-/// return its bytes.
+/// return its bytes, or why it did not link.
 ///
 /// **Without DWARF, and with `.symtab`**, as `[profile.toyos]` links every Rust
 /// guest binary: the archive brings its Rust libraries' DWARF into every case
 /// and nothing reads it, while `symbolize` and the loader's dlopen binding read
 /// `.symtab`.
-pub fn link_toyos(objects: &[PathBuf], name: &str) -> Vec<u8> {
+pub fn link_toyos(objects: &[PathBuf], name: &str) -> Result<Vec<u8>, String> {
     let c = c_sysroot();
     let out = scratch(name, ".elf");
     let output = Command::new(&c.clang)
@@ -130,14 +133,15 @@ pub fn link_toyos(objects: &[PathBuf], name: &str) -> Vec<u8> {
     for object in objects {
         let _ = fs::remove_file(object);
     }
-    // One line: the harness reads a failed link for the symbol a declared case
-    // stops on, and LLD reports each undefined symbol on a line of its own.
-    assert!(
-        output.status.success(),
-        "clang could not link {name}: {}",
-        String::from_utf8_lossy(&output.stderr).lines().collect::<Vec<_>>().join(" | "),
-    );
+    // One line, as a failed compile's is: each case's refusal is one line of
+    // the corpus compile's list and of `NOT_RUN`'s report.
+    if !output.status.success() {
+        return Err(format!(
+            "clang could not link {name}: {}",
+            String::from_utf8_lossy(&output.stderr).lines().collect::<Vec<_>>().join(" | ")
+        ));
+    }
     let linked = fs::read(&out).unwrap_or_else(|e| panic!("read {}: {e}", out.display()));
     let _ = fs::remove_file(&out);
-    linked
+    Ok(linked)
 }

tests/common/libcchildren.rs

@@ -40,8 +40,8 @@ pub fn libc_children() -> Result<(), String> {
     names.sort();
     let bins: Vec<(String, Vec<u8>)> = names
         .iter()
-        .map(|name| (name.clone(), compile::link_toyos(&[compile::compile_own_c(&dir, name)], name)))
-        .collect();
+        .map(|name| Ok((name.clone(), compile::link_toyos(&[compile::compile_own_c(&dir, name)?], name)?)))
+        .collect::<Result<_, String>>()?;
     let case = compile::repo_root().join("tests/images/libccase.toml");
     let rust_tests = compile::repo_root().join("tests/guest/rust-tests");
     let probes: Vec<(String, Vec<u8>)> =

tests/common/metal.rs

@@ -692,23 +692,29 @@ pub fn reached(
     out
 }
 
-/// Build one batch's image, and answer where it landed.
-fn build(
+/// What one batch's image is built from: its config with the job list derived
+/// onto it, and the files it puts on ROOT beside the config's own.
+pub struct Staged {
+    pub config: PathBuf,
+    pub files: Vec<(String, Vec<u8>)>,
+}
+
+/// Stage `batch` in `home`, its runner given `bound_ms` for the whole list:
+/// the bench's image and a guest row's boot of the same batch both start here.
+pub fn stage(
     root: &Path,
-    dir: &Path,
-    label: &str,
+    home: &Path,
     batch: &Batch,
+    bound_ms: u64,
     rust_bins: &[(String, Vec<u8>)],
-    quiet: bool,
-) -> Result<PathBuf, String> {
-    let home = at(dir, label);
-    std::fs::create_dir_all(&home).map_err(|e| format!("{}: {e}", home.display()))?;
+) -> Result<Staged, String> {
+    std::fs::create_dir_all(home).map_err(|e| format!("{}: {e}", home.display()))?;
 
     let committed = root.join(batch.config);
     let text = std::fs::read_to_string(&committed)
         .map_err(|e| format!("{}: {e}", committed.display()))?;
     let jobs: Vec<&str> = batch.jobs.iter().map(String::as_str).collect();
-    let derived = image::derive(&text, batch.bound_ms(), &jobs, &batch.links)
+    let derived = image::derive(&text, bound_ms, &jobs, &batch.links)
         .map_err(|why| format!("{}: {why}", committed.display()))?;
     // **The job list is in the file name, not only in the file.**
     // `build_test_image` memoizes ROOT on the config's *path*, so two runs whose
@@ -762,6 +768,20 @@ fn build(
         }
         extra.extend(reached(&reachable, &batch.jobs, rust_bins));
     }
+    Ok(Staged { config, files: extra })
+}
+
+/// Build one batch's image, and answer where it landed.
+fn build(
+    root: &Path,
+    dir: &Path,
+    label: &str,
+    batch: &Batch,
+    rust_bins: &[(String, Vec<u8>)],
+    quiet: bool,
+) -> Result<PathBuf, String> {
+    let home = at(dir, label);
+    let staged = stage(root, &home, batch, batch.bound_ms(), rust_bins)?;
 
     // The build a boot asked for, or — where it asked for none — the one its
     // arms imply: an actuator outside `kernel/src/params.rs` needs the kernel
@@ -796,9 +816,9 @@ fn build(
     let deadline = format!("{}{}", toyos_tco::DEADLINE_PARAM, batch.deadline_ms());
     let mut params: Vec<&str> = batch.params.clone();
     params.push(&deadline);
-    let mut plan = toyos_build::build::Plan::new(toyos_build::arch::Arch::X86_64, &config, features, &params);
+    let mut plan = toyos_build::build::Plan::new(toyos_build::arch::Arch::X86_64, &staged.config, features, &params);
     plan.machines = toyos_build::microcode::Machines::Benches;
-    let bytes = toyos_build::build::build_test_image(root, &plan, quiet, &extra);
+    let bytes = toyos_build::build::build_test_image(root, &plan, quiet, &staged.files);
     let image = home.join("image.img");
     std::fs::write(&image, &bytes).map_err(|e| format!("{}: {e}", image.display()))?;
     Ok(image)

tests/common/qemu.rs

@@ -177,6 +177,10 @@ static FASTEST_BOOT_MS: AtomicU32 = AtomicU32::new(u32::MAX);
 /// host pays them at 1×.
 const REFERENCE_BOOT_MS: u32 = 1424;
 
+/// The most [`host_scale`] pays a ceiling for a slow host, so one anomalous
+/// boot cannot disable every guard in the suite at once.
+pub const HOST_SCALE_CAP: u32 = 8;
+
 fn record_boot(took: Duration) {
     let ms = took.as_millis().min(u32::MAX as u128) as u32;
     FASTEST_BOOT_MS.fetch_min(ms, Ordering::SeqCst);
@@ -186,16 +190,15 @@ fn record_boot(took: Duration) {
 /// a fraction so that a 1.4× host is not rounded to 1.
 ///
 /// **Only ever upward**, since a ceiling that shrank would report wedges that
-/// are not there; and at most 8×, so one anomalous boot cannot disable every
-/// guard in the suite at once.
+/// are not there; and at most [`HOST_SCALE_CAP`].
 fn host_scale() -> (u32, u32) {
     let fastest = FASTEST_BOOT_MS.load(Ordering::SeqCst);
     // Before the first boot there is no measurement, and the sentinel must not
     // read as the slowest host imaginable.
     if fastest == u32::MAX || fastest <= REFERENCE_BOOT_MS {
         return (1, 1);
     }
-    (fastest.min(REFERENCE_BOOT_MS * 8), REFERENCE_BOOT_MS)
+    (fastest.min(REFERENCE_BOOT_MS * HOST_SCALE_CAP), REFERENCE_BOOT_MS)
 }
 
 /// The fastest boot seen, the reference, and the scale it produced — for a run's
@@ -407,7 +410,7 @@ pub const NEVER_ANNOUNCED: &str =
 const WINDOW_LINES: usize = 40;
 
 /// The last [`WINDOW_LINES`] lines of `said`, under the count of what was cut.
-fn window(said: &str) -> String {
+pub fn window(said: &str) -> String {
     let lines: Vec<&str> = said.lines().collect();
     let kept = lines.len().min(WINDOW_LINES);
     let head = if lines.len() > kept {
@@ -1410,14 +1413,14 @@ impl Drop for HeldCpus {
 /// Which of [`DECLARED_KERNEL_BUILDS`] this boot wants.
 ///
 /// **A parameter never decides a build.** Every actuator lives in the one test
-/// kernel, so asking for one selects that kernel and nothing more; the third
-/// build is asked for by name and by one test.
+/// kernel, so asking for one selects that kernel, which a boot may also name,
+/// and nothing more; the third build is asked for by name and by one test.
 fn kernel_of(options: &BootOptions) -> Vec<&'static str> {
     if options.kernel_params.is_empty() {
         return options.kernel_features.to_vec();
     }
     assert!(
-        options.kernel_features.is_empty(),
+        options.kernel_features.is_empty() || options.kernel_features == toyos_build::build::TEST_KERNEL,
         "a boot asking to arm {:?} also asks for the kernel build {:?}; an actuator is a \
          parameter and the test kernel carries all of them",
         options.kernel_params,
@@ -1836,6 +1839,52 @@ impl QemuInstance {
         Ok(said)
     }
 
+    /// Everything the guest says until QEMU exits, onto `log`, and `Ok` once
+    /// it has exited 0, which under `-no-reboot` a reset, a power-off and a
+    /// triple fault all do: what stopped the guest is the caller's to ask. Or
+    /// why it did not exit, as [`ceiling_verdict`] reads `ceiling` paid out
+    /// for this guest.
+    pub fn await_end(&mut self, log: &mut String, ceiling: Duration) -> Result<(), String> {
+        let ceiling = self.budget(ceiling);
+        let from = log.len();
+        let start = Instant::now();
+        let (mut last_line, mut said_waiting) = (start, start);
+        let mut lines = 0usize;
+        let mut dying: Option<String> = None;
+        loop {
+            match self.rx.recv_timeout(VERDICT_POLL) {
+                Ok(line) => {
+                    last_line = Instant::now();
+                    lines += 1;
+                    if dying.is_none() && super::serial::died(&line) == Some(super::serial::Died::Kernel) {
+                        dying = Some(line.clone());
+                    }
+                    log.push_str(&line);
+                    log.push('\n');
+                }
+                Err(RecvTimeoutError::Timeout) => {}
+                Err(RecvTimeoutError::Disconnected) => break,
+            }
+            if said_waiting.elapsed() >= WAIT_SAID {
+                said_waiting = Instant::now();
+                eprintln!(
+                    "  [qemu] the job list has run {}s of its {}s guard, the guest last heard {}s ago",
+                    start.elapsed().as_secs(),
+                    ceiling.as_secs(),
+                    last_line.elapsed().as_secs()
+                );
+            }
+            if let Some(why) = ceiling_verdict(dying.as_deref(), start.elapsed(), ceiling, last_line.elapsed(), lines) {
+                return Err(WaitVerdict::new(why, &[&log[from..]]).to_string());
+            }
+        }
+        let status = self.child.wait().map_err(|e| format!("QEMU could not be waited for: {e}"))?;
+        if !status.success() {
+            return Err(format!("QEMU exited {status}"));
+        }
+        Ok(())
+    }
+
     /// [`budget_smp`] for a host-side wait on this guest's own vCPU count.
     pub fn budget(&self, ceiling: Duration) -> Duration {
         budget_smp(ceiling, self.smp)