Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 30 additions & 19 deletions .github/workflows/guest-rows.yml
Original file line number Diff line number Diff line change
@@ -1,47 +1,47 @@
name: guest-rows
# What a dispatcher finds its own run by.
run-name: guest-rows ${{ inputs.rev }} ${{ inputs.filter }}
run-name: guest-rows ${{ github.ref_name }} ${{ inputs.filter }}

# Guest rows on demand: `cargo test --test toyos-build -- <args>` at `rev`, on
# the guest runner (`cargo run -- --ci rows`, build/ci.rs). The runner is a
# dedicated, otherwise idle host that takes one job at a time, so this queue is
# the only one its guests wait in. It keeps its workspace and its toolchain
# store (`~/.cache/toyos`) between jobs; a revision whose toolchain is not in
# the store builds it here first.
# Guest rows on demand: `cargo test --test toyos-build -- <args>` at the
# dispatched branch's commit, on the guest runner (`cargo run -- --ci rows`,
# build/ci.rs). The runner is a dedicated, otherwise idle host that takes one
# job at a time, so this queue is the only one its guests wait in. It keeps its
# workspace and its toolchain store (`~/.cache/toyos`) between jobs, and that
# is its only cache; a revision whose toolchain is not in the store builds it
# here first.
#
# `workflow_dispatch` alone, never callable, refused on main, with no
# permissions and no `actions/cache`: a dispatch needs write access to the
# repository, no pull request reaches this runner's label from this
# repository's workflows, and code that outlives a job on this machine never
# holds a token of main's cache scope (build/ci.rs's
# `only_a_dispatch_reaches_a_self_hosted_runner`). The job reads no secret.
#
# The host provides: x86-64 Linux with `/dev/kvm` the runner's user opens; QEMU
# at build/ci/qemu-version for both architectures, with the firmware
# descriptors of its x86-64 and AArch64 UEFI firmware; rustup's stable on
# `PATH`; git, a C and C++ toolchain, CMake and Python 3 for a toolchain build.
#
# `workflow_dispatch` alone, and never callable: a dispatch needs write access
# to the repository, and no pull request reaches this runner's label from this
# repository's workflows (build/ci.rs's `only_a_dispatch_reaches_a_self_hosted_runner`).
# The job reads no secret.
# The runner runs as a dedicated user with no sudo; job code can read that
# user's files, the runner's `.credentials` among them.
on:
workflow_dispatch:
inputs:
rev:
description: The commit to test, as a full hash
required: true
type: string
filter:
description: The harness's arguments after `--`, split at whitespace; empty runs the whole suite
required: false
type: string

permissions:
contents: read
permissions: {}

jobs:
rows:
if: github.ref != 'refs/heads/main'
runs-on: [self-hosted, toyos-guest]
# A wedge guard, wide enough for a toolchain the store does not hold.
timeout-minutes: 360
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
ref: ${{ inputs.rev }}
# `target/` is the warm build; `git clean -ffdx` would remove it.
clean: false
# The fork a toolchain the store lacks is built from; the build
Expand All @@ -54,3 +54,14 @@ jobs:
- env:
FILTER: ${{ inputs.filter }}
run: cargo run -- --ci rows "$FILTER"

# The serial logs of a run that was not green: only this run's, since
# the driver removed what earlier runs kept.
- name: serial logs
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: serial
path: target/red-run-serial/**/uart-*.log
if-no-files-found: warn
retention-days: 7
55 changes: 48 additions & 7 deletions build/ci.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ const USAGE: &str = "cargo run -- --ci <job>, where <job> is one of:
guest the guest suite, on the sysroot its job restored (nightly)
rows [<args>...] `cargo test --test toyos-build -- <args>...`, each split at
whitespace, on this host's own toolchain store, held to the
instrument as `guest` is (guest-rows.yml)
instrument as `guest` is, after removing the serial logs
earlier runs kept (guest-rows.yml)
release put the sysroot main's nightly restored up as its toolchain release
publish put main's SDK crates on crates.io (publish.yml)";

Expand Down Expand Up @@ -140,7 +141,12 @@ pub fn dispatch(root: &Path, args: &[String]) {
Job::Bootstrap => vec![step("this tree's toolchain", || release::bootstrap(root))],
Job::Image => image(root),
Job::Guest => guest(root, &suite_args(&["--jobs", "1"]), true),
Job::Rows(args) => guest(root, &suite_args(&args.iter().map(String::as_str).collect::<Vec<_>>()), false),
Job::Rows(args) => {
let suite = suite_args(&args.iter().map(String::as_str).collect::<Vec<_>>());
let mut steps = vec![step("no serial logs kept by an earlier run", || forget_kept_serial(root))];
steps.extend(guest(root, &suite, false));
steps
}
Job::Release => vec![step("main's toolchain release", || release::release(root))],
Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))],
};
Expand Down Expand Up @@ -1353,6 +1359,17 @@ fn guest(root: &Path, suite: &[String], install: bool) -> Vec<Step> {
steps
}

/// An empty `target/red-run-serial`: a runner that keeps its workspace
/// uploads only what this run kept.
fn forget_kept_serial(root: &Path) -> Result<String, String> {
let kept = root.join("target/red-run-serial");
match std::fs::remove_dir_all(&kept) {
Ok(()) => Ok(format!("removed {}", kept.display())),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok("none".to_string()),
Err(e) => Err(format!("remove {}: {e}", kept.display())),
}
}

/// The suite's own count line and every line naming a verdict worth reading
/// without the log: a failure.
fn verdicts(log: &str) -> String {
Expand Down Expand Up @@ -2007,17 +2024,29 @@ mod tests {
for called in text.lines().filter_map(|l| l.strip_prefix(" uses:")) {
assert!(called.trim().starts_with("./.github/workflows/"), "{name}: a job uses{called}");
}
for runner in text.lines().filter_map(|l| l.trim_start().strip_prefix("runs-on:")) {
if !hosted(runner) {
assert_eq!(on, ["workflow_dispatch"], "{name}: runs-on:{runner} is started by {on:?}");
self_hosted.push(name.clone());
}
let runners: Vec<&str> = text.lines().filter_map(|l| l.trim_start().strip_prefix("runs-on:")).collect();
if runners.iter().any(|runner| !hosted(runner)) {
assert_eq!(on, ["workflow_dispatch"], "{name}: a self-hosted job is started by {on:?}");
refuses_main_and_writes_nothing(&name, &text, runners.len());
self_hosted.push(name.clone());
}
}
assert_eq!(self_hosted, ["guest-rows.yml"], "the workflows that reach a self-hosted runner");
assert_eq!(seen, 6, "ci.yml, nightly.yml, publish.yml and guest-rows.yml, and guest.yml and toolchain.yml");
}

/// A self-hosted workflow's job never runs on main's ref, whose token
/// writes the caches every other ref restores; it holds no permission and
/// uses no cache action, since the runner's own workspace is its cache.
fn refuses_main_and_writes_nothing(name: &str, text: &str, jobs: usize) {
let permissions: Vec<&str> = text.lines().filter(|l| l.trim_start().starts_with("permissions:")).collect();
assert_eq!(permissions, ["permissions: {}"], "{name}: its permissions");
let caches = text.lines().filter(|l| l.contains("uses:") && l.contains("actions/cache"));
assert_eq!(caches.count(), 0, "{name} uses a cache action");
let conditions: Vec<&str> = text.lines().filter_map(|l| l.strip_prefix(" if:")).map(str::trim).collect();
assert_eq!(conditions, vec!["github.ref != 'refs/heads/main'"; jobs], "{name}: each job's condition");
}

#[test]
fn the_trigger_and_runner_readers_read_what_they_refuse() {
let block = "name: x\non:\n # why\n pull_request:\n branches: [main]\n merge_group:\n\njobs:\n";
Expand All @@ -2028,6 +2057,18 @@ mod tests {
for runner in [" [self-hosted, toyos-guest]", " toyos-guest", " ${{ inputs.runner }}", "", " ubuntu-24.04 # x"] {
assert!(!hosted(runner), "{runner:?}");
}
let good = "permissions: {}\njobs:\n rows:\n if: github.ref != 'refs/heads/main'\n runs-on: [self-hosted]\n";
refuses_main_and_writes_nothing("good", good, 1);
let refused = [
good.replace(" if: github.ref != 'refs/heads/main'\n", ""),
good.replace(" if: github.ref != 'refs/heads/main'", " if: always()"),
good.replace("permissions: {}", "permissions:\n contents: read"),
good.replace(" runs-on:", " permissions:\n actions: write\n runs-on:"),
format!("{good} steps:\n - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830\n"),
];
for text in refused {
assert!(std::panic::catch_unwind(|| refuses_main_and_writes_nothing("bad", &text, 1)).is_err(), "{text}");
}
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion build/ci/qemu-version
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# declaration the guest image is held to, never a fact read off it. The image
# digest is chosen to satisfy this line; changing either is a deliberate act:
# it says the instrument moved.
11.1.1
11.1.2
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
---
status: open
kind: tooling
opened: 2026-11-04
---

# `a_narrow_guest_does_not_overtake_a_wide_one_waiting` reads any holder of the queue as the wide guest waiting

`build/buildlock.rs`'s test starts the wide guest, then polls `guest-cpus/queue`
and starts the narrow guest once a `try_lock` of it fails, taking that as the
wide guest holding the queue while it waits. Under host load it failed with
`a narrow guest took the CPU a queued wide one waits for`. Its log shows both
guests *waiting for* the queue, the narrow one acquiring it first and the wide
one after, so the probe's failed `try_lock` was not the wide guest queued.
The run was `cargo run -- --ci host` at `2d29d98db` (#1173, a change that does not
touch `buildlock.rs`), host load average 22 to 28; the test passed alone three
times on the same tree: evidence `pr-1173/r1/r1-host-ci.log` and
`isolated-*.log` at ToyOSOrg/evidence `067de8d244ac7e8c517cf1039ec692f5241c097f`.

**Exit**: the test starts the narrow guest only once the wide guest itself
reports that it holds the queue, and stays green under load.
2 changes: 1 addition & 1 deletion issues/the-build-runs-host-tools-outside-rust-and-qemu.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ arrives and is not one. M4 and M5 are stages of `issues/toyos-builds-itself.md`.
| the toolchain's own `clang`, `llvm-ar`, `rust-lld` and `llvm-config`, built from `ToyOSOrg/llvm-project` | rustc links every guest binary with `rust-lld`; `clang` compiles the C corpus (`tests/common/compile.rs`) and, with `llvm-ar`, through `cc::Build` the C of every guest crate that has any (`build/clang.rs`, `build/build.rs`'s `GuestEnv`): doomgeneric, and ring's C and assembly in a test crate's `https_get` and `ring_kat`; rustc's bootstrap asks `llvm-config` how to link LLVM | admitted: our fork's C++, which ToyOS can one day build and run; no Rust tool compiles C, `cc::Build` archives with an `ar`, bootstrap reads LLVM through `llvm-config`, and `CLAUDE.md` links everything with `rust-lld` | M5: no host in the loop |
| `ovmf-generic`, `qemu-efi-aarch64` | the x86-64 and AArch64 UEFI firmware of CI's guest containers and of the guest runner's host (`build/firmware.rs`), packaged by Debian apart from QEMU | admitted: QEMU's own firmware, and no Rust firmware does its job | the instrument's QEMU carries its own firmware |
| `ca-certificates` | the trust store `git` and `curl` verify against in CI's containers | admitted: data both of them need | goes when neither runs there |
| Debian's `sid` image, Docker Hub's `sid-20260918` copied once to `ghcr.io/toyosorg/debian` and pulled with the run's token | the container of `guest.yml`'s suite, pinned to digest `a2aa4626…`, whose dated snapshot archive `deps` installs QEMU from; and, pulled anonymously from ECR Public's copy of Docker Hub's, of `nightly.yml`'s `portability-linux`, `sid` as it stands | admitted: the runner's Ubuntu 24.04 publishes QEMU 8.2.2 (Launchpad, noble, every pocket) and `build/ci/qemu-version` declares 11.1.1, which that snapshot archive carries | the runner's own OS carries the declared QEMU, or the build system builds it |
| Debian's `sid` image, Docker Hub's `sid-20260918` copied once to `ghcr.io/toyosorg/debian` and pulled with the run's token | the container of `guest.yml`'s suite, pinned to digest `a2aa4626…`, whose dated snapshot archive `deps` installs QEMU from; and, pulled anonymously from ECR Public's copy of Docker Hub's, of `nightly.yml`'s `portability-linux`, `sid` as it stands | admitted: the runner's Ubuntu 24.04 publishes QEMU 8.2.2 (Launchpad, noble, every pocket) and that snapshot archive carries QEMU 11.1.1; `build/ci/qemu-version` now declares 11.1.2, the guest runner's (`issues/the-nightly-is-frozen-so-nothing-runs-the-guest-matrix.md`) | the runner's own OS carries the declared QEMU, or the build system builds it |
| GitHub's Actions runner on the guest runner's host | takes `guest-rows.yml`'s jobs | admitted: the owner's ruling (2026-11-04) runs guest tests on a dedicated self-hosted runner; it is the agent every GitHub-hosted job already runs under | goes with GitHub Actions |
| Docker on the runner | Actions pulls and runs every `container:` image with it, those two jobs' | refused: Go, not C or C++ ToyOS can one day build and run, and Actions runs a `container:` with Docker alone | the container goes with the image row |
| `skopeo` | `skopeo copy --all --preserve-digests` copied Docker Hub's `sid-20260918` index, every manifest by its digest, to `ghcr.io/toyosorg/debian`, once, from a `run:` step of a workflow that ran on a branch and is not in `main`; a new pin is a new copy made the same way, and the copy it replaces is deleted | refused: Go, not C or C++ ToyOS can one day build and run | a Rust tool makes the copy, or the container goes with the image row |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,8 @@ writer, main's toolchain release, and the fresh-machine and macOS builds.
Guest rows run only where an agent dispatches them on the guest runner
(`guest-rows.yml`), so a regression in a row no PR selected goes unseen.

**Exit**: the nightly resumes. Held by the owner.
`build/ci/qemu-version` moved to 11.1.2, the guest runner's QEMU, while
`guest.yml`'s container still installs 11.1.1 from its snapshot archive, so its
instrument step reds until the container moves to an image carrying 11.1.2.

**Exit**: the nightly resumes, its guest container on 11.1.2. Held by the owner.
2 changes: 1 addition & 1 deletion tests/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ The mechanics live where the work is: profiles and shapes in `tests/common/`, re

## Where guests run

Guest rows run on the guest runner, a dedicated idle host that takes one job at a time: `gh workflow run guest-rows.yml -f rev=<pushed full hash> -f filter='<harness args>'`, found by its title in `gh run list --workflow guest-rows.yml` and read with `gh run view <id> --log`. A run that other work on its host shared ends `HOST NOT QUIET` with exit 3: neither red nor green, and re-run.
Guest rows run on the guest runner, a dedicated idle host that takes one job at a time: `gh workflow run guest-rows.yml --ref <pushed branch> -f filter='<harness args>'` runs the branch's head, found by its title in `gh run list --workflow guest-rows.yml` and read with `gh run view <id> --log`; a run that is not green leaves its serial logs as the run's `serial` artifact. The job refuses main, so test main's commit from a branch at it. A run that other work on its host shared ends `HOST NOT QUIET` with exit 3: neither red nor green, and re-run.

## Caveats that bite every agent

Expand Down
5 changes: 5 additions & 0 deletions tests/checks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,11 @@ mod checks {
hostload_checks::self_check()
}

#[test]
fn iowait_going_back() -> Result<(), String> {
hostload_checks::iowait_going_back()
}

/// What a suspend is worth to a verdict, staged rather than reasoned about.
///
/// `clock_checks::self_check` gates the detector; this gates what the suite
Expand Down
16 changes: 16 additions & 0 deletions tests/checks/hostload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,19 @@ pub fn self_check() -> Result<(), String> {
);
Ok(())
}

/// Two `/proc/stat` readings across which the kernel moved iowait to idle:
/// iowait went down by 20 ticks and idle up by 100.
pub fn iowait_going_back() -> Result<(), String> {
let before = hostload::proc_stat("cpu 100 0 50 1000 40 0 0 0 0 0\ncpu0 100 0 50 1000 40 0 0 0 0 0\n");
let after = hostload::proc_stat("cpu 110 0 55 1100 20 0 0 0 0 0\ncpu0 110 0 55 1100 20 0 0 0 0 0\n");
let (busy, total) = after.since(&before);
if (busy, total) != (hostload::ticks(15), hostload::ticks(95)) {
return Err(format!(
"busy {busy:?} of {total:?}, and it is {:?} of {:?}: iowait going down is not idle going down",
hostload::ticks(15),
hostload::ticks(95)
));
}
Ok(())
}
Loading