From 10c1821d33fc1290718bd0639e1e47973a2f7797 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 22:03:00 +0100 Subject: [PATCH 1/2] buildlock: a wait polls with a ceiling, and a would-be maker tries A 12-wide `cargo test --test toyos-build` waited 2611 s on a sysroot key lock its own process had finished with. Two defects, both reproduced: 1. macOS lets blocked shared flock waiters in one at a time. Measured on Darwin 27.0.0, across threads and across processes alike: of four shared waiters queued behind an exclusive holder, one is granted when it lets go, and each other only once the shared holder before it lets go, while a fresh non-blocking shared try succeeds at once. A build holds its uses (of a sysroot, of the worktree lock) for its whole length, so a waiter there waits on another build's whole length, and forever on one held to the end of the run. In the incident one waiter was admitted after the sysroot build, five more after 17:45:06, when the failing build ahead of them dropped its use, and the seventh was left behind a use never dropped. 2. `keyed_made` queued exclusively to make a key it had just found unmade. When another waiter made it meanwhile and its users kept their uses, the queued maker waited out every one of them. Every acquisition now polls `File::try_lock{,_shared}` with a pause doubling from 1 ms to 50 ms, and fails after six hours, past the 350 minutes the toolchain workflow allows, naming the holder. The macOS admission is what forces the poll: on a host that admits every compatible waiter, a blocking wait with a timed join would do. This replaces the blocking `flock` FFI and the heartbeat thread: the poll loop says "still waiting" itself. `keyed_building` is a try, and `keyed_made` goes back to waiting for a use when it fails. Tests: `waiters_on_a_failed_build_all_get_in` (a thread's make fails with eight waiting threads; all eight get and hold their use), `a_would_be_maker_does_not_wait_out_another_use` and `a_wait_past_its_deadline_names_the_holder`. Negative controls: the base code with the first test, and this change with kernel-blocking waits, both let in 2 of 8; the try reverted to a wait fails the second; the deadline check removed fails the third. Closes issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md. Co-Authored-By: Claude Opus 5.5 --- build/buildlock.rs | 341 +++++++++++------- ...inutes-on-its-own-processs-sysroot-lock.md | 34 -- 2 files changed, 209 insertions(+), 166 deletions(-) delete mode 100644 issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md diff --git a/build/buildlock.rs b/build/buildlock.rs index 835d0481882..d449010c134 100644 --- a/build/buildlock.rs +++ b/build/buildlock.rs @@ -44,20 +44,20 @@ use std::fs; use std::io::{self, Read, Seek, SeekFrom, Write}; -use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::keystore::Key; unsafe extern "C" { - fn flock(fd: i32, operation: i32) -> i32; fn kill(pid: i32, sig: i32) -> i32; } -const LOCK_SH: i32 = 1; -const LOCK_EX: i32 = 2; -const LOCK_NB: i32 = 4; +#[derive(Clone, Copy, PartialEq, Eq)] +enum Mode { + Shared, + Exclusive, +} /// How often a wait that is lasting repeats itself. /// @@ -69,6 +69,14 @@ const HEARTBEAT: Duration = Duration::from_secs(30); #[cfg(test)] const HEARTBEAT: Duration = Duration::from_millis(300); +/// How long any one acquisition waits before it fails, naming the holder: past +/// the 350 minutes `.github/workflows/toolchain.yml` allows the whole +/// toolchain, the longest phase any of these locks covers. +const CEILING: Duration = Duration::from_secs(6 * 60 * 60); + +/// The longest a waiter sleeps between two tries ([`take`]). +const MAX_PAUSE: Duration = Duration::from_millis(50); + const LOCK_DIR: &str = ".build-locks"; /// A held lock. Releasing it is closing the file. @@ -133,7 +141,7 @@ impl Held { } let dir = self.worktree_dir.clone(); self.without_shared(|| { - let _exclusive = acquire(&dir, LOCK_EX, phase, BUILD); + let _exclusive = acquire(&dir, Mode::Exclusive, phase, BUILD); if let Some(work) = decide() { act(work); } @@ -150,14 +158,14 @@ impl Held { } fn take_shared(&self) -> Guard { - acquire(&self.worktree_dir, LOCK_SH, &self.what, BUILD) + acquire(&self.worktree_dir, Mode::Shared, &self.what, BUILD) } } /// This worktree's build lock, exclusively: what a sysroot build holds while it /// writes the worktree's fork build directory, with the key's lock already held. pub fn worktree_exclusive(root: &Path, what: &str) -> Guard { - acquire(&root.join(LOCK_DIR), LOCK_EX, what, BUILD) + acquire(&root.join(LOCK_DIR), Mode::Exclusive, what, BUILD) } /// Exclusive lock over the shared cargo artifact paths. @@ -211,11 +219,17 @@ impl Keyed { } } -/// Make what `key` names: exclusive, and waited for by every other process that -/// wants the same key, which then finds it made. -fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Guard { - let lock = format!("{} lock", kind.name()); - exclusive(&keyed_lock_path(store, kind, key), &lock, &format!("building {} {key}", kind.name())) +/// Make what `key` names, if nobody is making or using it: exclusive, and +/// waited for by every other process that wants the same key, which then finds +/// it made. +fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Option { + let file = open_lock_file(&keyed_lock_path(store, kind, key)); + if !try_lock(&file, Mode::Exclusive) { + return None; + } + let mut guard = Guard { file, records_holder: true }; + write_note(&mut guard.file, ¬e_text(&format!("building {} {key}", kind.name()))); + Some(guard) } /// Use what `key` names: shared, so any number of builds use it at once, a @@ -224,14 +238,9 @@ fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Guard { pub(crate) fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { let path = keyed_lock_path(store, kind, key); let file = open_lock_file(&path); - if !try_lock(&file, LOCK_SH) { - let lock = format!("{} lock", kind.name()); - let what = format!("using {} {key}", kind.name()); - let holder = describe_holder(&path) - .unwrap_or_else(|| "held, but the holder left no readable note".to_string()); - announce(&lock, &what, &holder); - take_lock_announcing(&file, LOCK_SH, &path, &lock, &what); - } + let lock = format!("{} lock", kind.name()); + let what = format!("using {} {key}", kind.name()); + take(&file, Mode::Shared, &path, &lock, &what, UNNOTED, Instant::now() + CEILING); file.set_modified(SystemTime::now()).unwrap_or_else(|e| panic!("build lock: date {}: {e}", path.display())); Guard { file, records_holder: false } } @@ -240,6 +249,9 @@ pub(crate) fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { /// [`keyed_building`] while `defect`, which says why it is not whole, says it is /// not. A `make` that leaves it not whole is refused by that defect rather than /// run again. +/// +/// The making is tried, never waited for: a waiter queued to make it would +/// wait out every use of what another waiter made meanwhile. pub fn keyed_made( store: &Path, kind: Keyed, @@ -247,13 +259,23 @@ pub fn keyed_made( defect: impl Fn() -> Option, mut make: impl FnMut(), ) -> Guard { + let deadline = Instant::now() + CEILING; + let mut pause = Duration::from_millis(1); loop { let using = keyed_using(store, kind, key); if defect().is_none() { return using; } drop(using); - let _building = keyed_building(store, kind, key); + let Some(_building) = keyed_building(store, kind, key) else { + // Another is making it, or deciding to, as this one just was. + let path = keyed_lock_path(store, kind, key); + let holder = describe_holder(&path).unwrap_or_else(|| UNNOTED.to_string()); + assert!(Instant::now() < deadline, "build lock: gave up making {} {key} after {CEILING:?} — {holder}", kind.name()); + std::thread::sleep(pause); + pause = (pause * 2).min(MAX_PAUSE); + continue; + }; if defect().is_some() { make(); if let Some(defect) = defect() { @@ -267,7 +289,7 @@ pub fn keyed_made( /// a sweep holds while it removes one. pub fn keyed_idle(store: &Path, kind: Keyed, key: &Key) -> Option { let file = open_lock_file(&keyed_lock_path(store, kind, key)); - try_lock(&file, LOCK_EX).then_some(Guard { file, records_holder: false }) + try_lock(&file, Mode::Exclusive).then_some(Guard { file, records_holder: false }) } /// When `kind`'s `key` in `store` was last used or made. A key nothing has @@ -289,14 +311,7 @@ pub(crate) fn keyed_lock_path(store: &Path, kind: Keyed, key: &Key) -> PathBuf { /// can always be told who it is waiting for. fn exclusive(path: &Path, lock: &str, what: &str) -> Guard { let file = open_lock_file(path); - let start = Instant::now(); - if !try_lock(&file, LOCK_EX) { - let holder = describe_holder(path) - .unwrap_or_else(|| "held, but the holder left no readable note".to_string()); - announce(lock, what, &holder); - take_lock_announcing(&file, LOCK_EX, path, lock, what); - eprintln!("[build-lock] {what} acquired after {:.1?}", start.elapsed()); - } + take(&file, Mode::Exclusive, path, lock, what, UNNOTED, Instant::now() + CEILING); let mut guard = Guard { file, records_holder: true }; write_note(&mut guard.file, ¬e_text(what)); guard @@ -332,37 +347,18 @@ const BUILD: Lock = Lock { /// instead of overtaking it. `intent` is always taken before `state` and /// dropped as soon as `state` is held, so nothing ever waits on `intent` while /// holding `state`. -fn acquire(dir: &Path, op: i32, what: &str, lock: Lock) -> Guard { +fn acquire(dir: &Path, mode: Mode, what: &str, lock: Lock) -> Guard { let intent_path = dir.join("intent"); let state_path = dir.join("state"); let intent = open_lock_file(&intent_path); let state = open_lock_file(&state_path); - let label = format!("{}, {what}", if op == LOCK_EX { "exclusive" } else { "shared" }); - - let start = Instant::now(); - let mut waited = false; - - if !try_lock(&intent, op) { - announce(lock.name, &label, lock.queued_ahead); - waited = true; - take_lock_announcing(&intent, op, &intent_path, lock.name, &label); - } - if !try_lock(&state, op) { - if !waited { - let holder = describe_holder(&state_path) - .unwrap_or_else(|| lock.shared_holders.to_string()); - announce(lock.name, &label, &holder); - waited = true; - } - take_lock_announcing(&state, op, &state_path, lock.name, &label); - } + let label = format!("{}, {what}", if mode == Mode::Exclusive { "exclusive" } else { "shared" }); + let deadline = Instant::now() + CEILING; + take(&intent, mode, &intent_path, lock.name, &label, lock.queued_ahead, deadline); + take(&state, mode, &state_path, lock.name, &label, lock.shared_holders, deadline); drop(intent); - if waited { - eprintln!("[build-lock] acquired ({label}) after {:.1?}", start.elapsed()); - } - - let records_holder = op == LOCK_EX; + let records_holder = mode == Mode::Exclusive; let mut guard = Guard { file: state, records_holder }; if records_holder { write_note(&mut guard.file, ¬e_text(what)); @@ -370,47 +366,42 @@ fn acquire(dir: &Path, op: i32, what: &str, lock: Lock) -> Guard { guard } -/// An agent staring at silence kills and retries, which is the pathology this -/// module exists to remove — so a wait says what it is waiting for and, when -/// that can be established, who has it. -fn announce(lock: &str, label: &str, holder: &str) { - eprintln!("[build-lock] waiting for the {lock} ({label}) — {holder}"); -} +/// What a waiter is told of a holder that left no note it can read. +const UNNOTED: &str = "held, but the holder left no readable note"; -/// [`take_lock`], saying every 30 s that it is still waiting and who for. +/// Take `mode` of `file`, the lock at `path`, or fail at `deadline` naming the +/// holder. A wait says what it waits for and who has it, when that can be +/// established ([`describe_holder`], else `unnoted`), and says so again every +/// [`HEARTBEAT`]: an agent staring at silence kills and retries. /// -/// The kernel keeps the queue and a thread does the talking: nothing here polls -/// a lock, so `flock`'s own ordering is given up nowhere. The holder is re-read -/// each time, so the message follows the queue forward rather than naming the -/// process that was in front when the wait began. -fn take_lock_announcing(file: &fs::File, op: i32, path: &Path, lock: &str, label: &str) { - use std::sync::mpsc::{channel, RecvTimeoutError}; - - // A channel and not a polled flag: this runs on every *contended* - // acquisition, the artifact lock among them, and a flag checked every few - // milliseconds would put that granularity on the front of each one. The - // sender dropping wakes the thread at once and it never sleeps past the - // acquisition. - let (tx, rx) = channel::<()>(); - let heartbeat = { - let path = path.to_path_buf(); - let lock = lock.to_string(); - let label = label.to_string(); - std::thread::spawn(move || { - let began = Instant::now(); - while rx.recv_timeout(HEARTBEAT) == Err(RecvTimeoutError::Timeout) { - let holder = describe_holder(&path) - .unwrap_or_else(|| "the holder left no readable note".to_string()); - eprintln!( - "[build-lock] still waiting for the {lock} ({label}), {:.0?} so far — {holder}", - began.elapsed() - ); - } - }) - }; - take_lock(file, op, path); - drop(tx); - heartbeat.join().expect("the lock heartbeat panicked"); +/// A poll and not a blocking `flock`, which has no timed wait, and whose +/// shared waiters macOS lets in one at a time: measured on Darwin 27.0.0, of +/// shared waiters queued behind an exclusive holder, one is let in when it +/// lets go and each of the others only once the shared holder before it lets +/// go, though a new shared try succeeds at once. A build holds its uses for its +/// whole length, so one waiting there waits on every other build. +fn take(file: &fs::File, mode: Mode, path: &Path, lock: &str, label: &str, unnoted: &str, deadline: Instant) { + if try_lock(file, mode) { + return; + } + let holder = || describe_holder(path).unwrap_or_else(|| unnoted.to_string()); + eprintln!("[build-lock] waiting for the {lock} ({label}) — {}", holder()); + let began = Instant::now(); + let mut said = began; + let mut pause = Duration::from_millis(1); + while !try_lock(file, mode) { + let now = Instant::now(); + if now >= deadline { + panic!("build lock: gave up waiting for the {lock} ({label}) after {:.0?} — {}", began.elapsed(), holder()); + } + if now - said >= HEARTBEAT { + eprintln!("[build-lock] still waiting for the {lock} ({label}), {:.0?} so far — {}", began.elapsed(), holder()); + said = now; + } + std::thread::sleep(pause.min(deadline - now)); + pause = (pause * 2).min(MAX_PAUSE); + } + eprintln!("[build-lock] acquired the {lock} ({label}) after {:.1?}", began.elapsed()); } /// What the last exclusive holder of `path` recorded, if it is still running. @@ -478,33 +469,15 @@ fn open_lock_file(path: &Path) -> fs::File { .unwrap_or_else(|e| panic!("build lock: open {}: {e}", path.display())) } -fn take_lock(file: &fs::File, op: i32, path: &Path) { - loop { - // SAFETY: `file` owns the fd for the duration of the call and of the - // guard the caller builds from it. - if unsafe { flock(file.as_raw_fd(), op) } == 0 { - return; - } - let err = io::Error::last_os_error(); - if err.kind() == io::ErrorKind::Interrupted { - continue; - } - panic!("build lock: flock on {}: {err}", path.display()); - } -} - -fn try_lock(file: &fs::File, op: i32) -> bool { - loop { - // SAFETY: as in `take_lock`. - if unsafe { flock(file.as_raw_fd(), op | LOCK_NB) } == 0 { - return true; - } - let err = io::Error::last_os_error(); - match err.kind() { - io::ErrorKind::Interrupted => continue, - io::ErrorKind::WouldBlock => return false, - _ => panic!("build lock: flock: {err}"), - } +fn try_lock(file: &fs::File, mode: Mode) -> bool { + let tried = match mode { + Mode::Shared => file.try_lock_shared(), + Mode::Exclusive => file.try_lock(), + }; + match tried { + Ok(()) => true, + Err(fs::TryLockError::WouldBlock) => false, + Err(fs::TryLockError::Error(e)) => panic!("build lock: flock: {e}"), } } @@ -607,7 +580,7 @@ pub(crate) mod tests { /// Whether `root`'s worktree lock would keep a shared acquirer out now. pub(crate) fn keeps_out_shared(root: &Path) -> bool { - !try_lock(&open_lock_file(&worktree_lock_dir(root).join("state")), LOCK_SH) + !try_lock(&open_lock_file(&worktree_lock_dir(root).join("state")), Mode::Shared) } /// What `role` of [`child_role`] takes in `root`, held by a process of its own. @@ -672,7 +645,7 @@ pub(crate) mod tests { /// A fresh fd per probe: a successful `try_lock` *holds* what it took, and /// polling on one fd would itself be the thing keeping the writer out. fn intent_is_taken(root: &Path) -> bool { - !try_lock(&open_lock_file(&worktree_lock_dir(root).join("intent")), LOCK_SH) + !try_lock(&open_lock_file(&worktree_lock_dir(root).join("intent")), Mode::Shared) } fn note(root: &Path, line: &str) { @@ -724,7 +697,7 @@ pub(crate) mod tests { note(&root, "sh"); } "hold-sysroot-build" => { - let _building = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k1")); + let _building = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k1")).expect("nobody else has k1"); hold_until_released(); note(&root, "built"); } @@ -766,8 +739,8 @@ pub(crate) mod tests { let kid = held_elsewhere(&root, "hold-exclusive"); let state = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&state, LOCK_SH), "a build got in while an exclusive phase ran"); - assert!(!try_lock(&state, LOCK_EX), "two exclusive phases at once"); + assert!(!try_lock(&state, Mode::Shared), "a build got in while an exclusive phase ran"); + assert!(!try_lock(&state, Mode::Exclusive), "two exclusive phases at once"); let holder = describe_holder(&worktree_lock_dir(&root).join("state")) .expect("no holder note"); assert!( @@ -787,12 +760,12 @@ pub(crate) mod tests { assert!(appeared(&root.join("held"), Duration::from_secs(20)), "child never acquired"); let state = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&state, LOCK_EX), "the lock was not actually held"); + assert!(!try_lock(&state, Mode::Exclusive), "the lock was not actually held"); kid.kill().unwrap(); kid.wait().unwrap(); - assert!(try_lock(&state, LOCK_EX), "a SIGKILLed holder stranded the lock"); + assert!(try_lock(&state, Mode::Exclusive), "a SIGKILLed holder stranded the lock"); // And the note it left behind names a pid that is gone, so nobody is // sent to wait on it. assert_eq!(describe_holder(&worktree_lock_dir(&root).join("state")), None); @@ -803,10 +776,10 @@ pub(crate) mod tests { let root = scratch("shared"); let _mine = shared(&root, "parent"); let second = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(try_lock(&second, LOCK_SH), "two builds cannot run at once"); + assert!(try_lock(&second, Mode::Shared), "two builds cannot run at once"); drop(second); let third = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&third, LOCK_EX), "a clean got in while a build was running"); + assert!(!try_lock(&third, Mode::Exclusive), "a clean got in while a build was running"); } /// `flock` alone would let a stream of builds starve the rebuild they are @@ -858,7 +831,7 @@ pub(crate) mod tests { let builder = held_elsewhere(&root, "hold-sysroot-build"); assert!(keyed_idle(&root, Keyed::Sysroot, &Key::of(b"k1")).is_none(), "a sweep could remove a key being built"); - let other = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k2")); + let other = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k2")).expect("a process making another key held this one"); drop(other); let mut user = child(&root, "want-sysroot"); @@ -929,6 +902,110 @@ pub(crate) mod tests { ); } + /// **A build failing while others wait for its key lets every one of them + /// in, and none waits on another's use.** One thread's make of a sysroot + /// fails while eight others wait for the key; one of them makes it, and + /// every one then holds its use at once, as builds hold theirs for their + /// whole length. The waiters are threads of one process, as a test run's + /// builds are. + #[test] + fn waiters_on_a_failed_build_all_get_in() { + use std::sync::mpsc::channel; + const WAITERS: usize = 8; + let store = scratch("failed-build"); + let key = Key::of(b"a build that fails"); + let whole = Keyed::Sysroot.store(&store).join(&key); + let defect = { + let whole = whole.clone(); + move || (!whole.is_dir()).then(|| "not made".to_string()) + }; + + let (started, has_started) = channel(); + let (fail, failed) = channel::<()>(); + let failing = { + let (store, key, defect) = (store.to_path_buf(), key.clone(), defect.clone()); + std::thread::spawn(move || { + keyed_made(&store, Keyed::Sysroot, &key, defect, || { + started.send(()).unwrap(); + failed.recv().unwrap(); + panic!("the build failed"); + }) + }) + }; + has_started.recv_timeout(Duration::from_secs(20)).expect("the failing build never began"); + + let (used, uses) = channel(); + for _ in 0..WAITERS { + let (store, key, defect, whole, used) = (store.to_path_buf(), key.clone(), defect.clone(), whole.clone(), used.clone()); + std::thread::spawn(move || { + let using = keyed_made(&store, Keyed::Sysroot, &key, defect, || fs::create_dir_all(&whole).unwrap()); + used.send(using).unwrap(); + }); + } + // Long enough for every waiter to be waiting before the build fails; + // one that is not yet only makes this test weaker. + std::thread::sleep(Duration::from_millis(300)); + fail.send(()).unwrap(); + assert!(failing.join().is_err(), "the failing build did not fail"); + + let deadline = Instant::now() + Duration::from_secs(20); + let mut held = Vec::new(); + while held.len() < WAITERS { + match uses.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(using) => held.push(using), + Err(_) => break, + } + } + assert_eq!(held.len(), WAITERS, "of {WAITERS} builds waiting on a failed one, only {} got their sysroot in 20 s", held.len()); + } + + /// **A build that found the key unmade, while another made it, does not + /// wait out that one's use**: the key is whole and in use by another + /// process, and `defect` says it is not once, as it did for a build that + /// checked just before it was placed. + #[test] + fn a_would_be_maker_does_not_wait_out_another_use() { + let store = scratch("would-be-maker"); + let key = Key::of(b"made meanwhile"); + let user = sysroot_used_elsewhere(&store, &key); + let (tx, rx) = std::sync::mpsc::channel(); + { + let store = store.to_path_buf(); + std::thread::spawn(move || { + let checked = std::cell::Cell::new(false); + let defect = || (!checked.replace(true)).then(|| "placed just after this check".to_string()); + let using = keyed_made(&store, Keyed::Sysroot, &key, defect, || panic!("made twice")); + let _ = tx.send(using); + }); + } + let got = rx.recv_timeout(Duration::from_secs(20)); + user.release(); + drop(got.expect("a build that found the key unmade waited out another's use of it for 20 s")); + } + + /// **A wait that outlasts its deadline fails, naming the holder.** + #[test] + fn a_wait_past_its_deadline_names_the_holder() { + let root = scratch("deadline"); + let key = Key::of(b"never made"); + let _building = keyed_building(&root, Keyed::Sysroot, &key).expect("nobody else has the key"); + let path = keyed_lock_path(&root, Keyed::Sysroot, &key); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let file = open_lock_file(&path); + let waited = std::panic::catch_unwind(|| { + take(&file, Mode::Shared, &path, "sysroot lock", "using it", UNNOTED, Instant::now() + Duration::from_millis(500)) + }); + let _ = tx.send(waited.map_err(|e| e.downcast_ref::().cloned().unwrap_or_default())); + }); + let refusal = rx + .recv_timeout(Duration::from_secs(20)) + .expect("a wait with a 500 ms deadline had not ended 20 s later") + .expect_err("a wait on a lock nobody let go of succeeded"); + let holder = format!("held by pid {} (building sysroot {key})", std::process::id()); + assert!(refusal.contains("gave up") && refusal.contains(&holder), "the refusal does not name the holder: {refusal}"); + } + /// The defect this module exists for, staged so that it is not itself a /// race: a clean lands in the middle of a build in the same target /// directory, and the build's next write finds the directory gone. Run once diff --git a/issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md b/issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md deleted file mode 100644 index efd129eaf52..00000000000 --- a/issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-11-01 ---- - -# A test build waited forty minutes on its own process's sysroot lock - -One `cargo test --test toyos-build` run, twelve wide, of eight guest tests -at `929e73d6c` (#1077's branch, on main `190f3b164`), on a macOS host, with -the sysroot rebuilt for an SDK change: - -- 17:35:32: seven guest-binary builds wait for sysroot `f2cc0cfbcb345f2d`'s - lock, "held by pid 86112 (building sysroot …)", which is the test - process itself. -- 17:39:59 to 17:44: the sysroot is built, and six of the seven waiters go - on by 17:45:07. -- 17:45:06: another worker's guest build (`shm_commit`) fails to compile: - `cargo build failed`, the panic at `build/build.rs:494`. -- The seventh waiter, `stream_handles` for `stream_holders`, goes on - waiting, "the holder left no readable note", until the run is killed at - 2611 s. Nothing compiles in the meantime, and the process has no - children. - -The next run, with the compile error fixed, did not wait. Whether the -failed build is the cause is not established; it is the only event of -that run that the others did not have. The log, with its paths masked, is -[`r3/logs/qemu1.log`](https://github.com/ToyOSOrg/evidence/blob/9ad74c276117e0418c04b546764e4afbada6f166/r3/logs/qemu1.log). - -**Exit**: the wait reproduced and its cause fixed in `build/buildlock.rs` -or its callers, or the lock's acquisition shown unable to wait on a holder -in its own process. - -**Owner**: the orchestrator, who assigns the build track. From 5ee9f0878497507f1b548f520a043d5f5b01e548 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 21:56:39 +0100 Subject: [PATCH 2/2] review r1 of #1145: one poll loop, the poll recorded, the wedge closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `keyed_made`'s retry was a second wait loop: its own deadline and backoff, and silent while another's use kept its make out. It is now one attempt, "use it if whole, else try to make it", polled by the same `poll` as every other acquisition. So there is one deadline, one "waiting"/"still waiting"/"acquired" voice, and one ceiling, which `a_wait_past_its_deadline_names_the_holder` now tests through `keyed_made`. The scenario test now stages both outcomes of the first make: success with eight waiters (the incident: the sysroot was built), and failure. Under kernel-blocking uses, success admits 1 of 8. The poll is a workaround for macOS's admission of blocked shared waiters. `flock` having no timed wait would not force it alone: a blocking wait on a helper thread with a timed join would do on a host that admits every compatible waiter. Its costs and exit are recorded in issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md, and the comment at `poll` names that. Two comments were false: the tests' header said a single-process test proves nothing about flock (threads with their own descriptors contend), and a key's second user was said to wait "not on a timer". issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md is this defect. Its recurrences have the signature: after the sysroot build, one worker alone on `using sysroot … — the holder left no readable note`, the lock open only in the harness. Its exit condition holds. Sysroot, compiler and freestanding keys are taken with the worktree lock put down (`Held::without_shared`, sysroot.rs, compiler.rs). The LLVM and ToyOS-hosted clang keys are taken inside it by the documented order. Eight guest tests run 12 wide on a head whose sysroot was not yet made finished; that run is the PR's evidence. Co-Authored-By: Claude Opus 5.5 --- build/buildlock.rs | 194 +++++++++++------- ...-its-sysroot-can-wedge-on-its-own-locks.md | 51 ----- ...d-shared-flock-waiters-in-one-at-a-time.md | 36 ++++ 3 files changed, 151 insertions(+), 130 deletions(-) delete mode 100644 issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md create mode 100644 issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md diff --git a/build/buildlock.rs b/build/buildlock.rs index d449010c134..e25d683a7d4 100644 --- a/build/buildlock.rs +++ b/build/buildlock.rs @@ -74,7 +74,7 @@ const HEARTBEAT: Duration = Duration::from_millis(300); /// toolchain, the longest phase any of these locks covers. const CEILING: Duration = Duration::from_secs(6 * 60 * 60); -/// The longest a waiter sleeps between two tries ([`take`]). +/// The longest a waiter sleeps between two tries ([`poll`]). const MAX_PAUSE: Duration = Duration::from_millis(50); const LOCK_DIR: &str = ".build-locks"; @@ -236,13 +236,20 @@ fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Option { /// builder of it is waited for, and a sweep cannot remove it; and dated now, /// which a sweep reads as its last use. pub(crate) fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { + let (lock, what) = (format!("{} lock", kind.name()), format!("using {} {key}", kind.name())); + let path = keyed_lock_path(store, kind, key); + poll(&path, &lock, &what, UNNOTED, Instant::now() + CEILING, || keyed_try_using(store, kind, key)) +} + +/// [`keyed_using`], if nobody is making it now. +fn keyed_try_using(store: &Path, kind: Keyed, key: &Key) -> Option { let path = keyed_lock_path(store, kind, key); let file = open_lock_file(&path); - let lock = format!("{} lock", kind.name()); - let what = format!("using {} {key}", kind.name()); - take(&file, Mode::Shared, &path, &lock, &what, UNNOTED, Instant::now() + CEILING); + if !try_lock(&file, Mode::Shared) { + return None; + } file.set_modified(SystemTime::now()).unwrap_or_else(|e| panic!("build lock: date {}: {e}", path.display())); - Guard { file, records_holder: false } + Some(Guard { file, records_holder: false }) } /// What `key` names, held in use and whole: made by `make` under @@ -253,36 +260,41 @@ pub(crate) fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { /// The making is tried, never waited for: a waiter queued to make it would /// wait out every use of what another waiter made meanwhile. pub fn keyed_made( + store: &Path, + kind: Keyed, + key: &Key, + defect: impl Fn() -> Option, + make: impl FnMut(), +) -> Guard { + keyed_made_by(store, kind, key, defect, make, Instant::now() + CEILING) +} + +/// [`keyed_made`], failing at `deadline`. +fn keyed_made_by( store: &Path, kind: Keyed, key: &Key, defect: impl Fn() -> Option, mut make: impl FnMut(), + deadline: Instant, ) -> Guard { - let deadline = Instant::now() + CEILING; - let mut pause = Duration::from_millis(1); - loop { - let using = keyed_using(store, kind, key); - if defect().is_none() { - return using; + let (lock, what) = (format!("{} lock", kind.name()), format!("using {} {key}", kind.name())); + poll(&keyed_lock_path(store, kind, key), &lock, &what, UNNOTED, deadline, || { + if let Some(using) = keyed_try_using(store, kind, key) { + if defect().is_none() { + return Some(using); + } } - drop(using); - let Some(_building) = keyed_building(store, kind, key) else { - // Another is making it, or deciding to, as this one just was. - let path = keyed_lock_path(store, kind, key); - let holder = describe_holder(&path).unwrap_or_else(|| UNNOTED.to_string()); - assert!(Instant::now() < deadline, "build lock: gave up making {} {key} after {CEILING:?} — {holder}", kind.name()); - std::thread::sleep(pause); - pause = (pause * 2).min(MAX_PAUSE); - continue; - }; + let building = keyed_building(store, kind, key)?; if defect().is_some() { make(); if let Some(defect) = defect() { panic!("{} {key} was made, and is not whole: {defect}", kind.name()); } } - } + drop(building); + keyed_try_using(store, kind, key).filter(|_| defect().is_none()) + }) } /// What `key` names, exclusively and only if nobody is making or using it: what @@ -343,7 +355,7 @@ const BUILD: Lock = Lock { /// host, four shared churners kept an exclusive waiter out for the whole 5.5 s /// they ran — and the exclusive phases are exactly the long, silent ones an /// agent kills and retries. So an exclusive acquirer holds `intent` while it -/// queues for `state`, which makes later shared acquirers line up behind it +/// waits for `state`, which makes later shared acquirers wait behind it /// instead of overtaking it. `intent` is always taken before `state` and /// dropped as soon as `state` is held, so nothing ever waits on `intent` while /// holding `state`. @@ -369,27 +381,29 @@ fn acquire(dir: &Path, mode: Mode, what: &str, lock: Lock) -> Guard { /// What a waiter is told of a holder that left no note it can read. const UNNOTED: &str = "held, but the holder left no readable note"; -/// Take `mode` of `file`, the lock at `path`, or fail at `deadline` naming the -/// holder. A wait says what it waits for and who has it, when that can be -/// established ([`describe_holder`], else `unnoted`), and says so again every -/// [`HEARTBEAT`]: an agent staring at silence kills and retries. -/// -/// A poll and not a blocking `flock`, which has no timed wait, and whose -/// shared waiters macOS lets in one at a time: measured on Darwin 27.0.0, of -/// shared waiters queued behind an exclusive holder, one is let in when it -/// lets go and each of the others only once the shared holder before it lets -/// go, though a new shared try succeeds at once. A build holds its uses for its -/// whole length, so one waiting there waits on every other build. +/// Take `mode` of `file`, the lock at `path`, by `deadline` ([`poll`]). fn take(file: &fs::File, mode: Mode, path: &Path, lock: &str, label: &str, unnoted: &str, deadline: Instant) { - if try_lock(file, mode) { - return; + poll(path, lock, label, unnoted, deadline, || try_lock(file, mode).then_some(())); +} + +/// What `attempt` gets, tried until it gets it or `deadline` passes, which +/// fails naming the holder of `path`. A wait says what it waits for and who +/// has it, when that can be established ([`describe_holder`], else `unnoted`), +/// and says so again every [`HEARTBEAT`]: an agent staring at silence kills and +/// retries. +/// +/// A poll, not a blocking `flock`: macOS lets blocked shared waiters in one at +/// a time (`issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md`). +fn poll(path: &Path, lock: &str, label: &str, unnoted: &str, deadline: Instant, mut attempt: impl FnMut() -> Option) -> T { + if let Some(got) = attempt() { + return got; } let holder = || describe_holder(path).unwrap_or_else(|| unnoted.to_string()); eprintln!("[build-lock] waiting for the {lock} ({label}) — {}", holder()); let began = Instant::now(); let mut said = began; let mut pause = Duration::from_millis(1); - while !try_lock(file, mode) { + loop { let now = Instant::now(); if now >= deadline { panic!("build lock: gave up waiting for the {lock} ({label}) after {:.0?} — {}", began.elapsed(), holder()); @@ -400,8 +414,11 @@ fn take(file: &fs::File, mode: Mode, path: &Path, lock: &str, label: &str, unnot } std::thread::sleep(pause.min(deadline - now)); pause = (pause * 2).min(MAX_PAUSE); + if let Some(got) = attempt() { + eprintln!("[build-lock] acquired the {lock} ({label}) after {:.1?}", began.elapsed()); + return got; + } } - eprintln!("[build-lock] acquired the {lock} ({label}) after {:.1?}", began.elapsed()); } /// What the last exclusive holder of `path` recorded, if it is still running. @@ -489,9 +506,10 @@ pub(crate) mod tests { use std::time::Duration; use toyos_tmpdir::TempDir; - // Two processes are the point. `flock` is per open file description, so a - // single-process test would prove nothing about the thing that actually - // races in this tree. The child is this same test binary, re-run with one + // A lock is per open file description, so threads that each open the file + // contend as processes do. A child process is for what a thread cannot + // show: a holder that dies, and a lock whose release no copy of this + // process's descriptors delays (see [`Elsewhere`]). The child is this same test binary, re-run with one // `#[ignore]`d test selected by name and its role in the environment, so an // ordinary `cargo test` never runs the child half on its own. const ROLE: &str = "TOYOS_BUILDLOCK_TEST_ROLE"; @@ -821,8 +839,8 @@ pub(crate) mod tests { } /// **One key is built once, and two keys never meet.** A second process - /// wanting the key being built waits on the builder's lock — not on a - /// timer — and gets it only once the build is done; a process making + /// wanting the key being built waits for it, and gets it only once the + /// build is done; a process making /// another key is not held at all; and a sweep cannot take a key that /// somebody is making or using. #[test] @@ -902,18 +920,27 @@ pub(crate) mod tests { ); } - /// **A build failing while others wait for its key lets every one of them - /// in, and none waits on another's use.** One thread's make of a sysroot - /// fails while eight others wait for the key; one of them makes it, and - /// every one then holds its use at once, as builds hold theirs for their - /// whole length. The waiters are threads of one process, as a test run's - /// builds are. + /// **Builds waiting on a key's making all get in, whether the making + /// succeeds or fails, and none waits on another's use.** Eight threads + /// wait while one makes a sysroot; once it is made, by that one or, when + /// that one fails, by a waiter, every one holds its use at once, as builds + /// hold theirs for their whole length. The waiters are threads of one + /// process, as a test run's builds are. #[test] - fn waiters_on_a_failed_build_all_get_in() { + fn waiters_on_a_build_all_get_in() { + waiters_on_a_build_get_in(false); + waiters_on_a_build_get_in(true); + } + + const WAITERS: usize = 8; + + /// Whether all [`WAITERS`] waiters on a make that `fails` or not hold + /// their use within 20 s, the maker's own use held beside them. Asserted + /// here, before the store goes: waiters still waiting write in it. + fn waiters_on_a_build_get_in(fails: bool) { use std::sync::mpsc::channel; - const WAITERS: usize = 8; - let store = scratch("failed-build"); - let key = Key::of(b"a build that fails"); + let store = scratch("waited-build"); + let key = Key::of(b"a build waited on"); let whole = Keyed::Sysroot.store(&store).join(&key); let defect = { let whole = whole.clone(); @@ -921,20 +948,22 @@ pub(crate) mod tests { }; let (started, has_started) = channel(); - let (fail, failed) = channel::<()>(); - let failing = { - let (store, key, defect) = (store.to_path_buf(), key.clone(), defect.clone()); + let (finish, finished) = channel::<()>(); + let (used, uses) = channel(); + let first = { + let (store, key, defect, whole) = (store.to_path_buf(), key.clone(), defect.clone(), whole.clone()); std::thread::spawn(move || { - keyed_made(&store, Keyed::Sysroot, &key, defect, || { + let using = keyed_made(&store, Keyed::Sysroot, &key, defect, || { started.send(()).unwrap(); - failed.recv().unwrap(); - panic!("the build failed"); - }) + finished.recv().unwrap(); + assert!(!fails, "the build failed"); + fs::create_dir_all(&whole).unwrap(); + }); + using }) }; - has_started.recv_timeout(Duration::from_secs(20)).expect("the failing build never began"); + has_started.recv_timeout(Duration::from_secs(20)).expect("the first build never began"); - let (used, uses) = channel(); for _ in 0..WAITERS { let (store, key, defect, whole, used) = (store.to_path_buf(), key.clone(), defect.clone(), whole.clone(), used.clone()); std::thread::spawn(move || { @@ -942,11 +971,12 @@ pub(crate) mod tests { used.send(using).unwrap(); }); } - // Long enough for every waiter to be waiting before the build fails; + // Long enough for every waiter to be waiting before the build ends; // one that is not yet only makes this test weaker. std::thread::sleep(Duration::from_millis(300)); - fail.send(()).unwrap(); - assert!(failing.join().is_err(), "the failing build did not fail"); + finish.send(()).unwrap(); + let first = first.join(); + assert_eq!(first.is_err(), fails, "the first build did not end as staged"); let deadline = Instant::now() + Duration::from_secs(20); let mut held = Vec::new(); @@ -956,7 +986,9 @@ pub(crate) mod tests { Err(_) => break, } } - assert_eq!(held.len(), WAITERS, "of {WAITERS} builds waiting on a failed one, only {} got their sysroot in 20 s", held.len()); + let ended = if fails { "failed" } else { "succeeded" }; + assert_eq!(held.len(), WAITERS, "of {WAITERS} builds waiting on one that {ended}, only {} got their sysroot in 20 s", held.len()); + drop(first); } /// **A build that found the key unmade, while another made it, does not @@ -978,30 +1010,34 @@ pub(crate) mod tests { let _ = tx.send(using); }); } - let got = rx.recv_timeout(Duration::from_secs(20)); + // Within the holder's own 20 s, so the verdict is this test's. + let using = rx.recv_timeout(Duration::from_secs(10)).expect("a build that found the key unmade waited out another's use of it for 10 s"); user.release(); - drop(got.expect("a build that found the key unmade waited out another's use of it for 20 s")); + drop(using); } - /// **A wait that outlasts its deadline fails, naming the holder.** + /// **A wait that outlasts its deadline fails, naming the holder**: a build + /// wanting a key that another holds in the making for longer. #[test] fn a_wait_past_its_deadline_names_the_holder() { - let root = scratch("deadline"); + let store = scratch("deadline"); let key = Key::of(b"never made"); - let _building = keyed_building(&root, Keyed::Sysroot, &key).expect("nobody else has the key"); - let path = keyed_lock_path(&root, Keyed::Sysroot, &key); + let _building = keyed_building(&store, Keyed::Sysroot, &key).expect("nobody else has the key"); let (tx, rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let file = open_lock_file(&path); - let waited = std::panic::catch_unwind(|| { - take(&file, Mode::Shared, &path, "sysroot lock", "using it", UNNOTED, Instant::now() + Duration::from_millis(500)) + { + let (store, key) = (store.to_path_buf(), key.clone()); + std::thread::spawn(move || { + let waited = std::panic::catch_unwind(|| { + let defect = || Some("not made".to_string()); + keyed_made_by(&store, Keyed::Sysroot, &key, defect, || panic!("made under another's making"), Instant::now() + Duration::from_millis(500)) + }); + let _ = tx.send(waited.map(drop).map_err(|e| e.downcast_ref::().cloned().unwrap_or_default())); }); - let _ = tx.send(waited.map_err(|e| e.downcast_ref::().cloned().unwrap_or_default())); - }); + } let refusal = rx .recv_timeout(Duration::from_secs(20)) .expect("a wait with a 500 ms deadline had not ended 20 s later") - .expect_err("a wait on a lock nobody let go of succeeded"); + .expect_err("a wait on a key nobody let go of succeeded"); let holder = format!("held by pid {} (building sysroot {key})", std::process::id()); assert!(refusal.contains("gave up") && refusal.contains(&holder), "the refusal does not name the holder: {refusal}"); } diff --git a/issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md b/issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md deleted file mode 100644 index f209e2a073a..00000000000 --- a/issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-10 ---- - -# A suite that builds its own sysroot can wedge on its own locks - -`build/buildlock.rs` orders a sysroot key's lock before the worktree build lock, -and takes the key's lock with the worktree lock put down. One run of the guest -suite (`cargo test --test toyos-build`, 12 wide) on a head whose sysroot no -earlier build had made held both the other way round, inside one harness -process, and stopped: - -- at 11:11:53 every worker waited on `building sysroot 697f7bee2b30245e`, and - one built it; -- at 11:14:26 the waiters acquired it in turn, one began `clean crate targets - against changed external deps` and waited for the build lock exclusive, - `held by other builds in this tree`, while workers that held it shared for a - test binary waited on `using sysroot 697f7bee2b30245e — held, but the holder - left no readable note`; -- 450 s later six tests (`https_fetch`, `iommu_virtio_platform`, - `machine_shutdown`, `nested_nmi_is_loud`, `netstack_streams`, - `netstack_streams_e1000e`) had not started, `lsof` showed the key's lock file - open in the harness alone, and the run was stopped. The same head, its - sysroot made, ran 50 of 50 green in 49 s. - -`cargo run -- --ci host` ran beside it in the same worktree; whether it took -part is not established. - -It recurred on #836's round 4, the same shape on the host the other agents' -suites loaded: the harness built `fa7a35ac0091a05b`'s std by 14:35:56, ran its -other tests, and from 14:43:03 one worker alone waited on `using sysroot -fa7a35ac0091a05b — the holder left no readable note`, 930 s by 14:48:17, with -`lsof` showing the key's lock file open twice in the harness and nowhere else. -`--ci host` again ran beside it and had finished at 14:45:47. - -It recurred on #1052's round 2, with nothing beside it: right after a `cargo -clean`, `cargo test --test toyos-build -- netstack_holders_exported_to_children -stream_holders libc_children iommu_virtio_platform` (harness pid 73228) took -the build lock exclusive to build `3c919ba2399d86e1` at 20:29:49 and finished -it at 20:31:15. Three of its tests ran and passed; `stream_holders`'s worker, -whose `stream_handles` build had begun at 20:29:45, waited on `using sysroot -3c919ba2399d86e1` from 20:29:49, first "held by pid 73228 (building sysroot -3c919ba2399d86e1)" and after the build "the holder left no readable note", for -1,290 s by 20:51:19, when the run was stopped. Alone, the same row passed in 8 s -(PR #1052's round 2 evidence, `guest-r2.log` and `guest-r2b.log`). - -**Exit condition**: no path holds the worktree build lock while it waits on a -key's lock, and a suite started on a head whose sysroot is not yet made -finishes. diff --git a/issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md b/issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md new file mode 100644 index 00000000000..a16db6b539d --- /dev/null +++ b/issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md @@ -0,0 +1,36 @@ +--- +status: open +kind: tooling +opened: 2026-11-01 +--- + +# macOS lets blocked shared flock waiters in one at a time, so the build locks poll + +When an exclusive `flock` is released, macOS admits one of the shared waiters +queued behind it. Each of the others is admitted only when the shared holder +ahead of it releases, although a new non-blocking shared try succeeds at once. +This was measured on Darwin 27.0.0, across threads and across processes alike, +by a stand-alone [probe](https://github.com/ToyOSOrg/evidence/blob/b2a2e6ed4e4948703ab270566db21d3ead2f35c9/r1/probe/main.rs) +([output](https://github.com/ToyOSOrg/evidence/blob/b2a2e6ed4e4948703ab270566db21d3ead2f35c9/r1/probe/probe.log)). +A build holds its uses for its whole length, so a waiter blocked in the kernel +waits on another build's whole length, and forever on a use held to the end of +a run. That is how test runs waited 2611 s and more on a sysroot their own +process had made (#1145, whose negative controls block in the kernel and admit +1 of 8). + +So `build/buildlock.rs` polls every lock (`poll`): it tries, then sleeps, +starting at 1 ms and doubling up to 50 ms. Its costs: + +- up to 50 ms between a release and a long waiter's next try, on every host; +- no FIFO order among waiters of one mode. A new waiter, trying often, can + overtake long waiters, and only the 6 h ceiling bounds how often. The + artifact lock is the exposed one, with five waiters at once for over 100 s in + a 12-wide run. Exclusive-before-shared order still holds, through `intent`. + +**Exit**: one of these lets a wait block on an event again, with a timed join +or equivalent for its ceiling, and the poll go. +- An event-based primitive that admits every compatible waiter on every host + the build supports, and keeps flock's release on holder death. +- macOS admitting every compatible shared waiter, shown by the probe. + +**Owner**: the orchestrator, who assigns the build track.