diff --git a/build/buildlock.rs b/build/buildlock.rs index 835d0481882..e25d683a7d4 100644 --- a/build/buildlock.rs +++ b/build/buildlock.rs @@ -44,20 +44,20 @@ use std::fs; use std::io::{self, Read, Seek, SeekFrom, Write}; -use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::keystore::Key; unsafe extern "C" { - fn flock(fd: i32, operation: i32) -> i32; fn kill(pid: i32, sig: i32) -> i32; } -const LOCK_SH: i32 = 1; -const LOCK_EX: i32 = 2; -const LOCK_NB: i32 = 4; +#[derive(Clone, Copy, PartialEq, Eq)] +enum Mode { + Shared, + Exclusive, +} /// How often a wait that is lasting repeats itself. /// @@ -69,6 +69,14 @@ const HEARTBEAT: Duration = Duration::from_secs(30); #[cfg(test)] const HEARTBEAT: Duration = Duration::from_millis(300); +/// How long any one acquisition waits before it fails, naming the holder: past +/// the 350 minutes `.github/workflows/toolchain.yml` allows the whole +/// toolchain, the longest phase any of these locks covers. +const CEILING: Duration = Duration::from_secs(6 * 60 * 60); + +/// The longest a waiter sleeps between two tries ([`poll`]). +const MAX_PAUSE: Duration = Duration::from_millis(50); + const LOCK_DIR: &str = ".build-locks"; /// A held lock. Releasing it is closing the file. @@ -133,7 +141,7 @@ impl Held { } let dir = self.worktree_dir.clone(); self.without_shared(|| { - let _exclusive = acquire(&dir, LOCK_EX, phase, BUILD); + let _exclusive = acquire(&dir, Mode::Exclusive, phase, BUILD); if let Some(work) = decide() { act(work); } @@ -150,14 +158,14 @@ impl Held { } fn take_shared(&self) -> Guard { - acquire(&self.worktree_dir, LOCK_SH, &self.what, BUILD) + acquire(&self.worktree_dir, Mode::Shared, &self.what, BUILD) } } /// This worktree's build lock, exclusively: what a sysroot build holds while it /// writes the worktree's fork build directory, with the key's lock already held. pub fn worktree_exclusive(root: &Path, what: &str) -> Guard { - acquire(&root.join(LOCK_DIR), LOCK_EX, what, BUILD) + acquire(&root.join(LOCK_DIR), Mode::Exclusive, what, BUILD) } /// Exclusive lock over the shared cargo artifact paths. @@ -211,63 +219,89 @@ impl Keyed { } } -/// Make what `key` names: exclusive, and waited for by every other process that -/// wants the same key, which then finds it made. -fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Guard { - let lock = format!("{} lock", kind.name()); - exclusive(&keyed_lock_path(store, kind, key), &lock, &format!("building {} {key}", kind.name())) +/// Make what `key` names, if nobody is making or using it: exclusive, and +/// waited for by every other process that wants the same key, which then finds +/// it made. +fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Option { + let file = open_lock_file(&keyed_lock_path(store, kind, key)); + if !try_lock(&file, Mode::Exclusive) { + return None; + } + let mut guard = Guard { file, records_holder: true }; + write_note(&mut guard.file, ¬e_text(&format!("building {} {key}", kind.name()))); + Some(guard) } /// Use what `key` names: shared, so any number of builds use it at once, a /// builder of it is waited for, and a sweep cannot remove it; and dated now, /// which a sweep reads as its last use. pub(crate) fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { + let (lock, what) = (format!("{} lock", kind.name()), format!("using {} {key}", kind.name())); + let path = keyed_lock_path(store, kind, key); + poll(&path, &lock, &what, UNNOTED, Instant::now() + CEILING, || keyed_try_using(store, kind, key)) +} + +/// [`keyed_using`], if nobody is making it now. +fn keyed_try_using(store: &Path, kind: Keyed, key: &Key) -> Option { let path = keyed_lock_path(store, kind, key); let file = open_lock_file(&path); - if !try_lock(&file, LOCK_SH) { - let lock = format!("{} lock", kind.name()); - let what = format!("using {} {key}", kind.name()); - let holder = describe_holder(&path) - .unwrap_or_else(|| "held, but the holder left no readable note".to_string()); - announce(&lock, &what, &holder); - take_lock_announcing(&file, LOCK_SH, &path, &lock, &what); + if !try_lock(&file, Mode::Shared) { + return None; } file.set_modified(SystemTime::now()).unwrap_or_else(|e| panic!("build lock: date {}: {e}", path.display())); - Guard { file, records_holder: false } + Some(Guard { file, records_holder: false }) } /// What `key` names, held in use and whole: made by `make` under /// [`keyed_building`] while `defect`, which says why it is not whole, says it is /// not. A `make` that leaves it not whole is refused by that defect rather than /// run again. +/// +/// The making is tried, never waited for: a waiter queued to make it would +/// wait out every use of what another waiter made meanwhile. pub fn keyed_made( + store: &Path, + kind: Keyed, + key: &Key, + defect: impl Fn() -> Option, + make: impl FnMut(), +) -> Guard { + keyed_made_by(store, kind, key, defect, make, Instant::now() + CEILING) +} + +/// [`keyed_made`], failing at `deadline`. +fn keyed_made_by( store: &Path, kind: Keyed, key: &Key, defect: impl Fn() -> Option, mut make: impl FnMut(), + deadline: Instant, ) -> Guard { - loop { - let using = keyed_using(store, kind, key); - if defect().is_none() { - return using; + let (lock, what) = (format!("{} lock", kind.name()), format!("using {} {key}", kind.name())); + poll(&keyed_lock_path(store, kind, key), &lock, &what, UNNOTED, deadline, || { + if let Some(using) = keyed_try_using(store, kind, key) { + if defect().is_none() { + return Some(using); + } } - drop(using); - let _building = keyed_building(store, kind, key); + let building = keyed_building(store, kind, key)?; if defect().is_some() { make(); if let Some(defect) = defect() { panic!("{} {key} was made, and is not whole: {defect}", kind.name()); } } - } + drop(building); + keyed_try_using(store, kind, key).filter(|_| defect().is_none()) + }) } /// What `key` names, exclusively and only if nobody is making or using it: what /// a sweep holds while it removes one. pub fn keyed_idle(store: &Path, kind: Keyed, key: &Key) -> Option { let file = open_lock_file(&keyed_lock_path(store, kind, key)); - try_lock(&file, LOCK_EX).then_some(Guard { file, records_holder: false }) + try_lock(&file, Mode::Exclusive).then_some(Guard { file, records_holder: false }) } /// When `kind`'s `key` in `store` was last used or made. A key nothing has @@ -289,14 +323,7 @@ pub(crate) fn keyed_lock_path(store: &Path, kind: Keyed, key: &Key) -> PathBuf { /// can always be told who it is waiting for. fn exclusive(path: &Path, lock: &str, what: &str) -> Guard { let file = open_lock_file(path); - let start = Instant::now(); - if !try_lock(&file, LOCK_EX) { - let holder = describe_holder(path) - .unwrap_or_else(|| "held, but the holder left no readable note".to_string()); - announce(lock, what, &holder); - take_lock_announcing(&file, LOCK_EX, path, lock, what); - eprintln!("[build-lock] {what} acquired after {:.1?}", start.elapsed()); - } + take(&file, Mode::Exclusive, path, lock, what, UNNOTED, Instant::now() + CEILING); let mut guard = Guard { file, records_holder: true }; write_note(&mut guard.file, ¬e_text(what)); guard @@ -328,41 +355,22 @@ const BUILD: Lock = Lock { /// host, four shared churners kept an exclusive waiter out for the whole 5.5 s /// they ran — and the exclusive phases are exactly the long, silent ones an /// agent kills and retries. So an exclusive acquirer holds `intent` while it -/// queues for `state`, which makes later shared acquirers line up behind it +/// waits for `state`, which makes later shared acquirers wait behind it /// instead of overtaking it. `intent` is always taken before `state` and /// dropped as soon as `state` is held, so nothing ever waits on `intent` while /// holding `state`. -fn acquire(dir: &Path, op: i32, what: &str, lock: Lock) -> Guard { +fn acquire(dir: &Path, mode: Mode, what: &str, lock: Lock) -> Guard { let intent_path = dir.join("intent"); let state_path = dir.join("state"); let intent = open_lock_file(&intent_path); let state = open_lock_file(&state_path); - let label = format!("{}, {what}", if op == LOCK_EX { "exclusive" } else { "shared" }); - - let start = Instant::now(); - let mut waited = false; - - if !try_lock(&intent, op) { - announce(lock.name, &label, lock.queued_ahead); - waited = true; - take_lock_announcing(&intent, op, &intent_path, lock.name, &label); - } - if !try_lock(&state, op) { - if !waited { - let holder = describe_holder(&state_path) - .unwrap_or_else(|| lock.shared_holders.to_string()); - announce(lock.name, &label, &holder); - waited = true; - } - take_lock_announcing(&state, op, &state_path, lock.name, &label); - } + let label = format!("{}, {what}", if mode == Mode::Exclusive { "exclusive" } else { "shared" }); + let deadline = Instant::now() + CEILING; + take(&intent, mode, &intent_path, lock.name, &label, lock.queued_ahead, deadline); + take(&state, mode, &state_path, lock.name, &label, lock.shared_holders, deadline); drop(intent); - if waited { - eprintln!("[build-lock] acquired ({label}) after {:.1?}", start.elapsed()); - } - - let records_holder = op == LOCK_EX; + let records_holder = mode == Mode::Exclusive; let mut guard = Guard { file: state, records_holder }; if records_holder { write_note(&mut guard.file, ¬e_text(what)); @@ -370,47 +378,47 @@ fn acquire(dir: &Path, op: i32, what: &str, lock: Lock) -> Guard { guard } -/// An agent staring at silence kills and retries, which is the pathology this -/// module exists to remove — so a wait says what it is waiting for and, when -/// that can be established, who has it. -fn announce(lock: &str, label: &str, holder: &str) { - eprintln!("[build-lock] waiting for the {lock} ({label}) — {holder}"); +/// What a waiter is told of a holder that left no note it can read. +const UNNOTED: &str = "held, but the holder left no readable note"; + +/// Take `mode` of `file`, the lock at `path`, by `deadline` ([`poll`]). +fn take(file: &fs::File, mode: Mode, path: &Path, lock: &str, label: &str, unnoted: &str, deadline: Instant) { + poll(path, lock, label, unnoted, deadline, || try_lock(file, mode).then_some(())); } -/// [`take_lock`], saying every 30 s that it is still waiting and who for. +/// What `attempt` gets, tried until it gets it or `deadline` passes, which +/// fails naming the holder of `path`. A wait says what it waits for and who +/// has it, when that can be established ([`describe_holder`], else `unnoted`), +/// and says so again every [`HEARTBEAT`]: an agent staring at silence kills and +/// retries. /// -/// The kernel keeps the queue and a thread does the talking: nothing here polls -/// a lock, so `flock`'s own ordering is given up nowhere. The holder is re-read -/// each time, so the message follows the queue forward rather than naming the -/// process that was in front when the wait began. -fn take_lock_announcing(file: &fs::File, op: i32, path: &Path, lock: &str, label: &str) { - use std::sync::mpsc::{channel, RecvTimeoutError}; - - // A channel and not a polled flag: this runs on every *contended* - // acquisition, the artifact lock among them, and a flag checked every few - // milliseconds would put that granularity on the front of each one. The - // sender dropping wakes the thread at once and it never sleeps past the - // acquisition. - let (tx, rx) = channel::<()>(); - let heartbeat = { - let path = path.to_path_buf(); - let lock = lock.to_string(); - let label = label.to_string(); - std::thread::spawn(move || { - let began = Instant::now(); - while rx.recv_timeout(HEARTBEAT) == Err(RecvTimeoutError::Timeout) { - let holder = describe_holder(&path) - .unwrap_or_else(|| "the holder left no readable note".to_string()); - eprintln!( - "[build-lock] still waiting for the {lock} ({label}), {:.0?} so far — {holder}", - began.elapsed() - ); - } - }) - }; - take_lock(file, op, path); - drop(tx); - heartbeat.join().expect("the lock heartbeat panicked"); +/// A poll, not a blocking `flock`: macOS lets blocked shared waiters in one at +/// a time (`issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md`). +fn poll(path: &Path, lock: &str, label: &str, unnoted: &str, deadline: Instant, mut attempt: impl FnMut() -> Option) -> T { + if let Some(got) = attempt() { + return got; + } + let holder = || describe_holder(path).unwrap_or_else(|| unnoted.to_string()); + eprintln!("[build-lock] waiting for the {lock} ({label}) — {}", holder()); + let began = Instant::now(); + let mut said = began; + let mut pause = Duration::from_millis(1); + loop { + let now = Instant::now(); + if now >= deadline { + panic!("build lock: gave up waiting for the {lock} ({label}) after {:.0?} — {}", began.elapsed(), holder()); + } + if now - said >= HEARTBEAT { + eprintln!("[build-lock] still waiting for the {lock} ({label}), {:.0?} so far — {}", began.elapsed(), holder()); + said = now; + } + std::thread::sleep(pause.min(deadline - now)); + pause = (pause * 2).min(MAX_PAUSE); + if let Some(got) = attempt() { + eprintln!("[build-lock] acquired the {lock} ({label}) after {:.1?}", began.elapsed()); + return got; + } + } } /// What the last exclusive holder of `path` recorded, if it is still running. @@ -478,33 +486,15 @@ fn open_lock_file(path: &Path) -> fs::File { .unwrap_or_else(|e| panic!("build lock: open {}: {e}", path.display())) } -fn take_lock(file: &fs::File, op: i32, path: &Path) { - loop { - // SAFETY: `file` owns the fd for the duration of the call and of the - // guard the caller builds from it. - if unsafe { flock(file.as_raw_fd(), op) } == 0 { - return; - } - let err = io::Error::last_os_error(); - if err.kind() == io::ErrorKind::Interrupted { - continue; - } - panic!("build lock: flock on {}: {err}", path.display()); - } -} - -fn try_lock(file: &fs::File, op: i32) -> bool { - loop { - // SAFETY: as in `take_lock`. - if unsafe { flock(file.as_raw_fd(), op | LOCK_NB) } == 0 { - return true; - } - let err = io::Error::last_os_error(); - match err.kind() { - io::ErrorKind::Interrupted => continue, - io::ErrorKind::WouldBlock => return false, - _ => panic!("build lock: flock: {err}"), - } +fn try_lock(file: &fs::File, mode: Mode) -> bool { + let tried = match mode { + Mode::Shared => file.try_lock_shared(), + Mode::Exclusive => file.try_lock(), + }; + match tried { + Ok(()) => true, + Err(fs::TryLockError::WouldBlock) => false, + Err(fs::TryLockError::Error(e)) => panic!("build lock: flock: {e}"), } } @@ -516,9 +506,10 @@ pub(crate) mod tests { use std::time::Duration; use toyos_tmpdir::TempDir; - // Two processes are the point. `flock` is per open file description, so a - // single-process test would prove nothing about the thing that actually - // races in this tree. The child is this same test binary, re-run with one + // A lock is per open file description, so threads that each open the file + // contend as processes do. A child process is for what a thread cannot + // show: a holder that dies, and a lock whose release no copy of this + // process's descriptors delays (see [`Elsewhere`]). The child is this same test binary, re-run with one // `#[ignore]`d test selected by name and its role in the environment, so an // ordinary `cargo test` never runs the child half on its own. const ROLE: &str = "TOYOS_BUILDLOCK_TEST_ROLE"; @@ -607,7 +598,7 @@ pub(crate) mod tests { /// Whether `root`'s worktree lock would keep a shared acquirer out now. pub(crate) fn keeps_out_shared(root: &Path) -> bool { - !try_lock(&open_lock_file(&worktree_lock_dir(root).join("state")), LOCK_SH) + !try_lock(&open_lock_file(&worktree_lock_dir(root).join("state")), Mode::Shared) } /// What `role` of [`child_role`] takes in `root`, held by a process of its own. @@ -672,7 +663,7 @@ pub(crate) mod tests { /// A fresh fd per probe: a successful `try_lock` *holds* what it took, and /// polling on one fd would itself be the thing keeping the writer out. fn intent_is_taken(root: &Path) -> bool { - !try_lock(&open_lock_file(&worktree_lock_dir(root).join("intent")), LOCK_SH) + !try_lock(&open_lock_file(&worktree_lock_dir(root).join("intent")), Mode::Shared) } fn note(root: &Path, line: &str) { @@ -724,7 +715,7 @@ pub(crate) mod tests { note(&root, "sh"); } "hold-sysroot-build" => { - let _building = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k1")); + let _building = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k1")).expect("nobody else has k1"); hold_until_released(); note(&root, "built"); } @@ -766,8 +757,8 @@ pub(crate) mod tests { let kid = held_elsewhere(&root, "hold-exclusive"); let state = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&state, LOCK_SH), "a build got in while an exclusive phase ran"); - assert!(!try_lock(&state, LOCK_EX), "two exclusive phases at once"); + assert!(!try_lock(&state, Mode::Shared), "a build got in while an exclusive phase ran"); + assert!(!try_lock(&state, Mode::Exclusive), "two exclusive phases at once"); let holder = describe_holder(&worktree_lock_dir(&root).join("state")) .expect("no holder note"); assert!( @@ -787,12 +778,12 @@ pub(crate) mod tests { assert!(appeared(&root.join("held"), Duration::from_secs(20)), "child never acquired"); let state = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&state, LOCK_EX), "the lock was not actually held"); + assert!(!try_lock(&state, Mode::Exclusive), "the lock was not actually held"); kid.kill().unwrap(); kid.wait().unwrap(); - assert!(try_lock(&state, LOCK_EX), "a SIGKILLed holder stranded the lock"); + assert!(try_lock(&state, Mode::Exclusive), "a SIGKILLed holder stranded the lock"); // And the note it left behind names a pid that is gone, so nobody is // sent to wait on it. assert_eq!(describe_holder(&worktree_lock_dir(&root).join("state")), None); @@ -803,10 +794,10 @@ pub(crate) mod tests { let root = scratch("shared"); let _mine = shared(&root, "parent"); let second = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(try_lock(&second, LOCK_SH), "two builds cannot run at once"); + assert!(try_lock(&second, Mode::Shared), "two builds cannot run at once"); drop(second); let third = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&third, LOCK_EX), "a clean got in while a build was running"); + assert!(!try_lock(&third, Mode::Exclusive), "a clean got in while a build was running"); } /// `flock` alone would let a stream of builds starve the rebuild they are @@ -848,8 +839,8 @@ pub(crate) mod tests { } /// **One key is built once, and two keys never meet.** A second process - /// wanting the key being built waits on the builder's lock — not on a - /// timer — and gets it only once the build is done; a process making + /// wanting the key being built waits for it, and gets it only once the + /// build is done; a process making /// another key is not held at all; and a sweep cannot take a key that /// somebody is making or using. #[test] @@ -858,7 +849,7 @@ pub(crate) mod tests { let builder = held_elsewhere(&root, "hold-sysroot-build"); assert!(keyed_idle(&root, Keyed::Sysroot, &Key::of(b"k1")).is_none(), "a sweep could remove a key being built"); - let other = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k2")); + let other = keyed_building(&root, Keyed::Sysroot, &Key::of(b"k2")).expect("a process making another key held this one"); drop(other); let mut user = child(&root, "want-sysroot"); @@ -929,6 +920,128 @@ pub(crate) mod tests { ); } + /// **Builds waiting on a key's making all get in, whether the making + /// succeeds or fails, and none waits on another's use.** Eight threads + /// wait while one makes a sysroot; once it is made, by that one or, when + /// that one fails, by a waiter, every one holds its use at once, as builds + /// hold theirs for their whole length. The waiters are threads of one + /// process, as a test run's builds are. + #[test] + fn waiters_on_a_build_all_get_in() { + waiters_on_a_build_get_in(false); + waiters_on_a_build_get_in(true); + } + + const WAITERS: usize = 8; + + /// Whether all [`WAITERS`] waiters on a make that `fails` or not hold + /// their use within 20 s, the maker's own use held beside them. Asserted + /// here, before the store goes: waiters still waiting write in it. + fn waiters_on_a_build_get_in(fails: bool) { + use std::sync::mpsc::channel; + let store = scratch("waited-build"); + let key = Key::of(b"a build waited on"); + let whole = Keyed::Sysroot.store(&store).join(&key); + let defect = { + let whole = whole.clone(); + move || (!whole.is_dir()).then(|| "not made".to_string()) + }; + + let (started, has_started) = channel(); + let (finish, finished) = channel::<()>(); + let (used, uses) = channel(); + let first = { + let (store, key, defect, whole) = (store.to_path_buf(), key.clone(), defect.clone(), whole.clone()); + std::thread::spawn(move || { + let using = keyed_made(&store, Keyed::Sysroot, &key, defect, || { + started.send(()).unwrap(); + finished.recv().unwrap(); + assert!(!fails, "the build failed"); + fs::create_dir_all(&whole).unwrap(); + }); + using + }) + }; + has_started.recv_timeout(Duration::from_secs(20)).expect("the first build never began"); + + for _ in 0..WAITERS { + let (store, key, defect, whole, used) = (store.to_path_buf(), key.clone(), defect.clone(), whole.clone(), used.clone()); + std::thread::spawn(move || { + let using = keyed_made(&store, Keyed::Sysroot, &key, defect, || fs::create_dir_all(&whole).unwrap()); + used.send(using).unwrap(); + }); + } + // Long enough for every waiter to be waiting before the build ends; + // one that is not yet only makes this test weaker. + std::thread::sleep(Duration::from_millis(300)); + finish.send(()).unwrap(); + let first = first.join(); + assert_eq!(first.is_err(), fails, "the first build did not end as staged"); + + let deadline = Instant::now() + Duration::from_secs(20); + let mut held = Vec::new(); + while held.len() < WAITERS { + match uses.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(using) => held.push(using), + Err(_) => break, + } + } + let ended = if fails { "failed" } else { "succeeded" }; + assert_eq!(held.len(), WAITERS, "of {WAITERS} builds waiting on one that {ended}, only {} got their sysroot in 20 s", held.len()); + drop(first); + } + + /// **A build that found the key unmade, while another made it, does not + /// wait out that one's use**: the key is whole and in use by another + /// process, and `defect` says it is not once, as it did for a build that + /// checked just before it was placed. + #[test] + fn a_would_be_maker_does_not_wait_out_another_use() { + let store = scratch("would-be-maker"); + let key = Key::of(b"made meanwhile"); + let user = sysroot_used_elsewhere(&store, &key); + let (tx, rx) = std::sync::mpsc::channel(); + { + let store = store.to_path_buf(); + std::thread::spawn(move || { + let checked = std::cell::Cell::new(false); + let defect = || (!checked.replace(true)).then(|| "placed just after this check".to_string()); + let using = keyed_made(&store, Keyed::Sysroot, &key, defect, || panic!("made twice")); + let _ = tx.send(using); + }); + } + // Within the holder's own 20 s, so the verdict is this test's. + let using = rx.recv_timeout(Duration::from_secs(10)).expect("a build that found the key unmade waited out another's use of it for 10 s"); + user.release(); + drop(using); + } + + /// **A wait that outlasts its deadline fails, naming the holder**: a build + /// wanting a key that another holds in the making for longer. + #[test] + fn a_wait_past_its_deadline_names_the_holder() { + let store = scratch("deadline"); + let key = Key::of(b"never made"); + let _building = keyed_building(&store, Keyed::Sysroot, &key).expect("nobody else has the key"); + let (tx, rx) = std::sync::mpsc::channel(); + { + let (store, key) = (store.to_path_buf(), key.clone()); + std::thread::spawn(move || { + let waited = std::panic::catch_unwind(|| { + let defect = || Some("not made".to_string()); + keyed_made_by(&store, Keyed::Sysroot, &key, defect, || panic!("made under another's making"), Instant::now() + Duration::from_millis(500)) + }); + let _ = tx.send(waited.map(drop).map_err(|e| e.downcast_ref::().cloned().unwrap_or_default())); + }); + } + let refusal = rx + .recv_timeout(Duration::from_secs(20)) + .expect("a wait with a 500 ms deadline had not ended 20 s later") + .expect_err("a wait on a key nobody let go of succeeded"); + let holder = format!("held by pid {} (building sysroot {key})", std::process::id()); + assert!(refusal.contains("gave up") && refusal.contains(&holder), "the refusal does not name the holder: {refusal}"); + } + /// The defect this module exists for, staged so that it is not itself a /// race: a clean lands in the middle of a build in the same target /// directory, and the build's next write finds the directory gone. Run once diff --git a/issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md b/issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md deleted file mode 100644 index f209e2a073a..00000000000 --- a/issues/a-suite-that-builds-its-sysroot-can-wedge-on-its-own-locks.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-10 ---- - -# A suite that builds its own sysroot can wedge on its own locks - -`build/buildlock.rs` orders a sysroot key's lock before the worktree build lock, -and takes the key's lock with the worktree lock put down. One run of the guest -suite (`cargo test --test toyos-build`, 12 wide) on a head whose sysroot no -earlier build had made held both the other way round, inside one harness -process, and stopped: - -- at 11:11:53 every worker waited on `building sysroot 697f7bee2b30245e`, and - one built it; -- at 11:14:26 the waiters acquired it in turn, one began `clean crate targets - against changed external deps` and waited for the build lock exclusive, - `held by other builds in this tree`, while workers that held it shared for a - test binary waited on `using sysroot 697f7bee2b30245e — held, but the holder - left no readable note`; -- 450 s later six tests (`https_fetch`, `iommu_virtio_platform`, - `machine_shutdown`, `nested_nmi_is_loud`, `netstack_streams`, - `netstack_streams_e1000e`) had not started, `lsof` showed the key's lock file - open in the harness alone, and the run was stopped. The same head, its - sysroot made, ran 50 of 50 green in 49 s. - -`cargo run -- --ci host` ran beside it in the same worktree; whether it took -part is not established. - -It recurred on #836's round 4, the same shape on the host the other agents' -suites loaded: the harness built `fa7a35ac0091a05b`'s std by 14:35:56, ran its -other tests, and from 14:43:03 one worker alone waited on `using sysroot -fa7a35ac0091a05b — the holder left no readable note`, 930 s by 14:48:17, with -`lsof` showing the key's lock file open twice in the harness and nowhere else. -`--ci host` again ran beside it and had finished at 14:45:47. - -It recurred on #1052's round 2, with nothing beside it: right after a `cargo -clean`, `cargo test --test toyos-build -- netstack_holders_exported_to_children -stream_holders libc_children iommu_virtio_platform` (harness pid 73228) took -the build lock exclusive to build `3c919ba2399d86e1` at 20:29:49 and finished -it at 20:31:15. Three of its tests ran and passed; `stream_holders`'s worker, -whose `stream_handles` build had begun at 20:29:45, waited on `using sysroot -3c919ba2399d86e1` from 20:29:49, first "held by pid 73228 (building sysroot -3c919ba2399d86e1)" and after the build "the holder left no readable note", for -1,290 s by 20:51:19, when the run was stopped. Alone, the same row passed in 8 s -(PR #1052's round 2 evidence, `guest-r2.log` and `guest-r2b.log`). - -**Exit condition**: no path holds the worktree build lock while it waits on a -key's lock, and a suite started on a head whose sysroot is not yet made -finishes. diff --git a/issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md b/issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md deleted file mode 100644 index efd129eaf52..00000000000 --- a/issues/a-test-build-waited-forty-minutes-on-its-own-processs-sysroot-lock.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-11-01 ---- - -# A test build waited forty minutes on its own process's sysroot lock - -One `cargo test --test toyos-build` run, twelve wide, of eight guest tests -at `929e73d6c` (#1077's branch, on main `190f3b164`), on a macOS host, with -the sysroot rebuilt for an SDK change: - -- 17:35:32: seven guest-binary builds wait for sysroot `f2cc0cfbcb345f2d`'s - lock, "held by pid 86112 (building sysroot …)", which is the test - process itself. -- 17:39:59 to 17:44: the sysroot is built, and six of the seven waiters go - on by 17:45:07. -- 17:45:06: another worker's guest build (`shm_commit`) fails to compile: - `cargo build failed`, the panic at `build/build.rs:494`. -- The seventh waiter, `stream_handles` for `stream_holders`, goes on - waiting, "the holder left no readable note", until the run is killed at - 2611 s. Nothing compiles in the meantime, and the process has no - children. - -The next run, with the compile error fixed, did not wait. Whether the -failed build is the cause is not established; it is the only event of -that run that the others did not have. The log, with its paths masked, is -[`r3/logs/qemu1.log`](https://github.com/ToyOSOrg/evidence/blob/9ad74c276117e0418c04b546764e4afbada6f166/r3/logs/qemu1.log). - -**Exit**: the wait reproduced and its cause fixed in `build/buildlock.rs` -or its callers, or the lock's acquisition shown unable to wait on a holder -in its own process. - -**Owner**: the orchestrator, who assigns the build track. diff --git a/issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md b/issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md new file mode 100644 index 00000000000..a16db6b539d --- /dev/null +++ b/issues/macos-lets-blocked-shared-flock-waiters-in-one-at-a-time.md @@ -0,0 +1,36 @@ +--- +status: open +kind: tooling +opened: 2026-11-01 +--- + +# macOS lets blocked shared flock waiters in one at a time, so the build locks poll + +When an exclusive `flock` is released, macOS admits one of the shared waiters +queued behind it. Each of the others is admitted only when the shared holder +ahead of it releases, although a new non-blocking shared try succeeds at once. +This was measured on Darwin 27.0.0, across threads and across processes alike, +by a stand-alone [probe](https://github.com/ToyOSOrg/evidence/blob/b2a2e6ed4e4948703ab270566db21d3ead2f35c9/r1/probe/main.rs) +([output](https://github.com/ToyOSOrg/evidence/blob/b2a2e6ed4e4948703ab270566db21d3ead2f35c9/r1/probe/probe.log)). +A build holds its uses for its whole length, so a waiter blocked in the kernel +waits on another build's whole length, and forever on a use held to the end of +a run. That is how test runs waited 2611 s and more on a sysroot their own +process had made (#1145, whose negative controls block in the kernel and admit +1 of 8). + +So `build/buildlock.rs` polls every lock (`poll`): it tries, then sleeps, +starting at 1 ms and doubling up to 50 ms. Its costs: + +- up to 50 ms between a release and a long waiter's next try, on every host; +- no FIFO order among waiters of one mode. A new waiter, trying often, can + overtake long waiters, and only the 6 h ceiling bounds how often. The + artifact lock is the exposed one, with five waiters at once for over 100 s in + a 12-wide run. Exclusive-before-shared order still holds, through `intent`. + +**Exit**: one of these lets a wait block on an event again, with a timed join +or equivalent for its ceiling, and the poll go. +- An event-based primitive that admits every compatible waiter on every host + the build supports, and keeps flock's release on holder death. +- macOS admitting every compatible shared waiter, shown by the probe. + +**Owner**: the orchestrator, who assigns the build track.