diff --git a/build/ci.rs b/build/ci.rs index 32a8297bd99..7bad31e168c 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -840,6 +840,23 @@ pub(crate) const CONTROLS: &[Control] = &[ red(SDK, "transport-end-keeps-inflight", None, &[Fails( "transport::inflight::tests::an_end_answers_every_tag_once_and_a_late_completion_nothing", )]), + // The descriptor table's hold beside its writer. + red(SDK, "fd-hold-revives", None, &[ + Fails("fd::table::model::a_call_beside_a_close_holds_a_live_object_or_none"), + Fails("fd::table::model::two_calls_beside_a_close_let_the_object_go_once"), + Fails("fd::table::model::a_call_beside_a_dup2_holds_the_old_object_or_the_new"), + ]), + red(SDK, "fd-hold-unchecked", None, &[Fails( + "fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot", + )]), + red(SDK, "fd-publish-relaxed", None, &[ + Fails("fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot"), + Fails("fd::table::model::a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new"), + ]), + red(SDK, "fd-reuse-relaxed", None, &[ + Fails("fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot"), + Fails("fd::table::model::a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new"), + ]), ]; /// Whether a control's run showed its teeth. diff --git a/issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md b/issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md new file mode 100644 index 00000000000..c03c55963ae --- /dev/null +++ b/issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md @@ -0,0 +1,25 @@ +--- +status: open +kind: finding +opened: 2026-11-11 +--- + +# Calls on two served files meet on one process-wide lock + +A call through a descriptor holds its object without a lock (#1234). A served +file's object is an index into `files::owner`'s `OPEN` +(`sdk/toyos/src/files/owner.rs`), one `Lock` over every served file the +process holds. `held()` takes it in every call on a served file +(`on_fd_held`) to clone that file's `Arc`, and `release_held` takes it when +the descriptor table lets one go. So two threads reading two different served +files still meet on one lock word each call, which is the collision the +descriptor table's hold no longer has. The critical section is short, so a +collision costs a brief wait. Found by inspection in the review of #1234; +nothing has measured it. + +Owner: the SDK's served-file client (`toyos::files`). + +**Exit**: a call on one served file takes no lock that a call on another +served file takes, or a metal measurement of two threads on two served files +shows the lock costs nothing material, with that measurement cited here before +the file is deleted. diff --git a/issues/every-descriptor-call-takes-one-process-wide-lock-twice.md b/issues/every-descriptor-call-takes-one-process-wide-lock-twice.md deleted file mode 100644 index ab71522a6c3..00000000000 --- a/issues/every-descriptor-call-takes-one-process-wide-lock-twice.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-11-04 ---- - -# Every descriptor call takes one process-wide lock twice - -A `read` or `write` through a descriptor (`sdk/std/sys/pal/mod.rs`'s `on_fd`) -holds its object for the call through `toyos::fd::hold`. That takes the -process's one descriptor-table lock (`sdk/toyos/src/fd.rs`, `TABLE`) to acquire -the slot and again to let it go. Two threads of one process doing I/O on -different descriptors meet on that lock twice per call. - -The lock was a plain futex lock, so any collision slept in `FUTEX_WAIT` and -woke through `FUTEX_WAKE`. In the S0-a metal A/B (#1150, round 2), the pipe -benchmark's writer and reader threads (`osbench` `pipe.throughput.*`) made -about 125-150 futex waits a run on main. With the kernel no longer -serializing their pipe calls under its process lock, they made 128,000-153,000 -on the T14. That was much of the small-pipe-write regression measured there. - -`toyos::sync::lock_word` now spins briefly before sleeping, which turns a -collision into a short wait. Spinning bounds the cost of a collision; it does -not stop two threads meeting on a lock neither needs. - -**Exit**: a descriptor's hold for one call takes no lock that a call on another -descriptor takes. diff --git a/sdk/toyos/Cargo.toml b/sdk/toyos/Cargo.toml index 4dbfda621c4..8e3f39d26e1 100644 --- a/sdk/toyos/Cargo.toml +++ b/sdk/toyos/Cargo.toml @@ -95,6 +95,21 @@ transport-no-resume = [] # A look without the gate is believed when it names a violation, so a cursor # another holder moved under it ends the call. transport-ungated-violation = [] +# The descriptor table's controls (`src/fd/table.rs`), declared and run as the +# transport's are; `src/fd/table/model.rs` holds a call beside the writer. +# +# A call counts itself onto a slot whose count is 0, so it holds an object +# the close already let go. +fd-hold-revives = [] +# A call keeps its reference without looking at the descriptor again, so it +# holds whatever took the slot its descriptor named. +fd-hold-unchecked = [] +# The writer stores a new object's count `Relaxed`, so a call that counts +# itself on need not see the object written. +fd-publish-relaxed = [] +# The writer finds a slot free by a `Relaxed` load, so its write of a new +# object need not follow the last call's read of the old one. +fd-reuse-relaxed = [] [lints.rust] # The guest's target: its stream sinks are compiled only for it. diff --git a/sdk/toyos/src/fd.rs b/sdk/toyos/src/fd.rs index 3a82b0c54ab..711897b4afb 100644 --- a/sdk/toyos/src/fd.rs +++ b/sdk/toyos/src/fd.rs @@ -389,48 +389,70 @@ fn file_group(label: &str) -> Option { /// The table itself, in the copy that owns it. #[cfg(all(feature = "fd-table", not(test)))] mod owner { - use super::table::{Fault, Table}; + use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + + use super::table::{Fault, ObjectCell, Table, Writer}; use super::*; use crate::endow::Endowments; use crate::sync::Lock; use crate::OwnedHandle; - /// The table, and whether its endowed descriptors were installed yet. - static TABLE: Lock<(bool, Table)> = Lock::new((false, Table::new())); + type Descriptors = Table, MAX_FDS>; + type TableWriter<'a> = Writer<'a, AtomicU32, ObjectCell, MAX_FDS>; + + static TABLE: Descriptors = Table::new(); + /// Held by whoever changes what a descriptor names. + static WRITING: Lock<()> = Lock::new(()); + /// Whether the endowed descriptors were installed yet. + static ENDOWED: AtomicBool = AtomicBool::new(false); + + fn refusal(fault: Fault) -> Error { + match fault { + Fault::Refused(e) => e, + Fault::Broken => panic!("toyos::fd: the descriptor table is inconsistent"), + } + } - /// `f` on the table, whose first use installs every descriptor the - /// process was endowed at its number. Nothing panics under the lock: a - /// panic writes to stderr, a descriptor, which would wait on it forever. - fn table(f: impl FnOnce(&mut Table) -> Result) -> Result { + /// `f` on the table's writer, whose first use installs every descriptor + /// the process was endowed at its number. Nothing panics under the lock. + fn change(f: impl FnOnce(&mut TableWriter<'_>) -> Result) -> Result { let endowed = Endowments::get(); let mut refused = 0; let answer = { - let mut held = TABLE.lock(); - let (ready, table) = &mut *held; - let installed = match *ready { + let _held = WRITING.lock(); + // SAFETY: the one writer while `WRITING` is held. + let mut writer = unsafe { TABLE.writer() }; + let installed = match ENDOWED.load(Ordering::Relaxed) { true => Ok(()), false => { - *ready = true; - endow(table, endowed).map(|n| refused = n) + let installed = endow(&mut writer, endowed).map(|n| refused = n); + ENDOWED.store(true, Ordering::Release); + installed } }; - installed.and_then(|()| f(table)) + installed.and_then(|()| f(&mut writer)) }; - // Said once the lock is let go, since stderr is a descriptor. + // Said once the lock is let go: the write may block. for _ in 0..refused { let _ = crate::files::write(2, b"toyos::fd: an fd: endowment whose number was taken was refused\n"); } - answer.map_err(|fault| match fault { - Fault::Refused(e) => e, - Fault::Broken => panic!("toyos::fd: the descriptor table is inconsistent"), - }) + answer.map_err(refusal) + } + + /// The table for a call, which takes no lock once the endowed + /// descriptors are installed. + fn table() -> Result<&'static Descriptors, Error> { + if !ENDOWED.load(Ordering::Acquire) { + change(|_| Ok(()))?; + } + Ok(&TABLE) } /// Install each `fd:` endowment at its number, a socket's taken up as /// this process's holder first. One whose number another already took, /// or whose socket this process holds no room for, is refused, its handle /// closed: how many were. - fn endow(table: &mut Table, endowed: &Endowments) -> Result { + fn endow(table: &mut TableWriter<'_>, endowed: &Endowments) -> Result { let mut refused = 0; let mut files = alloc::collections::BTreeMap::new(); for label in endowed.labels() { @@ -485,16 +507,29 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_install(record: Record, inheritable: bool, floor: Fd) -> i32 { - table(|t| t.install(record, inheritable, floor)).unwrap_or_else(|e| { + change(|t| t.install(record, inheritable, floor)).unwrap_or_else(|e| { // SAFETY: refused, so the record is still the caller's, given here. unsafe { record.release() }; encode(e) }) } + /// A call whose descriptor named a free slot, held under the writer's + /// lock, where that is a finished close or a broken table. + #[cold] + #[inline(never)] + fn settled(fd: Fd) -> Result<(usize, Record), Error> { + change(|t| t.acquire(fd)) + } + #[no_mangle] unsafe extern "C" fn toyos_fd_acquire(fd: Fd, out: *mut Record) -> i32 { - match table(|t| t.acquire(fd)) { + let held = table().and_then(|t| match t.acquire(fd, |record| released(Some(record))) { + Ok(Some(held)) => Ok(held), + Ok(None) => settled(fd), + Err(fault) => Err(refusal(fault)), + }); + match held { Ok((token, object)) => { // SAFETY: the caller's live place. unsafe { out.write(object) }; @@ -505,14 +540,16 @@ mod owner { } #[no_mangle] - extern "C" fn toyos_fd_unuse(token: i32) { + unsafe extern "C" fn toyos_fd_unuse(token: i32) { let token = usize::try_from(token).expect("toyos::fd: a token no acquire answered"); - released(table(|t| t.unuse(token)).expect("toyos::fd: a token no acquire answered")); + // SAFETY: the caller's: a token `toyos_fd_acquire` answered, given + // back once. + released(unsafe { TABLE.unuse(token) }.expect("toyos::fd: a token no acquire answered")); } #[no_mangle] extern "C" fn toyos_fd_close(fd: Fd) -> i32 { - match table(|t| t.close(fd)) { + match change(|t| t.close(fd)) { Ok(object) => { released(object); 0 @@ -523,12 +560,12 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_dup(fd: Fd, floor: Fd, inheritable: bool) -> i32 { - table(|t| t.dup(fd, floor, inheritable)).unwrap_or_else(encode) + change(|t| t.dup(fd, floor, inheritable)).unwrap_or_else(encode) } #[no_mangle] extern "C" fn toyos_fd_dup2(old: Fd, new: Fd, inheritable: bool) -> i32 { - match table(|t| t.dup2(old, new, inheritable)) { + match change(|t| t.dup2(old, new, inheritable)) { Ok(displaced) => { released(displaced); new @@ -539,7 +576,7 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_replace(fd: Fd, record: Record) -> i32 { - match table(|t| t.replace(fd, record)) { + match change(|t| t.replace(fd, record)) { Ok(old) => { released(old); 0 @@ -554,12 +591,12 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_inheritable(fd: Fd) -> i32 { - table(|t| t.inheritable(fd)).map_or_else(encode, i32::from) + table().and_then(|t| t.inheritable(fd).map_err(refusal)).map_or_else(encode, i32::from) } #[no_mangle] extern "C" fn toyos_fd_set_inheritable(fd: Fd, inheritable: bool) -> i32 { - table(|t| t.set_inheritable(fd, inheritable)).map_or_else(encode, |()| 0) + change(|t| t.set_inheritable(fd, inheritable)).map_or_else(encode, |()| 0) } } diff --git a/sdk/toyos/src/fd/table.rs b/sdk/toyos/src/fd/table.rs index 941278113ff..ca5d98d995c 100644 --- a/sdk/toyos/src/fd/table.rs +++ b/sdk/toyos/src/fd/table.rs @@ -1,27 +1,93 @@ -//! The table's state, without its lock: every decision the table makes, -//! which a host tests. Nothing in it panics. - -use super::{Error, Fd, Record, MAX_FDS}; - -/// A descriptor's word: 0 for a free number, else its object's index plus -/// one, with [`INHERITABLE`] set for an inheritable one. The empty table is -/// all zeros, so it costs a program no initialised data. -type Word = u16; -const INHERITABLE: Word = 0x8000; -const _: () = assert!(MAX_FDS < INHERITABLE as usize); - -#[derive(Clone, Copy)] -struct Held { - object: Record, - /// The descriptors naming it and the calls holding it; 0 for a free one. - refs: u32, +//! The table's state: every decision the table makes, which a host tests and +//! `model` checks under loom. Nothing in it panics. +//! +//! A descriptor's word names an object's slot, and a slot counts its +//! references: the descriptors naming it and the calls holding it. +//! +//! - **A slot is free while its count is 0**, and only the [`Writer`] takes a +//! count off 0: it writes the object, then stores the count. Whoever holds a +//! reference reads the object, so nothing is written under a reader. +//! - **A call counts itself onto a slot only while the count is not 0**, and +//! keeps the reference only if the descriptor still names that slot. So the +//! object a call holds is one its descriptor named during the call, and a +//! slot is reused only after its last hold. +//! - **Whoever takes a count to 0 is answered the object**, read while it +//! still held its reference, and lets it go: once, after the last call. +//! - **A descriptor changes in one store**, so a call beside a `dup2` holds +//! the old object or the new, and a closed number is free at once. + +use core::cell::UnsafeCell; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicU32, Ordering}; + +use super::{Error, Fd}; + +#[cfg(test)] +mod model; + +/// A descriptor's word: 0 for a free number, else its object's slot plus one, +/// with [`INHERITABLE`] set for an inheritable one. The empty table is all +/// zeros, so it costs a program no initialised data. +const INHERITABLE: u32 = 1 << 31; + +/// A shared word of the table: an atomic, or a model's. +pub(super) trait Word { + fn load(&self, order: Ordering) -> u32; + fn store(&self, value: u32, order: Ordering); + fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result; + fn fetch_sub(&self, value: u32, order: Ordering) -> u32; } -const FREE: Held = Held { object: Record::FREE, refs: 0 }; +impl Word for AtomicU32 { + fn load(&self, order: Ordering) -> u32 { + AtomicU32::load(self, order) + } + fn store(&self, value: u32, order: Ordering) { + AtomicU32::store(self, value, order) + } + fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result { + AtomicU32::compare_exchange(self, current, new, success, failure) + } + fn fetch_sub(&self, value: u32, order: Ordering) -> u32 { + AtomicU32::fetch_sub(self, value, order) + } +} + +/// Where a slot keeps its object: a cell, or a model's, which checks each +/// access against the others. +pub(super) trait Place { + type Object: Copy; + + /// # Safety + /// The place was written, and no write is concurrent. + unsafe fn read(&self) -> Self::Object; + + /// # Safety + /// No other access is concurrent. + unsafe fn write(&self, object: Self::Object); +} + +pub(super) struct ObjectCell(UnsafeCell>); + +// SAFETY: reached only through `Place`, whose callers keep a write alone. +unsafe impl Sync for ObjectCell {} + +impl Place for ObjectCell { + type Object = O; + + unsafe fn read(&self) -> O { + // SAFETY: the caller's: written, and not being written. + unsafe { (*self.0.get()).assume_init() } + } + + unsafe fn write(&self, object: O) { + // SAFETY: the caller's: nothing else reaches the cell. + unsafe { (*self.0.get()).write(object) }; + } +} /// Why a table operation failed: refused, or the table found inconsistent, -/// which the owner panics on only once its lock is released, since the panic -/// writes to a descriptor. +/// which the owner panics on only once its lock is released. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Fault { Refused(Error), @@ -34,68 +100,193 @@ impl From for Fault { } } -/// A removal answers the object it released, for the caller to let go -/// outside the lock. -pub(super) struct Table { - fds: [Word; N], - objects: [Held; N], +/// An object's slot, on a cache line of its own: calls on two objects write +/// no line they share. +#[repr(align(64))] +struct Slot { + /// The slot's count of references. + refs: W, + object: P, +} + +pub(super) struct Table { + fds: [W; N], + slots: [Slot; N], } -impl Table { +impl Table, N> { pub(super) const fn new() -> Self { - Self { fds: [0; N], objects: [FREE; N] } + assert!(N < INHERITABLE as usize); + Self { + fds: [const { AtomicU32::new(0) }; N], + slots: [const { Slot { refs: AtomicU32::new(0), object: ObjectCell(UnsafeCell::new(MaybeUninit::uninit())) } }; N], + } + } +} + +impl Table { + fn word(&self, fd: Fd) -> Result<&W, Error> { + usize::try_from(fd).ok().and_then(|i| self.fds.get(i)).ok_or(Error::BadFd) + } + + /// The slot a descriptor's word names. + fn named(word: u32) -> Result { + if word == 0 { + return Err(Error::BadFd.into()); + } + ((word & !INHERITABLE) as usize).checked_sub(1).filter(|&slot| slot < N).ok_or(Fault::Broken) + } + + /// One reference more on `slot`, unless it is free. + fn reference(&self, slot: usize) -> Result { + let refs = &self.slots.get(slot).ok_or(Fault::Broken)?.refs; + let mut count = refs.load(Ordering::Acquire); + loop { + if count == 0 && !cfg!(feature = "fd-hold-revives") { + return Ok(false); + } + let more = count.checked_add(1).ok_or(Fault::Broken)?; + match refs.compare_exchange(count, more, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return Ok(true), + Err(now) => count = now, + } + } + } + + /// One reference fewer on `slot`: its object, once none is left. + /// + /// # Safety + /// The caller gives up a reference it holds on `slot`. + unsafe fn unref(&self, slot: usize) -> Result, Fault> { + let slot = self.slots.get(slot).ok_or(Fault::Broken)?; + // SAFETY: the caller's reference keeps the slot from the writer, and + // it is given up only below. + let object = unsafe { slot.object.read() }; + match slot.refs.fetch_sub(1, Ordering::AcqRel) { + 0 => Err(Fault::Broken), + 1 => Ok(Some(object)), + _ => Ok(None), + } + } + + /// `fd`'s object, held until [`Self::unuse`] of the token answered, or + /// `None` for a descriptor naming a free slot: a close in flight or a + /// broken table, which only [`Writer::acquire`] tells apart. An object + /// whose last reference a lost race left with this call goes to `let_go`. + pub(super) fn acquire(&self, fd: Fd, mut let_go: impl FnMut(P::Object)) -> Result, Fault> { + let word = self.word(fd)?; + loop { + let slot = Self::named(word.load(Ordering::Acquire))?; + if !self.reference(slot)? { + return Ok(None); + } + // SAFETY: the reference just taken. + let object = unsafe { self.slots[slot].object.read() }; + if cfg!(feature = "fd-hold-unchecked") || Self::named(word.load(Ordering::Acquire)) == Ok(slot) { + return Ok(Some((slot, object))); + } + // SAFETY: the reference just taken. + if let Some(last) = unsafe { self.unref(slot) }? { + let_go(last); + } + } } - /// `fd`'s index, object and flag. + /// A call done with the object `token` names: the object, if nothing + /// names it any more. + /// + /// # Safety + /// `token` is one this table's [`Self::acquire`] or [`Writer::acquire`] + /// answered, given back once. + pub(super) unsafe fn unuse(&self, token: usize) -> Result, Fault> { + // SAFETY: the call's reference. + unsafe { self.unref(token) } + } + + pub(super) fn inheritable(&self, fd: Fd) -> Result { + let word = self.word(fd)?.load(Ordering::Acquire); + Self::named(word).map(|_| word & INHERITABLE != 0) + } + + /// The table's one writer. + /// + /// # Safety + /// No other [`Writer`] of this table exists while this one does. + pub(super) unsafe fn writer(&self) -> Writer<'_, W, P, N> { + Writer(self) + } +} + +/// The right to change what the table's descriptors name. A removal answers +/// the object it released, for the caller to let go outside its lock. +pub(super) struct Writer<'a, W, P, const N: usize>(&'a Table); + +impl Writer<'_, W, P, N> { + /// `fd`'s index, slot and flag. fn entry(&self, fd: Fd) -> Result<(usize, usize, bool), Fault> { let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; - let word = *self.fds.get(i).ok_or(Error::BadFd)?; - let object = usize::from(word & !INHERITABLE).checked_sub(1).ok_or(match word { - 0 => Fault::Refused(Error::BadFd), - _ => Fault::Broken, - })?; - Ok((i, object, word & INHERITABLE != 0)) + let word = self.0.fds.get(i).ok_or(Error::BadFd)?.load(Ordering::Relaxed); + let slot = Table::::named(word)?; + // Only this writer takes a name's reference away. + if self.0.slots[slot].refs.load(Ordering::Relaxed) == 0 { + return Err(Fault::Broken); + } + Ok((i, slot, word & INHERITABLE != 0)) } - fn set(&mut self, i: usize, object: usize, inheritable: bool) -> Result<(), Fault> { - let index = Word::try_from(object).ok().filter(|&o| o < INHERITABLE - 1).ok_or(Fault::Broken)?; + fn set(&mut self, i: usize, slot: usize, inheritable: bool) -> Result<(), Fault> { + let index = u32::try_from(slot).ok().filter(|&s| s < INHERITABLE - 1).ok_or(Fault::Broken)?; let flag = if inheritable { INHERITABLE } else { 0 }; - *self.fds.get_mut(i).ok_or(Fault::Broken)? = (index + 1) | flag; + self.0.fds.get(i).ok_or(Fault::Broken)?.store((index + 1) | flag, Ordering::Release); Ok(()) } + fn is_free(&self, i: usize) -> bool { + self.0.fds.get(i).is_some_and(|word| word.load(Ordering::Relaxed) == 0) + } + fn lowest_free(&self, floor: Fd) -> Result { let floor = usize::try_from(floor).map_err(|_| Error::Full)?; - let above = self.fds.get(floor..).ok_or(Error::Full)?; - above.iter().position(|&w| w == 0).map(|i| floor + i).ok_or(Error::Full) + (floor..N).find(|&i| self.is_free(i)).ok_or(Error::Full) } - /// A free object holding `object`, referenced once. - fn hold(&mut self, object: Record) -> Result { - let slot = self.objects.iter().position(|h| h.refs == 0).ok_or(Error::Full)?; - self.objects[slot] = Held { object, refs: 1 }; - Ok(slot) + /// A free slot holding `object`, referenced once. + fn hold(&mut self, object: P::Object) -> Result { + // The load that finds a slot free is ordered after its last reader. + let free = if cfg!(feature = "fd-reuse-relaxed") { Ordering::Relaxed } else { Ordering::Acquire }; + let index = self.0.slots.iter().position(|slot| slot.refs.load(free) == 0).ok_or(Error::Full)?; + let slot = &self.0.slots[index]; + // SAFETY: a free slot is this writer's alone until its count is stored. + unsafe { slot.object.write(object) }; + let publish = if cfg!(feature = "fd-publish-relaxed") { Ordering::Relaxed } else { Ordering::Release }; + slot.refs.store(1, publish); + Ok(index) } - fn held(&mut self, object: usize) -> Result<&mut Held, Fault> { - self.objects.get_mut(object).filter(|h| h.refs != 0).ok_or(Fault::Broken) + /// One reference more on a slot a descriptor names. + fn reference(&mut self, slot: usize) -> Result<(), Fault> { + self.0.reference(slot)?.then_some(()).ok_or(Fault::Broken) } - fn reference(&mut self, object: usize) -> Result { - let held = self.held(object)?; - held.refs = held.refs.checked_add(1).ok_or(Fault::Broken)?; - Ok(held.object) + /// One reference fewer on a slot [`Self::entry`] answered, whose name + /// was just taken away. + fn unname(&mut self, slot: usize) -> Result, Fault> { + // SAFETY: the name's reference, which `entry` found counted. + unsafe { self.0.unref(slot) } } - /// A reference fewer on `object`: the object, once none is left. - fn unref(&mut self, object: usize) -> Result, Fault> { - let held = self.held(object)?; - held.refs -= 1; - Ok((held.refs == 0).then(|| core::mem::replace(held, FREE).object)) + /// `fd`'s object for a call, held as [`Table::acquire`] holds it. Under + /// the writer no slot a descriptor names is freed, so one named free is + /// a broken table. + pub(super) fn acquire(&mut self, fd: Fd) -> Result<(usize, P::Object), Fault> { + let (_, slot, _) = self.entry(fd)?; + self.reference(slot)?; + // SAFETY: the reference just taken. + Ok((slot, unsafe { self.0.slots[slot].object.read() })) } /// Name `object` by the lowest free descriptor at or above `floor`. - pub(super) fn install(&mut self, object: Record, inheritable: bool, floor: Fd) -> Result { + pub(super) fn install(&mut self, object: P::Object, inheritable: bool, floor: Fd) -> Result { let fd = self.lowest_free(floor)?; let slot = self.hold(object)?; self.set(fd, slot, inheritable)?; @@ -104,11 +295,8 @@ impl Table { /// Name `object` by `fd` exactly: `BadFd` for a number past the table /// or already open. - pub(super) fn install_at(&mut self, fd: Fd, object: Record, inheritable: bool) -> Result<(), Fault> { - let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; - if self.fds.get(i) != Some(&0) { - return Err(Error::BadFd.into()); - } + pub(super) fn install_at(&mut self, fd: Fd, object: P::Object, inheritable: bool) -> Result<(), Fault> { + let i = usize::try_from(fd).ok().filter(|&i| self.is_free(i)).ok_or(Error::BadFd)?; let slot = self.hold(object)?; self.set(i, slot, inheritable) } @@ -116,80 +304,64 @@ impl Table { /// An endowed alias at its exact number, without displacing an earlier /// endowment. Both descriptors share one object and one final release. pub(super) fn alias_at(&mut self, source: Fd, fd: Fd) -> Result<(), Fault> { - let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; - if self.fds.get(i) != Some(&0) { return Err(Error::BadFd.into()); } - let (_, object, _) = self.entry(source)?; - self.reference(object)?; - self.set(i, object, true) + let i = usize::try_from(fd).ok().filter(|&i| self.is_free(i)).ok_or(Error::BadFd)?; + let (_, slot, _) = self.entry(source)?; + self.reference(slot)?; + self.set(i, slot, true) } /// A second descriptor on `fd`'s object, the lowest free at or above /// `floor`. pub(super) fn dup(&mut self, fd: Fd, floor: Fd, inheritable: bool) -> Result { - let (_, object, _) = self.entry(fd)?; + let (_, slot, _) = self.entry(fd)?; let new = self.lowest_free(floor)?; - self.reference(object)?; - self.set(new, object, inheritable)?; + self.reference(slot)?; + self.set(new, slot, inheritable)?; Ok(new as Fd) } /// `new` on `old`'s object, whatever `new` named before: that object, /// if this was its last reference. `new == old` changes nothing. - pub(super) fn dup2(&mut self, old: Fd, new: Fd, inheritable: bool) -> Result, Fault> { - let (_, object, _) = self.entry(old)?; - let slot = usize::try_from(new).ok().filter(|&i| i < N).ok_or(Error::BadFd)?; + pub(super) fn dup2(&mut self, old: Fd, new: Fd, inheritable: bool) -> Result, Fault> { + let (_, slot, _) = self.entry(old)?; + let i = usize::try_from(new).ok().filter(|&i| i < N).ok_or(Error::BadFd)?; if new == old { return Ok(None); } let displaced = self.entry(new).ok(); - self.reference(object)?; - self.set(slot, object, inheritable)?; + self.reference(slot)?; + self.set(i, slot, inheritable)?; match displaced { - Some((_, displaced, _)) => self.unref(displaced), + Some((_, displaced, _)) => self.unname(displaced), None => Ok(None), } } /// Forget `fd`: its object, if this was its last reference. - pub(super) fn close(&mut self, fd: Fd) -> Result, Fault> { - let (i, object, _) = self.entry(fd)?; - *self.fds.get_mut(i).ok_or(Fault::Broken)? = 0; - self.unref(object) - } - - /// `fd`'s object, held until [`Self::unuse`] of the token answered. - pub(super) fn acquire(&mut self, fd: Fd) -> Result<(usize, Record), Fault> { - let (_, object, _) = self.entry(fd)?; - Ok((object, self.reference(object)?)) - } - - /// A call done with the object `token` names: the object, if nothing - /// names it any more. - pub(super) fn unuse(&mut self, token: usize) -> Result, Fault> { - self.unref(token) + pub(super) fn close(&mut self, fd: Fd) -> Result, Fault> { + let (i, slot, _) = self.entry(fd)?; + self.0.fds.get(i).ok_or(Fault::Broken)?.store(0, Ordering::Release); + self.unname(slot) } /// Make `fd` name `object` from here on, in place: the object it named, /// if this was its last reference. Its flag stays. - pub(super) fn replace(&mut self, fd: Fd, object: Record) -> Result, Fault> { + pub(super) fn replace(&mut self, fd: Fd, object: P::Object) -> Result, Fault> { let (i, old, inheritable) = self.entry(fd)?; let slot = self.hold(object)?; self.set(i, slot, inheritable)?; - self.unref(old) - } - - pub(super) fn inheritable(&self, fd: Fd) -> Result { - self.entry(fd).map(|(_, _, inheritable)| inheritable) + self.unname(old) } pub(super) fn set_inheritable(&mut self, fd: Fd, inheritable: bool) -> Result<(), Fault> { - let (i, object, _) = self.entry(fd)?; - self.set(i, object, inheritable) + let (i, slot, _) = self.entry(fd)?; + self.set(i, slot, inheritable) } } #[cfg(test)] mod tests { + use super::super::Record; use super::*; extern "C" fn nothing(_: u64) {} @@ -198,21 +370,34 @@ mod tests { Record { handle: toyos_abi::HANDLE_INVALID, tag: super::super::UNTAGGED, word, release: Some(nothing) } } - type T = Table<8>; + type T = Table, 8>; + type W<'a> = Writer<'a, AtomicU32, ObjectCell, 8>; + + /// These tests' one writer of `t`. + fn writer(t: &T) -> W<'_> { + // SAFETY: each test makes one. + unsafe { t.writer() } + } /// The word of what a removal released, which names it in these tests. fn w(released: Result, Fault>) -> Result, Fault> { released.map(|o| o.map(|o| o.word)) } + fn acquire(t: &T, fd: Fd) -> Result<(usize, Record), Fault> { + let held = t.acquire(fd, |_| panic!("a call with no race was left an object"))?; + Ok(held.expect("a call with no race found a free slot")) + } + /// The objects something still references. fn held(t: &T) -> usize { - t.objects.iter().filter(|h| h.refs != 0).count() + t.slots.iter().filter(|slot| slot.refs.load(Ordering::Relaxed) != 0).count() } #[test] fn descriptors_are_numbered_lowest_free_from_the_floor() { - let mut t = T::new(); + let t = T::new(); + let mut t = writer(&t); assert_eq!(t.install(o(1), false, 0), Ok(0)); assert_eq!(t.install(o(2), false, 0), Ok(1)); assert_eq!(t.install(o(3), false, 5), Ok(5)); @@ -226,43 +411,55 @@ mod tests { /// keeps a flag of its own. #[test] fn an_object_goes_with_its_last_descriptor_and_flags_are_per_descriptor() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let a = t.install(o(7), true, 0).unwrap(); let b = t.dup(a, 0, false).unwrap(); - assert_eq!((t.inheritable(a), t.inheritable(b)), (Ok(true), Ok(false))); + assert_eq!((table.inheritable(a), table.inheritable(b)), (Ok(true), Ok(false))); assert_eq!(t.set_inheritable(b, true), Ok(())); - assert_eq!((t.inheritable(a), t.inheritable(b)), (Ok(true), Ok(true))); + assert_eq!((table.inheritable(a), table.inheritable(b)), (Ok(true), Ok(true))); assert_eq!(t.set_inheritable(a, false), Ok(())); - assert_eq!((t.inheritable(a), t.inheritable(b)), (Ok(false), Ok(true)), "a flag is the descriptor's, not its object's"); + assert_eq!((table.inheritable(a), table.inheritable(b)), (Ok(false), Ok(true)), "a flag is the descriptor's, not its object's"); assert_eq!(w(t.close(a)), Ok(None), "released while a dup named it"); assert_eq!(w(t.close(b)), Ok(Some(7))); assert_eq!(w(t.close(b)), Err(Error::BadFd.into())); - assert_eq!(held(&t), 0, "an object outlived its descriptors"); + assert_eq!(held(&table), 0, "an object outlived its descriptors"); } /// A call holds its object: a close meanwhile makes the descriptor gone - /// at once, and the object goes when the call is done. + /// at once, its number free, and the object goes when the call is done. #[test] fn a_close_during_a_call_releases_the_object_when_the_call_is_done() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let fd = t.install(o(9), false, 0).unwrap(); - let (token, object) = t.acquire(fd).unwrap(); + let (token, object) = acquire(&table, fd).unwrap(); assert_eq!(object.word, 9); assert_eq!(w(t.close(fd)), Ok(None), "the object went under a call"); - assert!(matches!(t.acquire(fd), Err(Fault::Refused(Error::BadFd))), "a closed descriptor still answered"); - assert_eq!(w(t.unuse(token)), Ok(Some(9))); - assert_eq!(held(&t), 0, "an object outlived its call"); + assert!(matches!(acquire(&table, fd), Err(Fault::Refused(Error::BadFd))), "a closed descriptor still answered"); + assert_eq!(t.install(o(10), false, 0), Ok(fd), "a closed number waited for a call"); + let (other, object) = acquire(&table, fd).unwrap(); + assert_eq!(object.word, 10); + assert_ne!(other, token, "an object took the slot of one a call still holds"); + // SAFETY: each a token `acquire` answered, given back once. + unsafe { + assert_eq!(w(table.unuse(token)), Ok(Some(9))); + assert_eq!(w(table.unuse(other)), Ok(None)); + } + assert_eq!(w(t.close(fd)), Ok(Some(10))); + assert_eq!(held(&table), 0, "an object outlived its call"); } #[test] fn dup2_displaces_what_the_target_named_and_onto_itself_changes_nothing() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let a = t.install(o(1), false, 0).unwrap(); let b = t.install(o(2), false, 0).unwrap(); assert_eq!(w(t.dup2(a, b, true)), Ok(Some(2))); - assert_eq!(t.inheritable(b), Ok(true)); + assert_eq!(table.inheritable(b), Ok(true)); assert_eq!(w(t.dup2(a, a, true)), Ok(None)); - assert_eq!(t.inheritable(a), Ok(false), "dup2 onto itself set a flag"); + assert_eq!(table.inheritable(a), Ok(false), "dup2 onto itself set a flag"); assert_eq!(w(t.dup2(a, 6, false)), Ok(None), "an unused target"); assert_eq!(w(t.dup2(a, 8, false)), Err(Error::BadFd.into()), "a target past the table"); assert_eq!(w(t.dup2(7, 1, false)), Err(Error::BadFd.into())); @@ -270,15 +467,16 @@ mod tests { let last = t.close(fd).unwrap(); assert_eq!(last.is_some(), fd == 6, "descriptor {fd}"); } - assert_eq!(held(&t), 0, "an object outlived its descriptors"); + assert_eq!(held(&table), 0, "an object outlived its descriptors"); } #[test] fn replace_keeps_the_number_and_the_flag_and_releases_the_old_object() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let fd = t.install(o(1), true, 0).unwrap(); assert_eq!(w(t.replace(fd, o(2))), Ok(Some(1))); - assert_eq!(t.inheritable(fd), Ok(true)); + assert_eq!(table.inheritable(fd), Ok(true)); assert_eq!(w(t.close(fd)), Ok(Some(2))); assert_eq!(w(t.replace(fd, o(3))), Err(Error::BadFd.into())); } @@ -287,9 +485,10 @@ mod tests { /// open already is refused, and the table is unchanged. #[test] fn an_endowment_is_installed_at_its_number_or_refused() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); assert_eq!(t.install_at(3, o(1), true), Ok(())); - assert_eq!(t.inheritable(3), Ok(true)); + assert_eq!(table.inheritable(3), Ok(true)); assert_eq!(t.install_at(3, o(2), true), Err(Error::BadFd.into())); assert_eq!(t.install_at(8, o(2), true), Err(Error::BadFd.into())); assert_eq!(t.install_at(-1, o(2), true), Err(Error::BadFd.into())); @@ -299,25 +498,47 @@ mod tests { #[test] fn endowed_aliases_share_one_object_and_cannot_displace_an_endowment() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); t.install_at(3, o(1), true).unwrap(); t.alias_at(3, 0).unwrap(); - assert_eq!(held(&t), 1); + assert_eq!(held(&table), 1); assert_eq!(t.alias_at(3, 0), Err(Error::BadFd.into())); assert_eq!(w(t.close(3)), Ok(None)); assert_eq!(w(t.close(0)), Ok(Some(1))); } - /// A descriptor naming no object is a broken table, which the table - /// answers rather than panics on: the owner panics once its lock is let - /// go. + /// A call the writer answers holds its object as any call does. + #[test] + fn a_call_the_writer_answers_holds_its_object_until_it_is_done() { + let table = T::new(); + let mut t = writer(&table); + let fd = t.install(o(5), false, 0).unwrap(); + let (token, object) = t.acquire(fd).unwrap(); + assert_eq!(object.word, 5); + assert_eq!(w(t.close(fd)), Ok(None), "the object went under a call"); + assert!(matches!(t.acquire(fd), Err(Fault::Refused(Error::BadFd))), "a closed descriptor still answered"); + // SAFETY: a token the writer's `acquire` answered, given back once. + assert_eq!(w(unsafe { table.unuse(token) }), Ok(Some(5))); + assert_eq!(held(&table), 0, "an object outlived its call"); + } + + /// A descriptor naming no slot, or a free one under the writer, is a + /// broken table, which the table answers rather than panics on. A call + /// whose descriptor names a free slot is left to the writer. #[test] fn an_inconsistent_table_is_answered_not_panicked_on() { - let mut t = T::new(); - t.fds[2] = 1; + let table = T::new(); + let mut t = writer(&table); + table.fds[2].store(9, Ordering::Relaxed); + assert_eq!(acquire(&table, 2).map(|_| ()), Err(Fault::Broken)); + assert_eq!(w(t.close(2)), Err(Fault::Broken)); + table.fds[2].store(INHERITABLE, Ordering::Relaxed); + assert_eq!(table.inheritable(2), Err(Fault::Broken)); + table.fds[2].store(1, Ordering::Relaxed); + assert!(matches!(table.acquire(2, |_| panic!("a call with no race was left an object")), Ok(None))); assert_eq!(t.acquire(2).map(|_| ()), Err(Fault::Broken)); assert_eq!(w(t.close(2)), Err(Fault::Broken)); - t.fds[2] = INHERITABLE; - assert_eq!(t.inheritable(2), Err(Fault::Broken)); + assert_eq!(t.dup(2, 0, false), Err(Fault::Broken)); } } diff --git a/sdk/toyos/src/fd/table/model.rs b/sdk/toyos/src/fd/table/model.rs new file mode 100644 index 00000000000..497bb0a7541 --- /dev/null +++ b/sdk/toyos/src/fd/table/model.rs @@ -0,0 +1,230 @@ +//! Calls beside the writer on two CPUs, under loom: every interleaving it +//! explores of a hold against a close, a reuse and a `dup2`. Loom does not +//! explore load buffering. `build/ci.rs`'s `CONTROLS` names the cases each +//! control reds. +//! +//! cargo test -p toyos --lib --features fd::table::model + +use core::sync::atomic::Ordering; + +use loom::sync::atomic::AtomicU32; +use loom::sync::{Arc, Mutex}; +use loom::thread; + +use super::{Error, Fault, Fd, Place, Slot, Table, Word, Writer}; + +impl Word for AtomicU32 { + fn load(&self, order: Ordering) -> u32 { + AtomicU32::load(self, order) + } + fn store(&self, value: u32, order: Ordering) { + AtomicU32::store(self, value, order) + } + fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result { + AtomicU32::compare_exchange(self, current, new, success, failure) + } + fn fetch_sub(&self, value: u32, order: Ordering) -> u32 { + AtomicU32::fetch_sub(self, value, order) + } +} + +/// A slot's object, which loom fails on a read beside a write or one not +/// ordered after it. +struct Tracked(loom::cell::UnsafeCell); + +impl Place for Tracked { + type Object = usize; + + unsafe fn read(&self) -> usize { + // SAFETY: the caller's. + self.0.with(|object| unsafe { *object }) + } + + unsafe fn write(&self, object: usize) { + // SAFETY: the caller's. + self.0.with_mut(|place| unsafe { *place = object }); + } +} + +const N: usize = 2; +/// The objects a case installs, each named by its index. +const OBJECTS: usize = 3; + +struct World { + table: Table, + writing: Mutex<()>, + /// How often each object was let go. + gone: [AtomicU32; OBJECTS], +} + +impl World { + fn new() -> Arc { + Arc::new(Self { + table: Table { + fds: core::array::from_fn(|_| AtomicU32::new(0)), + slots: core::array::from_fn(|_| Slot { + refs: AtomicU32::new(0), + object: Tracked(loom::cell::UnsafeCell::new(usize::MAX)), + }), + }, + writing: Mutex::new(()), + gone: core::array::from_fn(|_| AtomicU32::new(0)), + }) + } + + fn writer(&self, f: impl FnOnce(&mut Writer<'_, AtomicU32, Tracked, N>) -> Result) -> Result { + let _held = self.writing.lock().unwrap(); + // SAFETY: the one writer while `writing` is held. + f(&mut unsafe { self.table.writer() }) + } + + fn write(&self, f: impl FnOnce(&mut Writer<'_, AtomicU32, Tracked, N>) -> Result) -> R { + self.writer(f).expect("the writer was refused") + } + + fn let_go(&self, object: usize) { + assert_eq!(self.gone[object].fetch_add(1, Ordering::SeqCst), 0, "object {object} was let go twice"); + } + + fn released(&self, object: Option) { + if let Some(object) = object { + self.let_go(object); + } + } + + fn install(&self, object: usize, floor: Fd) -> Fd { + self.write(|w| w.install(object, false, floor)) + } + + fn close(&self, fd: Fd) { + self.released(self.write(|w| w.close(fd))); + } + + /// A call's hold of `fd`: its token and object. + fn hold(&self, fd: Fd) -> Option<(usize, usize)> { + let held = match self.table.acquire(fd, |object| self.let_go(object)) { + Ok(Some(held)) => Ok(held), + Ok(None) => self.writer(|w| w.acquire(fd)), + Err(fault) => Err(fault), + }; + match held { + Ok(held) => Some(held), + Err(Fault::Refused(Error::BadFd)) => None, + Err(fault) => panic!("a call answered {fault:?}"), + } + } + + /// The end of the call [`Self::hold`] answered, whose object was not let + /// go under it. + fn done(&self, (token, object): (usize, usize)) { + assert_eq!(self.gone[object].load(Ordering::SeqCst), 0, "a call held object {object} after it was let go"); + // SAFETY: a token `hold` answered, given back once. + self.released(unsafe { self.table.unuse(token) }.expect("a call's own token")); + } + + /// One call on `fd`: the object it held. + fn call(&self, fd: Fd) -> Option { + let held = self.hold(fd)?; + self.done(held); + Some(held.1) + } + + /// Every object in `objects` was let go once. + fn all_gone(&self, objects: impl IntoIterator) { + for object in objects { + assert_eq!(self.gone[object].load(Ordering::SeqCst), 1, "object {object} was not let go"); + } + } +} + +#[test] +fn a_call_beside_a_close_holds_a_live_object_or_none() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + caller.join().unwrap(); + world.all_gone([0]); + }); +} + +/// Two calls give their references up beside each other and the close. +#[test] +fn two_calls_beside_a_close_let_the_object_go_once() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let held = world.hold(fd).expect("an open descriptor"); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + world.done(held); + caller.join().unwrap(); + world.all_gone([0]); + }); +} + +/// The close frees the descriptor's slot, another descriptor's object takes +/// it and is closed in turn, so a call that counted itself onto the slot can +/// be left the object its own descriptor never named. +#[test] +fn a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + let other = world.install(1, fd + 1); + world.close(other); + let held = caller.join().unwrap(); + assert!(matches!(held, None | Some(0)), "a call on a closed descriptor held {held:?}"); + world.all_gone([0, 1]); + }); +} + +#[test] +fn a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + assert_eq!(world.install(1, 0), fd); + caller.join().unwrap(); + world.close(fd); + world.all_gone([0, 1]); + }); +} + +/// A descriptor is open throughout a `dup2` onto it. +#[test] +fn a_call_beside_a_dup2_holds_the_old_object_or_the_new() { + loom::model(|| { + let world = World::new(); + let old = world.install(0, 0); + let new = world.install(1, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(new) + }); + world.released(world.write(|w| w.dup2(old, new, false))); + let held = caller.join().unwrap(); + assert!(matches!(held, Some(0 | 1)), "a call beside a dup2 held {held:?}"); + world.all_gone([1]); + world.close(old); + world.close(new); + world.all_gone([0]); + }); +}