diff --git a/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md b/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md index 33350fbedfd..23f273e90fd 100644 --- a/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md +++ b/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md @@ -68,6 +68,25 @@ where the row measured what it means, it is behind as before. That neither thread slept in the short rounds is an inference from the timings; the row counts no parks, so a run's p50 is not this row's value. +#1253 translates a futex wait's word once per call and checks it with one +load of its space's change count, where every check took the space's lock +and walked its tables again (wakeperf: 60 to 63 ns a translation, two to four +times a round trip). Its speed-up is not yet measured on metal. + +- **Owed on metal:** the T14's whole `testcases`, `testcases-watchdog` and + `shared-debug` boots of main after #1253. Main must pass every member, + among them `futex_wake_counts` (whose `orphaned_by_unmap` arm is a wait on + a frame an unmap frees), `kill_ends_every_wait`, `exit_wait_storm`, + `std_threading`, `std_threads`, `shm_release_reclaims` and + `tlb_shootdown_waits`. +- **Owed on metal:** an A/B of main before and after #1253, on the T14 and + the AMD bench: `osbench sched`, `time`, `files` and `ipc`, four boots a + side in rotated order, each boot's p50, compared by an exact two-sided + Mann-Whitney test; and Linux's `osbench sched` on each bench. Main after + must read `pingpong.futex.same_cpu` and `pingpong.futex.cross_core` lower + or not separated, the latter only from boots whose round trips parked, and + no other row 5% or more worse unless the code #1253 changed explains it. + **Exit**: `pingpong.*` within Linux's on both benches in a labelled current-state comparison, or each remaining difference measured to its parts and justified; and `pingpong.futex.*` reports the parks of each round, so a diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 65e662cfea7..e860ed8c6d2 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -21,7 +21,8 @@ //! waits on the lock, whose spin answers the shootdown, and finds it gone. use alloc::sync::Arc; -use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use core::cell::Cell; +use core::sync::atomic::{fence, AtomicU64, Ordering::{Acquire, Relaxed}}; use kernel::vm::{self, Access, Entry, Kind, Prot}; use toyos_abi::syscall::SyscallError; @@ -82,7 +83,8 @@ impl At { } /// What a space holds, read without its lock: by `SYS_SYSINFO`, by a -/// process's statistics, and by its end's record. +/// process's statistics, and by its end's record; and how often its leaves +/// changed, by a futex wait ([`Translation`]). #[derive(Default)] pub struct Usage { /// Private frames populated now. @@ -94,6 +96,10 @@ pub struct Usage { /// Faults served, and the time they took. faults: AtomicU64, fault_ns: AtomicU64, + /// Changes to a leaf a CPU may have cached: every one is shot down, and + /// counted under the lock before its shootdown, so before a frame it + /// gave up is freed. + shot_down: AtomicU64, /// Changes to the space's leaves that reached another CPU by interrupt, /// for the test kernel's `SYS_DEBUG`. #[cfg(feature = "test-actuators")] @@ -190,6 +196,52 @@ impl Space { pub fn release_object(&self, at: UserAddr, len: u64, object: &SharedMemObject) { self.lock().release_object(at, len, object); } + + /// The frame behind `va`, populated where its range allows `access` + /// ([`AddressSpace::reach`]), as a translation a holder can ask about + /// later without the lock. + pub fn translate(space: PageTables, va: u64, access: Access) -> Option { + let (phys, at) = { + let mut held = space.lock(); + let phys = held.reach(va, access)?; + // Counted under this hold: a change made before it is in `at`, + // and one after it moves the count. + (phys, space.usage.shot_down.load(Relaxed)) + }; + Some(Translation { space, va, phys, at: Cell::new(at) }) + } +} + +/// A user address's frame as a hold of its space's lock found it. +pub struct Translation { + space: PageTables, + va: u64, + phys: u64, + /// The space's count of changes when `va` was last found mapping `phys`. + at: Cell, +} + +impl Translation { + pub fn phys(&self) -> u64 { + self.phys + } + + /// Whether the address still maps the frame, which is then not freed: no + /// leaf of the space has changed since it was last found there, or a hold + /// of the lock finds it there again. A load before this one is ordered + /// before it, so a value read from the frame was read while this held. + pub fn holds(&self) -> bool { + fence(Acquire); + if self.space.usage.shot_down.load(Relaxed) == self.at.get() { + return true; + } + let held = self.space.lock(); + let found = held.get(self.va).is_some_and(|(phys, _)| phys == self.phys); + if found { + self.at.set(self.space.usage.shot_down.load(Relaxed)); + } + found + } } pub struct AddressSpace { @@ -379,7 +431,7 @@ impl AddressSpace { let inventory = &self.inventory; let unlinked = self.tables.unlink(start, end, |s, e| inventory.backed(s, e)); if !unlinked.is_empty() { - self.tables.shoot_down(start, end, &unlinked); + self.shoot_down(start, end, &unlinked); } self.tables.free(unlinked); Err(SyscallError::ResourceExhausted) @@ -526,7 +578,7 @@ impl AddressSpace { Leaf::Held(_) | Leaf::Empty => None, }); if present { - self.tables.shoot_down(start, end, &Unlinked::none()); + self.shoot_down(start, end, &Unlinked::none()); } let mut freed = 0; let inventory = &self.inventory; @@ -684,6 +736,7 @@ impl AddressSpace { /// Every CPU has dropped what it cached of `[start, end)` and of the /// tables `unlinked` took out. fn shoot_down(&self, start: u64, end: u64, unlinked: &Unlinked) { + self.usage.shot_down.fetch_add(1, Relaxed); let _reached = self.tables.shoot_down(start, end, unlinked); #[cfg(feature = "test-actuators")] if _reached { diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 69cd1b45df9..eaa15eb2231 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -942,9 +942,9 @@ pub fn thread_sched(pid: Pid, tid: Tid) -> Option { table.get(pid)?.threads.get(&tid)?.sched().cloned() } -/// The physical address of a futex word, its page populated like any other user access. Alignment matters: the scheduler dereferences this directly on every wake check, so an unaligned word could straddle into the next physical page. -/// Not a lease: nothing pins the frame, and a sibling's `munmap` can free it under a parked wait. Made safe by an unmap ending the waits on the frames it frees (`sched::futex::revoke_frames`), and `scheduler::futex_wait` re-deriving the translation on every check. -fn futex_word(addr: UserAddr) -> Option { +/// A futex word's frame, its page populated like any other user access. Alignment matters: the scheduler reads the word through the frame, so an unaligned word could straddle into the next physical page. +/// Not a lease: nothing pins the frame, and a sibling's `munmap` can free it under a parked wait. Made safe by an unmap ending the waits on the frames it frees (`sched::futex::revoke_frames`) after it counts the change the translation's `holds` reads. +fn futex_word(addr: UserAddr) -> Option { if !addr.raw().is_multiple_of(4) { return None; } @@ -962,12 +962,12 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { }; // Physical, so a futex in shared memory works across processes. - let Some(phys_addr) = futex_word(addr) else { + let Some(word) = futex_word(addr) else { return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); }; // 0 covers both woken and never-blocked (the caller re-checks the word either way); 1 means only the caller's own deadline ended it. - match scheduler::futex_wait(addr, phys_addr, expected, deadline) { + match scheduler::futex_wait(&word, expected, deadline) { scheduler::FutexEnd::Changed => 0, scheduler::FutexEnd::Timeout => 1, } @@ -975,10 +975,10 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { /// Wake up to `count` threads blocked on the same physical address as `addr`. pub fn futex_wake(addr: UserAddr, count: u64) -> u64 { - let Some(phys_addr) = futex_word(addr) else { + let Some(word) = futex_word(addr) else { return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); }; - scheduler::futex_wake(phys_addr, count as usize) + scheduler::futex_wake(word.phys(), count as usize) } /// Wake processes blocked on reading from a pipe that now has data. diff --git a/kernel/src/sched/futex.rs b/kernel/src/sched/futex.rs index 2dbdd006e3d..40dd1ae0176 100644 --- a/kernel/src/sched/futex.rs +++ b/kernel/src/sched/futex.rs @@ -2,15 +2,14 @@ //! address so every process mapping a word shares it. use crate::watch::Watch; -use crate::DirectMap; // A collision only lengthens the walk: a futex wake is bounded by the waiter's token, not by which bucket it landed in. const FUTEX_BUCKETS: usize = 64; static FUTEX_WATCH: [Watch; FUTEX_BUCKETS] = [const { Watch::new() }; FUTEX_BUCKETS]; -/// The watch for a futex word. -pub fn watch_of(addr: DirectMap) -> &'static Watch { - &FUTEX_WATCH[(addr.phys() >> 2) as usize % FUTEX_BUCKETS] +/// The watch for the futex word at `phys`. +pub fn watch_of(phys: u64) -> &'static Watch { + &FUTEX_WATCH[(phys >> 2) as usize % FUTEX_BUCKETS] } /// End every futex wait on a word in one of `frames`, ascending 4 KiB frames diff --git a/kernel/src/scheduler.rs b/kernel/src/scheduler.rs index 42437e6d6f2..b755e49d73f 100644 --- a/kernel/src/scheduler.rs +++ b/kernel/src/scheduler.rs @@ -254,43 +254,26 @@ pub fn set_current_rt(enable: bool) { /// Block on a futex word unless it already changed, and answer which of the /// two things the ABI names ended the wait. #[track_caller] -pub fn futex_wait( - addr: crate::UserAddr, - phys_addr: DirectMap, - expected: u32, - deadline: Deadline, -) -> FutexEnd { +pub fn futex_wait(word: &crate::mm::space::Translation, expected: u32, deadline: Deadline) -> FutexEnd { let parkable = Parkable::at_entry(); - // Re-translated rather than trusted: `munmap` clears the entry before - // walking futex buckets, so a changed translation means this arm is stale. - let read = || { - let Some(pt) = current_address_space() else { - return true; - }; - let same_frame = pt.lock().get(addr.raw()).is_some_and(|(now, _)| now == phys_addr.phys()); - if !same_frame { - return true; - } - // SAFETY: `same_frame` re-translated `addr` and found it still names - // `phys_addr`'s frame, so this is a live, mapped, syscall-checked - // 4-byte-aligned word; volatile because this predicate may run more - // than once. - let word = unsafe { phys_addr.as_ptr::().read_volatile() }; - word != expected + let phys = word.phys(); + let changed = || { + // SAFETY: a frame the user's checked, 4-byte-aligned word translated + // to, read through the direct map, which maps every frame a user page + // can hold; volatile because this predicate may run more than once. + // Whether the frame was still the word's when read is `holds`, after + // the read. + let now = unsafe { DirectMap::from_phys(phys).as_ptr::().read_volatile() }; + // An address that no longer maps the frame is a wait the unmap's + // revoke may have missed: the word moved, and the caller re-checks it. + now != expected || !word.holds() }; - let _ = watch::wait_until( - &parkable, - futex::watch_of(phys_addr), - phys_addr.phys(), - WaitClass::Futex, - deadline, - read, - ); + let _ = watch::wait_until(&parkable, futex::watch_of(phys), phys, WaitClass::Futex, deadline, changed); // `wait_until` answers `Ok(())` for a satisfied predicate and an expired // deadline alike, so the word itself is what tells them apart. With no // deadline the word or its frame changed, or a kill ended it, whose answer // never reaches Ring 3. - if deadline.is_never() || read() { + if deadline.is_never() || changed() { FutexEnd::Changed } else { FutexEnd::Timeout @@ -306,9 +289,9 @@ pub enum FutexEnd { Timeout, } -/// Wake up to `count` waiters on this futex word, and answer how many. -pub fn futex_wake(phys_addr: DirectMap, count: usize) -> u64 { - futex::watch_of(phys_addr).post_n(phys_addr.phys(), count) as u64 +/// Wake up to `count` waiters on the futex word at `phys`, and answer how many. +pub fn futex_wake(phys: u64, count: usize) -> u64 { + futex::watch_of(phys).post_n(phys, count) as u64 } /// Set a thread's kill bit and ask its CPU for a safe point; returns at once. diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index e129405bf9c..6a0faf362cd 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -45,12 +45,10 @@ fn asked(access: Access) -> Asked { } } -/// Translate a user virtual address to its direct-map address, populating its -/// page if needed; `pub(crate)` because the futex word outlives its syscall. -pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Option { - let space = unlocked_space(); - let phys = space.lock().reach(addr.raw(), asked(access))?; - Some(crate::mm::DirectMap::from_phys(phys)) +/// Translate a user virtual address to its frame, populating its page if +/// needed; `pub(crate)` because the futex word outlives its syscall. +pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Option { + crate::mm::space::Space::translate(unlocked_space(), addr.raw(), asked(access)) } /// The current space, for a reach made holding no lock.