diff --git a/build/ci.rs b/build/ci.rs index ec8f0826a60..0960af8c987 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -566,6 +566,21 @@ pub(crate) const CONTROLS: &[Control] = &[ Fails("an_acknowledged_flush_postdates_the_page_table_write"), Fails("one_serve_answers_two_concurrent_shootdowns"), ]), + red(KERNEL_LOOM, "shootdown-take-relaxed", Some("tlb_shootdown"), &[ + Fails("an_answered_ask_flushed_its_range_after_the_write"), + ]), + red(KERNEL_LOOM, "shootdown-answer-relaxed", Some("tlb_shootdown"), &[ + Fails("an_answered_ask_flushed_its_range_after_the_write"), + ]), + red(KERNEL_LOOM, "shootdown-ask-range-late", Some("tlb_shootdown"), &[ + Fails("an_answered_ask_flushed_its_range_after_the_write"), + ]), + red(KERNEL_LOOM, "shootdown-ask-stale-answers", Some("tlb_shootdown"), &[ + Fails("a_mailbox_reused_waits_for_its_new_answer"), + ]), + red(KERNEL_LOOM, "shootdown-ask-many-as-one", Some("tlb_shootdown"), &[ + Fails("two_asks_of_one_cpu_are_each_answered_for_their_own_page"), + ]), red(KERNEL_LOOM, "tlb-tags-enter-relaxed", Some("tlb_tags"), &[ Fails("a_change_reaches_a_runner_or_the_entering_cpu_flushes"), ]), diff --git a/issues/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md b/issues/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md index aa2c283e9a0..3110c98744a 100644 --- a/issues/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md +++ b/issues/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md @@ -11,16 +11,24 @@ that space (`kernel/src/arch/x86_64/tlb.rs`, `user_change`, over `kernel/src/tlb_tags.rs`). A CPU that switched away keeps the space's entries under PCID, since every CR3 load sets NOFLUSH, so it is not reached; it flushes every user translation before it next enters a space that changed -since its own last such flush. The initiator flushes its own entries by page -for up to 64 pages, and a change no other CPU runs sends no IPI. AArch64 -broadcasts its invalidations in hardware and needs none of this. Every other -shootdown, a kernel-half change and a PCID reclaim among them, still reaches -every CPU, and a reached CPU still answers with a flush of every user -translation, not of the range. +since its own last such flush. A change no other CPU runs sends no IPI. +Each CPU reached is asked the change's range, from the initiator's own +mailbox, and answers that initiator alone: it flushes the range by page, up +to 64 pages, or every user translation when the range is longer or it was +asked by more than one initiator at once. So a CPU no change asks owes no +flush, and the initiator owes itself none: it sends its IPIs first and then +flushes its own entries by page while they fly. Only a flush of every user +translation counts as the CPU's last such flush. AArch64 broadcasts its +invalidations in hardware and needs none of this. Every other shootdown, a +kernel-half change and a PCID reclaim among them, still reaches every CPU, +which answers with a flush of every user translation. What holds it: `kernel/loom/tests/tlb_tags.rs` checks the pairing a CPU entering a space and a change reading who runs it rest on, and reds with the -pairing's fence removed; `tlb_shootdown_waits` shows on the test kernel that +pairing's fence removed; `kernel/loom/tests/tlb_shootdown.rs` checks that an +answered ask flushed its range after the write, alone or with a second +initiator's, and that a mailbox reused waits for its new answer, each red +under its mutation; `tlb_shootdown_waits` shows on the test kernel that a change waits for a CPU running a thread of the space and for none when no other CPU runs one, each half red under its mutation. @@ -40,5 +48,22 @@ four boots against five Linux runs) the T14 read `mem.munmap_part.4k` at (3,341 against 2,279 ns), each resolved, and `mem.mprotect.4k` ahead (816 against 1,019 ns); the AMD bench is ahead on all three. +- **Owed on metal** for the asks above (a reached CPU flushing only the + change's range, the initiator owing itself nothing): + - The T14's `shared-debug` boot (the actuator kernel), where main must pass + `test_rs_tlb_shootdown_waits`: a change still waits for every CPU running + the space. + - An A/B on both benches of `osbench_system` and `osbench_mem`, main from + before and after. Main after must read `mem.munmap_touched_remote.4k` + faster, and `mem.munmap_part.4k`, `mem.mprotect.4k` and the multi-thread + `mem.malloc_churn.*` rows no slower. Until then the gain is unmeasured. + - With it, each bench's cycles per flush, one page by `INVPCID` (the T14) + or `INVLPG` (the AMD bench, which has no PCID) against one flush of every + user translation. From those, the number of pages at which flushing one + by one costs as much as a full flush; the reached CPU's limit, 64, is + lowered if that number is well below it, or the reason not to is + recorded. Branches `memperf-sd-ab-base` and `memperf-sd-ab-head` carry a + measurement-only probe that logs both; they were staged and never booted. + **Exit**: that case, on each proving machine with PCID, red with the flush on entry removed; and the T14's munmap and `vm_protect` figures against Linux. diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 0889eacac62..ba55f85b60c 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -50,6 +50,21 @@ reap-raise-relaxed = [] shootdown-serve-relaxed = [] # `src/shootdown.rs`'s `served` reads `Relaxed`, and `counters_round` reds. shootdown-served-relaxed = [] +# `src/shootdown.rs`'s `Asks::serve` takes its asks `Relaxed`, and +# `tlb_shootdown` reds. +shootdown-take-relaxed = [] +# `src/shootdown.rs`'s `Asks::serve` answers `Relaxed`, and `tlb_shootdown` +# reds. +shootdown-answer-relaxed = [] +# `src/shootdown.rs`'s `Asks::ask` writes its range after asking, and +# `tlb_shootdown` reds. +shootdown-ask-range-late = [] +# `src/shootdown.rs`'s `Asks::ask` leaves the last ask's answers standing, and +# `tlb_shootdown` reds. +shootdown-ask-stale-answers = [] +# `src/shootdown.rs`'s `Asks::serve` answers two asks with one's range, and +# `tlb_shootdown` reds. +shootdown-ask-many-as-one = [] # `src/tlb_tags.rs`'s `enter` loses its fence, and `tlb_tags` reds. tlb-tags-enter-relaxed = [] # `src/log/shard.rs`'s publishing store goes `Relaxed`, and `log_record` reds. diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index f9d8c0a3446..184e55e05de 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -86,6 +86,29 @@ shootdown-serve-relaxed = [] # Never on by default and never reachable from a kernel build, which declares # the same name only so `cfg` checking knows it. shootdown-served-relaxed = [] +# The negative controls for a user change's ask (`shootdown.rs`'s `Asks`), +# each of which `tlb_shootdown.rs` must red: +# +# cargo test --manifest-path kernel/loom/Cargo.toml --features \ +# --test tlb_shootdown +# +# `shootdown-take-relaxed`: the target takes its asks `Relaxed`, so its flush +# no longer follows the asking CPU's page-table write. +# `shootdown-answer-relaxed`: the target answers `Relaxed`, so an asking CPU +# can read its answer before the flush it stands for. +# `shootdown-ask-range-late`: the asking CPU writes its range after asking, so +# a target can flush a range it was not asked. +# `shootdown-ask-stale-answers`: an ask leaves the last ask's answers standing, +# so a reused mailbox is answered before its targets have flushed. +# `shootdown-ask-many-as-one`: an answer to two asks at once flushes one range. +# +# Never on by default and never reachable from a kernel build, which declares +# the same names only so `cfg` checking knows them. +shootdown-take-relaxed = [] +shootdown-answer-relaxed = [] +shootdown-ask-range-late = [] +shootdown-ask-stale-answers = [] +shootdown-ask-many-as-one = [] # The negative control for the CPU roster's publication. It makes # `smp_roster.rs`'s `commit` publish the count with a `Relaxed` store, so a # reader can see a count over a slot the commit has not filled, and diff --git a/kernel/loom/tests/tlb_shootdown.rs b/kernel/loom/tests/tlb_shootdown.rs index 8c394c83076..1203af4c7bb 100644 --- a/kernel/loom/tests/tlb_shootdown.rs +++ b/kernel/loom/tests/tlb_shootdown.rs @@ -41,7 +41,7 @@ use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; -use kernel_loom::shootdown::{Reach, Shootdown}; +use kernel_loom::shootdown::{Asks, CpuSet, Flush, Reach, Shootdown}; use loom::sync::Arc; use loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; @@ -126,7 +126,8 @@ impl Machine { } } - /// One serve of vector 0xFE's body on cpu 1. + /// One serve on cpu 1: vector 0xFE's body serves when owed, and serving + /// regardless explores every schedule that one does and more. fn serve(&self) { self.shootdown.serve(1, |reach| self.flush(reach)); } @@ -221,49 +222,6 @@ fn one_serve_answers_two_concurrent_shootdowns() { }); } -/// A CPU that is waiting for an acknowledgement has not stopped giving them. -/// -/// **Not an interleaving question, so loom explores nothing here** — the -/// schedule below is written out because it is the one two CPUs took on -/// 2026-08-07, and it is the whole defect. `loom::model` is still what wraps it: -/// this crate compiles `shootdown.rs` against loom's atomics, which panic -/// outside a model, and the property under test is the protocol's shape rather -/// than any edge between its atomics. -/// -/// `IF` is clear for the whole of a syscall (`arch::syscall`'s `MSR_FMASK`), so -/// neither CPU below can take the other's IPI. Each one's own wait is the only -/// thing left that can answer the other, and until this test existed the wait -/// asked without answering: both spun until `ACK_TIMEOUT_NS` killed them, as a -/// double kernel panic and as seven wide-phase test failures sharing one line. -#[test] -fn an_initiator_answers_while_it_waits() { - model(|| { - let s = two_cpus(); - - // cpu 0 unmaps and answers for itself; cpu 1 has not issued yet, so the - // generation cpu 0 publishes is its own. - let g0 = s.issue(Reach::Narrow); - s.serve(0, |_| {}); - - // cpu 1 unmaps while cpu 0 is between its own `serve` and its wait. - let g1 = s.issue(Reach::Narrow); - - // cpu 0's wait is not satisfied — cpu 1 has not flushed — and that turn - // is nevertheless where cpu 1's answer comes from. - assert!(!s.wait_turn(0, 1, g0, |_| {})); - assert!( - s.served(0, g1), - "cpu 0 is waiting for cpu 1 and has stopped answering it, which is \ - a wait neither CPU can leave", - ); - - // And cpu 1's own wait then has an answer to find. - s.serve(1, |_| {}); - assert!(s.wait_turn(1, 0, g1, |_| {})); - assert!(s.wait_turn(0, 1, g0, |_| {})); - }); -} - /// A serve that an interrupt nests inside another publishes the later /// generation, and the outer serve, finishing after it, must not take it back. /// @@ -397,8 +355,9 @@ fn a_serve_nested_in_an_unfinished_wide_one_answers_wide() { /// kernel-half one, which it answers wide once: a user-half change keeps the /// kernel's global translations. /// -/// **Not an interleaving question**, as `an_initiator_answers_while_it_waits` -/// is not; reds when a serve answers wide with no wide generation owed. +/// **Not an interleaving question**: `loom::model` wraps it because this +/// crate's atomics are loom's. Reds when a serve answers wide with no wide +/// generation owed. #[test] fn only_a_kernel_half_shootdown_asks_a_wide_answer() { model(|| { @@ -429,3 +388,168 @@ fn two_cpus() -> Shootdown { shootdown.size(2); shootdown } + +/// A user change's range asked of the CPUs running its space ([`Asks`]): cpu 1 +/// is the target and holds a translation of each of two pages, page 0 and +/// page 1, as one cell each. A range answer re-walks the pages it names, an +/// answer to more than one ask at once every page. +struct Asked { + asks: Asks, + pte: [AtomicU64; 2], + tlb: [AtomicU64; 2], +} + +impl Asked { + fn new() -> Arc { + let asks = Asks::new(); + asks.size(3); + Arc::new(Self { + asks, + pte: [AtomicU64::new(MAPPED), AtomicU64::new(MAPPED)], + tlb: [AtomicU64::new(MAPPED), AtomicU64::new(MAPPED)], + }) + } + + fn rewalk(&self, page: usize) { + self.tlb[page].store(self.pte[page].load(Relaxed), Relaxed); + } + + /// cpu 1's answers, `SERVES` of them. + fn take_interrupts(&self) { + for _ in 0..SERVES { + loom::thread::yield_now(); + self.asks.serve(1, |what| match what { + Flush::Range { start, end, .. } => (start..end.min(2)).for_each(|page| self.rewalk(page as usize)), + Flush::User => (0..2).for_each(|page| self.rewalk(page)), + }); + } + } + + /// `me` unmaps `page` and asks cpu 1 to flush it; what cpu 1 holds of the + /// page once it has answered, `None` when the bounded poll ran out. + fn unmap_and_ask(&self, me: usize, page: usize) -> Option { + self.pte[page].store(UNMAPPED, Relaxed); + let mut targets = CpuSet::default(); + targets[0] = 1 << 1; + self.asks.ask(me, 0, page as u64, page as u64 + 1, &targets); + for _ in 0..POLLS { + if self.asks.answered(me, 1) { + return Some(self.tlb[page].load(Relaxed)); + } + loom::thread::yield_now(); + } + None + } +} + +/// An answered ask means cpu 1 has re-walked the page asked after the unmap. +/// +/// Reds when the target takes its asks `Relaxed` +/// (`shootdown-take-relaxed`), so its flush no longer follows the write; when +/// it answers `Relaxed` (`shootdown-answer-relaxed`), so the asking CPU can +/// read its answer before the flush; and when the asking CPU writes its range +/// after asking (`shootdown-ask-range-late`), so the target can flush a range +/// it was never asked. +#[test] +fn an_answered_ask_flushed_its_range_after_the_write() { + acknowledged_model(|| { + let m = Asked::new(); + let target = { + let m = m.clone(); + loom::thread::spawn(move || m.take_interrupts()) + }; + let answer = m.unmap_and_ask(0, 0); + if let Some(reachable) = answer { + assert_eq!(reachable, UNMAPPED, "cpu 1 answered the ask while still holding the page asked"); + } + target.join().unwrap(); + answer.is_some() + }); +} + +/// Two CPUs ask cpu 1 at once, each for its own page: an answer to one alone +/// flushes its range, an answer to both every page, and each asking CPU finds +/// its own page gone once answered, whichever way its ask was taken. +/// +/// Reds when an answer to two asks at once flushes one of their ranges +/// (`shootdown-ask-many-as-one`). +#[test] +fn two_asks_of_one_cpu_are_each_answered_for_their_own_page() { + acknowledged_model(|| { + let m = Asked::new(); + let target = { + let m = m.clone(); + loom::thread::spawn(move || m.take_interrupts()) + }; + let second = { + let m = m.clone(); + loom::thread::spawn(move || m.unmap_and_ask(2, 1)) + }; + let first = m.unmap_and_ask(0, 0); + if let Some(reachable) = first { + assert_eq!(reachable, UNMAPPED, "cpu 0 was answered while cpu 1 still held page 0"); + } + let second = second.join().unwrap(); + if let Some(reachable) = second { + assert_eq!(reachable, UNMAPPED, "cpu 2 was answered while cpu 1 still held page 1"); + } + target.join().unwrap(); + first.is_some() || second.is_some() + }); +} + +/// A CPU asks again only once every CPU it asked has answered, and its next +/// ask waits for answers of its own: its earlier answers do not stand for it. +/// +/// Reds when an ask leaves the previous ask's answers standing +/// (`shootdown-ask-stale-answers`): the mailbox reused would be answered at +/// once, while cpu 1 still holds the second page. +#[test] +fn a_mailbox_reused_waits_for_its_new_answer() { + acknowledged_model(|| { + let m = Asked::new(); + let target = { + let m = m.clone(); + loom::thread::spawn(move || { + m.take_interrupts(); + m.take_interrupts(); + }) + }; + let mut answered = false; + if m.unmap_and_ask(0, 0).is_some() { + let again = m.unmap_and_ask(0, 1); + if let Some(reachable) = again { + assert_eq!(reachable, UNMAPPED, "cpu 0's second ask was answered while cpu 1 still held page 1"); + } + answered = again.is_some(); + } + target.join().unwrap(); + answered + }); +} + +/// A serve answers each CPU asking it once, with that CPU's own range: two +/// CPUs that ask each other each find their answer once the other has +/// served, and a second serve finds nothing left to answer. **Not an +/// interleaving question.** The order in which a waiting CPU answers before +/// it reads its own answer is `arch::tlb::wait_for`'s, which loom does not +/// compile; its doc states it and inspection holds it. +#[test] +fn each_asker_is_answered_once_with_its_own_range() { + model(|| { + let asks = Asks::new(); + asks.size(2); + let (mut to_one, mut to_zero) = (CpuSet::default(), CpuSet::default()); + to_one[0] = 1 << 1; + to_zero[0] = 1; + asks.ask(0, 0, 0, 1, &to_one); + asks.ask(1, 0, 1, 2, &to_zero); + // cpu 0's serve answers cpu 1 with cpu 1's range. + assert!(asks.serve(0, |what| assert_eq!(what, Flush::Range { tag: 0, start: 1, end: 2 }))); + assert!(!asks.answered(0, 1)); + assert!(asks.answered(1, 0), "cpu 0 served and did not answer cpu 1"); + assert!(asks.serve(1, |_| {})); + assert!(asks.answered(0, 1)); + assert!(!asks.serve(0, |_| {}), "an ask was answered twice"); + }); +} diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 7e19396c517..89e80009546 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -353,7 +353,9 @@ pub fn flush_tlb_all() { } /// Flush this CPU's translation of the user page `va` in the space tagged -/// `tag`, which this CPU runs. +/// `tag`: by the tag where PCIDs are on; otherwise in the space this CPU runs, +/// since without them loading a root drops every translation of the one it +/// replaces (SDM Vol. 3A §4.10.4.1, APM Vol. 2 §5.5). pub fn flush_user_page(tag: u16, va: u64) { match pcid_active() { Some(have) => crate::arch::cpu::invpcid(have, Invpcid::Address, tag, va), @@ -1072,8 +1074,9 @@ impl UserTables { } /// Every CPU has dropped what it cached of `[start, end)` and of the - /// tables `unlinked` took out before this returns: every CPU's user - /// translations go, paging-structure caches with them. + /// tables `unlinked` took out before this returns. Where tables went, + /// every reached CPU's user translations go, paging-structure caches with + /// them; otherwise a short range goes page by page. /// Answers whether it reached another CPU. pub fn shoot_down(&self, start: u64, end: u64, unlinked: &Unlinked) -> bool { // A table taken away is cached apart from the leaves: by page is not diff --git a/kernel/src/arch/x86_64/tlb.rs b/kernel/src/arch/x86_64/tlb.rs index a001fcdef23..bccb91d5995 100644 --- a/kernel/src/arch/x86_64/tlb.rs +++ b/kernel/src/arch/x86_64/tlb.rs @@ -8,8 +8,10 @@ //! //! **A change to one user space's leaves reaches only the CPUs running it** //! ([`user_change`], over `crate::tlb_tags`): a CPU that ran it before flushes -//! when it next enters it, if it changed since. Every other shootdown targets -//! every other CPU the machine brought up. +//! when it next enters it, if it changed since. Each CPU reached is asked for +//! the change's range alone, and flushes that range, or every user +//! translation where it was asked more than one at once or the range is long. +//! Every other shootdown targets every other CPU the machine brought up. //! //! **A user-half change keeps global translations**: its flush is `INVPCID` //! type 3 or a `CR3` reload, and only a kernel-half change, whose leaves are @@ -18,19 +20,23 @@ use core::sync::atomic::{AtomicU64, Ordering}; -use crate::shootdown::{Generation, Reach, Shootdown}; +use crate::shootdown::{Asks, CpuSet, Flush, Reach, Shootdown}; use super::{apic, percpu}; use crate::smp; static SHOOTDOWN: Shootdown = Shootdown::new(); +/// The ranges [`user_change`] asks of the CPUs running a space. +static ASKS: Asks = Asks::new(); + /// Which CPUs run each user space, by its PCID, and when each changed. static USER: crate::tlb_tags::Tags = crate::tlb_tags::Tags::new(); /// An answer for each of the `cpus` CPUs the firmware names. pub fn size(cpus: usize) { SHOOTDOWN.size(cpus); + ASKS.size(cpus); USER.size(usize::from(kernel::pcid::MAX_USER_PCID) + 1, cpus); } @@ -52,8 +58,8 @@ pub fn switching(to: u16, load: impl FnOnce()) { USER.left(cpu, switched.from); } -/// Pages a change flushes one at a time on the initiator; past this, its -/// whole space. +/// Pages a change's flush takes one at a time; past this, every user +/// translation. const BY_PAGE: u64 = 64; /// The user space tagged `tag` changed the leaves of `[start, end)`, already @@ -63,54 +69,52 @@ const BY_PAGE: u64 = 64; /// Answers whether it reached another CPU. pub fn user_change(origin: Origin, tag: u16, start: u64, end: u64) -> bool { let me = percpu::cpu_id() as usize; - let mut targets = [0u64; 4]; + let mut targets = CpuSet::default(); let mut beyond = false; USER.changed(usize::from(tag), |cpu| match targets.get_mut(cpu / 64) { Some(word) => *word |= 1 << (cpu % 64), None => beyond = true, }); - // A CPU past what the targets hold reaches every other, as before. - let Some(mine) = targets.get(me / 64).copied() else { - shootdown(origin, Half::User); - return true; - }; - // This CPU first: it may be running the space. - if mine & (1 << (me % 64)) != 0 { - if (end - start) / crate::mm::PAGE_SIZE <= BY_PAGE { - (start..end).step_by(crate::mm::PAGE_SIZE as usize).for_each(|va| crate::mm::paging::flush_user_page(tag, va)); - } else { - flush(Reach::Narrow); - } - targets[me / 64] &= !(1 << (me % 64)); - } - if beyond { - // More CPUs than the targets hold: every one of them, as before. + // A CPU past what a set holds reaches every other, and this one, by a + // machine-wide shootdown. + if beyond || me >= crate::shootdown::ASKABLE { shootdown(origin, Half::User); return true; } + // This CPU may be running the space too. + let here = targets[me / 64] & 1 << (me % 64) != 0; + targets[me / 64] &= !(1 << (me % 64)); if targets.iter().all(|&word| word == 0) || !smp::answering() { + if here { + flush_asked(Flush::Range { tag, start, end }); + } return false; } ISSUED[origin as usize].fetch_add(1, Ordering::Relaxed); let began = crate::clock::nanos_since_boot(); - let generation = SHOOTDOWN.issue(Reach::Narrow); - let each = |f: &mut dyn FnMut(u32)| { - for (w, &word) in targets.iter().enumerate() { - let mut bits = word; - while bits != 0 { - f((w * 64 + bits.trailing_zeros() as usize) as u32); - bits &= bits - 1; - } - } - }; - each(&mut |cpu| apic::tlb_ipi_to(cpu)); - each(&mut |cpu| wait_for(me, cpu, generation)); + ASKS.ask(me, tag, start, end, &targets); + crate::shootdown::for_each(&targets, |cpu| apic::tlb_ipi_to(cpu as u32)); + // Its own flush goes while the IPIs fly. + if here { + flush_asked(Flush::Range { tag, start, end }); + } + crate::shootdown::for_each(&targets, |cpu| wait_for(cpu, || ASKS.answered(me, cpu))); let took = crate::clock::nanos_since_boot().saturating_sub(began); WAIT_NS.fetch_add(took, Ordering::Relaxed); MAX_NS.fetch_max(took, Ordering::Relaxed); true } +/// What an ask has this CPU flush. +fn flush_asked(what: Flush) { + match what { + Flush::Range { tag, start, end } if (end - start) / crate::mm::PAGE_SIZE <= BY_PAGE => { + (start..end).step_by(crate::mm::PAGE_SIZE as usize).for_each(|va| crate::mm::paging::flush_user_page(tag, va)); + } + Flush::Range { .. } | Flush::User => flush(Reach::Narrow), + } +} + use crate::invalidation::{Half, Origin}; /// Issuer-side census; `irq_census`'s `tlb` column is the receiver side, and a @@ -170,10 +174,8 @@ pub fn shootdown(origin: Origin, half: Half) { // This CPU answers itself locally instead of by self-IPI. SHOOTDOWN.serve(me, answer(Answerer::Initiator)); apic::tlb_ipi(); - for cpu in 0..cpus { - if cpu as usize != me { - wait_for(me, cpu, generation); - } + for cpu in (0..cpus as usize).filter(|&cpu| cpu != me) { + wait_for(cpu, || SHOOTDOWN.served(cpu, generation)); } let took = crate::clock::nanos_since_boot().saturating_sub(began); WAIT_NS.fetch_add(took, Ordering::Relaxed); @@ -215,12 +217,21 @@ pub fn bench() { ); } +/// Spin until `cpu` has `answered`, answering meanwhile whatever this CPU +/// owes: an initiator inside an `IrqGuard`, a handler or the tick's pass takes +/// no IPI, so two initiators waiting on each other would deadlock. Answer +/// first, then ask: the answer is what a sibling waiting on this CPU needs. +/// /// Never logs: `drivers::serial`'s lock under its `IrqGuard` would deadlock a /// target that cannot answer while blocked on it. -fn wait_for(me: usize, cpu: u32, generation: Generation) { +fn wait_for(cpu: usize, answered: impl Fn() -> bool) { let mut spins = 0u32; let mut deadline = None; - while !SHOOTDOWN.wait_turn(me, cpu as usize, generation, answer(Answerer::Initiator)) { + loop { + answer_owed(Answerer::Initiator); + if answered() { + return; + } core::hint::spin_loop(); spins += 1; if spins == SPINS_PER_DEADLINE_CHECK { @@ -229,8 +240,7 @@ fn wait_for(me: usize, cpu: u32, generation: Generation) { match deadline { None => deadline = Some(now.saturating_add(crate::time::DEAF_CPU.nanos())), Some(at) if now >= at => panic!( - "tlb: cpu {cpu} has not flushed for generation {generation:?} in {}ns — \ - it is not taking interrupts", + "tlb: cpu {cpu} has not flushed in {}ns — it is not taking interrupts", crate::time::DEAF_CPU.nanos(), ), Some(_) => {} @@ -269,10 +279,26 @@ fn answer(by: Answerer) -> impl FnOnce(Reach) { } } -/// Vector 0xFE's whole body: flush this CPU and say which generation it covers. -pub fn serve_ipi() { +/// [`answer`] for an ask. +fn answer_ask(by: Answerer) -> impl FnOnce(Flush) { + move |what| { + flush_asked(what); + stage_ack_delay(by); + } +} + +/// Answer every generation and every ask this CPU owes. +fn answer_owed(by: Answerer) { let cpu = percpu::cpu_id() as usize; - SHOOTDOWN.serve(cpu, answer(Answerer::Asked)); + SHOOTDOWN.serve_if_owed(cpu, answer(by)); + ASKS.serve(cpu, answer_ask(by)); +} + +/// Vector 0xFE's whole body: answer whatever this CPU owes, a generation, an +/// ask, or both. Each is visible here: the IPI's send is ordered after the +/// write that asks it (`apic::send`). +pub fn serve_ipi() { + answer_owed(Answerer::Asked); } /// Answers a pending shootdown without taking a lock or allocating, so it is @@ -286,8 +312,7 @@ pub fn poll() { return; } super::clocksource::serve_here(); - let cpu = percpu::cpu_id() as usize; - SHOOTDOWN.serve_if_owed(cpu, answer(Answerer::Asked)); + answer_owed(Answerer::Asked); } /// Settle every shootdown issued before this CPU could answer one; called diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 9ce1b2d795f..b321a7d14a4 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -1,11 +1,21 @@ -//! A generation one CPU issues and every other acknowledges, with no hardware -//! in it: a TLB shootdown's, and a counters read's round. -//! Compiled a second time into `kernel/loom/` against loom's atomics, so this file names nothing of the kernel but -//! [`CpuSlice`], which `kernel/loom/` compiles beside it. -//! The read must happen before the answer, or a target could publish a generation its answer does not cover; -//! and an initiator that sees a generation served sees everything that answer wrote. -//! A generation is narrow or wide, and an answer reaches as far as the widest generation it covers: one -//! answer covers every generation owed, so the reach is the target's to read and never the IPI's. +//! Two protocols with no hardware in them, by which one CPU asks others and +//! waits for their answers. +//! +//! [`Shootdown`]: a generation one CPU issues and every other acknowledges: a +//! machine-wide TLB shootdown's, and a counters read's round. The read must +//! happen before the answer, or a target could publish a generation its answer +//! does not cover; and an initiator that sees a generation served sees +//! everything that answer wrote. A generation is narrow or wide, and an answer +//! reaches as far as the widest generation it covers: one answer covers every +//! generation owed, so the reach is the target's to read and never the IPI's. +//! +//! [`Asks`]: one user space's range a CPU asks the CPUs it names to flush, +//! each answering that CPU alone, so a CPU not asked owes nothing and the +//! asking CPU owes itself nothing. +//! +//! Compiled a second time into `kernel/loom/` against loom's atomics, so this +//! file names nothing of the kernel but [`CpuSlice`], which `kernel/loom/` +//! compiles beside it. #[cfg(not(feature = "loom"))] use core::sync::atomic::{AtomicU64, Ordering}; @@ -108,7 +118,10 @@ impl Shootdown { /// Does `cpu` owe anyone a flush? pub fn owes(&self, cpu: usize) -> bool { - // A hint: `serve` carries the real ordering, so a stale `false` costs nothing. + // `serve` carries the ordering of the flush. A poll that reads a stale + // `false` answers on a later turn; the interrupt handler, which gets + // no later turn, reads what the IPI asks because its send is ordered + // after the issue (`arch::tlb::serve_ipi`). self.requested.load(Ordering::Relaxed) > self.answers[cpu].flushed.load(Ordering::Relaxed) } @@ -125,18 +138,165 @@ impl Shootdown { } owed } +} + +// Acquire: makes the flush that follows see the asking CPU's page-table write +// and its range. +#[cfg(not(feature = "shootdown-take-relaxed"))] +const TAKE: Ordering = Ordering::Acquire; +// Relaxed arm: kernel-loom's negative-control mutation, never selected by a kernel build. +#[cfg(feature = "shootdown-take-relaxed")] +const TAKE: Ordering = Ordering::Relaxed; + +// Release: an asking CPU that reads its answer follows the flush before it. +#[cfg(not(feature = "shootdown-answer-relaxed"))] +const ANSWER: Ordering = Ordering::Release; +// Relaxed arm: kernel-loom's negative-control mutation, never selected by a kernel build. +#[cfg(feature = "shootdown-answer-relaxed")] +const ANSWER: Ordering = Ordering::Relaxed; + +/// A set of CPUs, a bit each, of the first [`ASKABLE`]: as many as a user +/// change asks, past which it reaches every CPU by a [`Shootdown`]. +pub type CpuSet = [u64; WORDS]; +const WORDS: usize = 4; +pub const ASKABLE: usize = 64 * WORDS; - /// One turn of an initiator's wait for `cpu`: answer first, then ask. - pub fn wait_turn( - &self, - me: usize, - cpu: usize, - generation: Generation, - flush: impl FnOnce(Reach), - ) -> bool { - // An initiator inside an `IrqGuard`, a handler or the tick's pass takes no IPI, so this CPU must serve itself or two initiators waiting on each other deadlock. - self.serve_if_owed(me, flush); - // Order is the fix: serving first is what publishes the generation a concurrent sibling is waiting on. - self.served(cpu, generation) +/// What one CPU asks: the range of the space it names. +struct Ask { + tag: AtomicU64, + start: AtomicU64, + end: AtomicU64, +} + +/// One CPU's side of [`Asks`]. +struct Mailbox { + /// A bit per CPU asking this one. + asking: [AtomicU64; WORDS], + /// What this CPU asks. + ask: Ask, + /// A bit per CPU that has answered this one's ask. + answered: [AtomicU64; WORDS], +} + +/// What an answer must flush: the one range it was asked, or, asked more than +/// one at once, every user translation. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Flush { + Range { tag: u16, start: u64, end: u64 }, + User, +} + +pub struct Asks { + boxes: CpuSlice, +} + +impl Asks { + /// Must stay `const`: the kernel's instance is a `static`, [`Self::size`]d + /// before an AP is attempted. + #[cfg(not(feature = "loom"))] + pub const fn new() -> Self { + Self { boxes: CpuSlice::new() } + } + + // Not `const`: loom's atomics have no const constructor. + #[allow(clippy::new_without_default)] + #[cfg(feature = "loom")] + pub fn new() -> Self { + Self { boxes: CpuSlice::new() } + } + + /// A mailbox for each of the first [`ASKABLE`] of the `cpus` CPUs the + /// firmware names. + pub fn size(&self, cpus: usize) { + let zero = || core::array::from_fn(|_| AtomicU64::new(0)); + self.boxes.size(cpus.min(ASKABLE), |_| Mailbox { + asking: zero(), + ask: Ask { tag: AtomicU64::new(0), start: AtomicU64::new(0), end: AtomicU64::new(0) }, + answered: zero(), + }); + } + + /// `me` asks each CPU `targets` holds, a bit per CPU, to flush + /// `[start, end)` of the space tagged `tag`. The caller's precondition, + /// on which the protocol rests: every CPU `me` asked last has answered, so + /// no CPU still reads its range or will answer it but the ones this asks. + pub fn ask(&self, me: usize, tag: u16, start: u64, end: u64, targets: &CpuSet) { + let own = &self.boxes[me]; + // Kernel-loom's negative controls, never selected by a kernel build, + // leave the last ask's answers standing and write the range late. + if !cfg!(feature = "shootdown-ask-stale-answers") { + own.answered.iter().for_each(|word| word.store(0, Ordering::Relaxed)); + } + let late = cfg!(feature = "shootdown-ask-range-late"); + if !late { + own.ask.store(tag, start, end); + } + for_each(targets, |cpu| { + // Release: the target that takes this bit reads the range and the + // page-table write behind it. + self.boxes[cpu].asking[me / 64].fetch_or(1 << (me % 64), Ordering::Release); + }); + if late { + own.ask.store(tag, start, end); + } + } + + /// Has `cpu` answered `me`'s ask? + pub fn answered(&self, me: usize, cpu: usize) -> bool { + // Acquire: pairs with `serve`'s Release, so the answer's flush comes first. + self.boxes[me].answered[cpu / 64].load(Ordering::Acquire) & 1 << (cpu % 64) != 0 + } + + /// `cpu`'s whole side: take every ask, flush as `flush` is told, and + /// answer each asking CPU. Whether anything was asked. + pub fn serve(&self, cpu: usize, flush: impl FnOnce(Flush)) -> bool { + let Some(own) = self.boxes.get(cpu) else { return false }; + let mut taken = CpuSet::default(); + for (word, asking) in taken.iter_mut().zip(&own.asking) { + // A stale 0 leaves the bit for a later serve; the interrupt + // handler reads it because the IPI's send is ordered after it. + if asking.load(Ordering::Relaxed) != 0 { + *word = asking.swap(0, TAKE); + } + } + let mut asker = None; + let mut asked = 0; + for_each(&taken, |one| { + asker = Some(one); + asked += 1; + }); + let Some(asker) = asker else { return false }; + // Kernel-loom's negative control answers many as one, never selected by a kernel build. + let one = asked == 1 || cfg!(feature = "shootdown-ask-many-as-one"); + flush(if one { self.boxes[asker].ask.load() } else { Flush::User }); + for_each(&taken, |asker| { + self.boxes[asker].answered[cpu / 64].fetch_or(1 << (cpu % 64), ANSWER); + }); + true + } +} + +impl Ask { + fn store(&self, tag: u16, start: u64, end: u64) { + self.tag.store(u64::from(tag), Ordering::Relaxed); + self.start.store(start, Ordering::Relaxed); + self.end.store(end, Ordering::Relaxed); + } + + fn load(&self) -> Flush { + // Every tag fits: it was stored from a `u16`. + let tag = self.tag.load(Ordering::Relaxed) as u16; + Flush::Range { tag, start: self.start.load(Ordering::Relaxed), end: self.end.load(Ordering::Relaxed) } + } +} + +/// Each CPU `set` holds, a bit per CPU. +pub fn for_each(set: &CpuSet, mut f: impl FnMut(usize)) { + for (w, &word) in set.iter().enumerate() { + let mut bits = word; + while bits != 0 { + f(w * 64 + bits.trailing_zeros() as usize); + bits &= bits - 1; + } } } diff --git a/kernel/src/sync.rs b/kernel/src/sync.rs index 5313b1b4f22..44919f83a15 100644 --- a/kernel/src/sync.rs +++ b/kernel/src/sync.rs @@ -145,7 +145,8 @@ impl Lock { // no IPI, and without this would deadlock an initiator that holds // the lock. // With `IF` set the IPI can land inside this poll's own serve, - // which `Shootdown::serve` survives. + // which `Shootdown::serve` survives, and `Asks::serve` too: each + // bit is answered by whichever serve took it. crate::arch::tlb::poll(); spins += 1; if spins == next_warn {