diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index a4ae53cb4e0..49c164f8c5e 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -19,7 +19,35 @@ lacks: to the placement code as one inside it. - **Wake-affine vs spread**: on wake, decide between placing a task near its waker (shared data stays warm — pipe/IPC pairs) and away from it - (parallelism). Every wake still ignores the waker. + (parallelism). The first half is in `CpuSched::toward_its_waker`: + - Every wake a thread posts names its CPU; one a device interrupt's + handler, a deadline or a retire posts names none, and so does one a + thread posts once it has begun leaving its process (`thread_exit`, + `process::leave`): a parent woken by each child's exit, its join or its + teardown's closes would otherwise follow children it never talks to + again. The thread's task carries that as one bit, set once; it is the + thread's lifecycle, not placement accounting. + - A run of `WAKER_RUN` (8) or more wakes from one other LLC moves the + task to a sleeping CPU there that its mask names, that no kick is on + its way to and that is off the waker's own core. A pair split across + LLCs shares one after 8 round trips; on one core it would read about + as slow as split. + A split pair costs about 25% on the AMD bench's `ipc.reqrep.8` and + `fs.disk.read_4k_at`. + - Only a wake that finds the task parked is counted. A server with a + client in each LLC therefore follows the far one when the near one's + requests keep finding it running: 5 of 40 runs of the simulator's + `server_of_two_llcs` move it once, whatever the run length from 8 to + 12, and none moves it back at ten times the rounds; a run of 4 does. + Parked wakes from both LLCs, independent and even, move it on about one + in 510. No metal row has a server with clients in both LLCs. + - The task's home CPU makes the move after its own wake, so a move costs + two wakes in series. + - Spreading a waker's partners for parallelism is still undecided. + - **Owner ruling, 2026-11-01:** one u16 of per-task placement state is + admitted, kept by the CPU that owns the task: the LLC of its last waker, + which holds the run's count beside it. It is the single exception to the + 2026-10-28 scheduling model's "no per-task state". - **An idle-state input**: see `issues/the-scheduler-and-the-clocks-never-talk.md`. - **Bounded scans.** Each choice walks the rings it needs whole. A steal probe reads the machine's offering mask first, so a machine with nothing to diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index f81eca19d5f..2e3bd52b7d9 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -533,6 +533,21 @@ impl CpuSched { self.leave_fair_for_rt(now); } + /// The CPU a wake the running task posts names as its waker + /// ([`WakeCause::waker`]): this one, unless nothing runs here or the + /// running thread is leaving its process. A dying waker is no partner + /// for [`Self::toward_its_waker`] to follow: a parent woken by each + /// child's exit would chase children it never talks to again. + pub fn waker(&self) -> Option { + self.running.as_ref().filter(|task| !task.is_leaving()).map(|_| self.id) + } + + /// The running thread has begun leaving its process: no wake it posts + /// from now on names a waker. + pub fn current_leaves(&mut self) { + self.running.as_mut().expect("a thread leaving its process is running").begin_leaving(); + } + /// `SYS_RT_ENTER` on the running task — permanent RT, as opposed to the /// bounded window a waker lends. The privilege gate lives at the syscall /// layer. @@ -1178,7 +1193,49 @@ impl CpuSched { let mut task = entry.task.wake(self.id, cause, entry.class, now); self.trace(env, now, TraceKind::Wake { task: key, name: task.ext().name() }); task.refresh_affinity(); - self.place(task, env, now); + match self.toward_its_waker(&mut task, cause, env) { + Some(dst) => self.hand_off(task, dst, env, now), + None => self.place(task, env, now), + } + } + + /// **A woken partner follows its waker's LLC.** A wake from another LLC + /// whose run from it is at or past [`WAKER_RUN`] goes to a CPU of that + /// LLC the task's mask names, [`Doorbell::sleeping_unrung`] and off the + /// waker's own core, nearest the waker and quickest to wake. A waker that + /// has parked leaves its SMT sibling looking like a free core, and a pair + /// on one core contends for it: on the AMD bench a request/reply pair + /// there reads about as slow as one split across LLCs. Only a wake that finds the task parked + /// reaches here, so only those are counted: one that finds it running or + /// about to park is a flag the task consumes unseen. A request/reply pair + /// split across LLCs shares one after [`WAKER_RUN`] round trips. A server + /// with a client in each LLC follows the far one when the near one's + /// requests keep finding it running; once moved, both are counted. The + /// run is not reset by a move, so a task stolen out of the LLC it was + /// woken from goes back on that LLC's next wake. A real-time task keeps + /// the RT band's placement, and a stopped or killed one [`Self::place`]'s. + fn toward_its_waker, P: PreemptGuard>( + &self, + task: &mut ReadyTask, + cause: WakeCause, + env: Env<'_, H, P>, + ) -> Option { + let waker = cause.waker?; + let topology = env.cpus.topology(); + let llc = topology.llc_index(waker); + let run = task.count_waker_llc(llc); + if run < WAKER_RUN + || llc == topology.llc_index(self.id) + || task.is_rt() + || task.shared().stop_pending() + || task.shared().kill_pending() + { + return None; + } + let mask = task.affinity(); + env.cpus.idle_in_llc(waker, topology.position(waker), |cpu| { + mask.contains(cpu.id()) && cpu.doorbell().sleeping_unrung() && topology.level(cpu.id(), waker) != Some(Level::Core) + }) } fn handle_adopt, P: PreemptGuard>( @@ -2634,6 +2691,12 @@ pub struct CpuHandles { offering: Box<[AtomicU64]>, } +/// How many parked wakes in a row from one other LLC move a task into it +/// ([`CpuSched::toward_its_waker`]). A split pair pays this many split round +/// trips once; a task whose parked wakes come from two LLCs independently +/// and evenly moves on about one in 2^(n+1) - 2, 1 in 510 at 8. +pub const WAKER_RUN: u16 = 8; + /// The CPUs an idle CPU is chosen among, nearest first: `me`'s LLC, then the /// rest of its package, then the machine. A choice stays in the first group /// with a candidate, so locality decides before core idleness does. @@ -2754,29 +2817,42 @@ impl CpuHandles { /// go to the first at or after topology position `from`, which is how /// consecutive choices rotate. pub fn idle_near(&self, me: CpuId, from: usize, usable: impl Fn(&CpuHandle) -> bool) -> Option { - for (inner, outer) in IDLE_GROUPS { - // The best of each tier, as (exit, cpu): strictly less replaces, so the first of equals stays. - let mut free_core: Option<(u64, CpuId)> = None; - let mut shared_core: Option<(u64, CpuId)> = None; - for cpu in self.topology.ring(me, inner, outer, from) { - let handle = self.get(cpu); - if !usable(handle) { - continue; - } - let exit = handle.idle_exit(); - let tier = if self.busy_siblings(cpu) == 0 { &mut free_core } else { &mut shared_core }; - if tier.is_none_or(|(best, _)| exit < best) { - *tier = Some((exit, cpu)); - } - if free_core.is_some_and(|(best, _)| best == 0) { - break; - } + IDLE_GROUPS.into_iter().find_map(|(inner, outer)| self.idle_among(me, inner, outer, from, &usable)) + } + + /// [`Self::idle_near`] inside `me`'s LLC alone. + pub fn idle_in_llc(&self, me: CpuId, from: usize, usable: impl Fn(&CpuHandle) -> bool) -> Option { + self.idle_among(me, None, Level::Llc, from, &usable) + } + + /// The best CPU `usable` accepts in one of [`IDLE_GROUPS`]' rings: a free + /// core before a shared one, then the quickest to wake. + fn idle_among( + &self, + me: CpuId, + inner: Option, + outer: Level, + from: usize, + usable: &impl Fn(&CpuHandle) -> bool, + ) -> Option { + // The best of each tier, as (exit, cpu): strictly less replaces, so the first of equals stays. + let mut free_core: Option<(u64, CpuId)> = None; + let mut shared_core: Option<(u64, CpuId)> = None; + for cpu in self.topology.ring(me, inner, outer, from) { + let handle = self.get(cpu); + if !usable(handle) { + continue; } - if let Some((_, cpu)) = free_core.or(shared_core) { - return Some(cpu); + let exit = handle.idle_exit(); + let tier = if self.busy_siblings(cpu) == 0 { &mut free_core } else { &mut shared_core }; + if tier.is_none_or(|(best, _)| exit < best) { + *tier = Some((exit, cpu)); + } + if free_core.is_some_and(|(best, _)| best == 0) { + break; } } - None + free_core.or(shared_core).map(|(_, cpu)| cpu) } /// Where a new task goes: the CPU in `mask` still answering whose queue it would wait @@ -3124,7 +3200,10 @@ mod tests { shared: &Arc>>, reason: WakeReason, ) { - let cause = WakeCause::new(reason); + self.post_claimed_cause(cpu, shared, WakeCause::new(reason)); + } + + fn post_claimed_cause(&self, cpu: CpuId, shared: &Arc>>, cause: WakeCause) { let slot = shared .wake_node() .claim() @@ -4596,6 +4675,144 @@ mod tests { w.abandon(); } + /// A task parked on cpu0, in [`two_llcs`]' LLC 0, every other CPU + /// asleep. LLC 1 is cpu1, cpu3, cpu5 and cpu7. + fn parked_in_llc_0(w: &mut World, mask: CpuMask) -> Arc>> { + let job = Arc::new(Job::new(JobKey(1000 + w.next_key))); + let (_, task) = w.spawn_masked(C0, RtState::default(), job, mask); + w.run_a_pass(C0); + w.park_running(C0); + for cpu in 1..8 { + w.run_a_pass(CpuId(cpu)); + } + task + } + + /// Wake the task parked on cpu0 as the thread running on `waker` would, + /// and let cpu0 take the wake. + fn woken_from(w: &mut World, task: &Arc>>, waker: u32) { + assert_eq!(task.claim_wake(), Claim::Parked(C0), "the task is parked on cpu0"); + w.post_claimed_cause(C0, task, WakeCause::new(WakeReason::Woken).from(Some(CpuId(waker)))); + w.run_a_pass(C0); + } + + /// Wake `task` `times` from `waker`, each time letting it run at home on + /// cpu0 and park again. + fn woken_at_home(w: &mut World, task: &Arc>>, waker: u32, times: u16) { + for _ in 0..times { + woken_from(w, task, waker); + assert_eq!(task.state(), TaskState::Running(C0), "a wake from cpu{waker} short of the run left it home"); + w.park_running(C0); + } + } + + /// **The wake that makes [`WAKER_RUN`] in a row from another LLC moves + /// the task into it**, onto a sleeping CPU there. + #[test] + fn a_run_of_wakes_from_another_llc_moves_the_task_into_it() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &task, 1, WAKER_RUN - 1); + woken_from(&mut w, &task, 1); + let state = task.state(); + w.abandon(); + assert!( + matches!(state, TaskState::InTransit(cpu) if cpu.0 % 2 == 1), + "the run's last wake handed it to a CPU of LLC 1, and it is {state:?}", + ); + } + + /// **A wake from home starts the run again**: a task woken from both + /// LLCs never makes [`WAKER_RUN`] in a row from the other one. + #[test] + fn a_wake_from_home_breaks_the_run() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + for _ in 0..3 { + woken_at_home(&mut w, &task, 1, WAKER_RUN - 1); + woken_at_home(&mut w, &task, 2, 1); + } + w.abandon(); + } + + /// **The move stays inside the mask**: a task whose mask names only LLC + /// 0 stays home however often LLC 1 wakes it. + #[test] + fn a_task_masked_out_of_its_wakers_llc_stays_home() { + let mut w = World::on(two_llcs()); + let llc_0 = CpuMask::from_words(&[0b0101_0101], 8).unwrap(); + let task = parked_in_llc_0(&mut w, llc_0); + woken_at_home(&mut w, &task, 1, 2 * WAKER_RUN); + w.abandon(); + } + + /// **The move stays off the waker's core**: a waker that has parked + /// leaves its SMT sibling asleep and first in the scan, and the task goes + /// to the other core of the waker's LLC instead. + #[test] + fn a_move_stays_off_the_wakers_core() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &task, 1, WAKER_RUN - 1); + woken_from(&mut w, &task, 1); + let state = task.state(); + w.abandon(); + assert!( + matches!(state, TaskState::InTransit(cpu) if cpu.0 == 3 || cpu.0 == 7), + "the task went to cpu3 or cpu7, off cpu1's core with cpu5, and it is {state:?}", + ); + } + + /// **A thread leaving its process names no waker**: the thread running + /// on cpu1 wakes the task one short of [`WAKER_RUN`], then begins + /// leaving, and a full run of wakes it posts on its way out leaves the + /// task home. + #[test] + fn a_leaving_threads_wakes_leave_the_task_home() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + w.spawn(C1); + w.run_a_pass(C1); + let woken_by_cpu1 = |w: &mut World, times| { + for _ in 0..times { + assert_eq!(task.claim_wake(), Claim::Parked(C0), "the task is parked on cpu0"); + let cause = WakeCause::new(WakeReason::Woken).from(w.cpus[1].waker()); + w.post_claimed_cause(C0, &task, cause); + w.run_a_pass(C0); + assert_eq!(task.state(), TaskState::Running(C0), "a wake from cpu1 moved the task"); + w.park_running(C0); + } + }; + assert_eq!(w.cpus[1].waker(), Some(C1), "cpu1's running thread names itself"); + woken_by_cpu1(&mut w, WAKER_RUN - 1); + w.cpus[1].current_leaves(); + woken_by_cpu1(&mut w, WAKER_RUN); + w.abandon(); + } + + /// **Two runs ending in one drain go to two CPUs**: the first move rings + /// its CPU, and a rung sleeper is passed over, as a wake forward passes + /// it. + #[test] + fn two_runs_ending_in_one_drain_go_to_two_cpus() { + let mut w = World::on(two_llcs()); + let first = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &first, 1, WAKER_RUN - 1); + let second = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &second, 1, WAKER_RUN - 1); + for task in [&first, &second] { + assert_eq!(task.claim_wake(), Claim::Parked(C0), "the task is parked on cpu0"); + w.post_claimed_cause(C0, task, WakeCause::new(WakeReason::Woken).from(Some(CpuId(1)))); + } + w.run_a_pass(C0); + let states = [first.state(), second.state()]; + w.abandon(); + match states { + [TaskState::InTransit(a), TaskState::InTransit(b)] if a != b && a.0 % 2 == 1 && b.0 % 2 == 1 => {} + _ => panic!("the two moves did not go to two CPUs of LLC 1: {states:?}"), + } + } + /// **One forward per idle sibling**: wakes drained in one pass are handed /// to a sibling only while it has no kick coming, so a burst does not pile /// onto the first sleeper — the rest queue here. diff --git a/kernel/pure/sched/task.rs b/kernel/pure/sched/task.rs index d2d7d4a6209..46dda7ce3f7 100644 --- a/kernel/pure/sched/task.rs +++ b/kernel/pure/sched/task.rs @@ -59,6 +59,9 @@ pub struct Standing { pub struct WakeCause { pub reason: WakeReason, pub boost: Option, + /// The CPU whose running thread posted this wake. `None` for a wake from + /// a device interrupt, a deadline or a retire, which no thread posted. + pub waker: Option, } impl WakeCause { @@ -66,6 +69,7 @@ impl WakeCause { Self { reason, boost: None, + waker: None, } } @@ -73,9 +77,15 @@ impl WakeCause { Self { reason, boost: Some(BoostWindow { until }), + waker: None, } } + /// This wake, posted by the thread running on `waker`, if one is. + pub fn from(self, waker: Option) -> Self { + Self { waker, ..self } + } + /// RT and boost wakes must preempt the target promptly; ordinary wakes /// ride the target's next safe point. pub fn urgency(&self) -> crate::sched::mailbox::Urgency { @@ -785,6 +795,32 @@ pub struct TaskAccounting { /// a raw context pointer taken before a container move stays valid. pub struct Task(Box>); +/// The LLC a task's last wakes came from +/// ([`crate::sched::topology::Topology::llc_index`]) and how many came from +/// it in a row, in one `u16`: the LLC in the low ten bits and the run, +/// saturating, in the high six. A run of 0 is no wake counted yet. The one +/// per-task placement state the scheduling model admits. +#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] +pub struct WakerStreak(u16); + +impl WakerStreak { + const LLC_BITS: u32 = 10; + const LLC: u16 = (1 << Self::LLC_BITS) - 1; + const MAX_RUN: u16 = u16::MAX >> Self::LLC_BITS; + + /// Count a wake from `llc` and answer the run it extends or starts. + pub fn count(&mut self, llc: u16) -> u16 { + assert!(llc <= Self::LLC, "an LLC index past {}", Self::LLC); + let run = self.0 >> Self::LLC_BITS; + let run = if run > 0 && self.0 & Self::LLC == llc { (run + 1).min(Self::MAX_RUN) } else { 1 }; + self.0 = run << Self::LLC_BITS | llc; + run + } +} + +// Every LLC index is the position of a CPU a mask can name. +const _: () = assert!(crate::sched::affinity::MAX_CPUS <= 1 << WakerStreak::LLC_BITS); + struct TaskInner { key: TaskKey, shared: Arc>>, @@ -797,6 +833,11 @@ struct TaskInner { /// loaded at. Changed only by [`Task::refresh_affinity`]. affinity: CpuMask, affinity_version: u64, + /// Where this task's recent wakes came from. Kept by the owner CPU, as + /// the mask copy is. + waker_streak: WakerStreak, + /// Set once the thread has begun leaving its process, and never cleared. + leaving: bool, /// When the current residency began: enqueue time while ready, dispatch /// time while running, park time while blocked. One field, because a task /// is in exactly one state. @@ -1015,6 +1056,8 @@ impl TaskBuilder { acct: TaskAccounting::default(), affinity: self.mask, affinity_version: 0, + waker_streak: WakerStreak::default(), + leaving: false, since: now, adopt_node: MailboxNode::new(), ext: Some(self.ext), @@ -1052,6 +1095,12 @@ impl ReadyTask { self.0.refresh_affinity() } + /// Count a wake whose waker ran in `llc`, and answer how many wakes in a + /// row have come from it. + pub(crate) fn count_waker_llc(&mut self, llc: u16) -> u16 { + self.0 .0.waker_streak.count(llc) + } + pub(crate) fn standing_mut(&mut self) -> &mut Standing { &mut self.0 .0.standing } @@ -1119,6 +1168,15 @@ impl RunningTask { self.0.refresh_affinity() } + /// The thread has begun leaving its process, for good. + pub(crate) fn begin_leaving(&mut self) { + self.0 .0.leaving = true; + } + + pub fn is_leaving(&self) -> bool { + self.0 .0.leaving + } + pub(crate) fn standing_mut(&mut self) -> &mut Standing { &mut self.0 .0.standing } diff --git a/kernel/pure/sched/topology.rs b/kernel/pure/sched/topology.rs index 7e6e05b2072..b4926c00df1 100644 --- a/kernel/pure/sched/topology.rs +++ b/kernel/pure/sched/topology.rs @@ -136,6 +136,13 @@ impl Topology { self.keys.len() } + /// A number for the LLC `cpu` is in, the same for every CPU of that LLC + /// and different for every other: the position of its first CPU in the + /// topology's order. + pub fn llc_index(&self, cpu: CpuId) -> u16 { + u16::try_from(self.places[cpu.0 as usize].llc.start).expect("a CPU mask names at most 1024 CPUs") + } + pub fn keys(&self, cpu: CpuId) -> CpuKeys { self.keys[cpu.0 as usize] } diff --git a/kernel/sim/src/hw_impl.rs b/kernel/sim/src/hw_impl.rs index 22b851904cd..822ca7281d3 100644 --- a/kernel/sim/src/hw_impl.rs +++ b/kernel/sim/src/hw_impl.rs @@ -122,6 +122,11 @@ impl SimHw { self.with(|s| core::mem::replace(&mut s.poster, cpu)) } + /// The CPU whose step is posting, as a kernel wake names its waker. + pub fn poster(&self) -> Option { + self.with(|s| s.poster) + } + fn cpu(state: &SimHwState) -> usize { state .current diff --git a/kernel/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs index 99ac6ebf5d7..c502454cf30 100644 --- a/kernel/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -1195,6 +1195,67 @@ pub fn turn_taking_pair() -> Scenario { /// How many turns each of [`turn_taking_pair`]'s threads takes. pub const TURN_TAKING_ROUNDS: usize = 8; +/// **A request/reply pair split across the AMD bench's two LLCs**: one thread +/// on cpu0 and its partner on cpu8, each waking the other and parking, with +/// balancing off. Only wake placement can bring them together. +pub fn split_pair() -> Scenario { + let turn = MS / 10; + let wake = |queue| Op::Wake { queue, all: false, boost: None }; + let block = |queue| Op::Block { queue, deadline: None }; + scenario( + "split_pair", + 16, + vec![queue(WaitClass::Io), queue(WaitClass::Io)], + vec![process( + "pair", + vec![0, 1], + vec![ + Script::looping(vec![Op::Run(turn), wake(1), block(0)], WAKE_LLC_ROUNDS), + Script::looping(vec![block(1), Op::Run(turn), wake(0)], WAKE_LLC_ROUNDS), + ], + )], + ) + .with_placement(PlacementShape::Listed(&[0, 8])) + .with_topology(Shape::amd_bench()) + .with_balance(Balance::None) +} + +/// **One server woken from both LLCs in turn**: a server on cpu0 answers a +/// client pinned to cpu1, in its LLC, and one pinned to cpu9, in the other, +/// with balancing off, each taking `rounds` turns. A request that finds the +/// server running is never counted, and which client's request finds it +/// parked is the run's timing: following each wake's LLC would move it back +/// and forth. +pub fn server_of_two_llcs(rounds: usize) -> Scenario { + let turn = MS / 10; + let wake = |queue| Op::Wake { queue, all: false, boost: None }; + let block = |queue| Op::Block { queue, deadline: None }; + let client = |cpu: u64, reply| Script::looping(vec![Op::SetAffinity(1 << cpu), Op::Run(turn), wake(0), block(reply)], rounds); + let mut scenario = scenario( + "server_of_two_llcs", + 16, + vec![queue(WaitClass::Io), queue(WaitClass::Io), queue(WaitClass::Io)], + vec![process( + "served", + vec![0, 1, 2], + vec![ + Script::looping(vec![block(0), Op::Run(turn), wake(1), block(0), Op::Run(turn), wake(2)], rounds), + client(1, 1), + client(9, 2), + ], + )], + ) + .with_placement(PlacementShape::Listed(&[0, 1, 9])) + .with_topology(Shape::amd_bench()) + .with_balance(Balance::None); + scenario.max_steps *= rounds.div_ceil(WAKE_LLC_ROUNDS); + scenario +} + +/// How many turns [`split_pair`]'s threads take, and the rounds +/// [`server_of_two_llcs`] is measured at. +pub const WAKE_LLC_ROUNDS: usize = 24; + /// **Masks under stealing**, every thread spawned on cpu0 of four with the /// shipped balance policy: two threads pinned to cpu0, two masked to cpu1 and /// cpu2, one that moves itself between halves of the machine as it runs, and diff --git a/kernel/sim/src/vm.rs b/kernel/sim/src/vm.rs index 83074e2ce3b..dc7a6b30746 100644 --- a/kernel/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -2007,7 +2007,8 @@ fn wake( WakeCause::boosted(WakeReason::Woken, until) } None => WakeCause::new(WakeReason::Woken), - }; + } + .from(hw.poster()); let env = Poster { cpus: handles, kicker: hw, preempt: &SimPreempt }; if all { q.queue.post(cause, &env); diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index 8dd83d6eff3..436df6731c6 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -485,6 +485,38 @@ fn a_turn_taking_pair_is_never_forwarded_off_its_cpu() { }); } +/// **A pair split across LLCs is brought into one**: on +/// `scenarios::split_pair`, with balancing off, the one move is a wake +/// following its waker's LLC, and after it every wake stays inside that LLC. +#[test] +fn a_split_pair_moves_once_into_one_llc() { + sweep(&scenarios::split_pair(), 40, |outcome| { + assert_eq!(outcome.migrations, 1, "the pair did not settle into one LLC with one move: {}", outcome.report()); + }); +} + +/// **A server with a client in each LLC seldom follows them, and never +/// back**: on `scenarios::server_of_two_llcs` at most a quarter of the runs +/// move the server, and no run moves it twice, even at ten times the rounds. +/// A server moves when one client's requests keep finding it running, so +/// that only the other's are counted; after the move both are, and nothing +/// moves it back. A run half as long as `WAKER_RUN` already moves it more +/// often than the bound, and following every wake moves it hundreds of times +/// over these seeds. +#[test] +fn a_server_with_a_client_in_each_llc_seldom_follows_them() { + const SEEDS: u64 = 40; + sweep(&scenarios::server_of_two_llcs(10 * scenarios::WAKE_LLC_ROUNDS), SEEDS, |outcome| { + assert!(outcome.migrations <= 1, "the server moved {} times in one run: {}", outcome.migrations, outcome.report()); + }); + let mut moves = 0; + sweep(&scenarios::server_of_two_llcs(scenarios::WAKE_LLC_ROUNDS), SEEDS, |outcome| { + assert!(outcome.migrations <= 1, "the server moved {} times in one run: {}", outcome.migrations, outcome.report()); + moves += outcome.migrations; + }); + assert!(moves <= SEEDS / 4, "the server followed its clients' LLCs: {moves} moves over {SEEDS} runs"); +} + /// **A wake due ahead of the running task waits out [`MIN_SLICE_NS`] and a /// hop, not a request** — on one CPU, against one rival process, every wake /// but one per run. diff --git a/kernel/src/arch/aarch64/percpu.rs b/kernel/src/arch/aarch64/percpu.rs index 3a97e5930f3..1bf91623552 100644 --- a/kernel/src/arch/aarch64/percpu.rs +++ b/kernel/src/arch/aarch64/percpu.rs @@ -33,6 +33,10 @@ pub struct PerCpu { current_tid: AtomicU32, current_pid: AtomicU32, preempt_count: AtomicU32, + /// Non-zero while this CPU runs a device interrupt's arm, written only by + /// `trap`'s `irq`: a wake posted there is the device's, not the + /// interrupted thread's. + irq_depth: AtomicU32, need_resched: AtomicU8, fault_state: AtomicU8, /// Timer interrupts taken at EL1, which only count and ask for a pass. @@ -81,6 +85,7 @@ pub fn alloc(cpu_id: u32) -> &'static PerCpu { current_tid: AtomicU32::new(u32::MAX), current_pid: AtomicU32::new(u32::MAX), preempt_count: AtomicU32::new(0), + irq_depth: AtomicU32::new(0), need_resched: AtomicU8::new(0), fault_state: AtomicU8::new(CpuFaultState::Normal as u8), kernel_timer_fires: AtomicU32::new(0), @@ -128,6 +133,20 @@ pub fn cpu_id() -> u32 { this().cpu_id } +/// Whether this CPU is inside a device interrupt's arm. +pub fn in_device_irq() -> bool { + this().irq_depth.load(Relaxed) != 0 +} + +/// Bracket a device interrupt's arm, as [`in_device_irq`] reads it. +pub fn irq_depth_up() { + this().irq_depth.fetch_add(1, Relaxed); +} + +pub fn irq_depth_down() { + this().irq_depth.fetch_sub(1, Relaxed); +} + /// `None` means idle. pub fn current_tid() -> Option { match this().current_tid.load(Relaxed) { diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 1aa14ff2599..e8eda01cf4b 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -212,6 +212,7 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { _ => {} } percpu::preempt_count_up(); + percpu::irq_depth_up(); let resched = match intid { irqchip::SGI_KICK => { percpu::irq_took(Source::Kick); @@ -243,6 +244,7 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { } }, }; + percpu::irq_depth_down(); percpu::preempt_count_down(); irqchip::end(intid); if resched { diff --git a/kernel/src/arch/x86_64/idt/device_irq.rs b/kernel/src/arch/x86_64/idt/device_irq.rs index 9f2824fb654..e86733c932c 100644 --- a/kernel/src/arch/x86_64/idt/device_irq.rs +++ b/kernel/src/arch/x86_64/idt/device_irq.rs @@ -1,6 +1,6 @@ //! Shared device interrupt entry: saves scratch GPRs, brackets the handler -//! with the percpu preempt count, and on Ring 3 return runs the -//! deferred-preempt epilogue. +//! with the percpu preempt count and interrupt depth, and on Ring 3 return +//! runs the deferred-preempt epilogue. //! //! IF stays 0 through the entry (interrupt gate; handlers never sti), satisfying //! `kernel_exit_to_user_check`'s IF=0-on-entry contract without an explicit cli. @@ -34,6 +34,7 @@ macro_rules! device_irq_entry { "push r11", "push rbp", "add dword ptr gs:[{preempt_count}], 1", + "add dword ptr gs:[{irq_depth}], 1", // Ring 0 entry has unknown rsp alignment; align via the rbp save. "mov rbp, rsp", "and rsp, -16", @@ -45,6 +46,7 @@ macro_rules! device_irq_entry { "mov rdi, [rbp + {cs}]", #[cfg(feature = "mask-windows")] "call {windows_leaving}", + "sub dword ptr gs:[{irq_depth}], 1", "sub dword ptr gs:[{preempt_count}], 1", "test dword ptr [rbp + {cs}], 3", "jz 1f", @@ -67,6 +69,7 @@ macro_rules! device_irq_entry { cs = const $cs, exit_to_user = sym crate::arch::idt::kernel_exit_to_user_check, preempt_count = const $crate::arch::percpu::OFF_PREEMPT_COUNT, + irq_depth = const $crate::arch::percpu::OFF_IRQ_DEPTH, #[cfg(feature = "mask-windows")] windows_entered = sym super::device_irq::windows_entered, #[cfg(feature = "mask-windows")] diff --git a/kernel/src/arch/x86_64/percpu.rs b/kernel/src/arch/x86_64/percpu.rs index 879b2a2584f..d2102abc9fe 100644 --- a/kernel/src/arch/x86_64/percpu.rs +++ b/kernel/src/arch/x86_64/percpu.rs @@ -115,6 +115,10 @@ pub struct PerCpu { /// Non-zero inside this CPU's NMI handler, written only by `arch::idt::nmi`'s entry; IST2 isn't re-entrant, so this proves no second NMI lands on it. nmi_active: u32, ap_token: u32, + /// Non-zero while this CPU runs a device interrupt's handler, written only + /// by `device_irq_entry` through `gs:`: a wake posted there is the + /// device's, not the interrupted thread's. + irq_depth: AtomicU32, /// Interrupt deliveries, one counter per `irq_census::Source`; written only by `irq_census::irq_took!`, kept last so a new source moves nothing else. pub irq_counts: [AtomicU64; crate::irq_census::Source::COUNT], } @@ -203,6 +207,7 @@ pub(crate) const OFF_LAST_ARMED_TICKS: u32 = offset_of!(PerCpu, last_armed_ticks const OFF_LOG_SHARD: u32 = offset_of!(PerCpu, log_shard) as u32; /// Opened/closed by `arch::syscall`'s and `arch::idt`'s naked stubs; `preempt` is the Rust half. pub(crate) const OFF_PREEMPT_COUNT: u32 = offset_of!(PerCpu, preempt_count) as u32; +pub(crate) const OFF_IRQ_DEPTH: u32 = offset_of!(PerCpu, irq_depth) as u32; /// Set by the timer ISR, cleared by the deferred-preempt epilogue — `preempt`. pub(crate) const OFF_NEED_RESCHED: u32 = offset_of!(PerCpu, need_resched) as u32; /// Read by `preempt::enable`'s slow path to decline rescheduling a faulting CPU. @@ -376,6 +381,7 @@ fn alloc_percpu(cpu_id: u32) -> *mut PerCpu { log_shard: log::shard_for(cpu_id) as *const log::Shard as u64, nmi_active: 0, ap_token: 0, + irq_depth: AtomicU32::new(0), irq_counts: [const { AtomicU64::new(0) }; crate::irq_census::Source::COUNT], }, ); @@ -596,6 +602,11 @@ pub fn ap_token() -> u32 { gs::read_u32::() } +/// Whether this CPU is inside a device interrupt's handler. +pub fn in_device_irq() -> bool { + gs::read_u32::() != 0 +} + /// Read the Tid of the thread currently running on this CPU. None means idle. pub fn current_tid() -> Option { let raw = gs::read_u32::(); diff --git a/kernel/src/log/console.rs b/kernel/src/log/console.rs index 97525599ccb..9e53c2f0ac5 100644 --- a/kernel/src/log/console.rs +++ b/kernel/src/log/console.rs @@ -102,7 +102,7 @@ pub fn post_wake() { // One read-modify-write of `klogd`'s own word and no lock: it claims a parked or committing `klogd` // and flags a running one, whose next commit refuses — the post a producer holding any lock may make. irq_off(|guard| { - notify(shared, WakeCause::new(WakeReason::Woken), cpus(), &HW, guard); + notify(shared, WakeCause::new(WakeReason::Woken).from(crate::sched::driver::waker()), cpus(), &HW, guard); }); } diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 8e5ee1e8625..417866443a4 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -902,6 +902,7 @@ fn teardown(pid: Pid, tid: Tid, end: End, mark: i32, process_data: &Arc) { + crate::sched::driver::current_leaves(); let (pid, tid) = (current_process(), current_tid()); crate::mm::paging::activate_kernel(); let process_data = process_data(); @@ -965,6 +966,7 @@ pub fn exit(end: End) -> ! { /// Exit the current thread. If this is the main thread, it is the process's exit. For a child thread, frees its own mappings and leaves. pub fn thread_exit(code: i32) -> ! { + crate::sched::driver::current_leaves(); let process_pid = current_process(); let tid = current_tid(); let route = { diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index adc95d2098d..d4e4dc6dc46 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -711,6 +711,21 @@ pub fn current_shared() -> Option> { try_with_cpu(|cpu| cpu.running().map(|t| t.shared().clone())).flatten() } +/// The CPU a wake posted here names as its waker ([`CpuSched::waker`]); none +/// inside a device interrupt's handler, which posts for the device and not +/// for the thread it interrupted, or inside a pass. +pub fn waker() -> Option { + if percpu::in_device_irq() { + return None; + } + try_with_cpu(CpuSched::waker).flatten() +} + +/// The running thread has begun leaving its process ([`CpuSched::current_leaves`]). +pub fn current_leaves() { + with_cpu(CpuSched::current_leaves); +} + /// The running task's mask, which a thread or process it spawns starts with; every CPU off-task, as at boot. pub fn current_mask() -> CpuMask { current_shared().map_or(CpuMask::ANY, |shared| shared.mask().load().0) diff --git a/kernel/src/watch.rs b/kernel/src/watch.rs index 8d34310feb5..95cfd6a445e 100644 --- a/kernel/src/watch.rs +++ b/kernel/src/watch.rs @@ -29,7 +29,7 @@ pub use kernel::sched::park::Cancel; use crate::hw::HW; use crate::inbox::PollEntry; -use crate::sched::driver::{cpus, preempt_off}; +use crate::sched::driver::{cpus, preempt_off, waker}; use crate::sched::payload::{KMsg, KShared, KernelLock, TaskHandle}; use crate::sync::Masked; use crate::scheduler::Parkable; @@ -90,7 +90,7 @@ impl Watch { fn post_as(&self, cause: WakeCause) { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.post(cause, &env); + self.0.post(cause.from(waker()), &env); }); } @@ -99,7 +99,7 @@ impl Watch { pub fn post_n(&self, token: u64, limit: usize) -> usize { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.post_n(token, limit, WakeCause::new(WakeReason::Woken), &env) + self.0.post_n(token, limit, WakeCause::new(WakeReason::Woken).from(waker()), &env) }) } @@ -109,7 +109,7 @@ impl Watch { pub fn revoke(&self, ended: impl Fn(u64) -> bool) -> usize { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.revoke(ended, WakeCause::new(WakeReason::Woken), &env) + self.0.revoke(ended, WakeCause::new(WakeReason::Woken).from(waker()), &env) }) } } @@ -127,7 +127,7 @@ impl IrqWatch { pub fn post_in_place(&self) { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.post_in_place(WakeCause::new(WakeReason::Woken), &env); + self.0.post_in_place(WakeCause::new(WakeReason::Woken).from(waker()), &env); }); } }