From 48d2f04b5be250b5147b5724173d2b1314e9aeac Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 03:02:52 +0100 Subject: [PATCH 1/5] sched: a woken partner follows its waker's LLC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wake went back to the CPU its task parked on, and spawn placement does not weigh LLCs, so a request/reply pair that started split across an LLC boundary stayed split. On the AMD bench such a pair reads about 25% slower: ipc.reqrep.8 at 8.4–8.5 µs against 6.7–6.9 µs, and fs.disk.read_4k_at at 11.6–11.7 µs against 9.7 µs. Every wake a thread posts now names the CPU it ran on (WakeCause.waker). A wake posted from a device interrupt's handler, a deadline or a retire names none. The kernel tells a handler apart by a per-CPU interrupt depth, which device_irq_entry and aarch64's irq bracket. A poll's completion runs in its poster's context, so a client's send that completes a server's inbox poll names the client's CPU. The woken task's owner CPU keeps the LLC of the task's last waker, the one u16 of per-task placement state the owner admitted on 2026-11-01. A wake from another LLC that the previous wake also came from goes to a sleeping CPU of the waker's LLC that the task's mask names, nearest the waker and quickest to wake. So a split pair shares an LLC after its second round trip, and a server woken from two LLCs in turn stays put. Real-time, stopped and killed tasks keep their own placement. The simulator's wakes name their posting CPU. - split_pair: on the AMD shape, a pair started on cpu0 and cpu8 moves exactly once in every one of 40 seeds. - server_of_two_llcs: a server answering a client pinned in each LLC moves 36 times over 40 seeds, at most 3 in one run. - Following every wake's LLC moves it about 220 times over the same seeds. Three host tests state the rule, including that the move stays inside the mask. Controls, each run and restored in place: - Dropping the rule reds split_pair and the moves-once test. - Dropping the previous-waker condition reds the server scenario and both host tests that keep a task home. - Ignoring the mask reds the mask test. Co-Authored-By: Claude Opus 5.5 --- ...-has-no-migration-cost-or-wake-affinity.md | 13 +- kernel/pure/sched/cpu.rs | 121 +++++++++++++++++- kernel/pure/sched/task.rs | 21 +++ kernel/pure/sched/topology.rs | 7 + kernel/sim/src/hw_impl.rs | 5 + kernel/sim/src/scenarios.rs | 56 ++++++++ kernel/sim/src/vm.rs | 3 +- kernel/sim/tests/policy.rs | 24 ++++ kernel/src/arch/aarch64/percpu.rs | 19 +++ kernel/src/arch/aarch64/trap.rs | 2 + kernel/src/arch/x86_64/idt/device_irq.rs | 7 +- kernel/src/arch/x86_64/percpu.rs | 11 ++ kernel/src/log/console.rs | 2 +- kernel/src/sched/driver.rs | 8 ++ kernel/src/watch.rs | 12 +- 15 files changed, 298 insertions(+), 13 deletions(-) diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index a4ae53cb4e0..5098d539ee7 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -19,7 +19,18 @@ lacks: to the placement code as one inside it. - **Wake-affine vs spread**: on wake, decide between placing a task near its waker (shared data stays warm — pipe/IPC pairs) and away from it - (parallelism). Every wake still ignores the waker. + (parallelism). Branch `sched-wake-llc` does the first half. + - Every wake a thread posts names its CPU. + - A task woken twice in a row from one other LLC moves to a sleeping CPU of + that LLC, so a pair split across LLCs shares one. + - A server woken from two LLCs in turn stays put. + - Measured on the AMD bench (ab3): a split pair costs about 25% on + `ipc.reqrep.8` and `fs.disk.read_4k_at`. + - Spreading a waker's partners for parallelism is still undecided. + - **Owner ruling, 2026-11-01:** the one u16 of per-task placement state is + admitted: the LLC of the task's last waker, kept by the CPU that owns the + task. It is the single exception to the 2026-10-28 scheduling model's "no + per-task state". - **An idle-state input**: see `issues/the-scheduler-and-the-clocks-never-talk.md`. - **Bounded scans.** Each choice walks the rings it needs whole. A steal probe reads the machine's offering mask first, so a machine with nothing to diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index f81eca19d5f..837d342de67 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -1178,7 +1178,48 @@ impl CpuSched { let mut task = entry.task.wake(self.id, cause, entry.class, now); self.trace(env, now, TraceKind::Wake { task: key, name: task.ext().name() }); task.refresh_affinity(); - self.place(task, env, now); + match self.toward_its_waker(&mut task, cause, env, now) { + Some(dst) => self.hand_off(task, dst, env, now), + None => self.place(task, env, now), + } + } + + /// **A woken partner follows its waker's LLC.** A wake from another LLC + /// whose previous wake came from that LLC too goes to a sleeping CPU of + /// the waker's LLC that the task's mask names, nearest the waker and + /// quickest to wake, so a request/reply pair split across LLCs shares one + /// after its second round trip, and no message crosses the LLCs again. + /// A task woken from two LLCs in turn, such as a server with a client in + /// each, never moves, because no two consecutive wakes name one other + /// LLC. Every wake that names a waker records that waker's LLC. A + /// real-time task keeps the RT band's placement, and a stopped or killed + /// one [`Self::place`]'s. + fn toward_its_waker, P: PreemptGuard>( + &self, + task: &mut ReadyTask, + cause: WakeCause, + env: Env<'_, H, P>, + now: Nanos, + ) -> Option { + let waker = cause.waker?; + let topology = env.cpus.topology(); + let llc = topology.llc_index(waker); + let again = task.swap_waker_llc(llc) == Some(llc); + if !again + || llc == topology.llc_index(self.id) + || task.is_rt() + || task.shared().stop_pending() + || task.shared().kill_pending() + { + return None; + } + let mask = task.affinity(); + env.cpus.idle_near(waker, topology.position(waker), |cpu| { + topology.llc_index(cpu.id()) == llc + && mask.contains(cpu.id()) + && cpu.doorbell().sleeping() + && cpu.answering(now) + }) } fn handle_adopt, P: PreemptGuard>( @@ -3124,7 +3165,10 @@ mod tests { shared: &Arc>>, reason: WakeReason, ) { - let cause = WakeCause::new(reason); + self.post_claimed_cause(cpu, shared, WakeCause::new(reason)); + } + + fn post_claimed_cause(&self, cpu: CpuId, shared: &Arc>>, cause: WakeCause) { let slot = shared .wake_node() .claim() @@ -4596,6 +4640,79 @@ mod tests { w.abandon(); } + /// A task parked on cpu0, in [`two_llcs`]' LLC 0, every other CPU + /// asleep. LLC 1 is cpu1, cpu3, cpu5 and cpu7. + fn parked_in_llc_0(w: &mut World, mask: CpuMask) -> Arc>> { + let job = Arc::new(Job::new(JobKey(1000 + w.next_key))); + let (_, task) = w.spawn_masked(C0, RtState::default(), job, mask); + w.run_a_pass(C0); + w.park_running(C0); + for cpu in 1..8 { + w.run_a_pass(CpuId(cpu)); + } + task + } + + /// Wake the task parked on cpu0 as the thread running on `waker` would, + /// and let cpu0 take the wake. + fn woken_from(w: &mut World, task: &Arc>>, waker: u32) { + assert_eq!(task.claim_wake(), Claim::Parked(C0), "the task is parked on cpu0"); + w.post_claimed_cause(C0, task, WakeCause::new(WakeReason::Woken).from(Some(CpuId(waker)))); + w.run_a_pass(C0); + } + + /// **A task woken twice in a row from another LLC moves into it**, onto a + /// sleeping CPU there; one wake from it is not enough. + #[test] + fn a_task_woken_twice_from_another_llc_moves_into_it() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_from(&mut w, &task, 1); + let first = task.state(); + w.park_running(C0); + woken_from(&mut w, &task, 1); + let second = task.state(); + w.abandon(); + assert_eq!(first, TaskState::Running(C0), "one wake from LLC 1 left it at home"); + assert!( + matches!(second, TaskState::InTransit(cpu) if cpu.0 % 2 == 1), + "the second wake from LLC 1 handed it to a CPU of LLC 1, and it is {second:?}", + ); + } + + /// **A task woken from two LLCs in turn stays home**: no two consecutive + /// wakes name one other LLC. + #[test] + fn a_task_woken_from_two_llcs_in_turn_stays_home() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + let mut states = Vec::new(); + for waker in [1, 2, 3, 4, 5] { + woken_from(&mut w, &task, waker); + states.push(task.state()); + w.park_running(C0); + } + w.abandon(); + assert!(states.iter().all(|&s| s == TaskState::Running(C0)), "it moved: {states:?}"); + } + + /// **The move stays inside the mask**: a task whose mask names only LLC + /// 0 stays home however often LLC 1 wakes it. + #[test] + fn a_task_masked_out_of_its_wakers_llc_stays_home() { + let mut w = World::on(two_llcs()); + let llc_0 = CpuMask::from_words(&[0b0101_0101], 8).unwrap(); + let task = parked_in_llc_0(&mut w, llc_0); + let mut states = Vec::new(); + for _ in 0..3 { + woken_from(&mut w, &task, 1); + states.push(task.state()); + w.park_running(C0); + } + w.abandon(); + assert!(states.iter().all(|&s| s == TaskState::Running(C0)), "it left its mask: {states:?}"); + } + /// **One forward per idle sibling**: wakes drained in one pass are handed /// to a sibling only while it has no kick coming, so a burst does not pile /// onto the first sleeper — the rest queue here. diff --git a/kernel/pure/sched/task.rs b/kernel/pure/sched/task.rs index d2d7d4a6209..de593e1e348 100644 --- a/kernel/pure/sched/task.rs +++ b/kernel/pure/sched/task.rs @@ -59,6 +59,9 @@ pub struct Standing { pub struct WakeCause { pub reason: WakeReason, pub boost: Option, + /// The CPU whose running thread posted this wake. `None` for a wake from + /// a device interrupt, a deadline or a retire, which no thread posted. + pub waker: Option, } impl WakeCause { @@ -66,6 +69,7 @@ impl WakeCause { Self { reason, boost: None, + waker: None, } } @@ -73,9 +77,15 @@ impl WakeCause { Self { reason, boost: Some(BoostWindow { until }), + waker: None, } } + /// This wake, posted by the thread running on `waker`, if one is. + pub fn from(self, waker: Option) -> Self { + Self { waker, ..self } + } + /// RT and boost wakes must preempt the target promptly; ordinary wakes /// ride the target's next safe point. pub fn urgency(&self) -> crate::sched::mailbox::Urgency { @@ -797,6 +807,11 @@ struct TaskInner { /// loaded at. Changed only by [`Task::refresh_affinity`]. affinity: CpuMask, affinity_version: u64, + /// The LLC ([`crate::sched::topology::Topology::llc_index`]) of the CPU + /// whose thread posted this task's last wake that named one. Kept by the + /// owner CPU, as the mask copy is. The one per-task placement state the + /// scheduling model admits (owner, 2026-11-01). + last_waker_llc: Option, /// When the current residency began: enqueue time while ready, dispatch /// time while running, park time while blocked. One field, because a task /// is in exactly one state. @@ -1015,6 +1030,7 @@ impl TaskBuilder { acct: TaskAccounting::default(), affinity: self.mask, affinity_version: 0, + last_waker_llc: None, since: now, adopt_node: MailboxNode::new(), ext: Some(self.ext), @@ -1052,6 +1068,11 @@ impl ReadyTask { self.0.refresh_affinity() } + /// Record `llc` as the last waker's, and answer the one it replaces. + pub(crate) fn swap_waker_llc(&mut self, llc: u16) -> Option { + self.0 .0.last_waker_llc.replace(llc) + } + pub(crate) fn standing_mut(&mut self) -> &mut Standing { &mut self.0 .0.standing } diff --git a/kernel/pure/sched/topology.rs b/kernel/pure/sched/topology.rs index 7e6e05b2072..b4926c00df1 100644 --- a/kernel/pure/sched/topology.rs +++ b/kernel/pure/sched/topology.rs @@ -136,6 +136,13 @@ impl Topology { self.keys.len() } + /// A number for the LLC `cpu` is in, the same for every CPU of that LLC + /// and different for every other: the position of its first CPU in the + /// topology's order. + pub fn llc_index(&self, cpu: CpuId) -> u16 { + u16::try_from(self.places[cpu.0 as usize].llc.start).expect("a CPU mask names at most 1024 CPUs") + } + pub fn keys(&self, cpu: CpuId) -> CpuKeys { self.keys[cpu.0 as usize] } diff --git a/kernel/sim/src/hw_impl.rs b/kernel/sim/src/hw_impl.rs index 22b851904cd..822ca7281d3 100644 --- a/kernel/sim/src/hw_impl.rs +++ b/kernel/sim/src/hw_impl.rs @@ -122,6 +122,11 @@ impl SimHw { self.with(|s| core::mem::replace(&mut s.poster, cpu)) } + /// The CPU whose step is posting, as a kernel wake names its waker. + pub fn poster(&self) -> Option { + self.with(|s| s.poster) + } + fn cpu(state: &SimHwState) -> usize { state .current diff --git a/kernel/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs index 99ac6ebf5d7..0d300a24c1f 100644 --- a/kernel/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -1195,6 +1195,62 @@ pub fn turn_taking_pair() -> Scenario { /// How many turns each of [`turn_taking_pair`]'s threads takes. pub const TURN_TAKING_ROUNDS: usize = 8; +/// **A request/reply pair split across the AMD bench's two LLCs**: one thread +/// on cpu0 and its partner on cpu8, each waking the other and parking, with +/// balancing off. Only wake placement can bring them together. +pub fn split_pair() -> Scenario { + let turn = MS / 10; + let wake = |queue| Op::Wake { queue, all: false, boost: None }; + let block = |queue| Op::Block { queue, deadline: None }; + scenario( + "split_pair", + 16, + vec![queue(WaitClass::Io), queue(WaitClass::Io)], + vec![process( + "pair", + vec![0, 1], + vec![ + Script::looping(vec![Op::Run(turn), wake(1), block(0)], WAKE_LLC_ROUNDS), + Script::looping(vec![block(1), Op::Run(turn), wake(0)], WAKE_LLC_ROUNDS), + ], + )], + ) + .with_placement(PlacementShape::Listed(&[0, 8])) + .with_topology(Shape::amd_bench()) + .with_balance(Balance::None) +} + +/// **One server woken from both LLCs in turn**: a server on cpu0 answers a +/// client pinned to cpu1, in its LLC, and one pinned to cpu9, in the other, +/// with balancing off. Following each wake's LLC would move the server on +/// every other request. +pub fn server_of_two_llcs() -> Scenario { + let turn = MS / 10; + let wake = |queue| Op::Wake { queue, all: false, boost: None }; + let block = |queue| Op::Block { queue, deadline: None }; + let client = |cpu: u64, reply| Script::looping(vec![Op::SetAffinity(1 << cpu), Op::Run(turn), wake(0), block(reply)], WAKE_LLC_ROUNDS); + scenario( + "server_of_two_llcs", + 16, + vec![queue(WaitClass::Io), queue(WaitClass::Io), queue(WaitClass::Io)], + vec![process( + "served", + vec![0, 1, 2], + vec![ + Script::looping(vec![block(0), Op::Run(turn), wake(1), block(0), Op::Run(turn), wake(2)], WAKE_LLC_ROUNDS), + client(1, 1), + client(9, 2), + ], + )], + ) + .with_placement(PlacementShape::Listed(&[0, 1, 9])) + .with_topology(Shape::amd_bench()) + .with_balance(Balance::None) +} + +/// How many turns [`split_pair`]'s and [`server_of_two_llcs`]' threads take. +pub const WAKE_LLC_ROUNDS: usize = 12; + /// **Masks under stealing**, every thread spawned on cpu0 of four with the /// shipped balance policy: two threads pinned to cpu0, two masked to cpu1 and /// cpu2, one that moves itself between halves of the machine as it runs, and diff --git a/kernel/sim/src/vm.rs b/kernel/sim/src/vm.rs index 83074e2ce3b..dc7a6b30746 100644 --- a/kernel/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -2007,7 +2007,8 @@ fn wake( WakeCause::boosted(WakeReason::Woken, until) } None => WakeCause::new(WakeReason::Woken), - }; + } + .from(hw.poster()); let env = Poster { cpus: handles, kicker: hw, preempt: &SimPreempt }; if all { q.queue.post(cause, &env); diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index 8dd83d6eff3..eff98843b4b 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -485,6 +485,30 @@ fn a_turn_taking_pair_is_never_forwarded_off_its_cpu() { }); } +/// **A pair split across LLCs is brought into one**: on +/// `scenarios::split_pair`, with balancing off, the one move is a wake +/// following its waker's LLC, and after it every wake stays inside that LLC. +#[test] +fn a_split_pair_moves_once_into_one_llc() { + sweep(&scenarios::split_pair(), 40, |outcome| { + assert_eq!(outcome.migrations, 1, "the pair did not settle into one LLC with one move: {}", outcome.report()); + }); +} + +/// **A server woken from two LLCs in turn mostly stays put**: on +/// `scenarios::server_of_two_llcs` the server moves only when two consecutive +/// wakes name the same other LLC, which the order the clients' requests +/// land in makes rare. Over 40 seeds it averages at most one move in a run +/// of 24 wakes (measured: 36 moves in all, at most 3 in one run). Following +/// every wake's LLC moves it about 220 times over the same seeds. +#[test] +fn a_server_woken_from_two_llcs_in_turn_does_not_follow_them() { + const SEEDS: u64 = 40; + let mut moves = 0; + sweep(&scenarios::server_of_two_llcs(), SEEDS, |outcome| moves += outcome.migrations); + assert!(moves <= SEEDS, "the server followed its clients' LLCs: {moves} moves over {SEEDS} runs"); +} + /// **A wake due ahead of the running task waits out [`MIN_SLICE_NS`] and a /// hop, not a request** — on one CPU, against one rival process, every wake /// but one per run. diff --git a/kernel/src/arch/aarch64/percpu.rs b/kernel/src/arch/aarch64/percpu.rs index 3a97e5930f3..1bf91623552 100644 --- a/kernel/src/arch/aarch64/percpu.rs +++ b/kernel/src/arch/aarch64/percpu.rs @@ -33,6 +33,10 @@ pub struct PerCpu { current_tid: AtomicU32, current_pid: AtomicU32, preempt_count: AtomicU32, + /// Non-zero while this CPU runs a device interrupt's arm, written only by + /// `trap`'s `irq`: a wake posted there is the device's, not the + /// interrupted thread's. + irq_depth: AtomicU32, need_resched: AtomicU8, fault_state: AtomicU8, /// Timer interrupts taken at EL1, which only count and ask for a pass. @@ -81,6 +85,7 @@ pub fn alloc(cpu_id: u32) -> &'static PerCpu { current_tid: AtomicU32::new(u32::MAX), current_pid: AtomicU32::new(u32::MAX), preempt_count: AtomicU32::new(0), + irq_depth: AtomicU32::new(0), need_resched: AtomicU8::new(0), fault_state: AtomicU8::new(CpuFaultState::Normal as u8), kernel_timer_fires: AtomicU32::new(0), @@ -128,6 +133,20 @@ pub fn cpu_id() -> u32 { this().cpu_id } +/// Whether this CPU is inside a device interrupt's arm. +pub fn in_device_irq() -> bool { + this().irq_depth.load(Relaxed) != 0 +} + +/// Bracket a device interrupt's arm, as [`in_device_irq`] reads it. +pub fn irq_depth_up() { + this().irq_depth.fetch_add(1, Relaxed); +} + +pub fn irq_depth_down() { + this().irq_depth.fetch_sub(1, Relaxed); +} + /// `None` means idle. pub fn current_tid() -> Option { match this().current_tid.load(Relaxed) { diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 1aa14ff2599..e8eda01cf4b 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -212,6 +212,7 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { _ => {} } percpu::preempt_count_up(); + percpu::irq_depth_up(); let resched = match intid { irqchip::SGI_KICK => { percpu::irq_took(Source::Kick); @@ -243,6 +244,7 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { } }, }; + percpu::irq_depth_down(); percpu::preempt_count_down(); irqchip::end(intid); if resched { diff --git a/kernel/src/arch/x86_64/idt/device_irq.rs b/kernel/src/arch/x86_64/idt/device_irq.rs index 9f2824fb654..e86733c932c 100644 --- a/kernel/src/arch/x86_64/idt/device_irq.rs +++ b/kernel/src/arch/x86_64/idt/device_irq.rs @@ -1,6 +1,6 @@ //! Shared device interrupt entry: saves scratch GPRs, brackets the handler -//! with the percpu preempt count, and on Ring 3 return runs the -//! deferred-preempt epilogue. +//! with the percpu preempt count and interrupt depth, and on Ring 3 return +//! runs the deferred-preempt epilogue. //! //! IF stays 0 through the entry (interrupt gate; handlers never sti), satisfying //! `kernel_exit_to_user_check`'s IF=0-on-entry contract without an explicit cli. @@ -34,6 +34,7 @@ macro_rules! device_irq_entry { "push r11", "push rbp", "add dword ptr gs:[{preempt_count}], 1", + "add dword ptr gs:[{irq_depth}], 1", // Ring 0 entry has unknown rsp alignment; align via the rbp save. "mov rbp, rsp", "and rsp, -16", @@ -45,6 +46,7 @@ macro_rules! device_irq_entry { "mov rdi, [rbp + {cs}]", #[cfg(feature = "mask-windows")] "call {windows_leaving}", + "sub dword ptr gs:[{irq_depth}], 1", "sub dword ptr gs:[{preempt_count}], 1", "test dword ptr [rbp + {cs}], 3", "jz 1f", @@ -67,6 +69,7 @@ macro_rules! device_irq_entry { cs = const $cs, exit_to_user = sym crate::arch::idt::kernel_exit_to_user_check, preempt_count = const $crate::arch::percpu::OFF_PREEMPT_COUNT, + irq_depth = const $crate::arch::percpu::OFF_IRQ_DEPTH, #[cfg(feature = "mask-windows")] windows_entered = sym super::device_irq::windows_entered, #[cfg(feature = "mask-windows")] diff --git a/kernel/src/arch/x86_64/percpu.rs b/kernel/src/arch/x86_64/percpu.rs index 879b2a2584f..d2102abc9fe 100644 --- a/kernel/src/arch/x86_64/percpu.rs +++ b/kernel/src/arch/x86_64/percpu.rs @@ -115,6 +115,10 @@ pub struct PerCpu { /// Non-zero inside this CPU's NMI handler, written only by `arch::idt::nmi`'s entry; IST2 isn't re-entrant, so this proves no second NMI lands on it. nmi_active: u32, ap_token: u32, + /// Non-zero while this CPU runs a device interrupt's handler, written only + /// by `device_irq_entry` through `gs:`: a wake posted there is the + /// device's, not the interrupted thread's. + irq_depth: AtomicU32, /// Interrupt deliveries, one counter per `irq_census::Source`; written only by `irq_census::irq_took!`, kept last so a new source moves nothing else. pub irq_counts: [AtomicU64; crate::irq_census::Source::COUNT], } @@ -203,6 +207,7 @@ pub(crate) const OFF_LAST_ARMED_TICKS: u32 = offset_of!(PerCpu, last_armed_ticks const OFF_LOG_SHARD: u32 = offset_of!(PerCpu, log_shard) as u32; /// Opened/closed by `arch::syscall`'s and `arch::idt`'s naked stubs; `preempt` is the Rust half. pub(crate) const OFF_PREEMPT_COUNT: u32 = offset_of!(PerCpu, preempt_count) as u32; +pub(crate) const OFF_IRQ_DEPTH: u32 = offset_of!(PerCpu, irq_depth) as u32; /// Set by the timer ISR, cleared by the deferred-preempt epilogue — `preempt`. pub(crate) const OFF_NEED_RESCHED: u32 = offset_of!(PerCpu, need_resched) as u32; /// Read by `preempt::enable`'s slow path to decline rescheduling a faulting CPU. @@ -376,6 +381,7 @@ fn alloc_percpu(cpu_id: u32) -> *mut PerCpu { log_shard: log::shard_for(cpu_id) as *const log::Shard as u64, nmi_active: 0, ap_token: 0, + irq_depth: AtomicU32::new(0), irq_counts: [const { AtomicU64::new(0) }; crate::irq_census::Source::COUNT], }, ); @@ -596,6 +602,11 @@ pub fn ap_token() -> u32 { gs::read_u32::() } +/// Whether this CPU is inside a device interrupt's handler. +pub fn in_device_irq() -> bool { + gs::read_u32::() != 0 +} + /// Read the Tid of the thread currently running on this CPU. None means idle. pub fn current_tid() -> Option { let raw = gs::read_u32::(); diff --git a/kernel/src/log/console.rs b/kernel/src/log/console.rs index 97525599ccb..9e53c2f0ac5 100644 --- a/kernel/src/log/console.rs +++ b/kernel/src/log/console.rs @@ -102,7 +102,7 @@ pub fn post_wake() { // One read-modify-write of `klogd`'s own word and no lock: it claims a parked or committing `klogd` // and flags a running one, whose next commit refuses — the post a producer holding any lock may make. irq_off(|guard| { - notify(shared, WakeCause::new(WakeReason::Woken), cpus(), &HW, guard); + notify(shared, WakeCause::new(WakeReason::Woken).from(crate::sched::driver::waker()), cpus(), &HW, guard); }); } diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index adc95d2098d..21bc5d1b271 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -711,6 +711,14 @@ pub fn current_shared() -> Option> { try_with_cpu(|cpu| cpu.running().map(|t| t.shared().clone())).flatten() } +/// The CPU a wake posted here names as its waker: this one, while a thread +/// runs on it outside a device interrupt's handler, which posts for the +/// device and not for the thread it interrupted. +pub fn waker() -> Option { + let thread = percpu::current_tid().is_some() && !percpu::in_device_irq(); + thread.then(|| CpuId(percpu::cpu_id())) +} + /// The running task's mask, which a thread or process it spawns starts with; every CPU off-task, as at boot. pub fn current_mask() -> CpuMask { current_shared().map_or(CpuMask::ANY, |shared| shared.mask().load().0) diff --git a/kernel/src/watch.rs b/kernel/src/watch.rs index 8d34310feb5..bf794edd097 100644 --- a/kernel/src/watch.rs +++ b/kernel/src/watch.rs @@ -29,7 +29,7 @@ pub use kernel::sched::park::Cancel; use crate::hw::HW; use crate::inbox::PollEntry; -use crate::sched::driver::{cpus, preempt_off}; +use crate::sched::driver::{cpus, preempt_off, waker}; use crate::sched::payload::{KMsg, KShared, KernelLock, TaskHandle}; use crate::sync::Masked; use crate::scheduler::Parkable; @@ -79,12 +79,12 @@ impl Watch { /// Something about the object changed: wake every thread waiting on it and /// fire every poll. pub fn post(&self) { - self.post_as(WakeCause::new(WakeReason::Woken)); + self.post_as(WakeCause::new(WakeReason::Woken).from(waker())); } /// The same, lending the poster's real-time window to whoever it wakes. pub fn post_boosted(&self, until: Nanos) { - self.post_as(WakeCause::boosted(WakeReason::Woken, until)); + self.post_as(WakeCause::boosted(WakeReason::Woken, until).from(waker())); } fn post_as(&self, cause: WakeCause) { @@ -99,7 +99,7 @@ impl Watch { pub fn post_n(&self, token: u64, limit: usize) -> usize { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.post_n(token, limit, WakeCause::new(WakeReason::Woken), &env) + self.0.post_n(token, limit, WakeCause::new(WakeReason::Woken).from(waker()), &env) }) } @@ -109,7 +109,7 @@ impl Watch { pub fn revoke(&self, ended: impl Fn(u64) -> bool) -> usize { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.revoke(ended, WakeCause::new(WakeReason::Woken), &env) + self.0.revoke(ended, WakeCause::new(WakeReason::Woken).from(waker()), &env) }) } } @@ -127,7 +127,7 @@ impl IrqWatch { pub fn post_in_place(&self) { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.post_in_place(WakeCause::new(WakeReason::Woken), &env); + self.0.post_in_place(WakeCause::new(WakeReason::Woken).from(waker()), &env); }); } } From 34c12a7ebecfeb7d04899a46272a5ba3cea2e959 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 10:18:13 +0100 Subject: [PATCH 2/5] review r1 of #1129: a task moves after eight wakes in a row from one LLC The rule moved a task on the second wake in a row from another LLC, and said a server with a client in each LLC never moved. It did: only a wake that finds the server parked reaches the rule, and which client's request that is depends on timing. The simulator's own server_of_two_llcs moved it 36 times over 40 seeds, and with independent, even arrivals a run of two moves it on 1 parked wake in 6. The u16 the owner admitted now holds the LLC in ten bits and the run's count in six (WakerStreak), and the wake that makes WAKER_RUN in a row from another LLC moves the task. With independent, even arrivals a run of k moves a server on 1 parked wake in 2^(k+1) - 2, checked by direct simulation: 1 in 6 at 2, 30 at 4, 126 at 6, 510 at 8. A split pair pays k split round trips once, about 1.7 us each on the AMD bench, so k = 8 costs a pair about 14 us and keeps a server's moves to one wake in 510. Measured in the simulator with the scenarios at 24 rounds (40 seeds, server moves in all / most in one run / runs that moved; then runs where the split pair moved exactly once): k=1 407/45/34 40 k=4 17/1/17 40 k=10 5/1/5 40 k=2 60/5/27 40 k=6 10/1/10 40 k=12 5/1/5 40 k=3 31/3/24 40 k=8 5/1/5 40 Past 8 the server's 5 moves stay put, so the streak no longer drives them. The simulator's clients alternate more than independent ones would, which is why it moves less than the formula; the formula is the bound the constant cites. The server test now requires at most one move per run and at most 10 over 40 seeds; k = 4 and k = 1 both red it. The move chose among sleepers that were answering, which accepts a CPU a kick is already on its way to, and the scan always starts at the waker. Two runs ending in one drain both went to the same CPU, which then forwarded one on. The move now takes only a sleeping CPU with no kick pending (Doorbell::sleeping_unrung, as a wake forward does), and scans the waker's LLC alone (CpuHandles::idle_in_llc) rather than every ring with the other LLCs filtered out. A host test ends two runs in one drain; the old predicate reds it. post and post_boosted named the waker before post_as disabled preemption, so a migration in between named the previous CPU; post_as names it now, as post_n, revoke and post_in_place do. The issue record describes main after landing and records the server's remaining move rate. Controls, each applied and reversed in place: the old predicate reds the two-run test; k = 1 and k = 4 red the server test; no rule reds split_pair, the run test and the two-run test; ignoring the mask reds the mask test and the server test. Co-Authored-By: Claude Opus 5.5 --- ...-has-no-migration-cost-or-wake-affinity.md | 30 ++-- kernel/pure/sched/cpu.rs | 160 +++++++++++------- kernel/pure/sched/task.rs | 43 ++++- kernel/sim/src/scenarios.rs | 7 +- kernel/sim/tests/policy.rs | 20 ++- kernel/src/watch.rs | 6 +- 6 files changed, 169 insertions(+), 97 deletions(-) diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index 5098d539ee7..12c9131c587 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -19,18 +19,26 @@ lacks: to the placement code as one inside it. - **Wake-affine vs spread**: on wake, decide between placing a task near its waker (shared data stays warm — pipe/IPC pairs) and away from it - (parallelism). Branch `sched-wake-llc` does the first half. - - Every wake a thread posts names its CPU. - - A task woken twice in a row from one other LLC moves to a sleeping CPU of - that LLC, so a pair split across LLCs shares one. - - A server woken from two LLCs in turn stays put. - - Measured on the AMD bench (ab3): a split pair costs about 25% on - `ipc.reqrep.8` and `fs.disk.read_4k_at`. + (parallelism). The first half is in `CpuSched::toward_its_waker`: + - Every wake a thread posts names its CPU; one a device interrupt's + handler, a deadline or a retire posts names none. + - The wake that makes `WAKER_RUN` (8) in a row from one other LLC moves + the task to a sleeping CPU there that its mask names and no kick is on + its way to, so a pair split across LLCs shares one after 8 round trips. + A split pair costs about 25% on the AMD bench's `ipc.reqrep.8` and + `fs.disk.read_4k_at`. + - A server whose parked wakes come from both LLCs still moves when 8 in a + row come from the other one: about one parked wake in 510 when they are + independent and even, and 5 moves in 40 runs of the simulator's + `server_of_two_llcs`. No metal row has a server with clients in both + LLCs. + - The task's home CPU makes the move after its own wake, so a move costs + two wakes in series. - Spreading a waker's partners for parallelism is still undecided. - - **Owner ruling, 2026-11-01:** the one u16 of per-task placement state is - admitted: the LLC of the task's last waker, kept by the CPU that owns the - task. It is the single exception to the 2026-10-28 scheduling model's "no - per-task state". + - **Owner ruling, 2026-11-01:** one u16 of per-task placement state is + admitted, kept by the CPU that owns the task: the LLC of its last waker, + which holds the run's count beside it. It is the single exception to the + 2026-10-28 scheduling model's "no per-task state". - **An idle-state input**: see `issues/the-scheduler-and-the-clocks-never-talk.md`. - **Bounded scans.** Each choice walks the rings it needs whole. A steal probe reads the machine's offering mask first, so a machine with nothing to diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index 837d342de67..ca474978093 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -1178,20 +1178,20 @@ impl CpuSched { let mut task = entry.task.wake(self.id, cause, entry.class, now); self.trace(env, now, TraceKind::Wake { task: key, name: task.ext().name() }); task.refresh_affinity(); - match self.toward_its_waker(&mut task, cause, env, now) { + match self.toward_its_waker(&mut task, cause, env) { Some(dst) => self.hand_off(task, dst, env, now), None => self.place(task, env, now), } } - /// **A woken partner follows its waker's LLC.** A wake from another LLC - /// whose previous wake came from that LLC too goes to a sleeping CPU of - /// the waker's LLC that the task's mask names, nearest the waker and - /// quickest to wake, so a request/reply pair split across LLCs shares one - /// after its second round trip, and no message crosses the LLCs again. - /// A task woken from two LLCs in turn, such as a server with a client in - /// each, never moves, because no two consecutive wakes name one other - /// LLC. Every wake that names a waker records that waker's LLC. A + /// **A woken partner follows its waker's LLC.** The wake that makes + /// [`WAKER_RUN`] in a row from one other LLC goes to a CPU of that LLC + /// the task's mask names, [`Doorbell::sleeping_unrung`], nearest the + /// waker and quickest to wake. Every wake that names a waker counts, and + /// only one that finds the task parked reaches here. So a request/reply + /// pair split across LLCs shares one after [`WAKER_RUN`] round trips, + /// and a server with clients in both LLCs still moves when [`WAKER_RUN`] + /// of its parked wakes in a row happen to come from the other one. A /// real-time task keeps the RT band's placement, and a stopped or killed /// one [`Self::place`]'s. fn toward_its_waker, P: PreemptGuard>( @@ -1199,13 +1199,12 @@ impl CpuSched { task: &mut ReadyTask, cause: WakeCause, env: Env<'_, H, P>, - now: Nanos, ) -> Option { let waker = cause.waker?; let topology = env.cpus.topology(); let llc = topology.llc_index(waker); - let again = task.swap_waker_llc(llc) == Some(llc); - if !again + let run = task.count_waker_llc(llc); + if run < WAKER_RUN || llc == topology.llc_index(self.id) || task.is_rt() || task.shared().stop_pending() @@ -1214,11 +1213,8 @@ impl CpuSched { return None; } let mask = task.affinity(); - env.cpus.idle_near(waker, topology.position(waker), |cpu| { - topology.llc_index(cpu.id()) == llc - && mask.contains(cpu.id()) - && cpu.doorbell().sleeping() - && cpu.answering(now) + env.cpus.idle_in_llc(waker, topology.position(waker), |cpu| { + mask.contains(cpu.id()) && cpu.doorbell().sleeping_unrung() }) } @@ -2675,6 +2671,12 @@ pub struct CpuHandles { offering: Box<[AtomicU64]>, } +/// How many wakes in a row from one other LLC move a task into it +/// ([`CpuSched::toward_its_waker`]). A split pair pays this many split round +/// trips once; a task whose wakes come from two LLCs independently and +/// evenly moves on about one parked wake in 2^(n+1) - 2, 1 in 510 at 8. +pub const WAKER_RUN: u16 = 8; + /// The CPUs an idle CPU is chosen among, nearest first: `me`'s LLC, then the /// rest of its package, then the machine. A choice stays in the first group /// with a candidate, so locality decides before core idleness does. @@ -2795,29 +2797,42 @@ impl CpuHandles { /// go to the first at or after topology position `from`, which is how /// consecutive choices rotate. pub fn idle_near(&self, me: CpuId, from: usize, usable: impl Fn(&CpuHandle) -> bool) -> Option { - for (inner, outer) in IDLE_GROUPS { - // The best of each tier, as (exit, cpu): strictly less replaces, so the first of equals stays. - let mut free_core: Option<(u64, CpuId)> = None; - let mut shared_core: Option<(u64, CpuId)> = None; - for cpu in self.topology.ring(me, inner, outer, from) { - let handle = self.get(cpu); - if !usable(handle) { - continue; - } - let exit = handle.idle_exit(); - let tier = if self.busy_siblings(cpu) == 0 { &mut free_core } else { &mut shared_core }; - if tier.is_none_or(|(best, _)| exit < best) { - *tier = Some((exit, cpu)); - } - if free_core.is_some_and(|(best, _)| best == 0) { - break; - } + IDLE_GROUPS.into_iter().find_map(|(inner, outer)| self.idle_among(me, inner, outer, from, &usable)) + } + + /// [`Self::idle_near`] inside `me`'s LLC alone. + pub fn idle_in_llc(&self, me: CpuId, from: usize, usable: impl Fn(&CpuHandle) -> bool) -> Option { + self.idle_among(me, None, Level::Llc, from, &usable) + } + + /// The best CPU `usable` accepts in one of [`IDLE_GROUPS`]' rings: a free + /// core before a shared one, then the quickest to wake. + fn idle_among( + &self, + me: CpuId, + inner: Option, + outer: Level, + from: usize, + usable: &impl Fn(&CpuHandle) -> bool, + ) -> Option { + // The best of each tier, as (exit, cpu): strictly less replaces, so the first of equals stays. + let mut free_core: Option<(u64, CpuId)> = None; + let mut shared_core: Option<(u64, CpuId)> = None; + for cpu in self.topology.ring(me, inner, outer, from) { + let handle = self.get(cpu); + if !usable(handle) { + continue; + } + let exit = handle.idle_exit(); + let tier = if self.busy_siblings(cpu) == 0 { &mut free_core } else { &mut shared_core }; + if tier.is_none_or(|(best, _)| exit < best) { + *tier = Some((exit, cpu)); } - if let Some((_, cpu)) = free_core.or(shared_core) { - return Some(cpu); + if free_core.is_some_and(|(best, _)| best == 0) { + break; } } - None + free_core.or(shared_core).map(|(_, cpu)| cpu) } /// Where a new task goes: the CPU in `mask` still answering whose queue it would wait @@ -4661,39 +4676,43 @@ mod tests { w.run_a_pass(C0); } - /// **A task woken twice in a row from another LLC moves into it**, onto a - /// sleeping CPU there; one wake from it is not enough. + /// Wake `task` `times` from `waker`, each time letting it run at home on + /// cpu0 and park again. + fn woken_at_home(w: &mut World, task: &Arc>>, waker: u32, times: u16) { + for _ in 0..times { + woken_from(w, task, waker); + assert_eq!(task.state(), TaskState::Running(C0), "a wake from cpu{waker} short of the run left it home"); + w.park_running(C0); + } + } + + /// **The wake that makes [`WAKER_RUN`] in a row from another LLC moves + /// the task into it**, onto a sleeping CPU there. #[test] - fn a_task_woken_twice_from_another_llc_moves_into_it() { + fn a_run_of_wakes_from_another_llc_moves_the_task_into_it() { let mut w = World::on(two_llcs()); let task = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &task, 1, WAKER_RUN - 1); woken_from(&mut w, &task, 1); - let first = task.state(); - w.park_running(C0); - woken_from(&mut w, &task, 1); - let second = task.state(); + let state = task.state(); w.abandon(); - assert_eq!(first, TaskState::Running(C0), "one wake from LLC 1 left it at home"); assert!( - matches!(second, TaskState::InTransit(cpu) if cpu.0 % 2 == 1), - "the second wake from LLC 1 handed it to a CPU of LLC 1, and it is {second:?}", + matches!(state, TaskState::InTransit(cpu) if cpu.0 % 2 == 1), + "the run's last wake handed it to a CPU of LLC 1, and it is {state:?}", ); } - /// **A task woken from two LLCs in turn stays home**: no two consecutive - /// wakes name one other LLC. + /// **A wake from home starts the run again**: a task woken from both + /// LLCs never makes [`WAKER_RUN`] in a row from the other one. #[test] - fn a_task_woken_from_two_llcs_in_turn_stays_home() { + fn a_wake_from_home_breaks_the_run() { let mut w = World::on(two_llcs()); let task = parked_in_llc_0(&mut w, CpuMask::ANY); - let mut states = Vec::new(); - for waker in [1, 2, 3, 4, 5] { - woken_from(&mut w, &task, waker); - states.push(task.state()); - w.park_running(C0); + for _ in 0..3 { + woken_at_home(&mut w, &task, 1, WAKER_RUN - 1); + woken_at_home(&mut w, &task, 2, 1); } w.abandon(); - assert!(states.iter().all(|&s| s == TaskState::Running(C0)), "it moved: {states:?}"); } /// **The move stays inside the mask**: a task whose mask names only LLC @@ -4703,14 +4722,31 @@ mod tests { let mut w = World::on(two_llcs()); let llc_0 = CpuMask::from_words(&[0b0101_0101], 8).unwrap(); let task = parked_in_llc_0(&mut w, llc_0); - let mut states = Vec::new(); - for _ in 0..3 { - woken_from(&mut w, &task, 1); - states.push(task.state()); - w.park_running(C0); + woken_at_home(&mut w, &task, 1, 2 * WAKER_RUN); + w.abandon(); + } + + /// **Two runs ending in one drain go to two CPUs**: the first move rings + /// its CPU, and a rung sleeper is passed over, as a wake forward passes + /// it. + #[test] + fn two_runs_ending_in_one_drain_go_to_two_cpus() { + let mut w = World::on(two_llcs()); + let first = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &first, 1, WAKER_RUN - 1); + let second = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &second, 1, WAKER_RUN - 1); + for task in [&first, &second] { + assert_eq!(task.claim_wake(), Claim::Parked(C0), "the task is parked on cpu0"); + w.post_claimed_cause(C0, task, WakeCause::new(WakeReason::Woken).from(Some(CpuId(1)))); } + w.run_a_pass(C0); + let states = [first.state(), second.state()]; w.abandon(); - assert!(states.iter().all(|&s| s == TaskState::Running(C0)), "it left its mask: {states:?}"); + match states { + [TaskState::InTransit(a), TaskState::InTransit(b)] if a != b && a.0 % 2 == 1 && b.0 % 2 == 1 => {} + _ => panic!("the two moves did not go to two CPUs of LLC 1: {states:?}"), + } } /// **One forward per idle sibling**: wakes drained in one pass are handed diff --git a/kernel/pure/sched/task.rs b/kernel/pure/sched/task.rs index de593e1e348..69df084d108 100644 --- a/kernel/pure/sched/task.rs +++ b/kernel/pure/sched/task.rs @@ -795,6 +795,32 @@ pub struct TaskAccounting { /// a raw context pointer taken before a container move stays valid. pub struct Task(Box>); +/// The LLC a task's last wakes came from +/// ([`crate::sched::topology::Topology::llc_index`]) and how many came from +/// it in a row, in one `u16`: the LLC in the low ten bits and the run, +/// saturating, in the high six. A run of 0 is no wake counted yet. The one +/// per-task placement state the scheduling model admits. +#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] +pub struct WakerStreak(u16); + +impl WakerStreak { + const LLC_BITS: u32 = 10; + const LLC: u16 = (1 << Self::LLC_BITS) - 1; + const MAX_RUN: u16 = u16::MAX >> Self::LLC_BITS; + + /// Count a wake from `llc` and answer the run it extends or starts. + pub fn count(&mut self, llc: u16) -> u16 { + assert!(llc <= Self::LLC, "an LLC index past {}", Self::LLC); + let run = self.0 >> Self::LLC_BITS; + let run = if run > 0 && self.0 & Self::LLC == llc { (run + 1).min(Self::MAX_RUN) } else { 1 }; + self.0 = run << Self::LLC_BITS | llc; + run + } +} + +// Every LLC index is the position of a CPU a mask can name. +const _: () = assert!(crate::sched::affinity::MAX_CPUS <= 1 << WakerStreak::LLC_BITS); + struct TaskInner { key: TaskKey, shared: Arc>>, @@ -807,11 +833,9 @@ struct TaskInner { /// loaded at. Changed only by [`Task::refresh_affinity`]. affinity: CpuMask, affinity_version: u64, - /// The LLC ([`crate::sched::topology::Topology::llc_index`]) of the CPU - /// whose thread posted this task's last wake that named one. Kept by the - /// owner CPU, as the mask copy is. The one per-task placement state the - /// scheduling model admits (owner, 2026-11-01). - last_waker_llc: Option, + /// Where this task's recent wakes came from. Kept by the owner CPU, as + /// the mask copy is. + waker_streak: WakerStreak, /// When the current residency began: enqueue time while ready, dispatch /// time while running, park time while blocked. One field, because a task /// is in exactly one state. @@ -1030,7 +1054,7 @@ impl TaskBuilder { acct: TaskAccounting::default(), affinity: self.mask, affinity_version: 0, - last_waker_llc: None, + waker_streak: WakerStreak::default(), since: now, adopt_node: MailboxNode::new(), ext: Some(self.ext), @@ -1068,9 +1092,10 @@ impl ReadyTask { self.0.refresh_affinity() } - /// Record `llc` as the last waker's, and answer the one it replaces. - pub(crate) fn swap_waker_llc(&mut self, llc: u16) -> Option { - self.0 .0.last_waker_llc.replace(llc) + /// Count a wake whose waker ran in `llc`, and answer how many wakes in a + /// row have come from it. + pub(crate) fn count_waker_llc(&mut self, llc: u16) -> u16 { + self.0 .0.waker_streak.count(llc) } pub(crate) fn standing_mut(&mut self) -> &mut Standing { diff --git a/kernel/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs index 0d300a24c1f..0e86e901f76 100644 --- a/kernel/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -1222,8 +1222,9 @@ pub fn split_pair() -> Scenario { /// **One server woken from both LLCs in turn**: a server on cpu0 answers a /// client pinned to cpu1, in its LLC, and one pinned to cpu9, in the other, -/// with balancing off. Following each wake's LLC would move the server on -/// every other request. +/// with balancing off. Which client's request finds the server parked is +/// the run's timing, so following each wake's LLC would move it back and +/// forth. pub fn server_of_two_llcs() -> Scenario { let turn = MS / 10; let wake = |queue| Op::Wake { queue, all: false, boost: None }; @@ -1249,7 +1250,7 @@ pub fn server_of_two_llcs() -> Scenario { } /// How many turns [`split_pair`]'s and [`server_of_two_llcs`]' threads take. -pub const WAKE_LLC_ROUNDS: usize = 12; +pub const WAKE_LLC_ROUNDS: usize = 24; /// **Masks under stealing**, every thread spawned on cpu0 of four with the /// shipped balance policy: two threads pinned to cpu0, two masked to cpu1 and diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index eff98843b4b..e2d55312e90 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -495,18 +495,20 @@ fn a_split_pair_moves_once_into_one_llc() { }); } -/// **A server woken from two LLCs in turn mostly stays put**: on -/// `scenarios::server_of_two_llcs` the server moves only when two consecutive -/// wakes name the same other LLC, which the order the clients' requests -/// land in makes rare. Over 40 seeds it averages at most one move in a run -/// of 24 wakes (measured: 36 moves in all, at most 3 in one run). Following -/// every wake's LLC moves it about 220 times over the same seeds. +/// **A server with a client in each LLC seldom follows them**: on +/// `scenarios::server_of_two_llcs` no run moves the server twice, and at most +/// a quarter of the runs move it at all. A run half as long as +/// `WAKER_RUN` already moves it more often than that, and following every +/// wake moves it hundreds of times over these seeds. #[test] -fn a_server_woken_from_two_llcs_in_turn_does_not_follow_them() { +fn a_server_with_a_client_in_each_llc_seldom_follows_them() { const SEEDS: u64 = 40; let mut moves = 0; - sweep(&scenarios::server_of_two_llcs(), SEEDS, |outcome| moves += outcome.migrations); - assert!(moves <= SEEDS, "the server followed its clients' LLCs: {moves} moves over {SEEDS} runs"); + sweep(&scenarios::server_of_two_llcs(), SEEDS, |outcome| { + assert!(outcome.migrations <= 1, "the server moved {} times in one run: {}", outcome.migrations, outcome.report()); + moves += outcome.migrations; + }); + assert!(moves <= SEEDS / 4, "the server followed its clients' LLCs: {moves} moves over {SEEDS} runs"); } /// **A wake due ahead of the running task waits out [`MIN_SLICE_NS`] and a diff --git a/kernel/src/watch.rs b/kernel/src/watch.rs index bf794edd097..95cfd6a445e 100644 --- a/kernel/src/watch.rs +++ b/kernel/src/watch.rs @@ -79,18 +79,18 @@ impl Watch { /// Something about the object changed: wake every thread waiting on it and /// fire every poll. pub fn post(&self) { - self.post_as(WakeCause::new(WakeReason::Woken).from(waker())); + self.post_as(WakeCause::new(WakeReason::Woken)); } /// The same, lending the poster's real-time window to whoever it wakes. pub fn post_boosted(&self, until: Nanos) { - self.post_as(WakeCause::boosted(WakeReason::Woken, until).from(waker())); + self.post_as(WakeCause::boosted(WakeReason::Woken, until)); } fn post_as(&self, cause: WakeCause) { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; - self.0.post(cause, &env); + self.0.post(cause.from(waker()), &env); }); } From 7c69daeea4f0a646cd8bfd59ba2b4596dd58e2ea Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 13:03:23 +0100 Subject: [PATCH 3/5] review r2 of #1129: only parked wakes are counted, and a server never moves back The PR said the simulator's server moved less than the formula and that past a run of 8 the streak no longer drove its moves. Both were wrong: at runs of 8, 10 and 12 it moved 5 times in 40 runs, more than the 3.8 the formula allows, and nothing but the rule can move it there. A trace of a moving seed shows why. All 24 requests from the near client (cpu1) found the server running and became flags it consumed unseen; every request that found it parked came from the far client (cpu9), so the far client alone made the run. Only a wake that finds the task parked reaches the owner with its cause (park::deliver), and the doc said the opposite. At ten times the rounds, 12 of 40 runs move the server, each once. After the move both clients' requests find it parked (71 counted from LLC 0, 63 from LLC 1 in the traced seed), and no run moves it back. A run of 4 does bounce it, 4 times in one run. The server test now holds both: at most one move per run at 240 rounds, and at most 10 moves over 40 runs at 24. Runs of 1 and 4 red the first sweep. The rule's doc now says that only parked wakes count, what that does to a server with a client in each LLC, and that a run at or past WAKER_RUN moves the task, so a task stolen out of the LLC it was woken from goes back on that LLC's next wake. The issue record says the same. Co-Authored-By: Claude Opus 5.5 --- ...-has-no-migration-cost-or-wake-affinity.md | 12 ++++---- kernel/pure/sched/cpu.rs | 28 ++++++++++--------- kernel/sim/src/scenarios.rs | 22 +++++++++------ kernel/sim/tests/policy.rs | 18 ++++++++---- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index 12c9131c587..0a81ae5ab0a 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -27,11 +27,13 @@ lacks: its way to, so a pair split across LLCs shares one after 8 round trips. A split pair costs about 25% on the AMD bench's `ipc.reqrep.8` and `fs.disk.read_4k_at`. - - A server whose parked wakes come from both LLCs still moves when 8 in a - row come from the other one: about one parked wake in 510 when they are - independent and even, and 5 moves in 40 runs of the simulator's - `server_of_two_llcs`. No metal row has a server with clients in both - LLCs. + - Only a wake that finds the task parked is counted. A server with a + client in each LLC therefore follows the far one when the near one's + requests keep finding it running: 5 of 40 runs of the simulator's + `server_of_two_llcs` move it once, whatever the run length from 8 to + 12, and none moves it back at ten times the rounds; a run of 4 does. + Parked wakes from both LLCs, independent and even, move it on about one + in 510. No metal row has a server with clients in both LLCs. - The task's home CPU makes the move after its own wake, so a move costs two wakes in series. - Spreading a waker's partners for parallelism is still undecided. diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index ca474978093..7490f07e3c3 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -1184,16 +1184,18 @@ impl CpuSched { } } - /// **A woken partner follows its waker's LLC.** The wake that makes - /// [`WAKER_RUN`] in a row from one other LLC goes to a CPU of that LLC - /// the task's mask names, [`Doorbell::sleeping_unrung`], nearest the - /// waker and quickest to wake. Every wake that names a waker counts, and - /// only one that finds the task parked reaches here. So a request/reply - /// pair split across LLCs shares one after [`WAKER_RUN`] round trips, - /// and a server with clients in both LLCs still moves when [`WAKER_RUN`] - /// of its parked wakes in a row happen to come from the other one. A - /// real-time task keeps the RT band's placement, and a stopped or killed - /// one [`Self::place`]'s. + /// **A woken partner follows its waker's LLC.** A wake from another LLC + /// whose run from it is at or past [`WAKER_RUN`] goes to a CPU of that + /// LLC the task's mask names, [`Doorbell::sleeping_unrung`], nearest the + /// waker and quickest to wake. Only a wake that finds the task parked + /// reaches here, so only those are counted: one that finds it running or + /// about to park is a flag the task consumes unseen. A request/reply pair + /// split across LLCs shares one after [`WAKER_RUN`] round trips. A server + /// with a client in each LLC follows the far one when the near one's + /// requests keep finding it running; once moved, both are counted. The + /// run is not reset by a move, so a task stolen out of the LLC it was + /// woken from goes back on that LLC's next wake. A real-time task keeps + /// the RT band's placement, and a stopped or killed one [`Self::place`]'s. fn toward_its_waker, P: PreemptGuard>( &self, task: &mut ReadyTask, @@ -2671,10 +2673,10 @@ pub struct CpuHandles { offering: Box<[AtomicU64]>, } -/// How many wakes in a row from one other LLC move a task into it +/// How many parked wakes in a row from one other LLC move a task into it /// ([`CpuSched::toward_its_waker`]). A split pair pays this many split round -/// trips once; a task whose wakes come from two LLCs independently and -/// evenly moves on about one parked wake in 2^(n+1) - 2, 1 in 510 at 8. +/// trips once; a task whose parked wakes come from two LLCs independently +/// and evenly moves on about one in 2^(n+1) - 2, 1 in 510 at 8. pub const WAKER_RUN: u16 = 8; /// The CPUs an idle CPU is chosen among, nearest first: `me`'s LLC, then the diff --git a/kernel/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs index 0e86e901f76..c502454cf30 100644 --- a/kernel/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -1222,15 +1222,16 @@ pub fn split_pair() -> Scenario { /// **One server woken from both LLCs in turn**: a server on cpu0 answers a /// client pinned to cpu1, in its LLC, and one pinned to cpu9, in the other, -/// with balancing off. Which client's request finds the server parked is -/// the run's timing, so following each wake's LLC would move it back and -/// forth. -pub fn server_of_two_llcs() -> Scenario { +/// with balancing off, each taking `rounds` turns. A request that finds the +/// server running is never counted, and which client's request finds it +/// parked is the run's timing: following each wake's LLC would move it back +/// and forth. +pub fn server_of_two_llcs(rounds: usize) -> Scenario { let turn = MS / 10; let wake = |queue| Op::Wake { queue, all: false, boost: None }; let block = |queue| Op::Block { queue, deadline: None }; - let client = |cpu: u64, reply| Script::looping(vec![Op::SetAffinity(1 << cpu), Op::Run(turn), wake(0), block(reply)], WAKE_LLC_ROUNDS); - scenario( + let client = |cpu: u64, reply| Script::looping(vec![Op::SetAffinity(1 << cpu), Op::Run(turn), wake(0), block(reply)], rounds); + let mut scenario = scenario( "server_of_two_llcs", 16, vec![queue(WaitClass::Io), queue(WaitClass::Io), queue(WaitClass::Io)], @@ -1238,7 +1239,7 @@ pub fn server_of_two_llcs() -> Scenario { "served", vec![0, 1, 2], vec![ - Script::looping(vec![block(0), Op::Run(turn), wake(1), block(0), Op::Run(turn), wake(2)], WAKE_LLC_ROUNDS), + Script::looping(vec![block(0), Op::Run(turn), wake(1), block(0), Op::Run(turn), wake(2)], rounds), client(1, 1), client(9, 2), ], @@ -1246,10 +1247,13 @@ pub fn server_of_two_llcs() -> Scenario { ) .with_placement(PlacementShape::Listed(&[0, 1, 9])) .with_topology(Shape::amd_bench()) - .with_balance(Balance::None) + .with_balance(Balance::None); + scenario.max_steps *= rounds.div_ceil(WAKE_LLC_ROUNDS); + scenario } -/// How many turns [`split_pair`]'s and [`server_of_two_llcs`]' threads take. +/// How many turns [`split_pair`]'s threads take, and the rounds +/// [`server_of_two_llcs`] is measured at. pub const WAKE_LLC_ROUNDS: usize = 24; /// **Masks under stealing**, every thread spawned on cpu0 of four with the diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index e2d55312e90..436df6731c6 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -495,16 +495,22 @@ fn a_split_pair_moves_once_into_one_llc() { }); } -/// **A server with a client in each LLC seldom follows them**: on -/// `scenarios::server_of_two_llcs` no run moves the server twice, and at most -/// a quarter of the runs move it at all. A run half as long as -/// `WAKER_RUN` already moves it more often than that, and following every -/// wake moves it hundreds of times over these seeds. +/// **A server with a client in each LLC seldom follows them, and never +/// back**: on `scenarios::server_of_two_llcs` at most a quarter of the runs +/// move the server, and no run moves it twice, even at ten times the rounds. +/// A server moves when one client's requests keep finding it running, so +/// that only the other's are counted; after the move both are, and nothing +/// moves it back. A run half as long as `WAKER_RUN` already moves it more +/// often than the bound, and following every wake moves it hundreds of times +/// over these seeds. #[test] fn a_server_with_a_client_in_each_llc_seldom_follows_them() { const SEEDS: u64 = 40; + sweep(&scenarios::server_of_two_llcs(10 * scenarios::WAKE_LLC_ROUNDS), SEEDS, |outcome| { + assert!(outcome.migrations <= 1, "the server moved {} times in one run: {}", outcome.migrations, outcome.report()); + }); let mut moves = 0; - sweep(&scenarios::server_of_two_llcs(), SEEDS, |outcome| { + sweep(&scenarios::server_of_two_llcs(scenarios::WAKE_LLC_ROUNDS), SEEDS, |outcome| { assert!(outcome.migrations <= 1, "the server moved {} times in one run: {}", outcome.migrations, outcome.report()); moves += outcome.migrations; }); From 8a1357642ad203d97b6d789e16bef4a3a9c0f665 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 15:09:07 +0100 Subject: [PATCH 4/5] sched: a woken partner's move stays off its waker's core ab6 on the AMD bench put every request/reply pair in one LLC, but often on one core: ipc.reqrep.8 ran on SMT siblings in 3 of 4 head boots (cpu3+cpu2, cpu11+cpu10 twice) and fs.disk.read_4k_at in 2 of 4 (cpu0+cpu1, cpu2+cpu3), against none in the base. On one core the pair reads about as slow as split across LLCs: ipc.reqrep.8 7.9-8.0 us against 6.9 on two cores of one LLC and 8.4 split, read_4k_at 11.7 us against 9.6-9.8 and 11.7. pipe.throughput.16384 read 3.7 GB/s against the base's 8.6, its writer and reader sharing one core. A request/reply waker has usually parked by the time its wake is placed, so its CPU reads asleep and busy_siblings counts its SMT sibling as a free core; the scan starts at the waker, so the sibling is first. The move now skips the waker's own core. A host test parks the waker's sibling first in the scan; without the check the task goes there (cpu5) and the test reds. Co-Authored-By: Claude Opus 5.5 --- ...-has-no-migration-cost-or-wake-affinity.md | 8 +++--- kernel/pure/sched/cpu.rs | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index 0a81ae5ab0a..da8071df8a9 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -22,9 +22,11 @@ lacks: (parallelism). The first half is in `CpuSched::toward_its_waker`: - Every wake a thread posts names its CPU; one a device interrupt's handler, a deadline or a retire posts names none. - - The wake that makes `WAKER_RUN` (8) in a row from one other LLC moves - the task to a sleeping CPU there that its mask names and no kick is on - its way to, so a pair split across LLCs shares one after 8 round trips. + - A run of `WAKER_RUN` (8) or more wakes from one other LLC moves the + task to a sleeping CPU there that its mask names, that no kick is on + its way to and that is off the waker's own core. A pair split across + LLCs shares one after 8 round trips; on one core it would read about + as slow as split. A split pair costs about 25% on the AMD bench's `ipc.reqrep.8` and `fs.disk.read_4k_at`. - Only a wake that finds the task parked is counted. A server with a diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index 7490f07e3c3..bf25ec9e6f4 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -1186,8 +1186,11 @@ impl CpuSched { /// **A woken partner follows its waker's LLC.** A wake from another LLC /// whose run from it is at or past [`WAKER_RUN`] goes to a CPU of that - /// LLC the task's mask names, [`Doorbell::sleeping_unrung`], nearest the - /// waker and quickest to wake. Only a wake that finds the task parked + /// LLC the task's mask names, [`Doorbell::sleeping_unrung`] and off the + /// waker's own core, nearest the waker and quickest to wake. A waker that + /// has parked leaves its SMT sibling looking like a free core, and a pair + /// on one core contends for it: on the AMD bench a request/reply pair + /// there reads about as slow as one split across LLCs. Only a wake that finds the task parked /// reaches here, so only those are counted: one that finds it running or /// about to park is a flag the task consumes unseen. A request/reply pair /// split across LLCs shares one after [`WAKER_RUN`] round trips. A server @@ -1216,7 +1219,7 @@ impl CpuSched { } let mask = task.affinity(); env.cpus.idle_in_llc(waker, topology.position(waker), |cpu| { - mask.contains(cpu.id()) && cpu.doorbell().sleeping_unrung() + mask.contains(cpu.id()) && cpu.doorbell().sleeping_unrung() && topology.level(cpu.id(), waker) != Some(Level::Core) }) } @@ -4728,6 +4731,23 @@ mod tests { w.abandon(); } + /// **The move stays off the waker's core**: a waker that has parked + /// leaves its SMT sibling asleep and first in the scan, and the task goes + /// to the other core of the waker's LLC instead. + #[test] + fn a_move_stays_off_the_wakers_core() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + woken_at_home(&mut w, &task, 1, WAKER_RUN - 1); + woken_from(&mut w, &task, 1); + let state = task.state(); + w.abandon(); + assert!( + matches!(state, TaskState::InTransit(cpu) if cpu.0 == 3 || cpu.0 == 7), + "the task went to cpu3 or cpu7, off cpu1's core with cpu5, and it is {state:?}", + ); + } + /// **Two runs ending in one drain go to two CPUs**: the first move rings /// its CPU, and a rung sleeper is passed over, as a wake forward passes /// it. From cebbdedd557fd1f15d28da0d1ecbd0087acd0e54 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 23:32:16 +0100 Subject: [PATCH 5/5] sched: a thread leaving its process names no waker A wake names the CPU of the thread that posted it, and a run of WAKER_RUN such wakes from another LLC moves the woken task there. A thread on its way out is no partner to follow: a parent woken by each child's exit, its join or its teardown's closes chased children that were already gone. On the AMD bench that cost work.build_shaped.j16 about 4% at any run length (ab8), and plausibly thread.spawn_join's +3-5% in ab5. The running task now carries one lifecycle bit, set once by `thread_exit` and `process::leave` before either posts anything, and `CpuSched::waker` answers no waker while it is set. The kernel's `driver::waker` asks it, outside a device interrupt's handler, so the pure scheduler owns who names a waker. Such a wake is neither counted nor breaks the run, as an interrupt's is not. Host test `a_leaving_threads_wakes_leave_the_task_home`: seven wakes from cpu1's staying thread, then a full run from it once it leaves, and the task stays home. With `waker` ignoring the bit, the eighth wake moves it and the test reds. Co-Authored-By: Claude Opus 5.5 --- ...-has-no-migration-cost-or-wake-affinity.md | 7 +++- kernel/pure/sched/cpu.rs | 42 +++++++++++++++++++ kernel/pure/sched/task.rs | 12 ++++++ kernel/src/process.rs | 2 + kernel/src/sched/driver.rs | 17 +++++--- 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/issues/placement-has-no-migration-cost-or-wake-affinity.md b/issues/placement-has-no-migration-cost-or-wake-affinity.md index da8071df8a9..49c164f8c5e 100644 --- a/issues/placement-has-no-migration-cost-or-wake-affinity.md +++ b/issues/placement-has-no-migration-cost-or-wake-affinity.md @@ -21,7 +21,12 @@ lacks: waker (shared data stays warm — pipe/IPC pairs) and away from it (parallelism). The first half is in `CpuSched::toward_its_waker`: - Every wake a thread posts names its CPU; one a device interrupt's - handler, a deadline or a retire posts names none. + handler, a deadline or a retire posts names none, and so does one a + thread posts once it has begun leaving its process (`thread_exit`, + `process::leave`): a parent woken by each child's exit, its join or its + teardown's closes would otherwise follow children it never talks to + again. The thread's task carries that as one bit, set once; it is the + thread's lifecycle, not placement accounting. - A run of `WAKER_RUN` (8) or more wakes from one other LLC moves the task to a sleeping CPU there that its mask names, that no kick is on its way to and that is off the waker's own core. A pair split across diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index bf25ec9e6f4..2e3bd52b7d9 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -533,6 +533,21 @@ impl CpuSched { self.leave_fair_for_rt(now); } + /// The CPU a wake the running task posts names as its waker + /// ([`WakeCause::waker`]): this one, unless nothing runs here or the + /// running thread is leaving its process. A dying waker is no partner + /// for [`Self::toward_its_waker`] to follow: a parent woken by each + /// child's exit would chase children it never talks to again. + pub fn waker(&self) -> Option { + self.running.as_ref().filter(|task| !task.is_leaving()).map(|_| self.id) + } + + /// The running thread has begun leaving its process: no wake it posts + /// from now on names a waker. + pub fn current_leaves(&mut self) { + self.running.as_mut().expect("a thread leaving its process is running").begin_leaving(); + } + /// `SYS_RT_ENTER` on the running task — permanent RT, as opposed to the /// bounded window a waker lends. The privilege gate lives at the syscall /// layer. @@ -4748,6 +4763,33 @@ mod tests { ); } + /// **A thread leaving its process names no waker**: the thread running + /// on cpu1 wakes the task one short of [`WAKER_RUN`], then begins + /// leaving, and a full run of wakes it posts on its way out leaves the + /// task home. + #[test] + fn a_leaving_threads_wakes_leave_the_task_home() { + let mut w = World::on(two_llcs()); + let task = parked_in_llc_0(&mut w, CpuMask::ANY); + w.spawn(C1); + w.run_a_pass(C1); + let woken_by_cpu1 = |w: &mut World, times| { + for _ in 0..times { + assert_eq!(task.claim_wake(), Claim::Parked(C0), "the task is parked on cpu0"); + let cause = WakeCause::new(WakeReason::Woken).from(w.cpus[1].waker()); + w.post_claimed_cause(C0, &task, cause); + w.run_a_pass(C0); + assert_eq!(task.state(), TaskState::Running(C0), "a wake from cpu1 moved the task"); + w.park_running(C0); + } + }; + assert_eq!(w.cpus[1].waker(), Some(C1), "cpu1's running thread names itself"); + woken_by_cpu1(&mut w, WAKER_RUN - 1); + w.cpus[1].current_leaves(); + woken_by_cpu1(&mut w, WAKER_RUN); + w.abandon(); + } + /// **Two runs ending in one drain go to two CPUs**: the first move rings /// its CPU, and a rung sleeper is passed over, as a wake forward passes /// it. diff --git a/kernel/pure/sched/task.rs b/kernel/pure/sched/task.rs index 69df084d108..46dda7ce3f7 100644 --- a/kernel/pure/sched/task.rs +++ b/kernel/pure/sched/task.rs @@ -836,6 +836,8 @@ struct TaskInner { /// Where this task's recent wakes came from. Kept by the owner CPU, as /// the mask copy is. waker_streak: WakerStreak, + /// Set once the thread has begun leaving its process, and never cleared. + leaving: bool, /// When the current residency began: enqueue time while ready, dispatch /// time while running, park time while blocked. One field, because a task /// is in exactly one state. @@ -1055,6 +1057,7 @@ impl TaskBuilder { affinity: self.mask, affinity_version: 0, waker_streak: WakerStreak::default(), + leaving: false, since: now, adopt_node: MailboxNode::new(), ext: Some(self.ext), @@ -1165,6 +1168,15 @@ impl RunningTask { self.0.refresh_affinity() } + /// The thread has begun leaving its process, for good. + pub(crate) fn begin_leaving(&mut self) { + self.0 .0.leaving = true; + } + + pub fn is_leaving(&self) -> bool { + self.0 .0.leaving + } + pub(crate) fn standing_mut(&mut self) -> &mut Standing { &mut self.0 .0.standing } diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 8e5ee1e8625..417866443a4 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -902,6 +902,7 @@ fn teardown(pid: Pid, tid: Tid, end: End, mark: i32, process_data: &Arc) { + crate::sched::driver::current_leaves(); let (pid, tid) = (current_process(), current_tid()); crate::mm::paging::activate_kernel(); let process_data = process_data(); @@ -965,6 +966,7 @@ pub fn exit(end: End) -> ! { /// Exit the current thread. If this is the main thread, it is the process's exit. For a child thread, frees its own mappings and leaves. pub fn thread_exit(code: i32) -> ! { + crate::sched::driver::current_leaves(); let process_pid = current_process(); let tid = current_tid(); let route = { diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 21bc5d1b271..d4e4dc6dc46 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -711,12 +711,19 @@ pub fn current_shared() -> Option> { try_with_cpu(|cpu| cpu.running().map(|t| t.shared().clone())).flatten() } -/// The CPU a wake posted here names as its waker: this one, while a thread -/// runs on it outside a device interrupt's handler, which posts for the -/// device and not for the thread it interrupted. +/// The CPU a wake posted here names as its waker ([`CpuSched::waker`]); none +/// inside a device interrupt's handler, which posts for the device and not +/// for the thread it interrupted, or inside a pass. pub fn waker() -> Option { - let thread = percpu::current_tid().is_some() && !percpu::in_device_irq(); - thread.then(|| CpuId(percpu::cpu_id())) + if percpu::in_device_irq() { + return None; + } + try_with_cpu(CpuSched::waker).flatten() +} + +/// The running thread has begun leaving its process ([`CpuSched::current_leaves`]). +pub fn current_leaves() { + with_cpu(CpuSched::current_leaves); } /// The running task's mask, which a thread or process it spawns starts with; every CPU off-task, as at boot.