From 8e4eeddc412b11be3852e8d41cf3a3019a84b608 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:13:37 +0100 Subject: [PATCH 01/24] idle: CPUs enter the deep states their firmware's _CST declares, chosen per idle period acpiserver reads each processor's _CST and _CSD and hands the kernel, through a new op on the acpi claim, only the states it may enter as they are: Intel FFH MWAIT states, and off Intel the SystemIO states the core traps. The kernel checks each against the CPU that will enter it (toyos-userbound::idle) and chooses, per idle period, the deepest state whose exit fits the held wake bound, whose residency fits before the next timer, and that the CPU's recent unforeseen wakes admit. A timer deadline is met by waking early by the state's exit. A wake forward prefers the shallowest sleeper. The processor _OSC commits ToyOS's claim (Intel bits 4 and 9) with no query: Table 1's bits say what OSPM is capable of (302223-007 section 2.1), and the T14's query answers 0x1 whatever is asked. A WakeLatency object, made through the new WAKE_LATENCY right, holds every idle CPU to waking within a bound; soundserver holds half a period while a stream plays. Co-Authored-By: Claude Opus 5.5 --- images/system.toml | 4 +- kernel/pure/cpu.rs | 21 ++ kernel/pure/sched/cpu.rs | 61 +++- kernel/pure/sched/idle.rs | 299 +++++++++++++++++++ kernel/pure/sched/mod.rs | 1 + kernel/src/arch/aarch64/acpi_mode.rs | 6 +- kernel/src/arch/aarch64/idle.rs | 20 ++ kernel/src/arch/aarch64/mod.rs | 1 + kernel/src/arch/x86_64/acpi_mode.rs | 8 + kernel/src/arch/x86_64/boot.rs | 1 + kernel/src/arch/x86_64/idle.rs | 92 ++++++ kernel/src/arch/x86_64/mod.rs | 3 + kernel/src/counters.rs | 2 + kernel/src/cpu_local.rs | 2 + kernel/src/drivers/acpi.rs | 8 +- kernel/src/hw.rs | 7 +- kernel/src/idle.rs | 231 +++++++++++++++ kernel/src/loader/mod.rs | 1 + kernel/src/main.rs | 1 + kernel/src/object/mod.rs | 3 + kernel/src/object/ops.rs | 31 +- kernel/src/object/wake_latency.rs | 35 +++ kernel/src/sched/driver.rs | 7 + kernel/src/syscall/device.rs | 11 + kernel/src/syscall/dispatch.rs | 3 +- kernel/src/syscall/proc.rs | 19 ++ lib/acpi/src/madt.rs | 8 +- lib/acpi/tests/corpus.rs | 4 +- lib/acpi/tests/fixtures.rs | 2 +- lib/manifest/src/lib.rs | 4 + lib/userbound/src/idle.rs | 261 +++++++++++++++++ lib/userbound/src/lib.rs | 1 + sdk/abi/src/acpi.rs | 63 ++++ sdk/abi/src/counters.rs | 15 +- sdk/abi/src/handle.rs | 17 +- sdk/abi/src/syscall.rs | 27 +- sdk/toyos/src/device.rs | 6 + sdk/toyos/src/syscap.rs | 14 + system/acpiserver/aml/src/exec.rs | 8 +- system/acpiserver/aml/src/lib.rs | 8 +- system/acpiserver/aml/src/object.rs | 7 +- system/acpiserver/aml/tests/walk.rs | 2 +- system/acpiserver/src/cst.rs | 413 +++++++++++++++++++++++++++ system/acpiserver/src/gpe.rs | 2 +- system/acpiserver/src/host.rs | 22 +- system/acpiserver/src/init.rs | 2 +- system/acpiserver/src/main.rs | 120 ++++++-- system/acpiserver/src/osc.rs | 59 ++-- system/soundserver/src/mix.rs | 34 ++- tests/images/metalcase.toml | 2 +- tests/images/metaldevicecase.toml | 2 +- tests/images/netcase.toml | 2 +- tests/images/testcases.toml | 2 +- 53 files changed, 1878 insertions(+), 107 deletions(-) create mode 100644 kernel/pure/sched/idle.rs create mode 100644 kernel/src/arch/aarch64/idle.rs create mode 100644 kernel/src/arch/x86_64/idle.rs create mode 100644 kernel/src/idle.rs create mode 100644 kernel/src/object/wake_latency.rs create mode 100644 lib/userbound/src/idle.rs create mode 100644 system/acpiserver/src/cst.rs diff --git a/images/system.toml b/images/system.toml index b9c2aeca941..1135b3ad838 100644 --- a/images/system.toml +++ b/images/system.toml @@ -115,11 +115,13 @@ receives = ["input"] devices = ["isa:0060,0064:1,12"] memory-floor-mib = 24 +# `wake-latency` holds every idle CPU to waking within half a period while a +# stream plays, so a CPU in a deep state does not make the mix late. [programs.soundserver] service = true serves = ["soundserver", "volume", "sound-output"] devices = ["pci:04:01:00/1af4:1059"] -syscap = ["rt"] +syscap = ["rt", "wake-latency"] # The Intel HD Audio driver: one process per HDA function the kernel does not # drive, each walking its own controller's codecs and offering soundserver the diff --git a/kernel/pure/cpu.rs b/kernel/pure/cpu.rs index 698f1772bc0..e09518c4c72 100644 --- a/kernel/pure/cpu.rs +++ b/kernel/pure/cpu.rs @@ -226,6 +226,16 @@ fn counts_smis(id: &Ident) -> bool { id.vendor == Vendor::Intel && id.family == 6 && SMI_COUNT_MODELS.contains(&id.model) } +/// Whether the CPU `id` names has `CStateBaseAddr` (MSRC001_0073), the +/// eight ports its core traps as C-state requests: the family the PPRs that +/// define it are for (55922, 55901, 55772: family 17h), on the CPU itself +/// and not under a hypervisor, whose emulation of the register nothing +/// documents. A CPU wrongly admitted is a `#GP` at its first idle, so a +/// family no document in hand defines it for is left out and keeps C1. +pub fn traps_cstate_ports(id: &Ident, cpuid_1_ecx: u32) -> bool { + id.vendor == Vendor::Amd && id.family == 0x17 && cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR == 0 +} + /// Why a CPU's topology leaves describe no place in a tree. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum TopologyRefusal { @@ -279,6 +289,17 @@ pub fn topology_keys(levels: &[[u32; 4]], caches: &[[u32; 4]]) -> Result { /// can tell a claim about the present from one a stopped CPU left behind. /// See [`CpuHandle::answering`]. last_pass: AtomicU64, + /// **How long this CPU takes to wake from the state it last chose to wait + /// in**, in nanoseconds, 0 for C1: what a wake forward weighs between + /// sleeping CPUs ([`CpuHandles::idle_near`]). Written before the CPU + /// waits, after it published SLEEPING, so a reader may see the previous + /// idle period's: a tiebreak, never a correctness input. + idle_exit: AtomicU64, /// The on-target counterpart to the simulator's invariants: the sim asserts /// what a pass *does*, this measures what a pass *costs*. /// @@ -2330,6 +2336,7 @@ impl CpuHandle { surplus: AtomicU32::new(0), weight: AtomicU64::new(0), last_pass: AtomicU64::new(0), + idle_exit: AtomicU64::new(0), #[cfg(feature = "sched-check")] pass_costs: PassCosts::new(), } @@ -2368,6 +2375,14 @@ impl CpuHandle { self.weight.store(weight, Ordering::Relaxed); } + pub fn idle_exit(&self) -> u64 { + self.idle_exit.load(Ordering::Relaxed) + } + + pub fn publish_idle_exit(&self, nanos: u64) { + self.idle_exit.store(nanos, Ordering::Relaxed); + } + /// Stamp the numbers above with the pass that published them. pub fn publish_pass(&self, now: Nanos) { self.last_pass.store(now.0, Ordering::Relaxed); @@ -2546,23 +2561,31 @@ impl CpuHandles { /// The CPU nearest `me` that `usable` accepts, preferring within each /// group of [`IDLE_GROUPS`] one whose SMT siblings all sleep too, so work - /// goes to a free physical core before it shares one with running work. - /// Ties go to the first at or after topology position `from`, which is how + /// goes to a free physical core before it shares one with running work, + /// and then the one quickest to wake ([`CpuHandle::idle_exit`]), so a wake + /// goes to a CPU waiting shallow and the deep sleepers stay asleep. Ties + /// go to the first at or after topology position `from`, which is how /// consecutive choices rotate. pub fn idle_near(&self, me: CpuId, from: usize, usable: impl Fn(&CpuHandle) -> bool) -> Option { for (inner, outer) in IDLE_GROUPS { - let mut shared_core = None; + // The best of each tier, as (exit, cpu): strictly less replaces, so the first of equals stays. + let (mut free_core, mut shared_core): (Option<(u64, CpuId)>, Option<(u64, CpuId)>) = (None, None); for cpu in self.topology.ring(me, inner, outer, from) { - if !usable(self.get(cpu)) { + let handle = self.get(cpu); + if !usable(handle) { continue; } - if self.busy_siblings(cpu) == 0 { - return Some(cpu); + let exit = handle.idle_exit(); + let tier = if self.busy_siblings(cpu) == 0 { &mut free_core } else { &mut shared_core }; + if tier.is_none_or(|(best, _)| exit < best) { + *tier = Some((exit, cpu)); + } + if free_core.is_some_and(|(best, _)| best == 0) { + break; } - shared_core.get_or_insert(cpu); } - if shared_core.is_some() { - return shared_core; + if let Some((_, cpu)) = free_core.or(shared_core) { + return Some(cpu); } } None @@ -4308,6 +4331,26 @@ mod tests { assert_eq!(handles.idle_near(me, 0, |h| asleep(h) && h.id() == CpuId(9)), None); } + /// **Among free cores of the nearest group, the one quickest to wake**: + /// rotation decides only between equals, and locality and a free core + /// still come before the wake cost. + #[test] + fn among_equally_near_free_cores_the_shallowest_sleeper_is_woken() { + let (handles, _consumers) = bare_handles(two_llcs()); + let me = CpuId(0); + let asleep = |h: &CpuHandle<_>| h.doorbell().sleeping(); + sleep(&handles, &[1, 2, 3, 4, 5, 6, 7]); + handles.get(CpuId(2)).publish_idle_exit(1_048_000); + handles.get(CpuId(6)).publish_idle_exit(253_000); + assert_eq!(handles.idle_near(me, 0, asleep), Some(CpuId(6)), "of free core {{2, 6}}, cpu6 wakes in 253 µs and cpu2 in 1048"); + handles.get(CpuId(6)).publish_idle_exit(1_048_000); + assert_eq!(handles.idle_near(me, 0, asleep), Some(CpuId(2)), "equals go to the rotation's first"); + let after_two = handles.topology().position(CpuId(2)) + 1; + assert_eq!(handles.idle_near(me, after_two, asleep), Some(CpuId(6)), "and rotate"); + wake_up(&handles, &[2, 6]); + assert_eq!(handles.idle_near(me, 0, asleep), Some(CpuId(4)), "cpu4 shares a busy core, but is in this LLC: locality before wake cost"); + } + /// **A spawn takes the least queue, then a sleeping CPU, then the CPU /// whose SMT siblings run least**, scanned from its rotating start. #[test] diff --git a/kernel/pure/sched/idle.rs b/kernel/pure/sched/idle.rs new file mode 100644 index 00000000000..50e0a88a6e6 --- /dev/null +++ b/kernel/pure/sched/idle.rs @@ -0,0 +1,299 @@ +//! Which idle state a CPU waits in, chosen once per idle period, and the +//! machine-wide bound on how long a waiting CPU may take to wake. +//! +//! A CPU's states are what its firmware declared, shallowest first, each with +//! its worst-case exit latency and the residency below which it saves nothing +//! ([`Cost`]). The shallowest wait, `hlt`'s C1, is not among them: it is always +//! permitted and is what [`choose`] answers `None` for. +//! +//! **The rule**: the deepest declared state whose exit latency is within the +//! held bound, whose residency fits before the CPU's next timer, and for which +//! fewer than half of the CPU's last [`WINDOW`] unforeseen wakes came sooner +//! than its residency ([`History`]). The timer horizon is exact; a kick, an IPI +//! or a device interrupt is not foreseen, and the history is the estimate of +//! those. A CPU that enters a state with a timer armed wakes `exit` early +//! ([`Choice::wake_early_by`]), so a known deadline is met rather than made late +//! by the state's exit. +//! +//! **A CPU the history alone holds shallow comes back to look** ([`Choice::revisit_after`]): +//! the history moves only on wakes, and a CPU that stopped being woken would +//! otherwise wait shallow for as long as nothing comes, which on a part whose +//! turbo counts a core in C1 as active costs every other core its clock. + +use alloc::collections::BTreeMap; + +/// The most states a CPU's declaration holds. +pub const MAX_STATES: usize = 8; + +/// How many unforeseen wakes [`History`] remembers per state. +pub const WINDOW: u32 = 8; + +/// Of the last [`WINDOW`] unforeseen wakes, how many may have come sooner than +/// a state's residency for it still to be chosen: fewer than half. A policy, +/// judged by metal: with it a CPU that turns to short idles pays at most this +/// many deep exits before it waits shallow. +pub const EARLY_ALLOWED: u32 = WINDOW / 2 - 1; + +/// One declared state's costs, in nanoseconds. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Cost { + /// The worst case from the wake to running again. + pub exit: u64, + /// The shortest idle for which the state is worth entering. + pub residency: u64, +} + +/// For each state, which of the CPU's last [`WINDOW`] unforeseen wakes came +/// sooner than its residency: bit 0 the latest. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct History { + early: [u8; MAX_STATES], +} + +impl History { + /// An idle period that a wake the CPU did not foresee ended after + /// `slept` nanoseconds. + pub fn unforeseen(&mut self, costs: &[Cost], slept: u64) { + for (bits, cost) in self.early.iter_mut().zip(costs) { + *bits = *bits << 1 | u8::from(slept < cost.residency); + } + } + + /// The states a new declaration replaced are not its states: what was + /// learned of them says nothing about these. + pub fn forget(&mut self) { + *self = Self::default(); + } + + fn admits(&self, state: usize) -> bool { + self.early[state].count_ones() <= EARLY_ALLOWED + } +} + +/// What [`choose`] picked. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Choice { + /// A declared state, or `None` for C1. + pub state: Option, + /// How much earlier than its deadline the CPU's timer must fire for the + /// deadline to be met. + pub wake_early_by: u64, + /// Where a deeper state fit the horizon and the bound and the history + /// alone kept the CPU from it: how soon to wake and choose again, that + /// state's residency. A wake then is a period that long, which the history + /// learns as one. + pub revisit_after: Option, +} + +/// The state to wait in. `horizon` is how long until the CPU's timer fires, +/// `None` where none is armed; `limit` the held wake bound, `None` where +/// nothing is held. +pub fn choose(costs: &[Cost], history: &History, horizon: Option, limit: Option) -> Choice { + let states = 0..costs.len().min(MAX_STATES); + let permitted = |state: usize| { + let cost = costs[state]; + limit.is_none_or(|limit| cost.exit <= limit) && horizon.is_none_or(|horizon| cost.residency <= horizon) + }; + let deepest_permitted = states.clone().rev().find(|&s| permitted(s)); + let state = states.rev().find(|&s| permitted(s) && history.admits(s)); + Choice { + state, + wake_early_by: match (state, horizon) { + (Some(state), Some(_)) => costs[state].exit, + _ => 0, + }, + revisit_after: deepest_permitted.filter(|&deepest| Some(deepest) != state).map(|deepest| costs[deepest].residency), + } +} + +/// The wake bounds held, as a multiset: the machine's bound is the smallest. +#[derive(Default, Debug)] +pub struct Bounds { + held: BTreeMap, +} + +impl Bounds { + pub const fn new() -> Self { + Self { held: BTreeMap::new() } + } + + pub fn hold(&mut self, bound: u64) { + *self.held.entry(bound).or_insert(0) += 1; + } + + /// Release one hold of `bound`, which must be held. + pub fn release(&mut self, bound: u64) { + let count = self.held.get_mut(&bound).unwrap_or_else(|| panic!("a wake bound of {bound} ns released and never held")); + *count -= 1; + if *count == 0 { + self.held.remove(&bound); + } + } + + /// The smallest bound held. + pub fn least(&self) -> Option { + self.held.keys().next().copied() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const US: u64 = 1000; + + /// The T14's two deep states as acpiserver declares them: exit = `_CST`'s + /// latency, residency twice it. + const T14: [Cost; 2] = [Cost { exit: 253 * US, residency: 506 * US }, Cost { exit: 1048 * US, residency: 2096 * US }]; + + fn pick(history: &History, horizon: Option, limit: Option) -> Option { + choose(&T14, history, horizon, limit).state + } + + #[test] + fn a_cpu_with_nothing_due_and_no_bound_waits_deepest() { + assert_eq!(pick(&History::default(), None, None), Some(1)); + } + + #[test] + fn the_timer_horizon_admits_only_what_fits_before_it() { + let h = History::default(); + assert_eq!(pick(&h, Some(5000 * US), None), Some(1)); + assert_eq!(pick(&h, Some(2096 * US), None), Some(1)); + assert_eq!(pick(&h, Some(2095 * US), None), Some(0)); + assert_eq!(pick(&h, Some(506 * US), None), Some(0)); + assert_eq!(pick(&h, Some(505 * US), None), None); + } + + #[test] + fn a_held_bound_excludes_every_state_slower_to_wake_than_it() { + let h = History::default(); + assert_eq!(pick(&h, None, Some(1048 * US)), Some(1)); + assert_eq!(pick(&h, None, Some(1047 * US)), Some(0)); + assert_eq!(pick(&h, None, Some(253 * US)), Some(0)); + assert_eq!(pick(&h, None, Some(252 * US)), None); + } + + #[test] + fn a_state_entered_before_a_timer_wakes_early_by_its_exit_and_without_one_does_not() { + let h = History::default(); + assert_eq!(choose(&T14, &h, Some(5000 * US), None), Choice { state: Some(1), wake_early_by: 1048 * US, revisit_after: None }); + assert_eq!(choose(&T14, &h, None, None), Choice { state: Some(1), wake_early_by: 0, revisit_after: None }); + assert_eq!(choose(&T14, &h, Some(100 * US), None), Choice { state: None, wake_early_by: 0, revisit_after: None }, "C1 owes no early wake"); + } + + /// IPC ping-pong: every idle ends in a few microseconds, by a kick, with + /// no timer armed. Without the history the CPU would pay the deepest exit + /// on each wake. + #[test] + fn short_unforeseen_idles_bring_the_cpu_to_c1_within_the_allowance() { + let mut h = History::default(); + let mut deep = 0; + for _ in 0..2 * WINDOW { + if pick(&h, None, None).is_some() { + deep += 1; + } + h.unforeseen(&T14, 10 * US); + } + assert_eq!(deep, EARLY_ALLOWED + 1, "a CPU turned to short idles pays at most the allowance and one more deep exit"); + assert_eq!(pick(&h, None, None), None); + } + + #[test] + fn idles_between_the_two_residencies_settle_on_the_shallower_state() { + let mut h = History::default(); + for _ in 0..WINDOW { + h.unforeseen(&T14, 1000 * US); + } + assert_eq!(pick(&h, None, None), Some(0)); + } + + #[test] + fn long_idles_after_short_ones_return_the_cpu_to_the_deepest_state() { + let mut h = History::default(); + for _ in 0..WINDOW { + h.unforeseen(&T14, 10 * US); + } + assert_eq!(pick(&h, None, None), None); + let mut waited = 0; + while pick(&h, None, None) != Some(1) { + h.unforeseen(&T14, 10_000 * US); + waited += 1; + } + assert_eq!(waited, WINDOW - EARLY_ALLOWED); + } + + #[test] + fn a_new_declaration_starts_with_nothing_learned() { + let mut h = History::default(); + for _ in 0..WINDOW { + h.unforeseen(&T14, 10 * US); + } + h.forget(); + assert_eq!(pick(&h, None, None), Some(1)); + } + + #[test] + fn no_declared_state_is_c1() { + assert_eq!(choose(&[], &History::default(), None, None), Choice { state: None, wake_early_by: 0, revisit_after: None }); + } + + /// Only the history keeps a CPU from a state the horizon and the bound + /// allow, so only then does it come back to look; the horizon and the + /// bound are exact and need no second look. + #[test] + fn a_cpu_held_shallow_by_its_history_alone_revisits_after_the_deeper_states_residency() { + let mut short = History::default(); + for _ in 0..WINDOW { + short.unforeseen(&T14, 10 * US); + } + assert_eq!(choose(&T14, &short, None, None).revisit_after, Some(2096 * US)); + assert_eq!(choose(&T14, &short, Some(1000 * US), None).revisit_after, Some(506 * US), "C2 alone fits the horizon"); + assert_eq!(choose(&T14, &short, None, Some(300 * US)).revisit_after, Some(506 * US), "C2 alone fits the bound"); + assert_eq!(choose(&T14, &History::default(), Some(1000 * US), None).revisit_after, None, "the horizon kept it from C3, not the history"); + assert_eq!(choose(&T14, &short, Some(100 * US), None).revisit_after, None); + } + + /// A CPU that was busy and then got nothing more to do is woken only by + /// its revisits, each a period of the residency it was asked to look + /// after: it reaches the deepest state within the window less the + /// allowance, and then stays. + #[test] + fn a_cpu_left_idle_after_short_idles_reaches_the_deepest_state_on_its_revisits_alone() { + let mut h = History::default(); + for _ in 0..WINDOW { + h.unforeseen(&T14, 10 * US); + } + let mut revisits = 0; + loop { + let choice = choose(&T14, &h, None, None); + let Some(after) = choice.revisit_after else { break }; + h.unforeseen(&T14, after); + revisits += 1; + } + assert_eq!(choose(&T14, &h, None, None).state, Some(1)); + assert_eq!(revisits, WINDOW - EARLY_ALLOWED); + } + + #[test] + fn the_least_bound_held_is_the_machines_and_goes_with_its_last_hold() { + let mut b = Bounds::default(); + assert_eq!(b.least(), None); + b.hold(1_450 * US); + b.hold(300 * US); + b.hold(300 * US); + assert_eq!(b.least(), Some(300 * US)); + b.release(300 * US); + assert_eq!(b.least(), Some(300 * US)); + b.release(300 * US); + assert_eq!(b.least(), Some(1_450 * US)); + b.release(1_450 * US); + assert_eq!(b.least(), None); + } + + #[test] + #[should_panic(expected = "released and never held")] + fn releasing_a_bound_never_held_is_a_kernel_bug() { + Bounds::default().release(1); + } +} diff --git a/kernel/pure/sched/mod.rs b/kernel/pure/sched/mod.rs index ac4abc72d8b..9eba8cd13c7 100644 --- a/kernel/pure/sched/mod.rs +++ b/kernel/pure/sched/mod.rs @@ -15,6 +15,7 @@ pub mod cpu; pub mod eevdf; pub mod hw; +pub mod idle; pub mod invariants; pub mod job; pub mod job_sched; diff --git a/kernel/src/arch/aarch64/acpi_mode.rs b/kernel/src/arch/aarch64/acpi_mode.rs index bc75642c8ab..123f6d4b269 100644 --- a/kernel/src/arch/aarch64/acpi_mode.rs +++ b/kernel/src/arch/aarch64/acpi_mode.rs @@ -14,7 +14,7 @@ pub fn release() { unreachable!("AArch64 has no ACPI row") } -/// Never called, as [`release`]: the four are reached only with a claim. +/// Never called, as [`release`]: the five are reached only with a claim. pub fn access(_row: &crate::isa::Row, _request: &mut toyos_abi::acpi::Access) -> Result<(), toyos_abi::syscall::SyscallError> { unreachable!("AArch64 has no ACPI row") } @@ -31,6 +31,10 @@ pub fn s5(_row: &crate::isa::Row, _word: u64) -> Result<(), toyos_abi::syscall:: unreachable!("AArch64 has no ACPI row") } +pub fn idle(_row: &crate::isa::Row, _declaration: &toyos_abi::acpi::IdleDeclaration) -> Result<(), toyos_abi::syscall::SyscallError> { + unreachable!("AArch64 has no ACPI row") +} + #[cfg(feature = "test-actuators")] pub fn debug_firmware_lock(_act: u64) -> u64 { toyos_abi::syscall::SyscallError::NotSupported.to_u64() diff --git a/kernel/src/arch/aarch64/idle.rs b/kernel/src/arch/aarch64/idle.rs new file mode 100644 index 00000000000..f6e2b39c63f --- /dev/null +++ b/kernel/src/arch/aarch64/idle.rs @@ -0,0 +1,20 @@ +//! AArch64's half of `crate::idle`: nothing deeper than `wfi` is declared +//! here, since no AArch64 machine hands the kernel a declaration +//! (`issues/aarch64-cpus-idle-only-in-wfi.md`), so every CPU reports a timer +//! that does not run in a deep state and keeps C1. + +use toyos_userbound::idle::{CpuIdle, Entry}; + +pub fn facts() -> CpuIdle { + CpuIdle::default() +} + +/// Never called: [`facts`] admits no state. +pub fn enter(_entry: Entry) { + unreachable!("AArch64 admits no idle state deeper than wfi") +} + +/// No processor UID names a CPU here: nothing declares one. +pub fn cpu_of(_uid: u32) -> Option { + None +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 5d78ea7d205..e17e5ba4d07 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -23,6 +23,7 @@ pub mod entropy; pub mod entry; pub mod fpu; pub mod hw; +pub mod idle; pub mod irqchip; pub mod paging; pub mod percpu; diff --git a/kernel/src/arch/x86_64/acpi_mode.rs b/kernel/src/arch/x86_64/acpi_mode.rs index d5cc0fc4780..741e92b52d4 100644 --- a/kernel/src/arch/x86_64/acpi_mode.rs +++ b/kernel/src/arch/x86_64/acpi_mode.rs @@ -548,6 +548,14 @@ pub fn s5(row: &isa::Row, word: u64) -> Result<(), SyscallError> { Ok(()) } +/// Take one processor's idle states from the claim's holder +/// (`crate::idle::declare`), under the claim's lock and [`HOLDER`], so one +/// declaration is published at a time. +pub fn idle(row: &isa::Row, declaration: &toyos_abi::acpi::IdleDeclaration) -> Result<(), SyscallError> { + let _holder = acting(row)?; + crate::idle::declare(declaration) +} + /// Clear the lock word's owner where the claim's holder holds it, and tell /// the firmware where it asked meanwhile. `orphaned` says why the holder did /// not give it back itself, for the log; empty where it did. diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index d1b28cc9301..fcfebe0485f 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -91,6 +91,7 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { // allows: a fault in any later phase then diagnoses instead of stopping in // a handler the firmware left behind. let madt = acpi::parse_madt(rsdp_addr).expect("ACPI: MADT not found"); + super::idle::note_uids(&madt.processor_uids); // Off the same tables as the MADT, and before the IDT below makes a panic // reportable: a panic that can be reported but not ended leaves the machine // holding its panel for a hand that may not be in the room. diff --git a/kernel/src/arch/x86_64/idle.rs b/kernel/src/arch/x86_64/idle.rs new file mode 100644 index 00000000000..c5ac484d647 --- /dev/null +++ b/kernel/src/arch/x86_64/idle.rs @@ -0,0 +1,92 @@ +//! x86-64's half of `crate::idle`: what this CPU can enter, read on it, the +//! two entries, and which CPU a processor's ACPI UID names. + +use core::arch::asm; + +use alloc::vec::Vec; +use toyos_userbound::idle::{CpuIdle, Entry}; + +use super::cpu; +use crate::sync::Lock; + +/// MSRC001_0073, `CStateBaseAddr` (AMD PPR 55922). +const CSTATE_BASE_ADDR: u32 = 0xC001_0073; + +/// The line `MONITOR` arms before an `MWAIT` state: one per CPU that nothing +/// writes, so the wait ends on an interrupt alone, as `hlt`'s does. `MWAIT` +/// enters no state without an armed monitor (SDM Vol. 2B, MWAIT). +#[repr(align(64))] +pub struct Line(#[allow(dead_code, reason = "the line MONITOR watches, which no code reads")] [u8; 64]); + +impl Line { + pub const fn new() -> Self { + Self([0; 64]) + } +} + +/// What this CPU can enter, read on it. +pub fn facts() -> CpuIdle { + let (signature, _, cpuid_1_ecx, _) = cpu::cpuid(1, 0); + let (cpuid_5_ecx, cpuid_5_edx) = if cpu::cpuid(0, 0).0 >= 5 { + let (_, _, ecx, edx) = cpu::cpuid(5, 0); + (ecx, edx) + } else { + (0, 0) + }; + let invariant_tsc = cpu::cpuid(0x8000_0000, 0).0 >= 0x8000_0007 && cpu::cpuid(0x8000_0007, 0).3 & 1 << 8 != 0; + let id = kernel::cpu::Ident::new(cpu::vendor(), signature); + let trapped_ports = kernel::cpu::traps_cstate_ports(&id, cpuid_1_ecx) + .then(|| cpu::rdmsr(CSTATE_BASE_ADDR) as u16) + .filter(|&base| base != 0); + CpuIdle { + monitor: cpuid_1_ecx & 1 << 3 != 0, + mwait_extensions: cpuid_5_ecx & 1 != 0, + mwait_substates: cpuid_5_edx, + always_running_timer: cpu::leaf_6().0 & 1 << 2 != 0, + invariant_tsc, + trapped_ports, + } +} + +/// Wait in `entry`'s state with interrupts unmasked, as [`super::hw::halt`] +/// waits in C1; called with them masked, and returns once an interrupt has +/// been taken. +/// +/// Both keep `sti; hlt`'s guarantee that a wake landing between the decision +/// and the wait is not slept through. `MWAIT` in the `STI` shadow: an interrupt +/// that "would be delivered to software (as if HLT had been executed)" exits +/// it (SDM Vol. 2B, MWAIT). A trapped port read: "Entry via IO read, +/// EFLAGS[IF]==0: The interrupt wakes the core", and with `IF` set it is taken +/// (AMD BKDG 52740 §2.5.3.2.5, the last of AMD's documents to describe this +/// interface's exit); in the shadow it is pending at the read either way. +pub fn enter(entry: Entry) { + #[cfg(feature = "mask-windows")] + crate::windows::irqs_unmasking(); + match entry { + Entry::Mwait { hint } => { + let line = &crate::cpu_local::here().arch.idle_line; + // SAFETY: arms a monitor on this CPU's own line, which it reads nothing through; ECX 0 and EDX 0 are MONITOR's only defined extensions and hints. + unsafe { asm!("monitor", in("rax") line as *const Line, in("ecx") 0u32, in("edx") 0u32, options(nostack, readonly)) }; + // SAFETY: the hint is one leaf 5 enumerates (`toyos_userbound::idle`), ECX 0 asks no extension; enables interrupts and waits, touching no memory. + unsafe { asm!("sti; mwait", in("eax") hint, in("ecx") 0u32, options(nomem, nostack)) }; + } + Entry::Port { port } => { + // SAFETY: a port this CPU's core traps as a C-state request (`toyos_userbound::idle`), which reaches no device; enables interrupts and waits. + unsafe { asm!("sti; in al, dx", in("dx") port, out("al") _, options(nomem, nostack)) }; + } + } +} + +/// Each enabled CPU's APIC ID and ACPI Processor UID, as the MADT names them. +static UIDS: Lock> = Lock::new(Vec::new()); + +/// Keep the MADT's processor UIDs, for [`cpu_of`]. +pub fn note_uids(uids: &[(u32, u32)]) { + *UIDS.lock() = uids.to_vec(); +} + +/// The CPU the ACPI Processor UID `uid` names, where this kernel runs one. +pub fn cpu_of(uid: u32) -> Option { + let apic = UIDS.lock().iter().find(|&&(_, u)| u == uid).map(|&(apic, _)| apic)?; + (0..crate::smp::cpu_count()).find(|&cpu| crate::smp::hardware_id(cpu) == apic) +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index 15ce32e035f..fc229b7890f 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -24,6 +24,7 @@ pub mod entry; pub mod fpu; pub mod hpet; pub mod hw; +pub mod idle; pub mod idt; pub mod iommu_unit; pub mod ioapic; @@ -122,6 +123,7 @@ pub unsafe fn percpu_fetch_add( /// This architecture's part of a CPU's `cpu_local` record. pub struct CpuLocal { pub counters: counters::Local, + pub idle_line: idle::Line, #[cfg(feature = "switch-witness")] pub hw: hw::Local, } @@ -130,6 +132,7 @@ impl CpuLocal { pub fn new() -> Self { Self { counters: counters::Local::new(), + idle_line: idle::Line::new(), #[cfg(feature = "switch-witness")] hw: hw::Local::new(), } diff --git a/kernel/src/counters.rs b/kernel/src/counters.rs index 3322aca6403..25945279c71 100644 --- a/kernel/src/counters.rs +++ b/kernel/src/counters.rs @@ -140,6 +140,8 @@ fn sample(me: usize) -> [u64; WORDS] { Counter::EnergyPerfBias => hardware.envelope.as_ref().map(|e| e.energy_perf_bias), Counter::FirmwareCalls => hardware.firmware.as_ref().map(|f| f.calls), Counter::FirmwareNanos => hardware.firmware.as_ref().map(|f| f.nanos), + Counter::DeepIdles => Some(crate::cpu_local::of(me as u32).idle.deep().0), + Counter::DeepIdleNanos => Some(crate::cpu_local::of(me as u32).idle.deep().1), }; if let Some(value) = value { words[1] |= 1 << counter as usize; diff --git a/kernel/src/cpu_local.rs b/kernel/src/cpu_local.rs index 7bc5c79c6bc..539d4ff0f82 100644 --- a/kernel/src/cpu_local.rs +++ b/kernel/src/cpu_local.rs @@ -17,6 +17,7 @@ pub struct CpuLocal { pub hw: crate::hw::Local, pub irq_census: crate::irq_census::Local, pub counters: crate::counters::Local, + pub idle: crate::idle::Local, #[cfg(feature = "mask-windows")] pub windows: crate::windows::Local, } @@ -32,6 +33,7 @@ pub fn size(capacity: usize) { hw: crate::hw::Local::new(), irq_census: crate::irq_census::Local::new(), counters: crate::counters::Local::new(), + idle: crate::idle::Local::new(), #[cfg(feature = "mask-windows")] windows: crate::windows::Local::new(), }))); diff --git a/kernel/src/drivers/acpi.rs b/kernel/src/drivers/acpi.rs index c9445ad13a2..54ccdb4454e 100644 --- a/kernel/src/drivers/acpi.rs +++ b/kernel/src/drivers/acpi.rs @@ -19,6 +19,8 @@ pub use toyos_acpi::{IoApicEntry, SourceOverride, TableError}; pub struct MadtInfo { pub apic_ids: Vec, + /// Each enabled CPU's APIC ID and ACPI Processor UID. + pub processor_uids: Vec<(u32, u32)>, pub io_apics: Vec, pub source_overrides: Vec, } @@ -223,14 +225,16 @@ pub fn parse_madt(rsdp_addr: u64) -> Option { }; let mut apic_ids = Vec::new(); + let mut processor_uids = Vec::new(); let mut io_apics = Vec::new(); let mut source_overrides = Vec::new(); for item in toyos_acpi::madt_entries(&madt) { match item { - Ok(MadtEntry::LocalApic { apic_id, enabled }) => { + Ok(MadtEntry::LocalApic { apic_id, uid, enabled }) => { if enabled { apic_ids.push(apic_id); + processor_uids.push((apic_id, uid)); } } Ok(MadtEntry::IoApic(entry)) => io_apics.push(entry), @@ -255,5 +259,5 @@ pub fn parse_madt(rsdp_addr: u64) -> Option { } log!("ACPI: MADT cpus={:?}", apic_ids); - Some(MadtInfo { apic_ids, io_apics, source_overrides }) + Some(MadtInfo { apic_ids, processor_uids, io_apics, source_overrides }) } diff --git a/kernel/src/hw.rs b/kernel/src/hw.rs index 06cd961dd9c..4510b519d83 100644 --- a/kernel/src/hw.rs +++ b/kernel/src/hw.rs @@ -75,16 +75,17 @@ impl Machine for KernelHw { crate::trace::record(ev); } - /// Diagnostic builds arm a periodic wake before halting so a quiescent CPU still reports. + /// Waits in the state `crate::idle` chooses. Diagnostic builds arm a + /// periodic wake first, so a quiescent CPU still reports. fn idle_wait(&self, token: SleepToken) { - let _consumed = token; + let armed = token.armed(); #[cfg(feature = "boot-actuators")] if crate::actuator::diag_tick() { irqchip::arm_within(DIAG_TICK_NS); } #[cfg(feature = "mask-windows")] crate::windows::halting(); - self.halt(); + crate::idle::wait(armed); #[cfg(feature = "mask-windows")] crate::windows::woken(); // **A CPU that is executing has a one-shot armed, and this is where diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs new file mode 100644 index 00000000000..e272e0e7958 --- /dev/null +++ b/kernel/src/idle.rs @@ -0,0 +1,231 @@ +//! Which state an idle CPU waits in: the states the `acpi` claim's holder +//! declared for it, checked at the boundary (`toyos_userbound::idle`), chosen +//! among once per idle period by `kernel::sched::idle::choose`, and the +//! machine-wide wake bound that [`WakeLatency`] holds set. +//! +//! **A CPU with nothing declared waits in C1**, `hlt`, as it always has: that +//! is the machine before its AML server has spoken, not a failure. +//! +//! **Everything here on the idle path is the CPU's own and lock-free**: the +//! declaration is a seqlock its CPU reads, the bound one atomic word, and what +//! the CPU learned of its wakes ([`Own`]) touched by that CPU alone, between +//! the pass's final interrupts-off check and its next pass. + +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicU64, Ordering}; + +use kernel::sched::hw::Nanos; +use kernel::sched::idle::{self, Bounds, Cost, History, MAX_STATES}; +use toyos_abi::acpi::IdleDeclaration; +use toyos_abi::syscall::SyscallError; +use toyos_userbound::idle::{Entry, Table}; + +use crate::seqlock::Published; +use crate::sync::Lock; + +/// A declaration's words: a generation and the count, then two per state. +const WORDS: usize = 1 + 2 * MAX_STATES; + +/// Tells one declaration of a CPU from the next, so the CPU forgets what it +/// learned of the states it replaced. +static GENERATION: AtomicU64 = AtomicU64::new(1); + +/// A CPU's part. +pub struct Local { + /// Published by [`declare`], under the claim's lock; read by the CPU. + table: Published, + /// What the CPU can enter, read by it at its first idle; `None` before. + facts: Lock>, + own: OwnCell, + /// Idle periods spent in a declared state, and their nanoseconds. + deep: AtomicU64, + deep_nanos: AtomicU64, +} + +impl Local { + pub fn new() -> Self { + Self { + table: Published::new(), + facts: Lock::new(None), + own: OwnCell(UnsafeCell::new(Own { history: History::default(), generation: 0 })), + deep: AtomicU64::new(0), + deep_nanos: AtomicU64::new(0), + } + } + + /// Idle periods this CPU spent in a declared state, and their nanoseconds. + pub fn deep(&self) -> (u64, u64) { + (self.deep.load(Ordering::Relaxed), self.deep_nanos.load(Ordering::Relaxed)) + } +} + +struct Own { + history: History, + /// The declaration [`Self::history`] learned under. + generation: u64, +} + +struct OwnCell(UnsafeCell); + +// SAFETY: reached only through `own`, by the CPU whose record holds it, from its idle path. +unsafe impl Sync for OwnCell {} + +/// This CPU's [`Own`]. +/// +/// # Safety +/// The caller is this CPU's idle path, which nothing re-enters: an interrupt +/// taken on an idle CPU runs no pass (`scheduler::do_preempt`) and no idle +/// code. +unsafe fn own() -> &'static mut Own { + // SAFETY: as the function's contract. + unsafe { &mut *crate::cpu_local::here().idle.own.0.get() } +} + +/// Read what this CPU can enter: called by each CPU as it first enters its +/// idle loop, before any declaration for it is checked. +pub fn init_here() { + *crate::cpu_local::here().idle.facts.lock() = Some(crate::arch::idle::facts()); +} + +/// Take a processor's declaration from the `acpi` claim's holder: `NotFound` +/// where its UID names no CPU this kernel runs, which a machine declaring +/// more processor objects than it has CPUs gives often and is not said; +/// `InvalidArgument`, said by name, where the CPU cannot enter it as declared. +/// The holder's lock is held: one declaration at a time. +pub fn declare(declaration: &IdleDeclaration) -> Result<(), SyscallError> { + let uid = declaration.processor_uid; + let cpu = crate::arch::idle::cpu_of(uid).ok_or(SyscallError::NotFound)?; + let local = &crate::cpu_local::of(cpu).idle; + let facts = (*local.facts.lock()).ok_or(SyscallError::NotFound)?; + let table = toyos_userbound::idle::table(declaration, &facts).map_err(|why| { + log!("idle: cpu{cpu}'s declaration (processor UID {uid}) refused, so it waits in C1: {why:?}"); + SyscallError::InvalidArgument + })?; + local.table.publish(encode(&table, GENERATION.fetch_add(1, Ordering::Relaxed))); + log!("idle: cpu{cpu} (processor UID {uid}) waits in C1 or {}", Said(&table)); + Ok(()) +} + +struct Said<'a>(&'a Table); + +impl core::fmt::Display for Said<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + for (i, state) in self.0.states().enumerate() { + let sep = if i == 0 { "" } else { ", " }; + match state.entry { + Entry::Mwait { hint } => write!(f, "{sep}MWAIT {hint:#x}")?, + Entry::Port { port } => write!(f, "{sep}port {port:#x}")?, + } + write!(f, " (exit {} µs, residency {} µs)", state.exit_ns / 1000, state.residency_ns / 1000)?; + } + if self.0.is_empty() { + f.write_str("nothing deeper")?; + } + Ok(()) + } +} + +const MWAIT: u64 = 1; +const PORT: u64 = 2; + +fn encode(table: &Table, generation: u64) -> [u64; WORDS] { + let mut words = [0; WORDS]; + words[0] = generation << 8 | table.len() as u64; + for (i, state) in table.states().enumerate() { + let (kind, address) = match state.entry { + Entry::Mwait { hint } => (MWAIT, u64::from(hint)), + Entry::Port { port } => (PORT, u64::from(port)), + }; + words[1 + 2 * i] = kind | address << 8; + // Microseconds, as declared: each fits its half. + words[2 + 2 * i] = state.exit_ns / 1000 | (state.residency_ns / 1000) << 32; + } + words +} + +/// A CPU's states as [`encode`] wrote them, entry and cost each. +struct Declared { + generation: u64, + len: usize, + entries: [Entry; MAX_STATES], + costs: [Cost; MAX_STATES], +} + +fn decode(words: &[u64; WORDS]) -> Declared { + let len = (words[0] & 0xFF) as usize; + let mut entries = [Entry::Mwait { hint: 0 }; MAX_STATES]; + let mut costs = [Cost { exit: 0, residency: 0 }; MAX_STATES]; + for i in 0..len { + let (head, cost) = (words[1 + 2 * i], words[2 + 2 * i]); + entries[i] = match head & 0xFF { + MWAIT => Entry::Mwait { hint: (head >> 8) as u32 }, + PORT => Entry::Port { port: (head >> 8) as u16 }, + kind => panic!("idle: a published state of kind {kind}, which encode writes none of"), + }; + costs[i] = Cost { exit: (cost & 0xFFFF_FFFF) * 1000, residency: (cost >> 32) * 1000 }; + } + Declared { generation: words[0] >> 8, len, entries, costs } +} + +/// The least bound held, in nanoseconds; `u64::MAX` is none. +static LIMIT: AtomicU64 = AtomicU64::new(u64::MAX); +static BOUNDS: Lock = Lock::new(Bounds::new()); + +/// Wait, with interrupts masked and the scheduler's timer armed for `armed`, +/// in the state this idle period chooses; returns once an interrupt has been +/// taken. The kernel's `Machine::idle_wait`. +pub fn wait(armed: Option) { + // SAFETY: this is the CPU's idle path (`Machine::idle_wait`). + let own = unsafe { own() }; + let local = &crate::cpu_local::here().idle; + let now = crate::clock::nanos_since_boot(); + // A declaration mid-publication is this period in C1, and the next reads it. + let declared = local.table.snapshot().map(|words| decode(&words)); + let (len, costs) = declared.as_ref().map_or((0, &[][..]), |d| (d.len, &d.costs[..d.len])); + if let Some(d) = declared.as_ref().filter(|d| d.generation != own.generation) { + own.history.forget(); + own.generation = d.generation; + } + let horizon = armed.map(|at| at.0.saturating_sub(now)); + let limit = Some(LIMIT.load(Ordering::Relaxed)).filter(|&l| l != u64::MAX); + let choice = idle::choose(costs, &own.history, horizon, limit); + crate::sched::driver::publish_idle_exit(choice.state.map_or(0, |state| costs[state].exit)); + // The deadline met rather than made late by the state's exit: the timer + // fires that much early, and the pass it brings finds nothing due and + // waits again, shallower. A CPU its history held shallow wakes to look + // again sooner still, where that is sooner. + let fires = armed.map(|at| at.0.saturating_sub(choice.wake_early_by)); + let early = fires.filter(|_| choice.wake_early_by > 0); + let revisit = choice.revisit_after.map(|after| now.saturating_add(after)); + if let Some(at) = early.into_iter().chain(revisit).min() { + crate::arch::irqchip::arm_within(at.saturating_sub(now)); + } + match (choice.state, &declared) { + (Some(state), Some(d)) => crate::arch::idle::enter(d.entries[state]), + _ => crate::arch::hw::halt(), + } + let slept = crate::clock::nanos_since_boot().saturating_sub(now); + if choice.state.is_some() { + local.deep.fetch_add(1, Ordering::Relaxed); + local.deep_nanos.fetch_add(slept, Ordering::Relaxed); + } + // The scheduler's own timer ended the period where it fired at or after + // the instant it was armed for; a revisit is a period the history learns. + let foreseen = fires.is_some_and(|fires| now + slept >= fires); + if !foreseen && len > 0 { + own.history.unforeseen(costs, slept); + } +} + +/// A [`WakeLatency`] object's bound, held until it goes. +pub fn hold(bound: u64) { + let mut bounds = BOUNDS.lock(); + bounds.hold(bound); + LIMIT.store(bounds.least().unwrap_or(u64::MAX), Ordering::Relaxed); +} + +pub fn release(bound: u64) { + let mut bounds = BOUNDS.lock(); + bounds.release(bound); + LIMIT.store(bounds.least().unwrap_or(u64::MAX), Ordering::Relaxed); +} diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index cf1e05b6d78..3d3f6e6e13f 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -474,6 +474,7 @@ pub fn spawn_supervisor(images: crate::bootimage::Objects) -> Pid { .union(Rights::JOBS) .union(Rights::FLOOR) .union(Rights::RT) + .union(Rights::WAKE_LATENCY) .union(Rights::LOG) .union(Rights::WAIT) .union(Rights::POWER) diff --git a/kernel/src/main.rs b/kernel/src/main.rs index a1c30c9a701..ce9eda9a7b5 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -62,6 +62,7 @@ mod loader; mod scheduler; mod sched; mod hw; +mod idle; mod iommu; mod preempt; mod counters; diff --git a/kernel/src/object/mod.rs b/kernel/src/object/mod.rs index 80cf0e199c0..e5c4a94ddac 100644 --- a/kernel/src/object/mod.rs +++ b/kernel/src/object/mod.rs @@ -33,6 +33,7 @@ mod releases; pub mod service; pub mod shm; pub mod syscap; +pub mod wake_latency; pub use handle::{HandleEntry, HandleError, HandleTable, Refusal}; pub(crate) use releases::{Released, Releases}; @@ -279,6 +280,8 @@ kobject! { immediate Job => job::JobObject, // Its one handle is its process's, and goes with it. immediate Notice => notice::NoticeObject, + // What it holds is released by its last reference: nothing waits on it. + immediate WakeLatency => wake_latency::WakeLatencyObject, } /// Objects whose last handle has gone, waiting for release with nothing held; diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 6fdd32aa937..9d26dc52673 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -58,6 +58,7 @@ pub fn initial_rights(object: &KObjectRef) -> Rights { KObjectRef::Null(_) => super::null::NullObject::RIGHTS, KObjectRef::Job(_) => super::job::JobObject::RIGHTS, KObjectRef::Notice(_) => super::notice::NoticeObject::RIGHTS, + KObjectRef::WakeLatency(_) => super::wake_latency::WakeLatencyObject::RIGHTS, } } @@ -216,7 +217,7 @@ pub fn pipe_read(object: &KObjectRef) -> Option<(&PipeRef, WaitClass)> { KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => None, } @@ -231,7 +232,7 @@ pub fn pipe_write(object: &KObjectRef) -> Option<(&PipeRef, WaitClass, Landing)> KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => None, } @@ -292,7 +293,7 @@ pub fn read_watch(object: &KObjectRef) -> Option { // Named unconditionally: the watch alone cannot enforce rights. KObjectRef::SysCap(_) => Some(WatchRef::Static(&crate::log::user::WATCH)), KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, } } @@ -303,7 +304,7 @@ pub fn watched_readable(object: &KObjectRef) { match object { KObjectRef::Notice(n) => n.listen(), KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) | KObjectRef::Device(_) - | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SharedMem(_) | KObjectRef::Connector(_) + | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SharedMem(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::File(_) | KObjectRef::Console(_) | KObjectRef::SysCap(_) | KObjectRef::Process(_) | KObjectRef::Null(_) | KObjectRef::Job(_) => {} } @@ -318,7 +319,7 @@ pub fn write_watch(object: &KObjectRef) -> Option { KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => None, } @@ -333,7 +334,7 @@ pub fn pipe_end_watch(object: &KObjectRef) -> Option { KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) - | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) + | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, } } @@ -347,7 +348,7 @@ pub fn other_end_gone(object: &KObjectRef) -> bool { KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) - | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) + | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => false, } } @@ -372,7 +373,7 @@ fn close_ends_polls(object: &KObjectRef) -> bool { }, KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => true, } } @@ -579,7 +580,7 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { }), KObjectRef::PipeWrite(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) => Some(SyscallError::PermissionDenied.to_u64()), } } @@ -610,7 +611,7 @@ pub fn try_write(object: &KObjectRef, buf: &UserBytes) -> Option { KObjectRef::Null(_) => Some(buf.len() as u64), KObjectRef::PipeRead(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SharedMem(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) => { Some(SyscallError::PermissionDenied.to_u64()) } @@ -685,7 +686,7 @@ pub fn fstat(object: &KObjectRef) -> Stat { node: 0, }, KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) => plain(FileType::Unknown), KObjectRef::Device(d) => plain(match d.class() { device_registry::DeviceType::Framebuffer => FileType::Framebuffer, @@ -745,7 +746,7 @@ pub fn has_data(object: &KObjectRef) -> bool { device_registry::DeviceType::Framebuffer => true, }, KObjectRef::PipeWrite(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) => false, } } @@ -758,7 +759,7 @@ pub fn read_posts_are_readiness(object: &KObjectRef) -> bool { KObjectRef::SysCap(_) => true, KObjectRef::Console(_) | KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::File(_) | KObjectRef::Device(_) - | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => false, } @@ -774,7 +775,7 @@ pub fn has_space(object: &KObjectRef) -> bool { KObjectRef::Console(_) => crate::log::console::has_room(), KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) => false, } } @@ -793,7 +794,7 @@ pub fn mark_tty(object: &KObjectRef) -> u64 { KObjectRef::Connection(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::SharedMem(_) - | KObjectRef::Connector(_) | KObjectRef::Namespace(_) + | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::Namespace(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => { SyscallError::InvalidArgument.to_u64() } diff --git a/kernel/src/object/wake_latency.rs b/kernel/src/object/wake_latency.rs new file mode 100644 index 00000000000..4d3ba21560b --- /dev/null +++ b/kernel/src/object/wake_latency.rs @@ -0,0 +1,35 @@ +//! A hold on how long any idle CPU takes to wake: while one lives, no CPU waits +//! in a state slower to leave than the least bound held (`crate::idle`). It is +//! made through a `SysCap` carrying `WAKE_LATENCY` and lasts as long as the +//! object, so its last handle's going, the holder's death among them, ends it. + +use alloc::sync::Arc; + +use toyos_abi::handle::Rights; + +use crate::mm::charge::Exhausted; + +use super::{KObjectVariant, ObjectCore}; + +pub struct WakeLatencyObject { + pub(super) core: ObjectCore, + bound_ns: u64, +} + +impl WakeLatencyObject { + /// It is held, not used: a handle may be passed on and duplicated, and + /// does nothing else. + pub const RIGHTS: Rights = Rights::DUP.union(Rights::TRANSFER); + + pub fn new(bound_ns: u64) -> Result, Exhausted> { + let object = Arc::new(Self { core: Self::new_core(0)?, bound_ns }); + crate::idle::hold(bound_ns); + Ok(object) + } +} + +impl Drop for WakeLatencyObject { + fn drop(&mut self) { + crate::idle::release(self.bound_ns); + } +} diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 5825912e406..7d718891075 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -679,6 +679,7 @@ pub fn enter_idle_loop() -> ! { } extern "C" fn idle_loop() -> ! { + crate::idle::init_here(); loop { // The first NMI; its handler stages the nested one. #[cfg(feature = "boot-actuators")] @@ -761,6 +762,12 @@ pub fn current_kill_pending() -> bool { try_with_cpu(|cpu| cpu.running().is_some_and(|t| t.shared().kill_pending())).unwrap_or(false) } +/// How long this CPU takes to wake from the state it is about to wait in, for +/// a wake forward choosing between sleepers (`CpuHandles::idle_near`). +pub fn publish_idle_exit(nanos: u64) { + cpus().get(current_cpu()).publish_idle_exit(nanos); +} + pub fn current_cpu() -> CpuId { CpuId(percpu::cpu_id()) } diff --git a/kernel/src/syscall/device.rs b/kernel/src/syscall/device.rs index 94b0e957fbb..a0bfdf9f20c 100644 --- a/kernel/src/syscall/device.rs +++ b/kernel/src/syscall/device.rs @@ -78,6 +78,13 @@ pub(super) fn sys_acpi(ctx: &SyscallContext, handle: RawHandle, op: u64, at: u64 }, _ => None, }; + let declaration = match op { + ops::IDLE => match ctx.copy_in::(UserAddr::new(at)) { + Ok(declaration) => Some(declaration), + Err(e) => return e.to_u64(), + }, + _ => None, + }; let pid = process::current_process(); // The holder's check and the act are one borrow of the row, as a read's are. let done = claim.isa(|row| { @@ -91,6 +98,10 @@ pub(super) fn sys_acpi(ctx: &SyscallContext, handle: RawHandle, op: u64, at: u64 } (ops::LOCK_RELEASE, _) => crate::arch::acpi_mode::lock_release(row).map(|()| 0), (ops::S5, _) => crate::arch::acpi_mode::s5(row, at).map(|()| 0), + (ops::IDLE, _) => match &declaration { + Some(declaration) => crate::arch::acpi_mode::idle(row, declaration).map(|()| 0), + None => unreachable!("the declaration is copied in for IDLE"), + }, _ => Err(SyscallError::InvalidArgument), } }); diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 078810ab35f..355e228073c 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -48,7 +48,7 @@ use super::machine::{ use super::machine::lower_sysinfo_bound; use super::proc::{ spawn_place, sys_endowments, sys_exit, sys_nanosleep, sys_process_stats, - sys_account_info, sys_job_create, sys_job_floor, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, + sys_account_info, sys_job_create, sys_job_floor, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, sys_wake_latency_hold, }; use super::vm::{ shared_image, sys_dlopen, sys_dlsym, sys_query_modules, sys_tls_alloc_block, sys_vm_decommit, sys_vm_discard, @@ -623,6 +623,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> _ => SyscallError::InvalidArgument.to_u64(), }, SYS_JOB_FLOOR => sys_job_floor(RawHandle(a1 as u32), RawHandle(a2 as u32), a3), + SYS_WAKE_LATENCY_HOLD => sys_wake_latency_hold(RawHandle(a1 as u32), a2), SYS_ACCOUNT_INFO => { if a3 < core::mem::size_of::() as u64 { return SyscallError::InvalidArgument.to_u64(); } let Some(addr) = UserAddr::checked(a2) else { return bad_addr }; diff --git a/kernel/src/syscall/proc.rs b/kernel/src/syscall/proc.rs index 6d521901464..a68427b32a7 100644 --- a/kernel/src/syscall/proc.rs +++ b/kernel/src/syscall/proc.rs @@ -125,6 +125,25 @@ pub(super) fn sys_rt_enter(syscap: RawHandle) -> u64 { 0 } +/// Hold every idle CPU to waking within `bound_ns`, gated on a `SysCap` +/// carrying [`Rights::WAKE_LATENCY`], and install the hold's handle. +pub(super) fn sys_wake_latency_hold(syscap: RawHandle, bound_ns: u64) -> u64 { + if let Err(e) = demand_syscap(syscap, Rights::WAKE_LATENCY) { + return e.refuse(); + } + if bound_ns == 0 { + return SyscallError::InvalidArgument.to_u64(); + } + let object = match crate::object::wake_latency::WakeLatencyObject::new(bound_ns) { + Ok(object) => crate::object::KObjectRef::WakeLatency(object), + Err(e) => return SyscallError::from(e).to_u64(), + }; + match process::with_process_data(|data| crate::object::ops::install(&mut data.handles, object)) { + Ok(h) => u64::from(h.0), + Err(e) => e.to_u64(), + } +} + /// Make a new job, gated on a `SysCap` carrying [`Rights::JOBS`], and install a handle to it. pub(super) fn sys_job_create(syscap: RawHandle) -> u64 { if let Err(e) = demand_syscap(syscap, Rights::JOBS) { diff --git a/lib/acpi/src/madt.rs b/lib/acpi/src/madt.rs index 5c87c337133..c2c2814d589 100644 --- a/lib/acpi/src/madt.rs +++ b/lib/acpi/src/madt.rs @@ -35,7 +35,9 @@ pub struct SourceOverride { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum MadtEntry { /// Type 0 (Table 5.21) or type 9 (Table 5.27); `enabled` is flags bit 0. - LocalApic { apic_id: u32, enabled: bool }, + /// `uid` is its ACPI Processor UID, which a processor object's `_UID` or + /// ProcID names it by (ACPI 6.5 §5.2.12.2). + LocalApic { apic_id: u32, uid: u32, enabled: bool }, IoApic(IoApicEntry), SourceOverride(SourceOverride), /// Type 0xB (Table 5.37): one CPU's GIC CPU interface; `enabled` is flags bit 0. @@ -115,6 +117,7 @@ impl Iterator for MadtEntries

{ // Table 5.21: ACPI Processor UID (2), APIC ID (3), Flags (4..8). (0, 8..) => MadtEntry::LocalApic { apic_id: u32::from(phys.byte(base + 3)), + uid: u32::from(phys.byte(base + 2)), enabled: u32le(phys, base + 4) & 1 != 0, }, // Table 5.22: I/O APIC ID (2), Address (4..8), GSI Base (8..12). @@ -130,9 +133,10 @@ impl Iterator for MadtEntries

{ gsi: u32le(phys, base + 4), flags: u16le(phys, base + 8), }), - // Table 5.27: X2APIC ID (4..8), Flags (8..12). + // Table 5.27: X2APIC ID (4..8), Flags (8..12), ACPI Processor UID (12..16). (9, 16..) => MadtEntry::LocalApic { apic_id: u32le(phys, base + 4), + uid: u32le(phys, base + 12), enabled: u32le(phys, base + 8) & 1 != 0, }, // Table 5.37: ACPI Processor UID (8..12), Flags (12..16), GICR Base diff --git a/lib/acpi/tests/corpus.rs b/lib/acpi/tests/corpus.rs index 64ea5b48ec9..77f6fa305f4 100644 --- a/lib/acpi/tests/corpus.rs +++ b/lib/acpi/tests/corpus.rs @@ -138,7 +138,7 @@ fn a_madt_entry_of_zero_length_halts_the_walk_instead_of_looping() { assert_eq!( madt_entries(&table).take(8).collect::>(), [ - Ok(MadtEntry::LocalApic { apic_id: 0, enabled: true }), + Ok(MadtEntry::LocalApic { apic_id: 0, uid: 0, enabled: true }), Err(MadtHalt { at: 8, declared: 0, list_len: 18 }), ], "the entry after the halt is unreachable: a list that lied cannot be resynchronised" @@ -181,7 +181,7 @@ fn a_trailing_byte_that_cannot_be_an_entry_header_ends_the_walk_quietly() { let table = find_table(Machine { regions }, RSDP_AT, b"APIC", MADT_ENTRIES).expect("MADT"); assert_eq!( madt_entries(&table).collect::>(), - [Ok(MadtEntry::LocalApic { apic_id: 3, enabled: true })] + [Ok(MadtEntry::LocalApic { apic_id: 3, uid: 0, enabled: true })] ); } diff --git a/lib/acpi/tests/fixtures.rs b/lib/acpi/tests/fixtures.rs index 7626d970fbe..89399770910 100644 --- a/lib/acpi/tests/fixtures.rs +++ b/lib/acpi/tests/fixtures.rs @@ -43,7 +43,7 @@ fn the_madt_names_the_two_cpus_that_boot_and_the_chip_that_interrupts_them() { let mut overrides = Vec::new(); for item in madt_entries(&madt) { match item.expect("a table QEMU published walks to its end") { - MadtEntry::LocalApic { apic_id, enabled } => { + MadtEntry::LocalApic { apic_id, enabled, .. } => { if enabled { cpus.push(apic_id); } diff --git a/lib/manifest/src/lib.rs b/lib/manifest/src/lib.rs index f0f137d34ab..d2ee73e9742 100644 --- a/lib/manifest/src/lib.rs +++ b/lib/manifest/src/lib.rs @@ -231,6 +231,10 @@ const SYSCAP_RIGHTS: &[(&str, Rights)] = &[ // a launcher's. Every other child joins its parent's job, so this is what // lets a program gain aggregate CPU by starting many. ("jobs", Rights::JOBS), + // Hold every idle CPU to waking within a bound: what an audio or other + // deadline-bound service holds while it has a deadline, paid for in the + // whole machine's power. + ("wake-latency", Rights::WAKE_LATENCY), ]; /// The whole right set a program's `syscap` list asks for. diff --git a/lib/userbound/src/idle.rs b/lib/userbound/src/idle.rs new file mode 100644 index 00000000000..1320ca05f48 --- /dev/null +++ b/lib/userbound/src/idle.rs @@ -0,0 +1,261 @@ +//! A processor's idle states as the `acpi` claim's holder declared them +//! (`toyos_abi::acpi::op::IDLE`), checked against the CPU that will enter +//! them before the kernel keeps any. +//! +//! **The holder is trusted platform code, and these are bug fences**: what a +//! state means is the firmware's and the holder's, and the holder hands over +//! only states that need nothing around their entry. What is checked here is +//! that the CPU can execute the entry and that it touches nothing: +//! +//! - an `MWAIT` hint names a sub-state CPUID leaf 5 enumerates, on a CPU with +//! MONITOR/MWAIT and its extensions (Intel SDM Vol. 2B, MWAIT, Table 1-3); +//! - a port is one the core itself traps as a C-state request and no device +//! sees: `CStateBaseAddr`'s eight (AMD PPR 55922, MSRC001_0073); +//! - the CPU keeps time in any state: its APIC timer always runs (CPUID.06H: +//! EAX[2]) and its TSC is invariant (CPUID.80000007H:EDX[8]). Without either a +//! deep state stops the clock or the one-shot, so a CPU lacking one keeps +//! only C1, whatever is declared. + +use toyos_abi::acpi::{IdleDeclaration, IdleEntry, IDLE_STATES}; + +/// What decides whether one CPU can enter a declared state, read on that CPU. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub struct CpuIdle { + /// CPUID.01H:ECX[3]. + pub monitor: bool, + /// CPUID.05H:ECX[0], MWAIT's extensions enumerated. + pub mwait_extensions: bool, + /// CPUID.05H:EDX: four bits of sub-state count per C-state, C0 lowest. + pub mwait_substates: u32, + /// CPUID.06H:EAX[2]. + pub always_running_timer: bool, + /// CPUID.80000007H:EDX[8]. + pub invariant_tsc: bool, + /// The first of the eight ports the core traps for C-state entry, where it + /// traps any: `CStateBaseAddr` on AMD, nonzero. + pub trapped_ports: Option, +} + +/// How a checked state is entered. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Entry { + Mwait { hint: u32 }, + Port { port: u16 }, +} + +/// One checked state, its costs in nanoseconds. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct State { + pub entry: Entry, + pub exit_ns: u64, + pub residency_ns: u64, +} + +/// A CPU's checked states, shallowest first. +/// +/// ```compile_fail,E0451 +/// let _ = toyos_userbound::idle::Table { states: [None; 8] }; +/// ``` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Table { + states: [Option; IDLE_STATES], +} + +impl Table { + /// No state but C1. + pub const EMPTY: Self = Self { states: [None; IDLE_STATES] }; + + pub fn states(&self) -> impl Iterator { + self.states.iter().map_while(Option::as_ref) + } + + pub fn len(&self) -> usize { + self.states().count() + } + + pub fn is_empty(&self) -> bool { + self.states[0].is_none() + } +} + +/// Why a declaration was not kept; `state` is its index in the declaration. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Refused { + /// More states than [`IDLE_STATES`]. + Count(u32), + /// The CPU's APIC timer stops in a deep state. + TimerStops, + /// The CPU's TSC is not invariant. + TscVaries, + /// An entry this ABI does not name, or a reserved byte set. + Malformed { state: usize }, + /// An `MWAIT` state on a CPU without MONITOR/MWAIT and its extensions. + NoMwait { state: usize }, + /// A hint with reserved bits, or a sub-state leaf 5 does not enumerate. + Hint { state: usize, hint: u64 }, + /// A port the core does not trap for C-state entry. + Port { state: usize, port: u64 }, + /// A state that claims to wake in no time. + NoLatency { state: usize }, +} + +/// `declaration` checked for the CPU `cpu` describes. +pub fn table(declaration: &IdleDeclaration, cpu: &CpuIdle) -> Result { + let count = declaration.count as usize; + if count > IDLE_STATES { + return Err(Refused::Count(declaration.count)); + } + let mut table = Table::EMPTY; + if count == 0 { + return Ok(table); + } + if !cpu.always_running_timer { + return Err(Refused::TimerStops); + } + if !cpu.invariant_tsc { + return Err(Refused::TscVaries); + } + for (state, declared) in declaration.states[..count].iter().enumerate() { + if declared.reserved != [0; 7] { + return Err(Refused::Malformed { state }); + } + let entry = match IdleEntry::from_raw(declared.entry).ok_or(Refused::Malformed { state })? { + IdleEntry::Mwait => { + if !cpu.monitor || !cpu.mwait_extensions { + return Err(Refused::NoMwait { state }); + } + Entry::Mwait { hint: mwait_hint(declared.address, cpu.mwait_substates).ok_or(Refused::Hint { state, hint: declared.address })? } + } + IdleEntry::Port => { + let trapped = cpu.trapped_ports.is_some_and(|base| (u64::from(base)..u64::from(base) + 8).contains(&declared.address)); + if !trapped { + return Err(Refused::Port { state, port: declared.address }); + } + Entry::Port { port: declared.address as u16 } + } + }; + if declared.latency_us == 0 { + return Err(Refused::NoLatency { state }); + } + table.states[state] = Some(State { + entry, + exit_ns: u64::from(declared.latency_us) * 1000, + residency_ns: u64::from(declared.residency_us) * 1000, + }); + } + Ok(table) +} + +/// `hint` where leaf 5 enumerates its sub-state: bits 7:4 the target C-state +/// less one (15 is C0), bits 3:0 the sub-state, the rest reserved (SDM Table +/// 1-3); `substates` holds C0's count in bits 3:0, C1's in 7:4, and so on. +fn mwait_hint(hint: u64, substates: u32) -> Option { + if hint > 0xFF { + return None; + } + let (target, sub) = ((hint >> 4) as u32, (hint & 0xF) as u32); + let nibble = target + 1; + if nibble >= 8 { + return None; + } + (sub < substates >> (4 * nibble) & 0xF).then_some(hint as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + use toyos_abi::acpi::IdleState; + + /// An Intel CPU whose leaf 5 counts sub-states C0 0, C1 2, C2 1, C3 2, + /// C4 2, C5 0, C6 0, C7 1: hint 0x31 (C4's second) and 0x60 (C7's first), + /// the T14's `_CST` hints, exist on it. + const INTEL: CpuIdle = CpuIdle { + monitor: true, + mwait_extensions: true, + mwait_substates: 0x1002_2120, + always_running_timer: true, + invariant_tsc: true, + trapped_ports: None, + }; + + /// A Zen 2's: MWAIT reaches C1 alone (PPR 55922, CPUID_Fn00000005_EDX + /// 0000_0011h), and the core traps 0x413 to 0x41A. + const ZEN_2: CpuIdle = CpuIdle { + monitor: true, + mwait_extensions: true, + mwait_substates: 0x11, + always_running_timer: true, + invariant_tsc: true, + trapped_ports: Some(0x413), + }; + + fn state(entry: IdleEntry, address: u64, latency_us: u32) -> IdleState { + IdleState { address, latency_us, residency_us: 2 * latency_us, entry: entry as u8, reserved: [0; 7] } + } + + fn declaring(states: &[IdleState]) -> IdleDeclaration { + let mut d = IdleDeclaration { processor_uid: 0, count: states.len() as u32, states: [state(IdleEntry::Mwait, 0, 0); IDLE_STATES] }; + d.states[..states.len()].copy_from_slice(states); + d + } + + #[test] + fn the_t14s_two_states_are_kept_as_declared() { + let t = table(&declaring(&[state(IdleEntry::Mwait, 0x31, 253), state(IdleEntry::Mwait, 0x60, 1048)]), &INTEL).unwrap(); + let mut states = [None; 2]; + for (slot, kept) in states.iter_mut().zip(t.states()) { + *slot = Some(*kept); + } + assert_eq!(t.len(), 2); + assert_eq!( + states.map(Option::unwrap), + [ + State { entry: Entry::Mwait { hint: 0x31 }, exit_ns: 253_000, residency_ns: 506_000 }, + State { entry: Entry::Mwait { hint: 0x60 }, exit_ns: 1_048_000, residency_ns: 2_096_000 }, + ] + ); + } + + #[test] + fn a_hint_leaf_5_does_not_enumerate_is_refused() { + // C3 has two sub-states, so 0x22 names a third; C6 has none; a target + // of 7 would be C8, past leaf 5; 0xF0 is C0; 0x100 sets a reserved bit. + for hint in [0x22, 0x50, 0x70, 0xF0, 0x100] { + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, hint, 10)]), &INTEL), Err(Refused::Hint { state: 0, hint }), "{hint:#x}"); + } + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 10)]), &ZEN_2), Err(Refused::Hint { state: 0, hint: 0x31 })); + } + + #[test] + fn the_amds_trapped_port_is_kept_and_any_other_refused() { + let kept = table(&declaring(&[state(IdleEntry::Port, 0x414, 400)]), &ZEN_2).unwrap(); + assert_eq!(kept.states().next().unwrap().entry, Entry::Port { port: 0x414 }); + for port in [0x412, 0x41B, 0x1816, 0x1_0414] { + assert_eq!(table(&declaring(&[state(IdleEntry::Port, port, 400)]), &ZEN_2), Err(Refused::Port { state: 0, port }), "{port:#x}"); + } + assert_eq!(table(&declaring(&[state(IdleEntry::Port, 0x414, 400)]), &INTEL), Err(Refused::Port { state: 0, port: 0x414 })); + } + + #[test] + fn a_cpu_whose_timer_or_clock_stops_keeps_c1_alone() { + let d = declaring(&[state(IdleEntry::Mwait, 0x31, 253)]); + assert_eq!(table(&d, &CpuIdle { always_running_timer: false, ..INTEL }), Err(Refused::TimerStops)); + assert_eq!(table(&d, &CpuIdle { invariant_tsc: false, ..INTEL }), Err(Refused::TscVaries)); + assert_eq!(table(&declaring(&[]), &CpuIdle::default()), Ok(Table::EMPTY), "nothing declared needs nothing of the CPU"); + } + + #[test] + fn a_malformed_declaration_is_refused_whole() { + let mut d = declaring(&[state(IdleEntry::Mwait, 0x31, 253)]); + d.count = 9; + assert_eq!(table(&d, &INTEL), Err(Refused::Count(9))); + let mut bad = state(IdleEntry::Mwait, 0x31, 253); + bad.entry = 3; + assert_eq!(table(&declaring(&[bad]), &INTEL), Err(Refused::Malformed { state: 0 })); + let mut bad = state(IdleEntry::Mwait, 0x31, 253); + bad.reserved[6] = 1; + assert_eq!(table(&declaring(&[bad]), &INTEL), Err(Refused::Malformed { state: 0 })); + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 0)]), &INTEL), Err(Refused::NoLatency { state: 0 })); + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 1)]), &CpuIdle { mwait_extensions: false, ..INTEL }), Err(Refused::NoMwait { state: 0 })); + } +} diff --git a/lib/userbound/src/lib.rs b/lib/userbound/src/lib.rs index d046e14cb9d..d7a97f0be45 100644 --- a/lib/userbound/src/lib.rs +++ b/lib/userbound/src/lib.rs @@ -25,6 +25,7 @@ #![forbid(unsafe_code)] pub mod fault; +pub mod idle; pub mod firmware; pub mod port; pub mod span; diff --git a/sdk/abi/src/acpi.rs b/sdk/abi/src/acpi.rs index 304d9355ed7..4febd670d99 100644 --- a/sdk/abi/src/acpi.rs +++ b/sdk/abi/src/acpi.rs @@ -29,6 +29,11 @@ //! ([`op::S5`]): what `\_S5` evaluates to is in the firmware's AML, and //! until a holder has supplied it [`crate::syscall::SYS_SHUTDOWN`] is refused //! [`crate::syscall::SyscallError::NotSupported`]. +//! +//! **So are the idle states deeper than C1, per processor** ([`op::IDLE`]): +//! what `_CST` names is in the AML too. The holder hands over only states the +//! kernel may enter as they are ([`IdleDeclaration`]); until it has, every CPU +//! waits in C1. crate::user_safe! { /// A run of ports a register block occupies; `len` 0 is no block. @@ -88,6 +93,11 @@ pub mod op { /// control to power the machine off. Once under a claim, and kept until /// the next claim's holder supplies its own. pub const S5: u64 = 3; + /// Declare one processor's idle states deeper than C1, an + /// [`IdleDeclaration`](super::IdleDeclaration) at the call's third word. + /// Each replaces what was declared for that processor before, by this + /// holder or an earlier one. + pub const IDLE: u64 = 4; pub const TAKEN: u64 = 0; pub const PENDING: u64 = 1; @@ -291,3 +301,56 @@ impl Access { Self { address, value, space: space as u8, width: width as u8, write: 1, refused: 0, memory_type: UNLISTED, reserved: [0; 3] } } } + +/// The most states an [`IdleDeclaration`] holds. +pub const IDLE_STATES: usize = 8; + +/// How a CPU enters an [`IdleState`]. +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IdleEntry { + /// `MWAIT` with [`IdleState::address`] as its hint (Intel 302223-007 + /// Table 4, the native C-state instruction). + Mwait = 1, + /// A byte read of the port [`IdleState::address`] names. + Port = 2, +} + +impl IdleEntry { + pub const fn from_raw(raw: u8) -> Option { + match raw { + 1 => Some(Self::Mwait), + 2 => Some(Self::Port), + _ => None, + } + } +} + +crate::user_safe! { + /// One idle state a CPU may enter as it is: no cache flush, bus-master + /// check or coordination with another CPU is owed around it. + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub struct IdleState { + /// The hint or the port, as [`Self::entry`] reads it. + pub address: u64, + /// The worst case from a wake to running again, in microseconds. + pub latency_us: u32, + /// The shortest idle period worth entering it for, in microseconds. + pub residency_us: u32, + /// An [`IdleEntry`]. + pub entry: u8, + pub reserved: [u8; 7], + } +} + +crate::user_safe! { + /// One processor's states, shallowest first: the processor named by its + /// ACPI Processor UID, as the MADT names it (ACPI 6.5 §5.2.12.2). + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub struct IdleDeclaration { + pub processor_uid: u32, + /// How many of [`Self::states`] are declared; 0 leaves the processor in C1. + pub count: u32, + pub states: [IdleState; IDLE_STATES], + } +} diff --git a/sdk/abi/src/counters.rs b/sdk/abi/src/counters.rs index d9526c30a3e..82bf72375fd 100644 --- a/sdk/abi/src/counters.rs +++ b/sdk/abi/src/counters.rs @@ -50,10 +50,15 @@ pub enum Counter { /// its return: the firmware's handler, where a write raises an interrupt /// to it. FirmwareNanos, + /// Idle periods the CPU waited in a state deeper than C1 its firmware + /// declared. + DeepIdles, + /// Nanoseconds those periods lasted, each from the choice to the wake. + DeepIdleNanos, } impl Counter { - pub const COUNT: usize = 10; + pub const COUNT: usize = 12; pub const ALL: [Counter; Self::COUNT] = [ Self::Stamp, Self::Smi, @@ -65,6 +70,8 @@ impl Counter { Self::EnergyPerfBias, Self::FirmwareCalls, Self::FirmwareNanos, + Self::DeepIdles, + Self::DeepIdleNanos, ]; pub const fn name(self) -> &'static str { @@ -79,6 +86,8 @@ impl Counter { Self::EnergyPerfBias => "energy_perf_bias", Self::FirmwareCalls => "firmware_calls", Self::FirmwareNanos => "firmware_nanos", + Self::DeepIdles => "deep_idles", + Self::DeepIdleNanos => "deep_idle_nanos", } } @@ -91,7 +100,9 @@ impl Counter { | Self::Kicks | Self::HwpRequest | Self::HwpRequestPkg - | Self::EnergyPerfBias => Rights::COUNTERS.union(Rights::TRACE), + | Self::EnergyPerfBias + | Self::DeepIdles + | Self::DeepIdleNanos => Rights::COUNTERS.union(Rights::TRACE), } } } diff --git a/sdk/abi/src/handle.rs b/sdk/abi/src/handle.rs index 0f96241d5dc..9f232ba02bf 100644 --- a/sdk/abi/src/handle.rs +++ b/sdk/abi/src/handle.rs @@ -78,10 +78,11 @@ pub enum ObjectKind { Null = 13, Job = 14, Notice = 15, + WakeLatency = 16, } impl ObjectKind { - pub const ALL: [ObjectKind; 16] = [ + pub const ALL: [ObjectKind; 17] = [ Self::PipeRead, Self::PipeWrite, Self::Connection, @@ -98,6 +99,7 @@ impl ObjectKind { Self::Null, Self::Job, Self::Notice, + Self::WakeLatency, ]; pub const fn from_word(word: u32) -> Option { @@ -238,10 +240,18 @@ impl Rights { /// /// [`SYS_JOB_FLOOR`]: crate::syscall::SYS_JOB_FLOOR pub const FLOOR: Rights = Rights(1 << 16); + /// On a `SysCap`: hold every idle CPU to waking within a bound + /// ([`SYS_WAKE_LATENCY_HOLD`]). What it costs is the whole machine's + /// power, every CPU kept out of its deeper states while the hold lives, + /// which is why it rides a bit of its own and not [`RT`](Self::RT)'s: + /// one takes CPU from other programs, this takes power from the machine. + /// + /// [`SYS_WAKE_LATENCY_HOLD`]: crate::syscall::SYS_WAKE_LATENCY_HOLD + pub const WAKE_LATENCY: Rights = Rights(1 << 17); /// Every bit that has a caller. A wider set than this is a bug in whoever /// composed it, not a right nobody uses. - pub const ALL: Rights = Rights(0x1_ffff); + pub const ALL: Rights = Rights(0x3_ffff); pub const fn from_bits(bits: u32) -> Option { if bits & !Self::ALL.0 == 0 { Some(Rights(bits)) } else { None } @@ -272,7 +282,7 @@ impl Rights { /// refusal saying which right was missing. impl core::fmt::Debug for Rights { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - const NAMES: [(Rights, &str); 17] = [ + const NAMES: [(Rights, &str); 18] = [ (Rights::DUP, "DUP"), (Rights::TRANSFER, "TRANSFER"), (Rights::READ, "READ"), @@ -290,6 +300,7 @@ impl core::fmt::Debug for Rights { (Rights::TRACE, "TRACE"), (Rights::JOBS, "JOBS"), (Rights::FLOOR, "FLOOR"), + (Rights::WAKE_LATENCY, "WAKE_LATENCY"), ]; if self.0 == 0 { return f.write_str("NONE"); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 3ee7518eb80..839262fd877 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -333,6 +333,11 @@ pub const SYS_SHM_SUB: u64 = 143; pub const SYS_JOB_FLOOR: u64 = 144; /// What a job's account holds. See [`account_info`]. pub const SYS_ACCOUNT_INFO: u64 = 145; +/// Hold every idle CPU to waking within a bound, gated by +/// [`Rights::WAKE_LATENCY`] on a `SysCap`. See [`wake_latency_hold`]. +/// +/// [`Rights::WAKE_LATENCY`]: crate::handle::Rights::WAKE_LATENCY +pub const SYS_WAKE_LATENCY_HOLD: u64 = 146; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -340,13 +345,13 @@ pub const SYS_ACCOUNT_INFO: u64 = 145; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 147; +pub const SYSCALL_PROFILE_BINS: usize = 148; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_ACCOUNT_INFO < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_WAKE_LATENCY_HOLD < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -1756,6 +1761,16 @@ pub fn job_floor(syscap: RawHandle, job: RawHandle, bytes: u64) -> Result<(), Sy check_unit(syscall(SYS_JOB_FLOOR, syscap.0 as u64, job.0 as u64, bytes, 0)) } +/// Hold every idle CPU to waking within `bound_ns` for as long as the +/// returned handle lives: no CPU waits in a state slower to leave than the +/// least bound held. Needs [`Rights::WAKE_LATENCY`]; a bound of 0 is +/// refused [`SyscallError::InvalidArgument`]. +/// +/// [`Rights::WAKE_LATENCY`]: crate::handle::Rights::WAKE_LATENCY +pub fn wake_latency_hold(syscap: RawHandle, bound_ns: u64) -> Result { + check(syscall(SYS_WAKE_LATENCY_HOLD, syscap.0 as u64, bound_ns, 0, 0)).map(|raw| RawHandle(raw as u32)) +} + /// What the account of the job `job` names holds, its handle carrying /// `READ`. pub fn account_info(job: RawHandle, info: &mut AccountInfo) -> Result<(), SyscallError> { @@ -2259,6 +2274,14 @@ pub fn acpi_s5(claim: RawHandle, slp_typ_a: u64) -> Result<(), SyscallError> { check_unit(syscall(SYS_ACPI, claim.0 as u64, crate::acpi::op::S5, slp_typ_a, 0)) } +/// Declare one processor's idle states ([`crate::acpi::op::IDLE`]). +/// `InvalidArgument` is a declaration the kernel will not enter on that +/// processor, `NotFound` a UID that names no CPU it runs, and its log line +/// says which; nothing of a refused one is kept. +pub fn acpi_idle(claim: RawHandle, declaration: &crate::acpi::IdleDeclaration) -> Result<(), SyscallError> { + check_unit(syscall(SYS_ACPI, claim.0 as u64, crate::acpi::op::IDLE, declaration as *const crate::acpi::IdleDeclaration as u64, 0)) +} + /// Sleep for the given number of nanoseconds. pub fn nanosleep(nanos: u64) { syscall(SYS_NANOSLEEP, nanos, 0, 0, 0); diff --git a/sdk/toyos/src/device.rs b/sdk/toyos/src/device.rs index 2e0bbe5acaa..8023b42a353 100644 --- a/sdk/toyos/src/device.rs +++ b/sdk/toyos/src/device.rs @@ -214,6 +214,12 @@ impl AcpiDev { pub fn s5(&self, slp_typ_a: u64) -> Result<(), SyscallError> { syscall::acpi_s5(self.as_handle(), slp_typ_a) } + + /// Declare one processor's idle states deeper than C1 + /// ([`syscall::acpi_idle`]). + pub fn idle(&self, declaration: &toyos_abi::acpi::IdleDeclaration) -> Result<(), SyscallError> { + syscall::acpi_idle(self.as_handle(), declaration) + } } impl AsHandle for AcpiDev { diff --git a/sdk/toyos/src/syscap.rs b/sdk/toyos/src/syscap.rs index d6dde469a87..d40b0265f5d 100644 --- a/sdk/toyos/src/syscap.rs +++ b/sdk/toyos/src/syscap.rs @@ -16,6 +16,10 @@ use crate::{AsHandle, OwnedHandle, RawHandle}; pub struct SysCap(pub(crate) OwnedHandle); +/// A bound on how long any idle CPU takes to wake, held until dropped +/// ([`SysCap::hold_wake_latency`]). +pub struct WakeLatencyHold(#[allow(dead_code, reason = "held for its drop, which closes the handle")] OwnedHandle); + impl SysCap { /// Mint the claim for a device class, as whichever typed wrapper the caller /// drives it through. @@ -91,6 +95,16 @@ impl SysCap { syscall::rt_enter(self.0.raw()) } + /// Hold every idle CPU to waking within `bound` until the returned hold + /// is dropped. Needs [`Rights::WAKE_LATENCY`]; a bound under a + /// nanosecond is refused `InvalidArgument`. + pub fn hold_wake_latency(&self, bound: core::time::Duration) -> Result { + let nanos = u64::try_from(bound.as_nanos()).unwrap_or(u64::MAX); + let raw = syscall::wake_latency_hold(self.0.raw(), nanos)?; + // SAFETY: the kernel installed this handle for this call and no other. + Ok(WakeLatencyHold(unsafe { OwnedHandle::from_raw(raw) })) + } + /// Power the machine off. /// /// **Returns only when refused**, because a shutdown that happened has no diff --git a/system/acpiserver/aml/src/exec.rs b/system/acpiserver/aml/src/exec.rs index 340f134ab21..ea7ab1c4a6b 100644 --- a/system/acpiserver/aml/src/exec.rs +++ b/system/acpiserver/aml/src/exec.rs @@ -825,7 +825,7 @@ impl<'a> Machine<'a> { // Processor, a Thermal Zone or a Power Resource. if !matches!( self.node_object(id)?, - Object::Scope | Object::Device | Object::Processor | Object::ThermalZone | Object::PowerResource + Object::Scope | Object::Device | Object::Processor(_) | Object::ThermalZone | Object::PowerResource ) { return Err(Error::Type("a Scope names an object that opens no scope (§19.6.120)")); } @@ -856,10 +856,10 @@ impl<'a> Machine<'a> { // DefProcessor := ProcessorOp PkgLength NameString ProcID // PblkAddr PblkLen TermList (ACPI 6.3A §20.2.5.2), parsed by // the owner's ruling to accept what real firmware ships. - body.byte()?; + let id = body.byte()?; body.dword()?; body.byte()?; - Object::Processor + Object::Processor(id) } _ => { // SystemLevel and ResourceOrder, which only OSPM's power @@ -1073,7 +1073,7 @@ impl<'a> Machine<'a> { let v = self.int_arg(f, c)?; let id = self.node_of(f, &t)?; // §19.6.94: a device, processor, or thermal zone. - if !matches!(self.node_object(id)?, Object::Device | Object::Processor | Object::ThermalZone) { + if !matches!(self.node_object(id)?, Object::Device | Object::Processor(_) | Object::ThermalZone) { return Err(Error::Type("Notify of an object that is not a device, processor or thermal zone (§19.6.94)")); } let path = self.path_of(id, None)?; diff --git a/system/acpiserver/aml/src/lib.rs b/system/acpiserver/aml/src/lib.rs index f34e07545f9..613e782524e 100644 --- a/system/acpiserver/aml/src/lib.rs +++ b/system/acpiserver/aml/src/lib.rs @@ -349,8 +349,10 @@ pub enum Kind { /// In this address space (Table 5.182). OperationRegion { space: u8 }, PowerResource, - /// The Processor object ACPI 6.4 deprecated (ACPI 6.3A §19.6.108). - Processor, + /// The Processor object ACPI 6.4 deprecated (ACPI 6.3A §19.6.108). `id` + /// is its ProcID, which the MADT's ACPI Processor UID names it by (ACPI + /// 6.5 §5.2.12.2). + Processor { id: u8 }, ThermalZone, BufferField, /// `\_SB` and the other scopes of §5.3.1: descended, and no device. @@ -404,7 +406,7 @@ impl Iterator for Walk<'_> { Object::Ref(_) | Object::Lazy(_) => Kind::Reference, Object::Scope => Kind::Scope, Object::Device => Kind::Device, - Object::Processor => Kind::Processor, + Object::Processor(id) => Kind::Processor { id: *id }, Object::ThermalZone => Kind::ThermalZone, Object::PowerResource => Kind::PowerResource, Object::Method(_) => Kind::Method, diff --git a/system/acpiserver/aml/src/object.rs b/system/acpiserver/aml/src/object.rs index c81041391b5..356cad25ae2 100644 --- a/system/acpiserver/aml/src/object.rs +++ b/system/acpiserver/aml/src/object.rs @@ -243,8 +243,9 @@ pub(crate) enum Object { /// A predefined scope such as `\_SB` (§5.3.1), typeless (§19.6.96). Scope, Device, - /// The Processor object ACPI 6.4 deprecated (ACPI 6.3A §19.6.108). - Processor, + /// The Processor object ACPI 6.4 deprecated (ACPI 6.3A §19.6.108), with + /// its ProcID. + Processor(u8), ThermalZone, PowerResource, Method(Kept), @@ -316,7 +317,7 @@ impl Object { Object::Mutex(_) => 9, Object::Region(_) => 10, Object::PowerResource => 11, - Object::Processor => 12, + Object::Processor(_) => 12, Object::ThermalZone => 13, Object::BufField(_) => 14, } diff --git a/system/acpiserver/aml/tests/walk.rs b/system/acpiserver/aml/tests/walk.rs index 47f07a0095f..e26a90f616c 100644 --- a/system/acpiserver/aml/tests/walk.rs +++ b/system/acpiserver/aml/tests/walk.rs @@ -168,7 +168,7 @@ fn every_object_is_walked_as_what_it_is() { assert_eq!(at("\\_SB_"), (1, Kind::Scope)); assert_eq!(at("\\_SB_.PCI0"), (2, Kind::Device)); assert_eq!(at("\\_SB_.PCI0._INI"), (3, Kind::Method)); - assert_eq!(at("\\_PR_.CPU0"), (2, Kind::Processor)); + assert_eq!(at("\\_PR_.CPU0"), (2, Kind::Processor { id: 1 })); assert_eq!(at("\\_TZ_.TZ00"), (2, Kind::ThermalZone)); assert_eq!(at("\\PWR0"), (1, Kind::PowerResource)); assert_eq!(at("\\STR0"), (1, Kind::String)); diff --git a/system/acpiserver/src/cst.rs b/system/acpiserver/src/cst.rs new file mode 100644 index 00000000000..b063dd49106 --- /dev/null +++ b/system/acpiserver/src/cst.rs @@ -0,0 +1,413 @@ +//! The idle states each processor's `_CST` names (ACPI 6.5 §8.4.1.1), handed +//! to the kernel as the states it may enter as they are +//! (`toyos_abi::acpi::op::IDLE`), and again on the processor's +//! `Notify(…, 0x81)`. +//! +//! **What a state means is ACPI's and its vendor's, and is decided here**; the +//! kernel checks only that its CPU can execute the entry. A state is handed +//! over where nothing is owed around its entry, and withheld by name where +//! something is: +//! +//! - type 1 is C1, which every CPU enters with `hlt` (§8.1.2) and is never +//! handed over; +//! - an Intel FFH state (302223-007 Table 4: vendor 1, class 2) is `MWAIT` +//! with Arg0 as its hint, withheld where Arg1 bit 1 asks bus-master +//! avoidance, which ToyOS does not track; any other FFH encoding has no +//! document in hand; +//! - a SystemIO state is a byte read of its register, handed over off Intel, +//! where the core traps it (the kernel checks which ports); on Intel it is a +//! chipset's P_LVLx read (§8.1.3) no machine ToyOS runs needs; +//! - a type-3 SystemIO state is withheld: for one OSPM keeps the caches +//! coherent, on a multiprocessor by flushing them before entry (§8.1.4), +//! which ToyOS does not, and no document in hand says the processor's own +//! hardware makes that unnecessary for it +//! (`issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md`); +//! - a state `_CSD` names for software coordination (Table 8.1, SW_ALL or +//! SW_ANY) is withheld: ToyOS coordinates no CPU's state with another's. +//! HW_ALL asks nothing of OSPM. +//! +//! **`_CST` gives no residency, so ToyOS's is a policy**: twice the state's +//! latency, which §8.4.1.1 says is the worst case to enter and exit, so a +//! state is entered only for an idle period it spends at least half of +//! asleep. Metal judges it. +//! +//! **`_LPI` is not read**, though a platform may declare both: OSPM uses it in +//! preference to `_CST` only once it has said in `\_SB._OSC` that it supports +//! processor containers, and never mixes the two (§8.4.3.5, §6.2.11.2), which +//! ToyOS does not say. + +use std::collections::BTreeMap; + +use toyos_abi::acpi::{IdleDeclaration, IdleEntry, IdleState, IDLE_STATES}; +use toyos_aml::{Kind, Value}; + +use crate::aml::{kind, Aml}; +use crate::host::{Declared, Kernel, Stopping, OWN}; +use crate::init::Names; +use crate::ledger::Ledger; + +/// ToyOS's residency for a `_CST` state, by its latency. +const RESIDENCY_PER_LATENCY: u32 = 2; + +/// A processor whose `_CST` is read, and the UID the MADT names it by. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Processor { + pub path: String, + pub uid: u32, + /// It has a `_CSD` beside its `_CST`. + pub csd: bool, +} + +/// Every processor with a `_CST`: each `Processor` object, by its ProcID, and +/// each `ACPI0007` device, by its `_UID` (§5.2.12.2), in walk order. +pub fn processors(aml: &mut Aml<'_, K>, names: &Names) -> Vec { + let mut found = Vec::new(); + for named in &names.objects { + if !names.paths.contains(&format!("{}._CST", named.path)) { + continue; + } + let uid = match named.kind { + Kind::Processor { id } => Some(u32::from(id)), + Kind::Device if crate::osc::processor_device(aml, names, &named.path) => { + match aml.interpreter.evaluate(&mut aml.host, &format!("{}._UID", named.path), &[]) { + Ok(Value::Integer(uid)) => u32::try_from(uid).ok(), + _ => None, + } + } + _ => continue, + }; + match uid { + Some(uid) => found.push(Processor { path: named.path.clone(), uid, csd: names.paths.contains(&format!("{}._CSD", named.path)) }), + None => println!("{OWN}{} has a _CST and no numeric _UID the MADT could name it by", named.path), + } + } + found +} + +/// What the processors' declarations came to. +#[derive(Default)] +pub struct Census { + pub kept: u64, + pub no_cpu: u64, + pub refused: u64, + /// `_CST` or `_CSD` refused or malformed, by why. + pub unread: Ledger, + /// Each state withheld, by why. + pub withheld: Ledger, + /// The states each kept declaration held, and how many held them. + pub shapes: BTreeMap, +} + +impl Census { + pub fn line(&self) -> String { + let shapes: Vec = self.shapes.iter().map(|(shape, n)| format!("{n} with {shape}")).collect(); + format!( + "acpiserver: the processors' idle states: {} kept, {} naming no CPU, {} refused by the kernel; kept: {}; withheld: {}; unread: {}", + self.kept, + self.no_cpu, + self.refused, + if shapes.is_empty() { "none".into() } else { shapes.join(", ") }, + if self.withheld.is_empty() { "none".into() } else { self.withheld.counts() }, + if self.unread.is_empty() { "none".into() } else { self.unread.counts() }, + ) + } +} + +/// Read `processor`'s `_CST` and `_CSD` and hand the kernel what of them it +/// may enter: on an Intel processor where `intel`. +pub fn declare(aml: &mut Aml<'_, K>, kernel: &K, processor: &Processor, intel: bool, census: &mut Census) -> Result<(), Stopping> { + let read = |aml: &mut Aml<'_, K>, method: &str| aml.interpreter.evaluate(&mut aml.host, &format!("{}.{method}", processor.path), &[]); + let states = match read(aml, "_CST").map_err(|why| kind(&why)).and_then(|v| cst(&v)) { + Ok(states) => states, + Err(why) => { + census.unread.see(&format!("_CST: {why}")); + println!("{OWN}{}._CST unread: {why}", processor.path); + return Ok(()); + } + }; + let software = if processor.csd { + match read(aml, "_CSD").map_err(|why| kind(&why)).and_then(|v| csd(&v)) { + Ok(software) => software, + Err(why) => { + census.unread.see(&format!("_CSD: {why}")); + println!("{OWN}{}._CSD unread, so none of its states is handed over: {why}", processor.path); + return Ok(()); + } + } + } else { + Vec::new() + }; + let (declaration, withheld) = translate(processor.uid, &states, &software, intel); + for why in withheld { + census.withheld.see(&why); + } + match kernel.idle(&declaration)? { + Declared::Kept => { + census.kept += 1; + *census.shapes.entry(shape(&declaration)).or_insert(0) += 1; + } + Declared::NoCpu => census.no_cpu += 1, + Declared::Refused => { + census.refused += 1; + println!("{OWN}{}'s idle states were refused by the kernel", processor.path); + } + } + Ok(()) +} + +fn shape(declaration: &IdleDeclaration) -> String { + let states: Vec = declaration.states[..declaration.count as usize] + .iter() + .map(|s| match IdleEntry::from_raw(s.entry) { + Some(IdleEntry::Mwait) => format!("MWAIT {:#x} {} µs", s.address, s.latency_us), + _ => format!("port {:#x} {} µs", s.address, s.latency_us), + }) + .collect(); + if states.is_empty() { "C1 alone".into() } else { states.join(" + ") } +} + +/// One `_CST` state: its Register, type and latency (Table 8.2). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct CState { + pub register: Register, + pub kind: u64, + pub latency: u64, +} + +/// A Generic Register descriptor's fields (§6.4.3.7). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Register { + pub space: u8, + pub bit_width: u8, + pub bit_offset: u8, + pub access_size: u8, + pub address: u64, +} + +const FFH: u8 = 0x7F; +const SYSTEM_IO: u8 = 1; + +/// The states `_CST` evaluated to, or why it is no `_CST`. +pub fn cst(value: &Value) -> Result, String> { + let Value::Package(elements) = value else { return Err("no Package".into()) }; + let (Some(Value::Integer(count)), states) = (elements.first(), &elements[1.min(elements.len())..]) else { + return Err("its first element is no Integer".into()); + }; + if *count != states.len() as u64 { + return Err(format!("its count is {count}, and {} states follow", states.len())); + } + states + .iter() + .enumerate() + .map(|(i, state)| match state { + Value::Package(s) => match &s[..] { + [Value::Buffer(register), Value::Integer(kind), Value::Integer(latency), Value::Integer(_power)] => { + Ok(CState { register: register_of(register).ok_or_else(|| format!("state {i}'s Register is no Generic Register descriptor"))?, kind: *kind, latency: *latency }) + } + _ => Err(format!("state {i} is no Package of a Buffer and three Integers")), + }, + _ => Err(format!("state {i} is no Package")), + }) + .collect() +} + +/// A resource template holding one Generic Register descriptor: tag 0x82, +/// length 12 (§6.4.3.7), then the end tag. +fn register_of(bytes: &[u8]) -> Option { + if bytes.len() < 15 || bytes[0] != 0x82 || u16::from_le_bytes([bytes[1], bytes[2]]) != 12 { + return None; + } + Some(Register { + space: bytes[3], + bit_width: bytes[4], + bit_offset: bytes[5], + access_size: bytes[6], + address: u64::from_le_bytes(bytes[7..15].try_into().ok()?), + }) +} + +/// The `_CST` indices `_CSD` names for software coordination (Table 8.3: +/// NumEntries 6, Revision 0, Domain, CoordType, NumProcessors, Index). +pub fn csd(value: &Value) -> Result, String> { + const SW_ALL: u64 = 0xFC; + const SW_ANY: u64 = 0xFD; + const HW_ALL: u64 = 0xFE; + let Value::Package(entries) = value else { return Err("no Package".into()) }; + let mut software = Vec::new(); + for (i, entry) in entries.iter().enumerate() { + let Value::Package(fields) = entry else { return Err(format!("entry {i} is no Package")) }; + let [Value::Integer(6), Value::Integer(0), Value::Integer(_), Value::Integer(coord), Value::Integer(_), Value::Integer(index)] = &fields[..] else { + return Err(format!("entry {i} is not six Integers of revision 0")); + }; + match *coord { + SW_ALL | SW_ANY => software.push(*index), + HW_ALL => {} + other => return Err(format!("entry {i}'s coordination type {other:#x} is none of Table 8.1's")), + } + } + Ok(software) +} + +/// What of `states` the kernel may enter as they are, and why each other was +/// withheld. +pub fn translate(uid: u32, states: &[CState], software: &[u64], intel: bool) -> (IdleDeclaration, Vec) { + let mut declaration = IdleDeclaration { processor_uid: uid, count: 0, states: [IdleState { address: 0, latency_us: 0, residency_us: 0, entry: 0, reserved: [0; 7] }; IDLE_STATES] }; + let mut withheld = Vec::new(); + for (index, state) in states.iter().enumerate() { + let entry = match enterable(state, software.contains(&(index as u64)), intel) { + Ok(Some(entry)) => entry, + Ok(None) => continue, + Err(why) => { + withheld.push(why); + continue; + } + }; + let Ok(latency_us) = u32::try_from(state.latency) else { + withheld.push(format!("a latency of {} µs, past a u32", state.latency)); + continue; + }; + if declaration.count as usize == IDLE_STATES { + withheld.push(format!("past the {IDLE_STATES} a declaration holds")); + continue; + } + declaration.states[declaration.count as usize] = IdleState { + address: state.register.address, + latency_us, + residency_us: latency_us.saturating_mul(RESIDENCY_PER_LATENCY), + entry: entry as u8, + reserved: [0; 7], + }; + declaration.count += 1; + } + (declaration, withheld) +} + +/// How `state` is entered, `None` for C1, or why ToyOS may not enter it as it is. +fn enterable(state: &CState, software: bool, intel: bool) -> Result, String> { + let r = state.register; + match state.kind { + 1 => return Ok(None), + 2 | 3 => {} + other => return Err(format!("type {other}, which Table 8.2 does not define")), + } + if state.latency == 0 { + return Err("a latency of 0".into()); + } + if software { + return Err("coordinated in software by _CSD".into()); + } + match r.space { + FFH if (r.bit_width, r.bit_offset) == (1, 2) => { + if r.address >> 32 != 0 { + Err("an Intel FFH state whose hint sets reserved bits 63:32".into()) + } else if r.access_size & 1 << 1 != 0 { + Err("an Intel FFH state asking bus-master avoidance".into()) + } else { + Ok(Some(IdleEntry::Mwait)) + } + } + FFH => Err(format!("FFH vendor {} class {}, an encoding no document in hand defines", r.bit_width, r.bit_offset)), + SYSTEM_IO if intel => Err("a SystemIO state on Intel, a chipset's P_LVLx read".into()), + SYSTEM_IO if state.kind == 3 => Err("a type-3 SystemIO state, whose caches OSPM keeps coherent (ACPI 6.5 §8.1.4)".into()), + SYSTEM_IO => Ok(Some(IdleEntry::Port)), + space => Err(format!("a state in space {space:#x}, which no C-state is entered through")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reg(space: u8, bit_width: u8, bit_offset: u8, access_size: u8, address: u64) -> Register { + Register { space, bit_width, bit_offset, access_size, address } + } + + fn state(register: Register, kind: u64, latency: u64) -> CState { + CState { register, kind, latency } + } + + /// The T14's `_CST` as its firmware answers ToyOS's processor `_OSC` + /// claim, decoded from its tables (#875's check outside the tree). + fn t14() -> Vec { + vec![state(reg(FFH, 1, 2, 1, 0x00), 1, 1), state(reg(FFH, 1, 2, 1, 0x31), 2, 253), state(reg(FFH, 1, 2, 1, 0x60), 3, 1048)] + } + + /// The AMD bench's. + fn amd() -> Vec { + vec![state(reg(FFH, 2, 2, 0, 0x00), 1, 1), state(reg(SYSTEM_IO, 8, 0, 1, 0x414), 2, 400), state(reg(SYSTEM_IO, 8, 0, 1, 0x415), 3, 350)] + } + + fn kept(d: &IdleDeclaration) -> Vec<(u8, u64, u32, u32)> { + d.states[..d.count as usize].iter().map(|s| (s.entry, s.address, s.latency_us, s.residency_us)).collect() + } + + #[test] + fn the_t14s_c2_and_c3_are_handed_over_as_mwait_and_its_c1_is_hlt() { + let (d, withheld) = translate(3, &t14(), &[], true); + assert_eq!(d.processor_uid, 3); + assert_eq!(kept(&d), [(IdleEntry::Mwait as u8, 0x31, 253, 506), (IdleEntry::Mwait as u8, 0x60, 1048, 2096)]); + assert!(withheld.is_empty(), "{withheld:?}"); + } + + #[test] + fn the_amds_c2_is_handed_over_and_its_type_3_state_withheld() { + let (d, withheld) = translate(0, &amd(), &[0], false); + assert_eq!(kept(&d), [(IdleEntry::Port as u8, 0x414, 400, 800)]); + assert_eq!(withheld, ["a type-3 SystemIO state, whose caches OSPM keeps coherent (ACPI 6.5 §8.1.4)"]); + } + + #[test] + fn each_state_ospm_owes_work_around_is_withheld_by_name() { + let withheld = |s: CState, software: bool, intel: bool| { + let (d, why) = translate(0, &[s], if software { &[0] } else { &[] }, intel); + assert_eq!(d.count, 0); + why + }; + assert_eq!(withheld(state(reg(FFH, 1, 2, 0b11, 0x31), 2, 253), false, true), ["an Intel FFH state asking bus-master avoidance"]); + assert_eq!(withheld(state(reg(FFH, 2, 2, 0, 0x10), 2, 50), false, false), ["FFH vendor 2 class 2, an encoding no document in hand defines"]); + assert_eq!(withheld(state(reg(FFH, 1, 1, 0, 0x1816), 2, 50), false, true), ["FFH vendor 1 class 1, an encoding no document in hand defines"]); + assert_eq!(withheld(state(reg(SYSTEM_IO, 8, 0, 0, 0x1816), 2, 253), false, true), ["a SystemIO state on Intel, a chipset's P_LVLx read"]); + assert_eq!(withheld(state(reg(SYSTEM_IO, 8, 0, 1, 0x414), 2, 400), true, false), ["coordinated in software by _CSD"]); + assert_eq!(withheld(state(reg(FFH, 1, 2, 1, 0x31), 4, 253), false, true), ["type 4, which Table 8.2 does not define"]); + assert_eq!(withheld(state(reg(FFH, 1, 2, 1, 0x31), 2, 0), false, true), ["a latency of 0"]); + assert_eq!(withheld(state(reg(FFH, 1, 2, 1, 1 << 32), 2, 10), false, true), ["an Intel FFH state whose hint sets reserved bits 63:32"]); + assert_eq!(withheld(state(reg(0, 8, 0, 1, 0x1000), 2, 10), false, false), ["a state in space 0x0, which no C-state is entered through"]); + } + + #[test] + fn more_states_than_a_declaration_holds_are_withheld_past_the_last() { + let many: Vec = (0..10).map(|i| state(reg(FFH, 1, 2, 1, 0x10 + i), 2, 10 + i)).collect(); + let (d, withheld) = translate(0, &many, &[], true); + assert_eq!(d.count as usize, IDLE_STATES); + assert_eq!(withheld.len(), 2); + } + + fn register_buffer(r: Register) -> Value { + let mut b = vec![0x82, 12, 0, r.space, r.bit_width, r.bit_offset, r.access_size]; + b.extend(r.address.to_le_bytes()); + b.extend([0x79, 0x00]); + Value::Buffer(b) + } + + #[test] + fn a_cst_package_decodes_and_a_malformed_one_is_refused() { + let s = |r, kind, latency| Value::Package(vec![register_buffer(r), Value::Integer(kind), Value::Integer(latency), Value::Integer(0)]); + let good = Value::Package(vec![Value::Integer(2), s(reg(FFH, 1, 2, 1, 0), 1, 1), s(reg(FFH, 1, 2, 1, 0x31), 2, 253)]); + assert_eq!(cst(&good), Ok(vec![state(reg(FFH, 1, 2, 1, 0), 1, 1), state(reg(FFH, 1, 2, 1, 0x31), 2, 253)])); + let miscounted = Value::Package(vec![Value::Integer(3), s(reg(FFH, 1, 2, 1, 0), 1, 1)]); + assert_eq!(cst(&miscounted), Err("its count is 3, and 1 states follow".into())); + let unregistered = Value::Package(vec![Value::Integer(1), Value::Package(vec![Value::Buffer(vec![0x86, 9, 0]), Value::Integer(2), Value::Integer(1), Value::Integer(0)])]); + assert_eq!(cst(&unregistered), Err("state 0's Register is no Generic Register descriptor".into())); + assert_eq!(cst(&Value::Integer(0)), Err("no Package".into())); + } + + #[test] + fn csd_names_software_coordinated_states_and_hw_all_none() { + let entry = |coord, index| Value::Package([6, 0, 0, coord, 2, index].map(Value::Integer).to_vec()); + // The AMD bench's: HW_ALL over each SMT pair, for C1. + assert_eq!(csd(&Value::Package(vec![entry(0xFE, 0)])), Ok(vec![])); + assert_eq!(csd(&Value::Package(vec![entry(0xFD, 1), entry(0xFC, 2)])), Ok(vec![1, 2])); + assert!(csd(&Value::Package(vec![entry(0xFB, 1)])).is_err()); + assert!(csd(&Value::Package(vec![Value::Package([5, 0, 0, 0xFE, 2].map(Value::Integer).to_vec())])).is_err()); + } +} diff --git a/system/acpiserver/src/gpe.rs b/system/acpiserver/src/gpe.rs index 3df2cfef92d..6d70ecf47cc 100644 --- a/system/acpiserver/src/gpe.rs +++ b/system/acpiserver/src/gpe.rs @@ -47,7 +47,7 @@ pub fn runtime(aml: &mut Aml<'_, K>, names: &Names, gpe0: Block, ec_g } } let mut wake = BTreeSet::new(); - for named in names.objects.iter().filter(|named| matches!(named.kind, Kind::Device | Kind::Processor | Kind::ThermalZone | Kind::PowerResource)) { + for named in names.objects.iter().filter(|named| matches!(named.kind, Kind::Device | Kind::Processor { .. } | Kind::ThermalZone | Kind::PowerResource)) { let prw = format!("{}._PRW", named.path); if !names.paths.contains(&prw) { continue; diff --git a/system/acpiserver/src/host.rs b/system/acpiserver/src/host.rs index 8d4cdeca73a..bc37f63e96f 100644 --- a/system/acpiserver/src/host.rs +++ b/system/acpiserver/src/host.rs @@ -92,7 +92,7 @@ use std::collections::BTreeMap; use std::fmt; use std::time::{Duration, Instant}; -use toyos_abi::acpi::{pci_address, Access, Refused, Space, Width, UNLISTED}; +use toyos_abi::acpi::{pci_address, Access, IdleDeclaration, Refused, Space, Width, UNLISTED}; use toyos_aml::{Address, Denied, Host, RuntimeLoad, Transfer}; use crate::devices::Ec; @@ -134,6 +134,16 @@ pub enum Take { Unusable, } +/// What the kernel did with a processor's idle states. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Declared { + Kept, + /// The processor's UID names no CPU the kernel runs. + NoCpu, + /// The CPU cannot enter them as declared; the kernel's log says why. + Refused, +} + /// The `acpi` claim as this server asks it, its mediated access and its SCI, /// and the power-off it asks of the supervisor. pub trait Kernel { @@ -148,6 +158,8 @@ pub trait Kernel { /// once under a claim: `false` where it is wider than the register's /// field, and the kernel kept nothing. fn s5(&self, slp_typ_a: u64) -> Result; + /// Hand the kernel one processor's idle states (`crate::cst`). + fn idle(&self, declaration: &IdleDeclaration) -> Result; /// Waits until `until`, serving the SCI as `sci::await_sleep` does for /// the fixed power button where the claim serves it and the controller's /// GPE `ec_gpe`: what latched and was taken. @@ -763,6 +775,8 @@ pub mod tests { pub stops_after: Cell>, /// The `SLP_TYPa` it was handed and kept. pub handed: Cell>, + /// Each processor's idle states it was handed, all kept. + pub declared: RefCell>, /// What each Sleep's wait takes, in turn; nothing once it runs out. /// Each served set a Sleep was asked to serve. pub sleeps: RefCell>, @@ -904,6 +918,12 @@ pub mod tests { Ok(true) } + fn idle(&self, declaration: &IdleDeclaration) -> Result { + self.stopping()?; + self.declared.borrow_mut().push(*declaration); + Ok(Declared::Kept) + } + fn latched(&self) -> Option { None } diff --git a/system/acpiserver/src/init.rs b/system/acpiserver/src/init.rs index 9795bd518f1..053cd7a8b94 100644 --- a/system/acpiserver/src/init.rs +++ b/system/acpiserver/src/init.rs @@ -84,7 +84,7 @@ impl Names { } fn is_device(kind: Kind) -> bool { - matches!(kind, Kind::Device | Kind::Processor) + matches!(kind, Kind::Device | Kind::Processor { .. }) } /// The scope an object's path is in. diff --git a/system/acpiserver/src/main.rs b/system/acpiserver/src/main.rs index b8fd855271c..f1cd5bf894a 100644 --- a/system/acpiserver/src/main.rs +++ b/system/acpiserver/src/main.rs @@ -44,9 +44,10 @@ //! is run as its method ([`run_query`]): a Notify of a press is a press. //! Elsewhere no query's method runs. //! -//! **Then `_OSC` is negotiated** ([`osc`]): `\_SB._OSC`, and each -//! processor's on an Intel processor, query then commit, claiming what ToyOS -//! honours; whatever a table they loaded added is initialised in turn. +//! **Then `_OSC` is negotiated** ([`osc`]): `\_SB._OSC` query then commit, +//! and each processor's on an Intel processor committed, claiming what ToyOS +//! honours; whatever a table they loaded added is initialised in turn. Then +//! each processor's idle states are handed to the kernel ([`cst`]). //! //! **Each SCI** is read off both blocks ([`sci::events`]): a press stops the //! machine through the supervisor, or is said and dropped on a machine with @@ -70,6 +71,7 @@ mod aml; mod battery; +mod cst; mod devices; mod ec; mod gpe; @@ -89,7 +91,7 @@ use toyos::ioport::{in16, in8, out16, out8}; use toyos::poller::{Poller, READABLE}; use toyos::power::{self, Mode, Refused, Stop}; use toyos::AcpiDev; -use toyos_abi::acpi::{Access, AcpiInfo, Block, FIXED_POWER_BUTTON}; +use toyos_abi::acpi::{Access, AcpiInfo, Block, IdleDeclaration, FIXED_POWER_BUTTON}; use toyos_abi::syscall::{DeviceType, SyscallError}; use acpiserver_api::{CONTROLLER_NONE, CONTROLLER_SERVED}; @@ -97,7 +99,7 @@ use aml::Aml; use battery::Power; use devices::{Ec, Queried}; use ec::Transaction; -use host::{port_in, Answer, Kernel, Stopping, Take, Unmade, OWN}; +use host::{port_in, Answer, Declared, Kernel, Stopping, Take, Unmade, OWN}; use init::{Census, Names}; use sci::{Bits, Event, Fixed, Served, Unserved, Unstopped, PM1_STATUS, PWRBTN}; @@ -143,6 +145,8 @@ struct Server<'a, K, P> { gpe_runs: BTreeMap, /// Each Notify delivered, by object and value. delivered: ledger::Ledger, + /// The processors whose `_CST` is read, which their Notify 0x81 reads again. + idle: Vec, } fn main() { @@ -159,6 +163,7 @@ fn main() { let (batteries, adapters) = server.arm_controller(names.as_ref()); if let Some(names) = server.negotiate(names) { println!("{}", server.census.line()); + server.declare_idle(&names); server.arm_gpes(&names); } server.sources = (batteries.clone(), adapters.clone()); @@ -266,6 +271,14 @@ impl Kernel for Claim<'_> { } } + fn idle(&self, declaration: &IdleDeclaration) -> Result { + match self.dev.idle(declaration) { + Err(SyscallError::NotFound) => Ok(Declared::NoCpu), + Err(SyscallError::InvalidArgument) => Ok(Declared::Refused), + answer => Self::answered("a processor's idle states", answer).map(|()| Declared::Kept), + } + } + fn latched(&self) -> Option { match self.dev.irq() { Ok(record) => Some(record.count), @@ -358,6 +371,7 @@ impl<'a, K: Kernel, P: Ports> Server<'a, K, P> { stop_asked: false, census: Census::default(), negotiated: osc::Negotiated::default(), + idle: Vec::new(), intel: toyos_abi::processor_is_intel(), sources: (Vec::new(), Vec::new()), gpe_runs: BTreeMap::new(), @@ -462,6 +476,29 @@ impl<'a, K: Kernel, P: Ports> Server<'a, K, P> { Some(names) } + /// Hand the kernel each processor's idle states ([`cst`]), once `_OSC` + /// has said what ToyOS does and the blocks it loaded stand. + fn declare_idle(&mut self, names: &Names) { + let Some(aml) = &mut self.aml else { return }; + self.idle = cst::processors(aml, names); + let mut census = cst::Census::default(); + for processor in &self.idle { + if cst::declare(aml, self.kernel, processor, self.intel, &mut census).is_err() { + return; + } + } + println!("{}", census.line()); + } + + /// `object`'s idle states read again and handed over, on its Notify 0x81. + fn redeclare_idle(&mut self, object: &str) { + let (Some(aml), Some(processor)) = (&mut self.aml, self.idle.iter().find(|p| p.path == object)) else { return }; + let mut census = cst::Census::default(); + if cst::declare(aml, self.kernel, processor, self.intel, &mut census).is_ok() { + println!("{OWN}{object}'s idle states read again: {}", census.line()); + } + } + /// Find the controller, and the buttons where they are control method /// devices, in the namespace the load kept; attach the controller to the /// AML host ([`devices::attach`]), enable its GPE and drain its backlog. What @@ -671,14 +708,16 @@ impl<'a, K: Kernel, P: Ports> Server<'a, K, P> { /// status change (0x80) reads the power sources again, and a battery's /// information change (0x81) finds them again; `\_SB`'s or a negotiated /// processor's 0x08 negotiates its `_OSC` again with the same claim - /// (§6.2.11); a processor's performance, idle or throttling change - /// (0x80, 0x81, 0x82, §8.4) is counted, since nothing reads those states - /// yet; anything else is served by nothing and counted. + /// (§6.2.11); a processor's idle change (0x81, §8.4.1.1) reads its + /// `_CST` again; its performance or throttling change (0x80, 0x82) is + /// counted, since nothing reads those states; anything else is served by + /// nothing and counted. fn deliver(&mut self, notified: std::collections::BTreeSet<(String, u64)>) { let (mut status, mut info) = (false, false); for (object, value) in notified { let source = self.sources.0.contains(&object) || self.sources.1.contains(&object); let processor = self.negotiated.processors.iter().any(|(path, _)| *path == object); + let idles = self.idle.iter().any(|p| p.path == object); let served = match value { 0x80 if source => { status = true; @@ -692,7 +731,11 @@ impl<'a, K: Kernel, P: Ports> Server<'a, K, P> { self.renegotiate(&object); "its _OSC negotiated again" } - 0x80..=0x82 if processor => "counted: nothing reads the processors' performance, idle or throttling states", + 0x81 if idles => { + self.redeclare_idle(&object); + "its idle states read again" + } + 0x80 | 0x82 if processor || idles => "counted: nothing reads the processors' performance or throttling states", _ => "served by nothing", }; if self.delivered.see(&format!("{object} {value:#x}")) { @@ -710,11 +753,16 @@ impl<'a, K: Kernel, P: Ports> Server<'a, K, P> { /// `object`'s `_OSC` negotiated again with its claim. fn renegotiate(&mut self, object: &str) { let Some(aml) = &mut self.aml else { return }; - let claim = if object == "\\_SB_" { &osc::PLATFORM } else { &osc::PROCESSOR }; let before = aml.interpreter.defined(); - match osc::negotiate(aml, &format!("{object}._OSC"), claim) { - Ok(answered) => println!("{OWN}{object}._OSC negotiated again: {answered:x?}"), - Err(why) => println!("{OWN}{object}._OSC was refused again: {why}"), + let path = format!("{object}._OSC"); + let answered = if object == "\\_SB_" { + osc::negotiate(aml, &path, &osc::PLATFORM).map(|answered| format!("{answered:x?}")) + } else { + osc::commit(aml, &path, &osc::PROCESSOR).map(|committed| format!("{committed:x?}")) + }; + match answered { + Ok(answered) => println!("{OWN}{path} negotiated again: {answered}"), + Err(why) => println!("{OWN}{path} was refused again: {why}"), } if let Ok(names) = Names::read(&aml.interpreter) { self.settle(before, names); @@ -1111,11 +1159,12 @@ mod tests { kernel } - /// §6.2.11: `\_SB._OSC` and, on an Intel processor, each processor's are - /// queried with what ToyOS claims, nothing, and committed with what the - /// query granted of it, nothing, though the firmware offered a bit; - /// the block the processor's commit loads from reserved memory loads, and - /// its device's `_INI` runs. On another vendor's processor its `_OSC` is + /// §6.2.11: `\_SB._OSC` is queried with what ToyOS claims, nothing, and + /// committed with what the query granted of it, nothing, though the + /// firmware offered a bit; on an Intel processor, each processor's is + /// committed with ToyOS's claim and never queried (302223-007 §2.1). The + /// block the processor's commit loads from reserved memory loads, and its + /// device's `_INI` runs. On another vendor's processor its `_OSC` is /// not evaluated, and a block in ACPI reclaim memory, which Table 15.6 /// calls neither reserved nor NVS, is refused and keeps nothing. #[test] @@ -1137,7 +1186,8 @@ mod tests { let platform: Vec = ["\\PUU", "\\PQ1", "\\PQ2", "\\PC1", "\\PC2"].map(&mut read).to_vec(); assert_eq!(platform, [1, 1, 0, 0, 0], "the platform's UUID, the query flag and nothing claimed, then the commit of nothing"); let processor: Vec = ["\\CUU", "\\CQ1", "\\CQ2", "\\CC1", "\\CC2"].map(&mut read).to_vec(); - assert_eq!(processor, if intel { [1, 1, 0, 0, 0] } else { [0; 5] }, "intel {intel}"); + let claimed = u64::from(osc::PROCESSOR.capabilities); + assert_eq!(processor, if intel { [1, 0, 0, 0, claimed] } else { [0; 5] }, "intel {intel}"); assert_eq!(read("\\NEWI"), if loaded { 0x77 } else { 0 }, "intel {intel}, block in type {block_type}"); assert_eq!((aml.host.loaded, aml.host.loads_refused), (u64::from(loaded), u64::from(intel && !loaded))); let answered = osc::Answered { query: 0x10, offered: 0, commit: Some((0, 0)) }; @@ -1147,6 +1197,38 @@ mod tests { } } + /// Each processor's `_CST` reaches the kernel by the UID the MADT names + /// it by, a `Processor` object's ProcID, holding the states deeper than + /// C1 as `_CST` names them: the T14's, as its firmware answers ToyOS's + /// processor `_OSC`. + #[test] + fn each_processors_cst_is_handed_to_the_kernel_by_its_uid() { + use crate::common::{buffer, cat, def_name, int, name, package, pkg}; + use toyos_abi::acpi::IdleEntry; + let register = |hint: u64| { + let mut descriptor = vec![0x82, 12, 0, 0x7F, 1, 2, 1]; + descriptor.extend(hint.to_le_bytes()); + descriptor.extend([0x79, 0x00]); + buffer(&int(descriptor.len() as u64), &descriptor) + }; + let state = |hint, kind, latency| package(&[register(hint), int(kind), int(latency), int(0)]); + let cst = def_name("_CST", &package(&[int(3), state(0x00, 1, 1), state(0x31, 2, 253), state(0x60, 3, 1048)])); + let processor = |path: &str, id: u8| cat(&[&[0x5B, 0x83], &pkg(&cat(&[&name(path), &[id], &0u32.to_le_bytes(), &[0x00], &cst]))]); + let kernel = crafted(&sealed(b"DSDT", &cat(&[&processor("\\_PR.CPU0", 2), &processor("\\_PR.CPU1", 5)])), &[]); + let info = AcpiInfo { rsdp: CRAFTED_RSDP, pm1_event: Block::NONE, gpe0: GPE0, flags: FIXED_POWER_BUTTON, reserved: [0; 3] }; + let mut server = Server::new(&kernel, Bytes::default(), info); + server.intel = true; + server.aml = load(&kernel, CRAFTED_RSDP).1; + let names = server.initialise().expect("the namespace stands"); + server.declare_idle(&names); + let declared = kernel.declared.borrow(); + assert_eq!(declared.iter().map(|d| d.processor_uid).collect::>(), [2, 5]); + for d in declared.iter() { + let states: Vec<(u8, u64, u32)> = d.states[..d.count as usize].iter().map(|s| (s.entry, s.address, s.latency_us)).collect(); + assert_eq!(states, [(IdleEntry::Mwait as u8, 0x31, 253), (IdleEntry::Mwait as u8, 0x60, 1048)]); + } + } + /// The POST port as a word field, `\\P80`, and a method `name` that writes /// `tag` to it: what the kernel's stand-in was asked to write there is /// which methods ran, in order. diff --git a/system/acpiserver/src/osc.rs b/system/acpiserver/src/osc.rs index b81936421ea..9f8f1a03ce5 100644 --- a/system/acpiserver/src/osc.rs +++ b/system/acpiserver/src/osc.rs @@ -1,7 +1,17 @@ //! `_OSC` negotiated (ACPI 6.5 §6.2.11; Intel 302223-007 §2.2): `\_SB._OSC` -//! under the platform-wide UUID (§6.2.11.2), then, on an Intel processor, -//! each processor's `_OSC` under Intel's, each a query with the Query flag -//! and then a commit of what the query granted (§6.2.11.1.1, §6.2.11.1.3). +//! under the platform-wide UUID (§6.2.11.2), a query with the Query flag and +//! then a commit of what the query granted (§6.2.11.1.1, §6.2.11.1.3); then, +//! on an Intel processor, each processor's `_OSC` under Intel's, committed +//! with what ToyOS claims and no query first. +//! +//! **A processor's bits are statements, not requests for control**: each of +//! 302223-007 Table 1 says what OSPM "is capable of", and "ACPI firmware will +//! interpret and record the set bits" (§2.1). ACPI 6.5 §6.2.11.1.1 "strongly +//! recommends" the query, for negotiating control; it does not require it, +//! and what the firmware does with a statement is in the objects it then +//! returns (`_CST` among them, [`crate::cst`]). A query some firmware answers +//! with the same bit whatever was asked, nothing included, would otherwise +//! decide what ToyOS says it does. //! //! **What ToyOS claims is one declaration a UUID** ([`PLATFORM`], //! [`PROCESSOR`]), each bit set only where ToyOS does what the bit promises, @@ -64,11 +74,14 @@ pub struct Claim { pub const PLATFORM: Claim = Claim { uuid: uuid(b"0811B06E-4A27-44F9-8D60-3CBBC22E7B48"), capabilities: 0 }; /// A processor's capabilities on Intel (302223-007 Table 1) ToyOS honours: -/// none. Bit 0: the kernel enables HWP itself +/// bit 4, each processor entering its C2 and C3 states independently, and +/// bit 9, native C-state instructions (`MWAIT`) for them +/// (`kernel/src/idle.rs`). Not bit 0: the kernel enables HWP itself /// (`kernel/src/arch/x86_64/control_regs.rs`), and writes no `IA32_PERF_CTL` -/// through `_PCT`. Bit 12: the kernel takes no HWP interrupt. Bits 8 and 9: -/// the kernel enters no `MWAIT` state but C1's, and reads no `_CST`. -pub const PROCESSOR: Claim = Claim { uuid: uuid(b"4077A616-290C-47BE-9EBD-D87058713953"), capabilities: 0 }; +/// through `_PCT`. Not bit 6: no C-state is coordinated in software through +/// `_CSD`. Not bit 8: C1 is `hlt`. Not bit 12: the kernel takes no HWP +/// interrupt. +pub const PROCESSOR: Claim = Claim { uuid: uuid(b"4077A616-290C-47BE-9EBD-D87058713953"), capabilities: 1 << 4 | 1 << 9 }; /// The first DWORD's bits (§6.2.11): passed, the Query flag; answered, the /// status. @@ -131,6 +144,20 @@ fn evaluate(aml: &mut Aml<'_, K>, path: &str, uuid: &[u8; 16], first: } } +/// What a processor's commit answered: its status, and what it acknowledged +/// of what was claimed. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Committed { + pub status: u32, + pub granted: u32, +} + +/// `path`'s `_OSC` committed with `claim` whole, no query first. +pub fn commit(aml: &mut Aml<'_, K>, path: &str, claim: &Claim) -> Result { + let [status, granted] = evaluate(aml, path, &claim.uuid, 0, claim.capabilities)?; + Ok(Committed { status, granted: granted & claim.capabilities }) +} + /// `path`'s `_OSC` negotiated under `claim`: the query, and the commit of /// what it granted that `claim` asked for. pub fn negotiate(aml: &mut Aml<'_, K>, path: &str, claim: &Claim) -> Result { @@ -148,11 +175,11 @@ pub fn negotiate(aml: &mut Aml<'_, K>, path: &str, claim: &Claim) -> pub struct Negotiated { pub platform: Option>, /// Each processor's, in walk order. - pub processors: Vec<(String, Result)>, + pub processors: Vec<(String, Result)>, } -/// `\_SB._OSC`, then each processor's `_OSC` where the processor is Intel's -/// (`intel`): every +/// `\_SB._OSC`, then each processor's `_OSC` committed where the processor is +/// Intel's (`intel`): every /// `Processor` object and every `ACPI0007` device (ACPI 6.5 §8.4) that names /// one, in the namespace's order. pub fn negotiate_all(aml: &mut Aml<'_, K>, names: &Names, intel: bool) -> Negotiated { @@ -169,7 +196,7 @@ pub fn negotiate_all(aml: &mut Aml<'_, K>, names: &Names, intel: bool .objects .iter() .filter(|named| match named.kind { - toyos_aml::Kind::Processor => true, + toyos_aml::Kind::Processor { .. } => true, toyos_aml::Kind::Device => processor_device(aml, names, &named.path), _ => false, }) @@ -185,9 +212,9 @@ pub fn negotiate_all(aml: &mut Aml<'_, K>, names: &Names, intel: bool } for processor in processors { let path = format!("{processor}._OSC"); - let answered = negotiate(aml, &path, &PROCESSOR); + let answered = commit(aml, &path, &PROCESSOR); match &answered { - Ok(answered) => println!("{OWN}{path} negotiated: {}", answered.said(PROCESSOR.capabilities)), + Ok(c) => println!("{OWN}{path} committed {:#x}: it answered {} and acknowledged {:#x}", PROCESSOR.capabilities, status(c.status), c.granted), Err(why) => println!("{OWN}{path} was refused: {why}"), } negotiated.processors.push((processor, answered)); @@ -196,9 +223,9 @@ pub fn negotiate_all(aml: &mut Aml<'_, K>, names: &Names, intel: bool } } let refused = negotiated.processors.iter().filter(|(_, answered)| answered.is_err()).count(); - let granted = negotiated.processors.iter().filter_map(|(_, answered)| answered.as_ref().ok()?.commit).fold(0, |all, (_, granted)| all | granted); + let granted = negotiated.processors.iter().filter_map(|(_, answered)| answered.as_ref().ok()).fold(0, |all, c| all | c.granted); println!( - "acpiserver: the processors' _OSC negotiated, Intel's: {} evaluated, query then commit, {refused} refused; granted {granted:#x} of {:#x}", + "acpiserver: the processors' _OSC committed, Intel's: {} evaluated, {refused} refused; acknowledged {granted:#x} of {:#x}", negotiated.processors.len(), PROCESSOR.capabilities ); @@ -207,7 +234,7 @@ pub fn negotiate_all(aml: &mut Aml<'_, K>, names: &Names, intel: bool /// Whether the device at `path` is a processor by its `_HID` (§8.4): /// ACPI0007. -fn processor_device(aml: &mut Aml<'_, K>, names: &Names, path: &str) -> bool { +pub fn processor_device(aml: &mut Aml<'_, K>, names: &Names, path: &str) -> bool { let hid = format!("{path}._HID"); names.paths.contains(&hid) && aml.interpreter.evaluate(&mut aml.host, &hid, &[]).ok().as_ref().and_then(crate::devices::id).as_deref() == Some("ACPI0007") } diff --git a/system/soundserver/src/mix.rs b/system/soundserver/src/mix.rs index 68eb18aac9a..38393108251 100644 --- a/system/soundserver/src/mix.rs +++ b/system/soundserver/src/mix.rs @@ -11,9 +11,11 @@ //! **What a period *is* was decided in `toyos-mixer`** before either loop ran: //! this file spends the answers and owns the effects around them. +use std::time::Duration; + use toyos::endow::Endowments; use toyos::poller::{Poller, READABLE}; -use toyos::syscap::SysCap; +use toyos::syscap::{SysCap, WakeLatencyHold}; use toyos_abi::syscall; use toyos_abi::RawHandle; use toyos_sound_output::{run, Run}; @@ -303,17 +305,27 @@ pub(crate) fn mix_thread( // Until this branch it was gated on the audio claim, which the dispatch's // own comment called out as not a privilege at all: whoever won the // first-come race for the sound card got the RT band with it. This is the - // `RT`-only capability the manifest's `syscap = ["rt"]` row asks the supervisor for, - // and soundserver is the only program in the tree that has one. Mixing on - // without the band would show up only as glitches, so a refusal is loud. - // Once: the band is the thread's, and the loop runs again on each output. - static RT: std::sync::Once = std::sync::Once::new(); - RT.call_once(|| { - let rt: SysCap = Endowments::get() + // capability the manifest's `syscap = ["rt", "wake-latency"]` row asks the + // supervisor for, and soundserver is the only program in the tree that has + // one. Mixing on without the band would show up only as glitches, so a + // refusal is loud. Once: the band is the thread's, and the loop runs + // again on each output. + static CAP: std::sync::OnceLock = std::sync::OnceLock::new(); + let cap = CAP.get_or_init(|| { + let cap: SysCap = Endowments::get() .take(toyos_abi::syscall::SYSCAP_LABEL) - .expect("the manifest declares this program `syscap = [\"rt\"]`"); - rt.enter_rt().expect("an RT capability refused the band it names"); + .expect("the manifest declares this program `syscap = [\"rt\", \"wake-latency\"]`"); + cap.enter_rt().expect("an RT capability refused the band it names"); + cap }); + // While a stream plays, every idle CPU wakes within half a period, which + // leaves the other half for the render and the rest of the wake chain: + // the period interrupt may land on one CPU and wake this thread on + // another. A policy until the realtime stage's audio workers carry their + // own deadline arithmetic; none is held while suspended, so the machine + // sleeps deep when nothing plays. + let wake_bound = Duration::from_nanos(period_nanos / 2); + let mut wake_hold: Option = None; let poller = Poller::new(64); let mut mix_f32 = vec![0.0f32; device_period_samples]; @@ -609,6 +621,7 @@ pub(crate) fn mix_thread( // started: the marker has to precede whatever the backend logs // about starting. say!("soundserver: resumed"); + wake_hold = Some(cap.hold_wake_latency(wake_bound).expect("the manifest declares this program `wake-latency`")); } backend.submit(idx, device_period_bytes); unplayed += 1; @@ -659,6 +672,7 @@ pub(crate) fn mix_thread( dll.reset(); backend.stop(); started = false; + wake_hold = None; say!("soundserver: suspended"); } diff --git a/tests/images/metalcase.toml b/tests/images/metalcase.toml index 8036f8dce52..d9d1f610f4d 100644 --- a/tests/images/metalcase.toml +++ b/tests/images/metalcase.toml @@ -34,7 +34,7 @@ devices = ["framebuffer"] service = true serves = ["soundserver", "volume", "sound-output"] devices = ["pci:04:01:00/1af4:1059"] -syscap = ["rt"] +syscap = ["rt", "wake-latency"] # The Intel HD Audio driver: one process per HDA function the kernel does not # drive, each walking its own controller's codecs and offering soundserver the diff --git a/tests/images/metaldevicecase.toml b/tests/images/metaldevicecase.toml index 65987801fee..d61a7fc8b68 100644 --- a/tests/images/metaldevicecase.toml +++ b/tests/images/metaldevicecase.toml @@ -21,7 +21,7 @@ syscap = ["logread"] [programs.soundserver] service = true serves = ["soundserver", "volume", "sound-output"] -syscap = ["rt"] +syscap = ["rt", "wake-latency"] # The Intel HD Audio driver: one process per HDA function the kernel does not # drive, each walking its own controller's codecs and offering soundserver the diff --git a/tests/images/netcase.toml b/tests/images/netcase.toml index 2b1a38d2d1c..a50e38f2fbe 100644 --- a/tests/images/netcase.toml +++ b/tests/images/netcase.toml @@ -31,7 +31,7 @@ devices = ["pci:02:00:00/1af4:1041"] service = true serves = ["soundserver", "volume", "sound-output"] devices = ["pci:04:01:00/1af4:1059"] -syscap = ["rt"] +syscap = ["rt", "wake-latency"] # A job reaches netstack through test-runner's namespace: # `netstack_holders_exported_to_children`'s jobs reach the harness through it. diff --git a/tests/images/testcases.toml b/tests/images/testcases.toml index 55cb249e956..41961cfb284 100644 --- a/tests/images/testcases.toml +++ b/tests/images/testcases.toml @@ -27,7 +27,7 @@ receives = ["power"] service = true serves = ["soundserver", "volume", "sound-output"] devices = ["pci:04:01:00/1af4:1059"] -syscap = ["rt"] +syscap = ["rt", "wake-latency"] # The Intel HD Audio driver: one process per HDA function the kernel does not # drive, each walking its own controller's codecs and offering soundserver the From b14bbde65cf638156cd7603988fb99bff6f24a8a Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:15:30 +0100 Subject: [PATCH 02/24] issues: the idle track records what is built, and its two gaps get records The AMD bench's type-3 state, withheld for want of a family 17h statement that its I/O C-states keep caches coherent, and AArch64's wfi-only idle each get a record with an exit. The AML track, the doorbell track and the scheduler/clocks track say what changed for them. Co-Authored-By: Claude Opus 5.5 --- issues/aarch64-cpus-idle-only-in-wfi.md | 36 ++++++++++++++++++ .../idle-cpus-enter-the-c-states-cst-names.md | 19 ++++++++-- ...-so-every-wake-of-an-idle-cpu-is-an-ipi.md | 11 ++++-- ...hheld-for-want-of-a-coherence-statement.md | 37 +++++++++++++++++++ ...the-scheduler-and-the-clocks-never-talk.md | 14 ++++--- ...ine-in-acpi-mode-and-interprets-its-aml.md | 18 +++++---- kernel/src/idle.rs | 4 +- 7 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 issues/aarch64-cpus-idle-only-in-wfi.md create mode 100644 issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md diff --git a/issues/aarch64-cpus-idle-only-in-wfi.md b/issues/aarch64-cpus-idle-only-in-wfi.md new file mode 100644 index 00000000000..d6e26af9999 --- /dev/null +++ b/issues/aarch64-cpus-idle-only-in-wfi.md @@ -0,0 +1,36 @@ +--- +status: open +kind: track +opened: 2026-10-28 +--- + +# AArch64 CPUs idle only in `wfi` + +On x86 the kernel enters the states a processor's `_CST` names +(`kernel/src/idle.rs`); on AArch64 every idle CPU waits in `wfi` +(`kernel/src/arch/aarch64/hw.rs`), and `arch::idle::facts` admits nothing +deeper (`kernel/src/arch/aarch64/idle.rs`). + +**The equivalent.** An Arm platform declares its idle states in `_LPI` under +its processor and processor-container devices (ACPI 6.5 §8.4.3), each entered +through PSCI `CPU_SUSPEND` with the `power_state` its FFH entry method names +(Arm DEN0022, DEN0048). A retention state comes back as `wfi` does. A +power-down state loses the core's context and needs a resume path. The +selection (`kernel::sched::idle`) and the `WakeLatency` bound are the +architecture's already; what is missing is the declaration's channel and +entry: + +- AArch64 has no `acpi` claim or AML server to hand a declaration over + (`kernel/src/arch/aarch64/acpi_mode.rs`); +- nothing decodes `_LPI` or processor containers, which claiming LPI support + in `\_SB._OSC` asserts (§6.2.11.2, §8.4.3.5); +- no `CPU_SUSPEND` entry, and no always-on timer check (the GTDT's flag) + standing for x86's ARAT. + +No fleet machine runs AArch64 on metal, and whether QEMU's `virt` declares an +`_LPI` is unread. + +**Exit**: an AArch64 machine whose firmware declares `_LPI` enters its +retention states through `CPU_SUSPEND`, chosen by the same selector, with a +metal or QEMU reading of the state entered and returned from. Power-down +states, with a resume path, follow once such a machine declares one. diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 6fb9d6d844f..c47968f6026 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -6,10 +6,21 @@ opened: 2026-09-29 # Idle CPUs enter the C-states `_CST` names -The kernel runs no `mwait` and reads no `_CST`; this track is being built on -branch `deep-idle`. ToyOS's cores never go below C1/C1E and its package -never leaves PC0, and its idle package draws 3.16 W against Linux's 1.18 to -1.57 W (#866, https://github.com/ToyOSOrg/ToyOS/pull/866#issuecomment-6119513916). +Built on branch `deep-idle`: acpiserver hands the kernel what each +processor's `_CST` names that it may enter as it is +(`system/acpiserver/src/cst.rs`), the kernel checks it against the CPU +(`lib/userbound/src/idle.rs`) and chooses a state each idle period +(`kernel/pure/sched/idle.rs`, `kernel/src/idle.rs`), and a `WakeLatency` +hold bounds the exit (`soundserver` holds half a period while it plays). +On the T14 that is `MWAIT 0x31` and `0x60` (253 and 1048 µs); on the AMD +bench port 0x414 (400 µs), its type-3 0x415 withheld +(`issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md`). +AArch64 keeps `wfi` (`issues/aarch64-cpus-idle-only-in-wfi.md`). What +remains is the exit below, read on metal. + +Before it, ToyOS's cores never went below C1/C1E and its package never left +PC0, and its idle package drew 3.16 W against Linux's 1.18 to 1.57 W (#866, +https://github.com/ToyOSOrg/ToyOS/pull/866#issuecomment-6119513916). What `hlt` alone costs a wake, firmware's C1E promotion included, is `issues/a-cpu-leaving-hlt-runs-its-first-microseconds-slow-under-c1e.md`. diff --git a/issues/idle-is-hlt-so-every-wake-of-an-idle-cpu-is-an-ipi.md b/issues/idle-is-hlt-so-every-wake-of-an-idle-cpu-is-an-ipi.md index 93f113f200c..c7b5e2ed701 100644 --- a/issues/idle-is-hlt-so-every-wake-of-an-idle-cpu-is-an-ipi.md +++ b/issues/idle-is-hlt-so-every-wake-of-an-idle-cpu-is-an-ipi.md @@ -6,8 +6,10 @@ opened: 2026-10-12 # Idle is `hlt`, so every wake of an idle CPU is an IPI -An idle CPU waits in `sti; hlt` (`kernel/src/arch/x86_64/hw.rs:21`), so the -only thing that ends its wait is an interrupt: a wake posted to it rings its +An idle CPU waits in `sti; hlt` (`kernel/src/arch/x86_64/hw.rs`) or in a +deep state its firmware declared, an `MWAIT` on a line nothing writes or a +trapped port read (`kernel/src/arch/x86_64/idle.rs`), so the only thing that +ends its wait is an interrupt: a wake posted to it rings its doorbell (`kernel/pure/sched/mailbox.rs:474-490`) and then sends a kick IPI (`kernel/src/arch/x86_64/apic.rs:256-260`), whose entry and handler run on the woken CPU before its `hlt` returns (`kernel/src/arch/x86_64/idt/kick.rs:3-9`). @@ -26,8 +28,9 @@ unless firmware disables them, gives the monitor line as 64 bytes, and has MWAIT support one C1 sub-state (PPR 55922 rev. 3.06, `CPUID_Fn00000001_ECX` bit 3, `CPUID_Fn00000005`). So the stage is backed on Intel by §11.10.4 and §11.10.6.4, and is Intel's alone: the AMD bench keeps `hlt` and the kick, and -whether it takes the doorbell is the orchestrator's to decide once that bench's -`_CST` is read. A 2020+ Intel CPU whose specification update names an erratum +whether it takes the doorbell is the orchestrator's to decide; that bench's +`_CST` reads C1 as MWAIT and its deeper states as port reads, which no store +ends. A 2020+ Intel CPU whose specification update names an erratum by which a store may not wake MWAIT keeps the kick. The T14's lists no MONITOR or MWAIT erratum (631123-034); Intel's Core Ultra 200V specification update is unread. Whether the T14 and the AMD bench enumerate MONITOR/MWAIT is ToyOS's diff --git a/issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md b/issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md new file mode 100644 index 00000000000..4344f4119ce --- /dev/null +++ b/issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md @@ -0,0 +1,37 @@ +--- +status: open +kind: defect +opened: 2026-10-28 +--- + +# The AMD bench's C3 is withheld for want of a coherence statement + +The AMD bench's `_CST` names three states: C1 as FFH `MWAIT 0`, C2 as a +read of port 0x414 (type 2, 400 µs) and C3 as a read of 0x415 (type 3, +350 µs). acpiserver hands the kernel C2 and withholds C3 +(`system/acpiserver/src/cst.rs`), so that bench idles no deeper than 0x414. + +**Why.** ACPI 6.5 §8.1.4 makes OSPM responsible for the caches' coherency in +a type-3 state, on a multiprocessor by flushing them before entry. ToyOS +does not flush: `WBINVD` on Zen 2 writes back and invalidates the whole +CCX's shared L3 under its running siblings. Neither PPR 55922, 55901, +55772, OSRR 56255 nor APM Vol. 2 says that an I/O-trapped C-state +(`CStateBaseAddr`, MSRC001_0073) keeps caches coherent on its own. AMD's +Family 16h BKDG (52740 §2.5.3.2.3) does say so of the same interface on that +family: a core in such a state either has its probes serviced or flushes +its L1 in hardware before CC6. That family's BIOS requirement declares its +I/O C-state type 2. It is another family's document, so it is not taken as +Zen 2's. + +The same firmware's `_LPI` declares 0x415 with no context to lose (18 and +350 µs wake for 0x414 and 0x415, 60 and 1000 µs residency). OSPM uses `_LPI` +only once `\_SB._OSC` has claimed support for processor containers, and +never mixed with `_CST` (§8.4.3.5). ToyOS claims no such support. + +**Exit**: either an AMD document for family 17h saying that its I/O C-states +keep caches coherent with no OS flush, after which `cst.rs` hands over a +type-3 SystemIO state on that family and the AMD's metal row reads 0x415 +entered; or ToyOS's `_LPI` support (`issues/aarch64-cpus-idle-only-in-wfi.md` +owes the same decode and processor containers) used on that bench in +place of `_CST`. **Mutation**: the type-3 rule removed, so 0x415 is handed +over and the host test in `cst.rs` reds. diff --git a/issues/the-scheduler-and-the-clocks-never-talk.md b/issues/the-scheduler-and-the-clocks-never-talk.md index 058e864b642..b0f53c0356f 100644 --- a/issues/the-scheduler-and-the-clocks-never-talk.md +++ b/issues/the-scheduler-and-the-clocks-never-talk.md @@ -19,12 +19,16 @@ Two couplings every production scheduler has and ours lacks: wake. A latency-sensitive wake should prefer a shallow-idle core; a throughput task can pay to wake a deep one. -Idle-state awareness waits for the states -`issues/idle-cpus-enter-the-c-states-cst-names.md` is building. The kernel -chooses no P-state: it declares HWP's bounds +Idle-state awareness has its first half: a wake forward prefers, among +equally near free cores, the one whose state wakes quickest +(`CpuHandles::idle_near`, with the exit each idle CPU publishes from +`kernel/src/idle.rs`). A latency-sensitive wake preferring a shallow core +over a nearer deep one, and a throughput task choosing to pay a deep one's +exit, wait for the explicit latency requests below, whose caller says which +it is. The kernel chooses no P-state: it declares HWP's bounds (`kernel/src/arch/x86_64/control_regs.rs`), and the CPU chooses within them. -Filed so the placement track (see `placement-has-no-migration-cost-or-wake-affinity.md`) -leaves room for an idle-state input when it designs wake placement. +The placement track (`placement-has-no-migration-cost-or-wake-affinity.md`) +takes the published exit as its idle-state input. ## Realtime exhaustion (owner decision J2, 2026-10-20; stage 4) diff --git a/issues/toyos-runs-the-machine-in-acpi-mode-and-interprets-its-aml.md b/issues/toyos-runs-the-machine-in-acpi-mode-and-interprets-its-aml.md index 1d69a95faa1..731a77b3e19 100644 --- a/issues/toyos-runs-the-machine-in-acpi-mode-and-interprets-its-aml.md +++ b/issues/toyos-runs-the-machine-in-acpi-mode-and-interprets-its-aml.md @@ -15,8 +15,9 @@ the kernel its sleep type, which the kernel powers the machine off with and has from nowhere else. It initialises the namespace, negotiates `_OSC`, serves the runtime GPEs and delivers Notifies (#875). What else ToyOS takes from them it takes another way: the loader asks UEFI for the root -bridges' windows `_CRS` would name (`loader/src/rootbridge.rs`). Nothing -reads `_CST`, which names a CPU's C-states. +bridges' windows `_CRS` would name (`loader/src/rootbridge.rs`). The +processors' `_CST`, which names their C-states, is the idle track's +(`issues/idle-cpus-enter-the-c-states-cst-names.md`). **Ruled** (owner, 2026-10-03, on the scout in #681's comment 5966817345): the switch to ACPI mode lands as stage 1, together with a userland server that @@ -485,8 +486,12 @@ ACPI Interface Specification (302223-007) for a processor's `_OSC` on Intel: - The server does OSPM's side of the protocol. It: - loads every block, static and loaded at run time; - initialises the namespace; - - negotiates `\_SB._OSC` and each processor's `_OSC` with only what ToyOS - honours; + - negotiates `\_SB._OSC`, query then commit, and commits each + processor's `_OSC` with only what ToyOS honours and no query (the + orchestrator's ruling of 2026-10-27, under the owner's direction for + deep idle that day: 302223-007 §2.1 makes a processor's bits statements + of what OSPM does, and ACPI 6.5 §6.2.11.1.1 only recommends the query; + the T14's processor query answers 0x1 whatever is asked); - runs every runtime GPE's method; - delivers every Notify. - On the T14 and the AMD bench, in the state the owner ruled, ACPI 6.5 @@ -503,9 +508,8 @@ Not in finished: - concurrent method execution, which §5.5.2 permits and does not require; - the PCI host bridges' `_OSC`; - sleep states; -- `_CST`, which no part of this track reads: idle CPUs entering the states - it names read it (`issues/idle-cpus-enter-the-c-states-cst-names.md`, - #870); +- `_CST`, which the idle track reads (`system/acpiserver/src/cst.rs`, + `issues/idle-cpus-enter-the-c-states-cst-names.md`); - `_PDC`, which Intel deprecated in favour of `_OSC` in ACPI 3.0 and which ACPI 6.5 does not define. diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index e272e0e7958..4e679e87aca 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -1,7 +1,7 @@ //! Which state an idle CPU waits in: the states the `acpi` claim's holder //! declared for it, checked at the boundary (`toyos_userbound::idle`), chosen //! among once per idle period by `kernel::sched::idle::choose`, and the -//! machine-wide wake bound that [`WakeLatency`] holds set. +//! machine-wide wake bound that `object::wake_latency` holds set. //! //! **A CPU with nothing declared waits in C1**, `hlt`, as it always has: that //! is the machine before its AML server has spoken, not a failure. @@ -217,7 +217,7 @@ pub fn wait(armed: Option) { } } -/// A [`WakeLatency`] object's bound, held until it goes. +/// A `WakeLatencyObject`'s bound, held until it goes. pub fn hold(bound: u64) { let mut bounds = BOUNDS.lock(); bounds.hold(bound); From 97ad7a5992409773957e10f484c78b75e6f0ee3b Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:22:05 +0100 Subject: [PATCH 03/24] tests: acpi_tables_loaded reads each bench's idle states and the T14's three blocks Under ToyOS's committed processor claim the T14's first processor loads three blocks, its _CST's among them (916, 459 and 1162 bytes, as the check outside the tree read them on its dump), and every processor's _CST hands over MWAIT 0x31 and 0x60; the AMD bench hands over port 0x414 and withholds its type-3 0x415. The row now holds each bench to that, and to no declaration refused by the kernel. Co-Authored-By: Claude Opus 5.5 --- tests/toyos.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index 0b7a11526cf..8b936da6736 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -6620,6 +6620,9 @@ struct Bench { /// The blocks the processors' `_OSC` load under ToyOS's claims, by /// signature and length. loaded: &'static [&'static str], + /// The end of the server's line for the processors' idle states: what + /// every declaration it handed over held, and what it withheld. + idle: &'static str, /// Each register the kernel keeps that this machine's AML writes: the /// server says it once, and its evaluation goes on. kept: &'static [At], @@ -6660,8 +6663,9 @@ fn said_writes(lines: &[&str], head: &str) -> Result, String> { } /// The T14: 14 blocks; its `\_S5`'s `SLP_TYPa` read by a byte scan of its -/// DSDT, with no interpreter; the first processor's commit loads one block, -/// of 459 bytes. Its controller's `_INI` writes its GPIO pad +/// DSDT, with no interpreter; the first processor's commit of ToyOS's claim +/// (bits 4 and 9) loads three blocks, its `_CST`'s among them, and every +/// processor's `_CST` then names `MWAIT 0x31` and `0x60` beyond C1. Its controller's `_INI` writes its GPIO pad /// `PAD_CFG_DW0_GPPC_A_13` at 0xFD6E0A70 (631120-002 §27.1.114), and its /// hotkey device's `_INI` writes MCHBAR's cTDP control and turbo activation /// ratio, at 5F50h and 5F54h, and PL1 at 59A0h, which the kernel keeps @@ -6671,7 +6675,8 @@ const T14: Bench = Bench { blocks: 14, refused: &[], s5: "power: S5 is PM1a 0x1804 with SLP_TYPa=7,", - loaded: &["SSDT, 459 bytes"], + loaded: &["SSDT, 916 bytes", "SSDT, 459 bytes", "SSDT, 1162 bytes"], + idle: " with MWAIT 0x31 253 µs + MWAIT 0x60 1048 µs; withheld: none; unread: none", kept: &[At::Mchbar(0x59A0)], first_written: &[At::Address(0xFD6E_0000), At::Mchbar(0x5000)], }; @@ -6680,7 +6685,9 @@ const T14: Bench = Bench { /// table defines, refused whole as the owner ruled /// (`issues/the-amd-benchs-ssdt8-scopes-a-controller-no-table-defines.md`); /// its `\_S5`'s `SLP_TYPa` as the check outside the tree evaluated it; no -/// processor names an `_OSC`. Its controller's `_REG(3, 1)` and its +/// processor names an `_OSC`; each `_CST` hands over its port 0x414 and +/// withholds its type-3 0x415 +/// (`issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md`). Its controller's `_REG(3, 1)` and its /// adapter's `_PSR` each reach `ALIB`, whose mailbox writes the root /// complex's 0xB8 and 0xBC through ECAM, in 00:00.0's page at 0xF8000000. const AMD: Bench = Bench { @@ -6689,6 +6696,7 @@ const AMD: Bench = Bench { refused: &[(9, "a name that does not resolve")], s5: "with SLP_TYPa=5,", loaded: &[], + idle: " with port 0x414 400 µs; withheld: a type-3 SystemIO state, whose caches OSPM keeps coherent (ACPI 6.5 §8.1.4) x16; unread: none", kept: &[], first_written: &[At::Address(0xF800_0000)], }; @@ -6704,8 +6712,9 @@ const AMD: Bench = Bench { /// the bench's AML writes said once, and each page of registers it writes /// said with the kernel's first write into it; every battery reading names /// the adapter online or offline; `\_SB._OSC` -/// negotiated, and on an Intel machine each processor's, none refused; every -/// block they load loaded, none refused. Every other CPU of the machine said +/// negotiated, and on an Intel machine each processor's committed, none +/// refused; every block they load loaded, none refused; every processor's +/// idle states the bench's, and none refused by the kernel. Every other CPU of the machine said /// how its range registers stand beside the boot processor's, and none holds /// registers that are on and not those. What it prints beside that is the /// first measurement of each: the load's time, the reads by address space, @@ -6793,12 +6802,17 @@ fn acpi_tables_on_metal(back: &metal::Readback) -> Result<(), String> { return Err(format!("this bench's processors name no _OSC: {processors}")); } } else { - let processors = log.must_say("acpiserver: the processors' _OSC negotiated, Intel's: ")?; + let processors = log.must_say("acpiserver: the processors' _OSC committed, Intel's: ")?; if !processors.contains(", 0 refused;") { return Err(format!("a processor's _OSC was refused: {processors}")); } eprintln!(" [acpi] {}", processors.trim()); } + let idle = log.must_say("acpiserver: the processors' idle states: ")?; + if !idle.contains(" 0 refused by the kernel;") || !idle.trim_end().ends_with(bench.idle) { + return Err(format!("the processors' idle states are not this bench's, or the kernel refused one: {idle}")); + } + eprintln!(" [acpi] {}", idle.trim()); let loads: Vec<&str> = lines.iter().filter_map(|l| l.split_once("acpiserver: a table loaded at run time: ").map(|(_, said)| said.trim())).collect(); if loads != bench.loaded { return Err(format!("the tables loaded at run time are {loads:?}, where this bench's claims load {:?}", bench.loaded)); From 2d6ab7d0649653e3e54245c5eba22a84784ad275 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:25:54 +0100 Subject: [PATCH 04/24] idle: a CPU keeps what it learned of its wakes in two atomics of its own The history is eight bytes, so it is one word its CPU alone reads and writes, with no hand-written Sync and no unsafe cell. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/sched/idle.rs | 9 ++++++++ kernel/src/idle.rs | 48 ++++++++++++--------------------------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/kernel/pure/sched/idle.rs b/kernel/pure/sched/idle.rs index 50e0a88a6e6..dda84e37691 100644 --- a/kernel/pure/sched/idle.rs +++ b/kernel/pure/sched/idle.rs @@ -65,6 +65,15 @@ impl History { *self = Self::default(); } + /// The whole history as one word, for a CPU to keep in an atomic of its own. + pub const fn bits(&self) -> u64 { + u64::from_le_bytes(self.early) + } + + pub const fn from_bits(bits: u64) -> Self { + Self { early: bits.to_le_bytes() } + } + fn admits(&self, state: usize) -> bool { self.early[state].count_ones() <= EARLY_ALLOWED } diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index 4e679e87aca..88a44d89830 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -8,10 +8,8 @@ //! //! **Everything here on the idle path is the CPU's own and lock-free**: the //! declaration is a seqlock its CPU reads, the bound one atomic word, and what -//! the CPU learned of its wakes ([`Own`]) touched by that CPU alone, between -//! the pass's final interrupts-off check and its next pass. +//! the CPU learned of its wakes two words that CPU alone writes. -use core::cell::UnsafeCell; use core::sync::atomic::{AtomicU64, Ordering}; use kernel::sched::hw::Nanos; @@ -36,7 +34,10 @@ pub struct Local { table: Published, /// What the CPU can enter, read by it at its first idle; `None` before. facts: Lock>, - own: OwnCell, + /// What the CPU learned of its wakes (`History::bits`), and the + /// declaration it learned it under. + history: AtomicU64, + learned_under: AtomicU64, /// Idle periods spent in a declared state, and their nanoseconds. deep: AtomicU64, deep_nanos: AtomicU64, @@ -47,7 +48,8 @@ impl Local { Self { table: Published::new(), facts: Lock::new(None), - own: OwnCell(UnsafeCell::new(Own { history: History::default(), generation: 0 })), + history: AtomicU64::new(0), + learned_under: AtomicU64::new(0), deep: AtomicU64::new(0), deep_nanos: AtomicU64::new(0), } @@ -59,28 +61,6 @@ impl Local { } } -struct Own { - history: History, - /// The declaration [`Self::history`] learned under. - generation: u64, -} - -struct OwnCell(UnsafeCell); - -// SAFETY: reached only through `own`, by the CPU whose record holds it, from its idle path. -unsafe impl Sync for OwnCell {} - -/// This CPU's [`Own`]. -/// -/// # Safety -/// The caller is this CPU's idle path, which nothing re-enters: an interrupt -/// taken on an idle CPU runs no pass (`scheduler::do_preempt`) and no idle -/// code. -unsafe fn own() -> &'static mut Own { - // SAFETY: as the function's contract. - unsafe { &mut *crate::cpu_local::here().idle.own.0.get() } -} - /// Read what this CPU can enter: called by each CPU as it first enters its /// idle loop, before any declaration for it is checked. pub fn init_here() { @@ -175,20 +155,19 @@ static BOUNDS: Lock = Lock::new(Bounds::new()); /// in the state this idle period chooses; returns once an interrupt has been /// taken. The kernel's `Machine::idle_wait`. pub fn wait(armed: Option) { - // SAFETY: this is the CPU's idle path (`Machine::idle_wait`). - let own = unsafe { own() }; let local = &crate::cpu_local::here().idle; + let mut history = History::from_bits(local.history.load(Ordering::Relaxed)); let now = crate::clock::nanos_since_boot(); // A declaration mid-publication is this period in C1, and the next reads it. let declared = local.table.snapshot().map(|words| decode(&words)); let (len, costs) = declared.as_ref().map_or((0, &[][..]), |d| (d.len, &d.costs[..d.len])); - if let Some(d) = declared.as_ref().filter(|d| d.generation != own.generation) { - own.history.forget(); - own.generation = d.generation; + if let Some(d) = declared.as_ref().filter(|d| d.generation != local.learned_under.load(Ordering::Relaxed)) { + history.forget(); + local.learned_under.store(d.generation, Ordering::Relaxed); } let horizon = armed.map(|at| at.0.saturating_sub(now)); let limit = Some(LIMIT.load(Ordering::Relaxed)).filter(|&l| l != u64::MAX); - let choice = idle::choose(costs, &own.history, horizon, limit); + let choice = idle::choose(costs, &history, horizon, limit); crate::sched::driver::publish_idle_exit(choice.state.map_or(0, |state| costs[state].exit)); // The deadline met rather than made late by the state's exit: the timer // fires that much early, and the pass it brings finds nothing due and @@ -213,8 +192,9 @@ pub fn wait(armed: Option) { // the instant it was armed for; a revisit is a period the history learns. let foreseen = fires.is_some_and(|fires| now + slept >= fires); if !foreseen && len > 0 { - own.history.unforeseen(costs, slept); + history.unforeseen(costs, slept); } + local.history.store(history.bits(), Ordering::Relaxed); } /// A `WakeLatencyObject`'s bound, held until it goes. From 187ea32793c28fdd4dfa532608cfe455caf8dec8 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:26:47 +0100 Subject: [PATCH 05/24] counters: the record tests carry the two idle counters; idle_near's tiers are two bindings Co-Authored-By: Claude Opus 5.5 --- kernel/pure/sched/cpu.rs | 3 ++- lib/inspect/src/kernel.rs | 4 ++-- sdk/abi/src/counters.rs | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index eff34408376..8b49317772e 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -2569,7 +2569,8 @@ impl CpuHandles { pub fn idle_near(&self, me: CpuId, from: usize, usable: impl Fn(&CpuHandle) -> bool) -> Option { for (inner, outer) in IDLE_GROUPS { // The best of each tier, as (exit, cpu): strictly less replaces, so the first of equals stays. - let (mut free_core, mut shared_core): (Option<(u64, CpuId)>, Option<(u64, CpuId)>) = (None, None); + let mut free_core: Option<(u64, CpuId)> = None; + let mut shared_core: Option<(u64, CpuId)> = None; for cpu in self.topology.ring(me, inner, outer, from) { let handle = self.get(cpu); if !usable(handle) { diff --git a/lib/inspect/src/kernel.rs b/lib/inspect/src/kernel.rs index 646fecaffbe..0b1f02937ee 100644 --- a/lib/inspect/src/kernel.rs +++ b/lib/inspect/src/kernel.rs @@ -67,7 +67,7 @@ mod tests { /// calls on the boot processor, which makes them. fn whole(cpu: u32) -> Record { let (calls, nanos) = if cpu == 0 { (Some(1), Some(15_968)) } else { (None, None) }; - record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3), Some(0x8000_2a04), Some(0x8000_ff01), Some(6), calls, nanos]) + record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3), Some(0x8000_2a04), Some(0x8000_ff01), Some(6), calls, nanos, Some(40), Some(2_000_000)]) } #[test] @@ -88,7 +88,7 @@ mod tests { /// `TRACE`: only what the record carries, and no zero in place of the rest. #[test] fn an_absent_counter_has_no_path() { - let got = render(&[record(0, false, [Some(10), None, None, None, None, None, None, None, None, None])]).unwrap(); + let got = render(&[record(0, false, [Some(10), None, None, None, None, None, None, None, None, None, None, None])]).unwrap(); let paths: Vec<&str> = got.keys().map(String::as_str).collect(); assert_eq!(paths, vec!["kernel.cpu.0.hardware_id", "kernel.cpu.0.stale", "kernel.cpu.0.stamp"]); } diff --git a/sdk/abi/src/counters.rs b/sdk/abi/src/counters.rs index 82bf72375fd..7e2b5ee5788 100644 --- a/sdk/abi/src/counters.rs +++ b/sdk/abi/src/counters.rs @@ -204,8 +204,8 @@ mod tests { #[test] fn every_shape_of_record_reads_back_as_written() { for r in [ - record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0), Some(0x8000_2a04), Some(6), Some(0), Some(2), Some(31_936)]), - record(true, [Some(9), None, None, None, Some(4), None, Some(0x8000_ff01), None, Some(0), None]), + record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0), Some(0x8000_2a04), Some(6), Some(0), Some(2), Some(31_936), Some(0), Some(7_000_000)]), + record(true, [Some(9), None, None, None, Some(4), None, Some(0x8000_ff01), None, Some(0), None, Some(12), None]), record(true, [None; Counter::COUNT]), ] { assert_eq!(Record::decode(&r.encode()), Ok(r)); @@ -215,7 +215,7 @@ mod tests { /// Absent is not zero: a zero that is there reads back there. #[test] fn a_present_zero_is_not_an_absent_counter() { - let r = record(false, [Some(5), Some(0), None, None, None, None, None, None, None, None]); + let r = record(false, [Some(5), Some(0), None, None, None, None, None, None, None, None, None, None]); assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Smi), Some(0)); assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Aperf), None); } @@ -232,7 +232,7 @@ mod tests { #[test] fn a_word_under_an_absent_counter_is_refused() { - let mut raw = record(false, [Some(1), None, None, None, None, None, None, None, None, None]).encode(); + let mut raw = record(false, [Some(1), None, None, None, None, None, None, None, None, None, None, None]).encode(); raw.0[16 + 8 * Counter::Mperf as usize] = 1; assert_eq!(Record::decode(&raw), Err(Undecodable::Absent(Counter::Mperf))); } From b392ba8a867abae0b92b1f1d45461566e5c1f01a Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:37:57 +0100 Subject: [PATCH 06/24] idle: the cost word is parenthesised Co-Authored-By: Claude Opus 5.5 --- kernel/src/idle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index 88a44d89830..4020f96ab87 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -118,7 +118,7 @@ fn encode(table: &Table, generation: u64) -> [u64; WORDS] { }; words[1 + 2 * i] = kind | address << 8; // Microseconds, as declared: each fits its half. - words[2 + 2 * i] = state.exit_ns / 1000 | (state.residency_ns / 1000) << 32; + words[2 + 2 * i] = (state.exit_ns / 1000) | ((state.residency_ns / 1000) << 32); } words } From 59dcd1f03df64a6360393390683913962ed2cea1 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:50:04 +0100 Subject: [PATCH 07/24] soundserver holds its bound through replace and take; cst ends with a stopping machine The hold is a value kept for its drop, which an assignment never read denies in the guest build. A _CST or _CSD evaluation refused because the machine is stopping ends the declarations, as the server's other walks do, rather than counting the processor unread. Co-Authored-By: Claude Opus 5.5 --- system/acpiserver/src/cst.rs | 2 ++ system/soundserver/src/mix.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/system/acpiserver/src/cst.rs b/system/acpiserver/src/cst.rs index b063dd49106..684797163a9 100644 --- a/system/acpiserver/src/cst.rs +++ b/system/acpiserver/src/cst.rs @@ -119,6 +119,7 @@ pub fn declare(aml: &mut Aml<'_, K>, kernel: &K, processor: &Processo let read = |aml: &mut Aml<'_, K>, method: &str| aml.interpreter.evaluate(&mut aml.host, &format!("{}.{method}", processor.path), &[]); let states = match read(aml, "_CST").map_err(|why| kind(&why)).and_then(|v| cst(&v)) { Ok(states) => states, + Err(_) if aml.host.stopping => return Err(Stopping), Err(why) => { census.unread.see(&format!("_CST: {why}")); println!("{OWN}{}._CST unread: {why}", processor.path); @@ -128,6 +129,7 @@ pub fn declare(aml: &mut Aml<'_, K>, kernel: &K, processor: &Processo let software = if processor.csd { match read(aml, "_CSD").map_err(|why| kind(&why)).and_then(|v| csd(&v)) { Ok(software) => software, + Err(_) if aml.host.stopping => return Err(Stopping), Err(why) => { census.unread.see(&format!("_CSD: {why}")); println!("{OWN}{}._CSD unread, so none of its states is handed over: {why}", processor.path); diff --git a/system/soundserver/src/mix.rs b/system/soundserver/src/mix.rs index 38393108251..7ebf1f3b774 100644 --- a/system/soundserver/src/mix.rs +++ b/system/soundserver/src/mix.rs @@ -621,7 +621,7 @@ pub(crate) fn mix_thread( // started: the marker has to precede whatever the backend logs // about starting. say!("soundserver: resumed"); - wake_hold = Some(cap.hold_wake_latency(wake_bound).expect("the manifest declares this program `wake-latency`")); + wake_hold.replace(cap.hold_wake_latency(wake_bound).expect("the manifest declares this program `wake-latency`")); } backend.submit(idx, device_period_bytes); unplayed += 1; @@ -672,7 +672,7 @@ pub(crate) fn mix_thread( dll.reset(); backend.stop(); started = false; - wake_hold = None; + drop(wake_hold.take()); say!("soundserver: suspended"); } From 7b78def41a9785d291710f68abfe57a74513d994 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 01:40:10 +0100 Subject: [PATCH 08/24] idle: review r1 and the first metal readings Review r1: - A port state's fence is the holder's own port policy (firmware::port), as every AML port read meets: no CStateBaseAddr read and no family gate, so every AMD family gets its declared states and no CPU can #GP on an MSR it lacks. - The cst header says why an Intel FFH state of type 3 is handed over: its only OSPM duty is bus-master avoidance (302223-007 Table 4), and the processor flushes what a C-state powers off as it enters it (SDM Vol. 3B section 20.19.6.4). - The boundary refuses a residency shorter than the exit, zero among them, which the early wake before a deadline needs. - A hold holds from when it is made: the idle CPU publishes its exit, fences and reads the bound again; the hold stores the bound, fences and wakes every CPU whose exit exceeds it. - The AArch64 header and the track's "built on branch" line say what is true. - The timer plan, the foreseen test and the declaration's words are pure and host-tested; the foreseen test has a tick's slack. The metal readings: - The T14's cpu.clock stayed at 3789 in 16 of 17 group starts where the always-deepest control reached 4189 in 10: a CPU the history held in C1 needed five quiet revisits at C3's residency, about 10 ms, longer than cpu.clock's 5 ms window. A CPU now revisits after the next deeper state's residency, and a revisit nothing interrupted clears the history of each state it covered: it climbs a state per residency. - The AMD's IPC round trips were 1.28-1.32 times A's: every shallow idle armed its revisit with arm_within, an uncached read and a write of its xAPIC timer. The pass has just armed the timer for the token's instant, and the revisit and the early wake are never later, so one write arms them. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 2 +- kernel/pure/cpu.rs | 20 --- kernel/pure/sched/idle.rs | 152 ++++++++++++++-- kernel/src/arch/aarch64/idle.rs | 12 +- kernel/src/arch/x86_64/acpi_mode.rs | 4 +- kernel/src/arch/x86_64/idle.rs | 16 +- kernel/src/idle.rs | 164 +++++++++--------- kernel/src/object/wake_latency.rs | 5 +- kernel/src/sched/driver.rs | 15 +- lib/userbound/src/idle.rs | 154 ++++++++++++---- sdk/abi/src/syscall.rs | 8 +- sdk/toyos/src/syscap.rs | 5 +- system/acpiserver/src/cst.rs | 16 +- 13 files changed, 386 insertions(+), 187 deletions(-) diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index c47968f6026..420d0782731 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -6,7 +6,7 @@ opened: 2026-09-29 # Idle CPUs enter the C-states `_CST` names -Built on branch `deep-idle`: acpiserver hands the kernel what each +Built (#1073): acpiserver hands the kernel what each processor's `_CST` names that it may enter as it is (`system/acpiserver/src/cst.rs`), the kernel checks it against the CPU (`lib/userbound/src/idle.rs`) and chooses a state each idle period diff --git a/kernel/pure/cpu.rs b/kernel/pure/cpu.rs index e09518c4c72..c636a03f82b 100644 --- a/kernel/pure/cpu.rs +++ b/kernel/pure/cpu.rs @@ -226,16 +226,6 @@ fn counts_smis(id: &Ident) -> bool { id.vendor == Vendor::Intel && id.family == 6 && SMI_COUNT_MODELS.contains(&id.model) } -/// Whether the CPU `id` names has `CStateBaseAddr` (MSRC001_0073), the -/// eight ports its core traps as C-state requests: the family the PPRs that -/// define it are for (55922, 55901, 55772: family 17h), on the CPU itself -/// and not under a hypervisor, whose emulation of the register nothing -/// documents. A CPU wrongly admitted is a `#GP` at its first idle, so a -/// family no document in hand defines it for is left out and keeps C1. -pub fn traps_cstate_ports(id: &Ident, cpuid_1_ecx: u32) -> bool { - id.vendor == Vendor::Amd && id.family == 0x17 && cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR == 0 -} - /// Why a CPU's topology leaves describe no place in a tree. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum TopologyRefusal { @@ -290,16 +280,6 @@ pub fn topology_keys(levels: &[[u32; 4]], caches: &[[u32; 4]]) -> Result, } @@ -103,15 +118,68 @@ pub fn choose(costs: &[Cost], history: &History, horizon: Option, limit: Op let cost = costs[state]; limit.is_none_or(|limit| cost.exit <= limit) && horizon.is_none_or(|horizon| cost.residency <= horizon) }; - let deepest_permitted = states.clone().rev().find(|&s| permitted(s)); - let state = states.rev().find(|&s| permitted(s) && history.admits(s)); + let state = states.clone().rev().find(|&s| permitted(s) && history.admits(s)); + let next_deeper = states.filter(|&s| state.is_none_or(|chosen| s > chosen)).find(|&s| permitted(s)); Choice { state, wake_early_by: match (state, horizon) { (Some(state), Some(_)) => costs[state].exit, _ => 0, }, - revisit_after: deepest_permitted.filter(|&deepest| Some(deepest) != state).map(|deepest| costs[deepest].residency), + revisit_after: next_deeper.map(|deeper| costs[deeper].residency), + } +} + +/// How early the scheduler's own timer may fire and still be the wake it was +/// armed for: a one-shot counts its span in whole ticks of its timer, rounding +/// down, so it fires up to a tick early. A policy far above one tick of either +/// architecture's timer, and far below any state's residency. +pub const FORESEEN_SLACK: u64 = 1_000; + +/// When the scheduler's own timer fires after this choice, where one is armed +/// for `armed`: its deadline, pulled in by the chosen state's exit. +pub fn deadline_fires(choice: &Choice, armed: Option) -> Option { + armed.map(|at| at.saturating_sub(choice.wake_early_by)) +} + +/// The instant to fire this CPU's timer at, sooner than the scheduler armed +/// it, for `choice` made at `now`: the deadline pulled in by the exit, or the +/// revisit, whichever is sooner. Never at or after `armed`, so writing it can +/// only move the timer earlier; `None` leaves it as the scheduler armed it. +pub fn wake_at(choice: &Choice, armed: Option, now: u64) -> Option { + let early = deadline_fires(choice, armed).filter(|_| choice.wake_early_by > 0); + let revisit = choice.revisit_after.map(|after| now.saturating_add(after)); + early.into_iter().chain(revisit).min().filter(|&at| armed.is_none_or(|armed| at < armed)) +} + +/// Whether a period that ended at `woke` was ended by the scheduler's own +/// timer, firing for `fires`: what the history must not learn from, since the +/// horizon already said it. +pub fn foreseen(fires: Option, woke: u64) -> bool { + fires.is_some_and(|fires| woke.saturating_add(FORESEEN_SLACK) >= fires) +} + +/// What ended an idle period. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Ended { + /// The scheduler's own timer, which the horizon already said. + Deadline, + /// The CPU's revisit, nothing else having come first. + Revisit, + /// A kick, an IPI or a device: what the history learns from. + Unforeseen, +} + +/// What ended an idle period chosen as `choice` at `now`, with the +/// scheduler's timer armed for `armed`, that ended at `woke`. +pub fn ended(choice: &Choice, armed: Option, now: u64, woke: u64) -> Ended { + let revisit = choice.revisit_after.map(|after| now.saturating_add(after)); + if foreseen(deadline_fires(choice, armed), woke) { + Ended::Deadline + } else if foreseen(revisit, woke) { + Ended::Revisit + } else { + Ended::Unforeseen } } @@ -256,7 +324,7 @@ mod tests { for _ in 0..WINDOW { short.unforeseen(&T14, 10 * US); } - assert_eq!(choose(&T14, &short, None, None).revisit_after, Some(2096 * US)); + assert_eq!(choose(&T14, &short, None, None).revisit_after, Some(506 * US), "the next deeper state is C2"); assert_eq!(choose(&T14, &short, Some(1000 * US), None).revisit_after, Some(506 * US), "C2 alone fits the horizon"); assert_eq!(choose(&T14, &short, None, Some(300 * US)).revisit_after, Some(506 * US), "C2 alone fits the bound"); assert_eq!(choose(&T14, &History::default(), Some(1000 * US), None).revisit_after, None, "the horizon kept it from C3, not the history"); @@ -264,24 +332,47 @@ mod tests { } /// A CPU that was busy and then got nothing more to do is woken only by - /// its revisits, each a period of the residency it was asked to look - /// after: it reaches the deepest state within the window less the - /// allowance, and then stays. + /// its revisits: it climbs a state per revisit, C2 after C2's residency + /// and C3 after C3's, and then stays. #[test] - fn a_cpu_left_idle_after_short_idles_reaches_the_deepest_state_on_its_revisits_alone() { + fn a_cpu_left_idle_after_short_idles_climbs_a_state_per_revisit() { let mut h = History::default(); for _ in 0..WINDOW { h.unforeseen(&T14, 10 * US); } - let mut revisits = 0; + let mut path = std::vec::Vec::new(); loop { let choice = choose(&T14, &h, None, None); + path.push(choice.state); let Some(after) = choice.revisit_after else { break }; - h.unforeseen(&T14, after); - revisits += 1; + h.quiet(&T14, after); } - assert_eq!(choose(&T14, &h, None, None).state, Some(1)); - assert_eq!(revisits, WINDOW - EARLY_ALLOWED); + assert_eq!(path, [None, Some(0), Some(1)]); + } + + /// A revisit clears only what it covered: a period of C2's residency says + /// nothing of an idle as long as C3's. + #[test] + fn a_quiet_period_clears_the_states_it_covers_and_no_deeper() { + let mut h = History::default(); + for _ in 0..WINDOW { + h.unforeseen(&T14, 10 * US); + } + h.quiet(&T14, 506 * US); + assert_eq!(choose(&T14, &h, None, None).state, Some(0)); + assert_eq!(choose(&T14, &h, None, None).revisit_after, Some(2096 * US)); + } + + #[test] + fn a_period_is_ended_by_the_deadline_the_revisit_or_neither() { + let held = Choice { state: None, wake_early_by: 0, revisit_after: Some(506 * US) }; + assert_eq!(ended(&held, Some(10_000 * US), 0, 10_000 * US), Ended::Deadline); + assert_eq!(ended(&held, Some(10_000 * US), 0, 506 * US), Ended::Revisit); + assert_eq!(ended(&held, None, 0, 506 * US - FORESEEN_SLACK), Ended::Revisit, "a tick short"); + assert_eq!(ended(&held, None, 0, 300 * US), Ended::Unforeseen); + let deep = Choice { state: Some(1), wake_early_by: 1048 * US, revisit_after: None }; + assert_eq!(ended(&deep, Some(10_000 * US), 0, 8952 * US), Ended::Deadline, "the early wake is the deadline's"); + assert_eq!(ended(&deep, None, 0, 9_000 * US), Ended::Unforeseen); } #[test] @@ -300,6 +391,33 @@ mod tests { assert_eq!(b.least(), None); } + #[test] + fn the_timer_fires_at_the_deadline_less_the_exit_or_the_revisit_whichever_is_sooner() { + let deep = Choice { state: Some(1), wake_early_by: 1048 * US, revisit_after: None }; + assert_eq!(deadline_fires(&deep, Some(10_000 * US)), Some(8952 * US)); + assert_eq!(wake_at(&deep, Some(10_000 * US), 0), Some(8952 * US)); + let held = Choice { state: Some(0), wake_early_by: 253 * US, revisit_after: Some(2096 * US) }; + assert_eq!(wake_at(&held, Some(10_000 * US), 1000 * US), Some(3096 * US), "the revisit is sooner"); + assert_eq!(wake_at(&held, Some(2_000 * US), 1000 * US), Some(1747 * US), "the deadline is sooner"); + let c1 = Choice { state: None, wake_early_by: 0, revisit_after: Some(506 * US) }; + assert_eq!(wake_at(&c1, None, 0), Some(506 * US), "a CPU with no timer still comes back to look"); + assert_eq!(deadline_fires(&c1, Some(10_000 * US)), Some(10_000 * US)); + assert_eq!(wake_at(&c1, Some(300 * US), 0), None, "a revisit after the deadline would move the timer later"); + let plain = Choice { state: None, wake_early_by: 0, revisit_after: None }; + assert_eq!(wake_at(&plain, Some(10_000 * US), 0), None, "nothing sooner than the scheduler armed"); + } + + #[test] + fn a_wake_is_the_schedulers_timer_from_a_tick_before_it_on_and_unforeseen_before_that() { + let fires = Some(5_000 * US); + assert!(foreseen(fires, 5_000 * US)); + assert!(foreseen(fires, 6_000 * US)); + assert!(foreseen(fires, 5_000 * US - FORESEEN_SLACK), "a one-shot that counted down a tick short"); + assert!(!foreseen(fires, 5_000 * US - FORESEEN_SLACK - 1)); + assert!(!foreseen(fires, 100 * US), "a kick long before the deadline"); + assert!(!foreseen(None, 5_000 * US), "with no timer armed, every wake is unforeseen"); + } + #[test] #[should_panic(expected = "released and never held")] fn releasing_a_bound_never_held_is_a_kernel_bug() { diff --git a/kernel/src/arch/aarch64/idle.rs b/kernel/src/arch/aarch64/idle.rs index f6e2b39c63f..6fe5a6d4b64 100644 --- a/kernel/src/arch/aarch64/idle.rs +++ b/kernel/src/arch/aarch64/idle.rs @@ -1,7 +1,7 @@ -//! AArch64's half of `crate::idle`: nothing deeper than `wfi` is declared -//! here, since no AArch64 machine hands the kernel a declaration -//! (`issues/aarch64-cpus-idle-only-in-wfi.md`), so every CPU reports a timer -//! that does not run in a deep state and keeps C1. +//! AArch64's half of `crate::idle`. Nothing declares an idle state on +//! AArch64 (`issues/aarch64-cpus-idle-only-in-wfi.md`): [`cpu_of`] names no +//! CPU, so no declaration is checked against [`facts`], and every CPU waits in +//! `wfi`. use toyos_userbound::idle::{CpuIdle, Entry}; @@ -9,9 +9,9 @@ pub fn facts() -> CpuIdle { CpuIdle::default() } -/// Never called: [`facts`] admits no state. +/// Never called: no declaration is kept. pub fn enter(_entry: Entry) { - unreachable!("AArch64 admits no idle state deeper than wfi") + unreachable!("AArch64 keeps no idle state deeper than wfi") } /// No processor UID names a CPU here: nothing declares one. diff --git a/kernel/src/arch/x86_64/acpi_mode.rs b/kernel/src/arch/x86_64/acpi_mode.rs index 741e92b52d4..a63c96f4087 100644 --- a/kernel/src/arch/x86_64/acpi_mode.rs +++ b/kernel/src/arch/x86_64/acpi_mode.rs @@ -553,7 +553,9 @@ pub fn s5(row: &isa::Row, word: u64) -> Result<(), SyscallError> { /// declaration is published at a time. pub fn idle(row: &isa::Row, declaration: &toyos_abi::acpi::IdleDeclaration) -> Result<(), SyscallError> { let _holder = acting(row)?; - crate::idle::declare(declaration) + // A port is one the holder's own AML could read through this claim. + let readable = |port| matches!(firmware::port(|port| pio::standing(port, ROW), port, Width::Byte, None), PortVerdict::Through(_)); + crate::idle::declare(declaration, readable) } /// Clear the lock word's owner where the claim's holder holds it, and tell diff --git a/kernel/src/arch/x86_64/idle.rs b/kernel/src/arch/x86_64/idle.rs index c5ac484d647..67dc25b9e43 100644 --- a/kernel/src/arch/x86_64/idle.rs +++ b/kernel/src/arch/x86_64/idle.rs @@ -9,9 +9,6 @@ use toyos_userbound::idle::{CpuIdle, Entry}; use super::cpu; use crate::sync::Lock; -/// MSRC001_0073, `CStateBaseAddr` (AMD PPR 55922). -const CSTATE_BASE_ADDR: u32 = 0xC001_0073; - /// The line `MONITOR` arms before an `MWAIT` state: one per CPU that nothing /// writes, so the wait ends on an interrupt alone, as `hlt`'s does. `MWAIT` /// enters no state without an armed monitor (SDM Vol. 2B, MWAIT). @@ -26,7 +23,7 @@ impl Line { /// What this CPU can enter, read on it. pub fn facts() -> CpuIdle { - let (signature, _, cpuid_1_ecx, _) = cpu::cpuid(1, 0); + let (_, _, cpuid_1_ecx, _) = cpu::cpuid(1, 0); let (cpuid_5_ecx, cpuid_5_edx) = if cpu::cpuid(0, 0).0 >= 5 { let (_, _, ecx, edx) = cpu::cpuid(5, 0); (ecx, edx) @@ -34,23 +31,18 @@ pub fn facts() -> CpuIdle { (0, 0) }; let invariant_tsc = cpu::cpuid(0x8000_0000, 0).0 >= 0x8000_0007 && cpu::cpuid(0x8000_0007, 0).3 & 1 << 8 != 0; - let id = kernel::cpu::Ident::new(cpu::vendor(), signature); - let trapped_ports = kernel::cpu::traps_cstate_ports(&id, cpuid_1_ecx) - .then(|| cpu::rdmsr(CSTATE_BASE_ADDR) as u16) - .filter(|&base| base != 0); CpuIdle { monitor: cpuid_1_ecx & 1 << 3 != 0, mwait_extensions: cpuid_5_ecx & 1 != 0, mwait_substates: cpuid_5_edx, always_running_timer: cpu::leaf_6().0 & 1 << 2 != 0, invariant_tsc, - trapped_ports, } } /// Wait in `entry`'s state with interrupts unmasked, as [`super::hw::halt`] -/// waits in C1; called with them masked, and returns once an interrupt has -/// been taken. +/// waits in C1; called with them masked, and returns once the CPU has left +/// it: for an interrupt, or from `MWAIT` for an event of the processor's own. /// /// Both keep `sti; hlt`'s guarantee that a wake landing between the decision /// and the wait is not slept through. `MWAIT` in the `STI` shadow: an interrupt @@ -71,7 +63,7 @@ pub fn enter(entry: Entry) { unsafe { asm!("sti; mwait", in("eax") hint, in("ecx") 0u32, options(nomem, nostack)) }; } Entry::Port { port } => { - // SAFETY: a port this CPU's core traps as a C-state request (`toyos_userbound::idle`), which reaches no device; enables interrupts and waits. + // SAFETY: a byte read of a port the acpi claim's holder may read itself (`toyos_userbound::idle`), which its firmware names a C-state entry; enables interrupts and waits. unsafe { asm!("sti; in al, dx", in("dx") port, out("al") _, options(nomem, nostack)) }; } } diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index 4020f96ab87..e8148dd3a03 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -9,21 +9,24 @@ //! **Everything here on the idle path is the CPU's own and lock-free**: the //! declaration is a seqlock its CPU reads, the bound one atomic word, and what //! the CPU learned of its wakes two words that CPU alone writes. +//! +//! **A bound holds from the moment its hold is made.** A CPU publishes the +//! exit of the state it is about to wait in, fences, and reads the bound +//! again; a new hold stores the bound, fences, and wakes every CPU whose +//! published exit exceeds it. So either the CPU sees the bound, or the hold +//! sees the CPU. -use core::sync::atomic::{AtomicU64, Ordering}; +use core::sync::atomic::{fence, AtomicU64, Ordering}; use kernel::sched::hw::Nanos; use kernel::sched::idle::{self, Bounds, Cost, History, MAX_STATES}; use toyos_abi::acpi::IdleDeclaration; use toyos_abi::syscall::SyscallError; -use toyos_userbound::idle::{Entry, Table}; +use toyos_userbound::idle::{Entry, Table, WORDS}; use crate::seqlock::Published; use crate::sync::Lock; -/// A declaration's words: a generation and the count, then two per state. -const WORDS: usize = 1 + 2 * MAX_STATES; - /// Tells one declaration of a CPU from the next, so the CPU forgets what it /// learned of the states it replaced. static GENERATION: AtomicU64 = AtomicU64::new(1); @@ -71,17 +74,18 @@ pub fn init_here() { /// where its UID names no CPU this kernel runs, which a machine declaring /// more processor objects than it has CPUs gives often and is not said; /// `InvalidArgument`, said by name, where the CPU cannot enter it as declared. -/// The holder's lock is held: one declaration at a time. -pub fn declare(declaration: &IdleDeclaration) -> Result<(), SyscallError> { +/// `readable` says whether the holder could read a port itself. The holder's +/// lock is held: one declaration at a time. +pub fn declare(declaration: &IdleDeclaration, readable: impl Fn(u16) -> bool) -> Result<(), SyscallError> { let uid = declaration.processor_uid; let cpu = crate::arch::idle::cpu_of(uid).ok_or(SyscallError::NotFound)?; let local = &crate::cpu_local::of(cpu).idle; let facts = (*local.facts.lock()).ok_or(SyscallError::NotFound)?; - let table = toyos_userbound::idle::table(declaration, &facts).map_err(|why| { + let table = toyos_userbound::idle::table(declaration, &facts, readable).map_err(|why| { log!("idle: cpu{cpu}'s declaration (processor UID {uid}) refused, so it waits in C1: {why:?}"); SyscallError::InvalidArgument })?; - local.table.publish(encode(&table, GENERATION.fetch_add(1, Ordering::Relaxed))); + local.table.publish(toyos_userbound::idle::words(&table, GENERATION.fetch_add(1, Ordering::Relaxed))); log!("idle: cpu{cpu} (processor UID {uid}) waits in C1 or {}", Said(&table)); Ok(()) } @@ -105,103 +109,97 @@ impl core::fmt::Display for Said<'_> { } } -const MWAIT: u64 = 1; -const PORT: u64 = 2; - -fn encode(table: &Table, generation: u64) -> [u64; WORDS] { - let mut words = [0; WORDS]; - words[0] = generation << 8 | table.len() as u64; - for (i, state) in table.states().enumerate() { - let (kind, address) = match state.entry { - Entry::Mwait { hint } => (MWAIT, u64::from(hint)), - Entry::Port { port } => (PORT, u64::from(port)), - }; - words[1 + 2 * i] = kind | address << 8; - // Microseconds, as declared: each fits its half. - words[2 + 2 * i] = (state.exit_ns / 1000) | ((state.residency_ns / 1000) << 32); - } - words -} - -/// A CPU's states as [`encode`] wrote them, entry and cost each. -struct Declared { - generation: u64, - len: usize, - entries: [Entry; MAX_STATES], - costs: [Cost; MAX_STATES], -} - -fn decode(words: &[u64; WORDS]) -> Declared { - let len = (words[0] & 0xFF) as usize; - let mut entries = [Entry::Mwait { hint: 0 }; MAX_STATES]; - let mut costs = [Cost { exit: 0, residency: 0 }; MAX_STATES]; - for i in 0..len { - let (head, cost) = (words[1 + 2 * i], words[2 + 2 * i]); - entries[i] = match head & 0xFF { - MWAIT => Entry::Mwait { hint: (head >> 8) as u32 }, - PORT => Entry::Port { port: (head >> 8) as u16 }, - kind => panic!("idle: a published state of kind {kind}, which encode writes none of"), - }; - costs[i] = Cost { exit: (cost & 0xFFFF_FFFF) * 1000, residency: (cost >> 32) * 1000 }; - } - Declared { generation: words[0] >> 8, len, entries, costs } -} - /// The least bound held, in nanoseconds; `u64::MAX` is none. static LIMIT: AtomicU64 = AtomicU64::new(u64::MAX); static BOUNDS: Lock = Lock::new(Bounds::new()); +fn limit() -> Option { + Some(LIMIT.load(Ordering::Relaxed)).filter(|&l| l != u64::MAX) +} + /// Wait, with interrupts masked and the scheduler's timer armed for `armed`, -/// in the state this idle period chooses; returns once an interrupt has been -/// taken. The kernel's `Machine::idle_wait`. +/// in the state this idle period chooses; returns once the CPU has left it, +/// for an interrupt or, from `MWAIT`, for an event of the processor's own +/// (SDM Vol. 2B, MWAIT), which is a short unforeseen period. The kernel's +/// `Machine::idle_wait`. pub fn wait(armed: Option) { let local = &crate::cpu_local::here().idle; let mut history = History::from_bits(local.history.load(Ordering::Relaxed)); let now = crate::clock::nanos_since_boot(); // A declaration mid-publication is this period in C1, and the next reads it. - let declared = local.table.snapshot().map(|words| decode(&words)); - let (len, costs) = declared.as_ref().map_or((0, &[][..]), |d| (d.len, &d.costs[..d.len])); - if let Some(d) = declared.as_ref().filter(|d| d.generation != local.learned_under.load(Ordering::Relaxed)) { - history.forget(); - local.learned_under.store(d.generation, Ordering::Relaxed); + let declared = local.table.snapshot().map(|words| toyos_userbound::idle::read(&words)); + let mut entries = [Entry::Mwait { hint: 0 }; MAX_STATES]; + let mut costs = [Cost { exit: 0, residency: 0 }; MAX_STATES]; + let mut len = 0; + if let Some(d) = &declared { + for (i, state) in d.states.iter().flatten().enumerate() { + entries[i] = state.entry; + costs[i] = Cost { exit: state.exit_ns, residency: state.residency_ns }; + len = i + 1; + } + if d.generation != local.learned_under.load(Ordering::Relaxed) { + history.forget(); + local.learned_under.store(d.generation, Ordering::Relaxed); + } } + let costs = &costs[..len]; let horizon = armed.map(|at| at.0.saturating_sub(now)); - let limit = Some(LIMIT.load(Ordering::Relaxed)).filter(|&l| l != u64::MAX); - let choice = idle::choose(costs, &history, horizon, limit); - crate::sched::driver::publish_idle_exit(choice.state.map_or(0, |state| costs[state].exit)); - // The deadline met rather than made late by the state's exit: the timer - // fires that much early, and the pass it brings finds nothing due and - // waits again, shallower. A CPU its history held shallow wakes to look - // again sooner still, where that is sooner. - let fires = armed.map(|at| at.0.saturating_sub(choice.wake_early_by)); - let early = fires.filter(|_| choice.wake_early_by > 0); - let revisit = choice.revisit_after.map(|after| now.saturating_add(after)); - if let Some(at) = early.into_iter().chain(revisit).min() { - crate::arch::irqchip::arm_within(at.saturating_sub(now)); + // Chosen under the bound read after the exit is published: a hold made + // meanwhile either is read here or sees this exit and wakes this CPU. + let mut bound = limit(); + let choice = loop { + let choice = idle::choose(costs, &history, horizon, bound); + let exit = choice.state.map_or(0, |state| costs[state].exit); + crate::sched::driver::publish_idle_exit(exit); + fence(Ordering::SeqCst); + let now_bound = limit(); + if now_bound.is_none_or(|b| exit <= b) { + break choice; + } + bound = now_bound; + }; + let armed = armed.map(|at| at.0); + // The pass just armed the timer for `armed`, or stopped it (`SleepToken`), + // and `wake_at` is never later: one write moves it earlier, with no read + // of the timer, which on an xAPIC is an uncached access on every idle. + if let Some(at) = idle::wake_at(&choice, armed, now) { + crate::arch::irqchip::arm_one_shot(at.saturating_sub(now)); } - match (choice.state, &declared) { - (Some(state), Some(d)) => crate::arch::idle::enter(d.entries[state]), - _ => crate::arch::hw::halt(), + match choice.state { + Some(state) => crate::arch::idle::enter(entries[state]), + None => crate::arch::hw::halt(), } - let slept = crate::clock::nanos_since_boot().saturating_sub(now); + // Awake: no hold has a reason to wake this CPU until it waits again. + crate::sched::driver::publish_idle_exit(0); + let woke = crate::clock::nanos_since_boot(); + let slept = woke.saturating_sub(now); if choice.state.is_some() { local.deep.fetch_add(1, Ordering::Relaxed); local.deep_nanos.fetch_add(slept, Ordering::Relaxed); } - // The scheduler's own timer ended the period where it fired at or after - // the instant it was armed for; a revisit is a period the history learns. - let foreseen = fires.is_some_and(|fires| now + slept >= fires); - if !foreseen && len > 0 { - history.unforeseen(costs, slept); + if len > 0 { + match idle::ended(&choice, armed, now, woke) { + idle::Ended::Deadline => {} + idle::Ended::Revisit => history.quiet(costs, slept), + idle::Ended::Unforeseen => history.unforeseen(costs, slept), + } } local.history.store(history.bits(), Ordering::Relaxed); } -/// A `WakeLatencyObject`'s bound, held until it goes. +/// A `WakeLatencyObject`'s bound, held until it goes: from when this returns +/// no CPU enters a state slower to leave than it, and each CPU waiting in one +/// has been sent a wake, which pays that state's exit once. pub fn hold(bound: u64) { - let mut bounds = BOUNDS.lock(); - bounds.hold(bound); - LIMIT.store(bounds.least().unwrap_or(u64::MAX), Ordering::Relaxed); + let least = { + let mut bounds = BOUNDS.lock(); + bounds.hold(bound); + let least = bounds.least().expect("a bound was just held"); + LIMIT.store(least, Ordering::Relaxed); + least + }; + fence(Ordering::SeqCst); + crate::sched::driver::wake_slower_than(least); } pub fn release(bound: u64) { diff --git a/kernel/src/object/wake_latency.rs b/kernel/src/object/wake_latency.rs index 4d3ba21560b..0460577d969 100644 --- a/kernel/src/object/wake_latency.rs +++ b/kernel/src/object/wake_latency.rs @@ -1,5 +1,6 @@ -//! A hold on how long any idle CPU takes to wake: while one lives, no CPU waits -//! in a state slower to leave than the least bound held (`crate::idle`). It is +//! A hold on how long any idle CPU takes to wake: from its making until it +//! goes, no CPU enters a state slower to leave than the least bound held, and +//! its making wakes each CPU waiting in one (`crate::idle::hold`). It is //! made through a `SysCap` carrying `WAKE_LATENCY` and lasts as long as the //! object, so its last handle's going, the holder's death among them, ends it. diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 7d718891075..95d25f666f7 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -763,11 +763,24 @@ pub fn current_kill_pending() -> bool { } /// How long this CPU takes to wake from the state it is about to wait in, for -/// a wake forward choosing between sleepers (`CpuHandles::idle_near`). +/// a wake forward choosing between sleepers (`CpuHandles::idle_near`) and for +/// a new wake bound ([`wake_slower_than`]); 0 once it is awake. pub fn publish_idle_exit(nanos: u64) { cpus().get(current_cpu()).publish_idle_exit(nanos); } +/// Ring and kick every other CPU whose published exit exceeds `bound`, so it +/// leaves its state and chooses again under the bound. +pub fn wake_slower_than(bound: u64) { + let me = current_cpu(); + for cpu in (0..cpus().len() as u32).map(CpuId).filter(|&cpu| cpu != me) { + let handle = cpus().get(cpu); + if handle.idle_exit() > bound && handle.poke() == Kick::Send { + HW.kick(cpu); + } + } +} + pub fn current_cpu() -> CpuId { CpuId(percpu::cpu_id()) } diff --git a/lib/userbound/src/idle.rs b/lib/userbound/src/idle.rs index 1320ca05f48..6b513413872 100644 --- a/lib/userbound/src/idle.rs +++ b/lib/userbound/src/idle.rs @@ -9,12 +9,20 @@ //! //! - an `MWAIT` hint names a sub-state CPUID leaf 5 enumerates, on a CPU with //! MONITOR/MWAIT and its extensions (Intel SDM Vol. 2B, MWAIT, Table 1-3); -//! - a port is one the core itself traps as a C-state request and no device -//! sees: `CStateBaseAddr`'s eight (AMD PPR 55922, MSRC001_0073); +//! - a port is one the holder could read itself through its claim: no port +//! the kernel keeps, protects or lent to another claim +//! (`crate::firmware::port`, asked by the caller), since a read the kernel +//! makes for the holder is one the holder's AML could make; //! - the CPU keeps time in any state: its APIC timer always runs (CPUID.06H: //! EAX[2]) and its TSC is invariant (CPUID.80000007H:EDX[8]). Without either a //! deep state stops the clock or the one-shot, so a CPU lacking one keeps -//! only C1, whatever is declared. +//! only C1, whatever is declared; +//! - a state's residency is no shorter than its exit: the CPU wakes `exit` +//! early before a deadline (`kernel::sched::idle`), which is a wait only +//! where the state fits a residency longer than that. +//! +//! The kernel keeps a CPU's table as words it publishes and its CPU reads +//! ([`words`], [`read`]). use toyos_abi::acpi::{IdleDeclaration, IdleEntry, IDLE_STATES}; @@ -31,9 +39,6 @@ pub struct CpuIdle { pub always_running_timer: bool, /// CPUID.80000007H:EDX[8]. pub invariant_tsc: bool, - /// The first of the eight ports the core traps for C-state entry, where it - /// traps any: `CStateBaseAddr` on AMD, nonzero. - pub trapped_ports: Option, } /// How a checked state is entered. @@ -93,14 +98,17 @@ pub enum Refused { NoMwait { state: usize }, /// A hint with reserved bits, or a sub-state leaf 5 does not enumerate. Hint { state: usize, hint: u64 }, - /// A port the core does not trap for C-state entry. + /// A port past 0xFFFF, or one the holder could not read itself. Port { state: usize, port: u64 }, /// A state that claims to wake in no time. NoLatency { state: usize }, + /// A residency shorter than the state's exit, zero among them. + Residency { state: usize }, } -/// `declaration` checked for the CPU `cpu` describes. -pub fn table(declaration: &IdleDeclaration, cpu: &CpuIdle) -> Result { +/// `declaration` checked for the CPU `cpu` describes; `readable` says whether +/// the holder could read a port itself. +pub fn table(declaration: &IdleDeclaration, cpu: &CpuIdle, readable: impl Fn(u16) -> bool) -> Result { let count = declaration.count as usize; if count > IDLE_STATES { return Err(Refused::Count(declaration.count)); @@ -126,17 +134,17 @@ pub fn table(declaration: &IdleDeclaration, cpu: &CpuIdle) -> Result { - let trapped = cpu.trapped_ports.is_some_and(|base| (u64::from(base)..u64::from(base) + 8).contains(&declared.address)); - if !trapped { - return Err(Refused::Port { state, port: declared.address }); - } - Entry::Port { port: declared.address as u16 } - } + IdleEntry::Port => match u16::try_from(declared.address) { + Ok(port) if readable(port) => Entry::Port { port }, + _ => return Err(Refused::Port { state, port: declared.address }), + }, }; if declared.latency_us == 0 { return Err(Refused::NoLatency { state }); } + if declared.residency_us < declared.latency_us { + return Err(Refused::Residency { state }); + } table.states[state] = Some(State { entry, exit_ns: u64::from(declared.latency_us) * 1000, @@ -146,6 +154,54 @@ pub fn table(declaration: &IdleDeclaration, cpu: &CpuIdle) -> Result [u64; WORDS] { + let mut words = [0; WORDS]; + words[0] = (generation << 8) | table.len() as u64; + for (i, state) in table.states().enumerate() { + let (kind, address) = match state.entry { + Entry::Mwait { hint } => (MWAIT, u64::from(hint)), + Entry::Port { port } => (PORT, u64::from(port)), + }; + words[1 + 2 * i] = kind | (address << 8); + // Microseconds, as declared: each fits its half. + words[2 + 2 * i] = (state.exit_ns / 1000) | ((state.residency_ns / 1000) << 32); + } + words +} + +/// What [`words`] wrote: the generation, and the states in order. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Published { + pub generation: u64, + pub states: [Option; IDLE_STATES], +} + +/// The words [`words`] wrote, read back. Words it did not write are a +/// kernel bug: the one publisher writes nothing else. +pub fn read(words: &[u64; WORDS]) -> Published { + let len = (words[0] & 0xFF) as usize; + assert!(len <= IDLE_STATES, "idle: a published table of {len} states, which no table holds"); + let mut states = [None; IDLE_STATES]; + for (i, slot) in states.iter_mut().enumerate().take(len) { + let (head, cost) = (words[1 + 2 * i], words[2 + 2 * i]); + let entry = match head & 0xFF { + MWAIT => Entry::Mwait { hint: (head >> 8) as u32 }, + PORT => Entry::Port { port: (head >> 8) as u16 }, + kind => panic!("idle: a published state of kind {kind}, which words writes none of"), + }; + *slot = Some(State { entry, exit_ns: (cost & 0xFFFF_FFFF) * 1000, residency_ns: (cost >> 32) * 1000 }); + } + Published { generation: words[0] >> 8, states } +} + /// `hint` where leaf 5 enumerates its sub-state: bits 7:4 the target C-state /// less one (15 is C0), bits 3:0 the sub-state, the rest reserved (SDM Table /// 1-3); `substates` holds C0's count in bits 3:0, C1's in 7:4, and so on. @@ -175,20 +231,24 @@ mod tests { mwait_substates: 0x1002_2120, always_running_timer: true, invariant_tsc: true, - trapped_ports: None, }; /// A Zen 2's: MWAIT reaches C1 alone (PPR 55922, CPUID_Fn00000005_EDX - /// 0000_0011h), and the core traps 0x413 to 0x41A. + /// 0000_0011h). const ZEN_2: CpuIdle = CpuIdle { monitor: true, mwait_extensions: true, mwait_substates: 0x11, always_running_timer: true, invariant_tsc: true, - trapped_ports: Some(0x413), }; + /// The holder's port policy as a test asks it: every port but the ones + /// the kernel keeps, here the 8259 pair's. + fn readable(port: u16) -> bool { + !matches!(port, 0x20 | 0x21 | 0xA0 | 0xA1) + } + fn state(entry: IdleEntry, address: u64, latency_us: u32) -> IdleState { IdleState { address, latency_us, residency_us: 2 * latency_us, entry: entry as u8, reserved: [0; 7] } } @@ -201,7 +261,7 @@ mod tests { #[test] fn the_t14s_two_states_are_kept_as_declared() { - let t = table(&declaring(&[state(IdleEntry::Mwait, 0x31, 253), state(IdleEntry::Mwait, 0x60, 1048)]), &INTEL).unwrap(); + let t = table(&declaring(&[state(IdleEntry::Mwait, 0x31, 253), state(IdleEntry::Mwait, 0x60, 1048)]), &INTEL, readable).unwrap(); let mut states = [None; 2]; for (slot, kept) in states.iter_mut().zip(t.states()) { *slot = Some(*kept); @@ -221,41 +281,65 @@ mod tests { // C3 has two sub-states, so 0x22 names a third; C6 has none; a target // of 7 would be C8, past leaf 5; 0xF0 is C0; 0x100 sets a reserved bit. for hint in [0x22, 0x50, 0x70, 0xF0, 0x100] { - assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, hint, 10)]), &INTEL), Err(Refused::Hint { state: 0, hint }), "{hint:#x}"); + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, hint, 10)]), &INTEL, readable), Err(Refused::Hint { state: 0, hint }), "{hint:#x}"); } - assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 10)]), &ZEN_2), Err(Refused::Hint { state: 0, hint: 0x31 })); + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 10)]), &ZEN_2, readable), Err(Refused::Hint { state: 0, hint: 0x31 })); } #[test] - fn the_amds_trapped_port_is_kept_and_any_other_refused() { - let kept = table(&declaring(&[state(IdleEntry::Port, 0x414, 400)]), &ZEN_2).unwrap(); + fn a_port_the_holder_may_read_is_kept_and_any_other_refused() { + let kept = table(&declaring(&[state(IdleEntry::Port, 0x414, 400)]), &ZEN_2, readable).unwrap(); assert_eq!(kept.states().next().unwrap().entry, Entry::Port { port: 0x414 }); - for port in [0x412, 0x41B, 0x1816, 0x1_0414] { - assert_eq!(table(&declaring(&[state(IdleEntry::Port, port, 400)]), &ZEN_2), Err(Refused::Port { state: 0, port }), "{port:#x}"); + for port in [0x20, 0xA1, 0x1_0414] { + assert_eq!(table(&declaring(&[state(IdleEntry::Port, port, 400)]), &ZEN_2, readable), Err(Refused::Port { state: 0, port }), "{port:#x}"); + } + } + + /// The early wake before a deadline is the exit; a state whose residency + /// is shorter would be woken from before it was worth entering. + #[test] + fn a_residency_shorter_than_the_exit_is_refused() { + let with = |latency_us, residency_us| IdleState { residency_us, ..state(IdleEntry::Mwait, 0x31, latency_us) }; + assert_eq!(table(&declaring(&[with(253, 252)]), &INTEL, readable), Err(Refused::Residency { state: 0 })); + assert_eq!(table(&declaring(&[with(253, 0)]), &INTEL, readable), Err(Refused::Residency { state: 0 })); + assert!(table(&declaring(&[with(253, 253)]), &INTEL, readable).is_ok()); + } + + #[test] + fn a_table_reads_back_from_its_words_whole() { + let t = table(&declaring(&[state(IdleEntry::Mwait, 0x31, 253), state(IdleEntry::Port, 0x414, 400)]), &INTEL, readable); + // Ports are no Intel state, but the boundary checks only what the CPU can do: a port is read on any. + let t = t.unwrap(); + let back = read(&words(&t, 7)); + assert_eq!(back.generation, 7); + let mut expected = [None; IDLE_STATES]; + for (slot, kept) in expected.iter_mut().zip(t.states()) { + *slot = Some(*kept); } - assert_eq!(table(&declaring(&[state(IdleEntry::Port, 0x414, 400)]), &INTEL), Err(Refused::Port { state: 0, port: 0x414 })); + assert_eq!(back.states, expected); + assert_eq!(read(&words(&Table::EMPTY, 9)), Published { generation: 9, states: [None; IDLE_STATES] }); } #[test] fn a_cpu_whose_timer_or_clock_stops_keeps_c1_alone() { let d = declaring(&[state(IdleEntry::Mwait, 0x31, 253)]); - assert_eq!(table(&d, &CpuIdle { always_running_timer: false, ..INTEL }), Err(Refused::TimerStops)); - assert_eq!(table(&d, &CpuIdle { invariant_tsc: false, ..INTEL }), Err(Refused::TscVaries)); - assert_eq!(table(&declaring(&[]), &CpuIdle::default()), Ok(Table::EMPTY), "nothing declared needs nothing of the CPU"); + assert_eq!(table(&d, &CpuIdle { always_running_timer: false, ..INTEL }, readable), Err(Refused::TimerStops)); + assert_eq!(table(&d, &CpuIdle { invariant_tsc: false, ..INTEL }, readable), Err(Refused::TscVaries)); + assert_eq!(table(&declaring(&[]), &CpuIdle::default(), readable), Ok(Table::EMPTY), "nothing declared needs nothing of the CPU"); } #[test] fn a_malformed_declaration_is_refused_whole() { let mut d = declaring(&[state(IdleEntry::Mwait, 0x31, 253)]); d.count = 9; - assert_eq!(table(&d, &INTEL), Err(Refused::Count(9))); + assert_eq!(table(&d, &INTEL, readable), Err(Refused::Count(9))); let mut bad = state(IdleEntry::Mwait, 0x31, 253); bad.entry = 3; - assert_eq!(table(&declaring(&[bad]), &INTEL), Err(Refused::Malformed { state: 0 })); + assert_eq!(table(&declaring(&[bad]), &INTEL, readable), Err(Refused::Malformed { state: 0 })); let mut bad = state(IdleEntry::Mwait, 0x31, 253); bad.reserved[6] = 1; - assert_eq!(table(&declaring(&[bad]), &INTEL), Err(Refused::Malformed { state: 0 })); - assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 0)]), &INTEL), Err(Refused::NoLatency { state: 0 })); - assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 1)]), &CpuIdle { mwait_extensions: false, ..INTEL }), Err(Refused::NoMwait { state: 0 })); + assert_eq!(table(&declaring(&[bad]), &INTEL, readable), Err(Refused::Malformed { state: 0 })); + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 0)]), &INTEL, readable), Err(Refused::NoLatency { state: 0 })); + assert_eq!(table(&declaring(&[state(IdleEntry::Mwait, 0x31, 1)]), &CpuIdle { mwait_extensions: false, ..INTEL }, readable), Err(Refused::NoMwait { state: 0 })); } } diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 839262fd877..5f435f1e4ad 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -1762,9 +1762,11 @@ pub fn job_floor(syscap: RawHandle, job: RawHandle, bytes: u64) -> Result<(), Sy } /// Hold every idle CPU to waking within `bound_ns` for as long as the -/// returned handle lives: no CPU waits in a state slower to leave than the -/// least bound held. Needs [`Rights::WAKE_LATENCY`]; a bound of 0 is -/// refused [`SyscallError::InvalidArgument`]. +/// returned handle lives: from when this returns no CPU enters a state slower +/// to leave than the least bound held, and each CPU waiting in one has been +/// sent a wake, which pays that state's exit once. Needs +/// [`Rights::WAKE_LATENCY`]; a bound of 0 is refused +/// [`SyscallError::InvalidArgument`]. /// /// [`Rights::WAKE_LATENCY`]: crate::handle::Rights::WAKE_LATENCY pub fn wake_latency_hold(syscap: RawHandle, bound_ns: u64) -> Result { diff --git a/sdk/toyos/src/syscap.rs b/sdk/toyos/src/syscap.rs index d40b0265f5d..f227c3e0f34 100644 --- a/sdk/toyos/src/syscap.rs +++ b/sdk/toyos/src/syscap.rs @@ -96,8 +96,9 @@ impl SysCap { } /// Hold every idle CPU to waking within `bound` until the returned hold - /// is dropped. Needs [`Rights::WAKE_LATENCY`]; a bound under a - /// nanosecond is refused `InvalidArgument`. + /// is dropped ([`syscall::wake_latency_hold`]: it holds from when this + /// returns). Needs [`Rights::WAKE_LATENCY`]; a bound under a nanosecond + /// is refused `InvalidArgument`. pub fn hold_wake_latency(&self, bound: core::time::Duration) -> Result { let nanos = u64::try_from(bound.as_nanos()).unwrap_or(u64::MAX); let raw = syscall::wake_latency_hold(self.0.raw(), nanos)?; diff --git a/system/acpiserver/src/cst.rs b/system/acpiserver/src/cst.rs index 684797163a9..f37e314aab0 100644 --- a/system/acpiserver/src/cst.rs +++ b/system/acpiserver/src/cst.rs @@ -15,13 +15,19 @@ //! avoidance, which ToyOS does not track; any other FFH encoding has no //! document in hand; //! - a SystemIO state is a byte read of its register, handed over off Intel, -//! where the core traps it (the kernel checks which ports); on Intel it is a -//! chipset's P_LVLx read (§8.1.3) no machine ToyOS runs needs; +//! where the core traps it as a C-state request (AMD PPR 55922, +//! MSRC001_0073); on Intel it is a chipset's P_LVLx read (§8.1.3) no machine +//! ToyOS runs needs; //! - a type-3 SystemIO state is withheld: for one OSPM keeps the caches //! coherent, on a multiprocessor by flushing them before entry (§8.1.4), //! which ToyOS does not, and no document in hand says the processor's own //! hardware makes that unnecessary for it -//! (`issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md`); +//! (`issues/the-amd-benchs-c3-is-withheld-for-want-of-a-coherence-statement.md`). +//! An Intel FFH state of any type is not: its entry is the vendor's native +//! instruction, whose only OSPM duty Intel encodes is bus-master avoidance +//! (302223-007 Table 4, withheld above), and the processor flushes what a +//! C-state powers off as it enters it (SDM Vol. 3B §20.19.6.4: data "may have +//! been flushed to memory during the process of entering the idle state"); //! - a state `_CSD` names for software coordination (Table 8.1, SW_ALL or //! SW_ANY) is withheld: ToyOS coordinates no CPU's state with another's. //! HW_ALL asks nothing of OSPM. @@ -229,7 +235,9 @@ fn register_of(bytes: &[u8]) -> Option { } /// The `_CST` indices `_CSD` names for software coordination (Table 8.3: -/// NumEntries 6, Revision 0, Domain, CoordType, NumProcessors, Index). +/// NumEntries 6, Revision 0, Domain, CoordType, NumProcessors, Index). Index +/// counts the states after `_CST`'s count from 0: §8.4.1.2's example names +/// its C2-type state, the second, Index 1. pub fn csd(value: &Value) -> Result, String> { const SW_ALL: u64 = 0xFC; const SW_ANY: u64 = 0xFD; From 5be58f4c4635ed9f94ea155e8256a1e36cf34a37 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 02:13:51 +0100 Subject: [PATCH 09/24] idle: the pass chooses the idle state and arms its wake in its one timer write The orchestrator's ruling of 2026-10-28: an idle cycle costs one timer write, as on main. Machine::idle_plan chooses the state in the pass, with the timer the CPU would sleep under; the plan's own wake, the early wake before a deadline or the revisit, joins the pass's timer plan through TimerPlan::no_later_than, and the SleepToken carries the plan to the wait, which touches no timer. The simulator plans C1 by default. The port fence's trust argument is stated at the boundary, with what a declared port that is no trapped C-state request costs: a read that returns at once, never a hang. osbench reads, around each crunch and malloc_churn row, each CPU's busy fraction, the clock it ran at (APERF/MPERF), its core's C6 and C7 residency (MSR_CORE_C6/C7_RESIDENCY, SDM Vol. 4 Table 2-20, now counters), the kernel's deep idles, and the CPUs the row's threads ran on. A host test holds that a spawn's placement reads no idle exit. The idle track records the deep states' exit cost with main's and Linux's figures, as ruled. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 18 +++ kernel/pure/cpu.rs | 27 +++- kernel/pure/sched/cpu.rs | 75 ++++++++++- kernel/pure/sched/hw.rs | 10 ++ kernel/pure/sched/idle.rs | 38 +++++- kernel/src/arch/aarch64/counters.rs | 2 +- kernel/src/arch/x86_64/counters.rs | 9 +- kernel/src/counters.rs | 11 +- kernel/src/hw.rs | 7 +- kernel/src/idle.rs | 124 ++++++++++-------- lib/inspect/src/kernel.rs | 4 +- lib/userbound/src/idle.rs | 9 +- sdk/abi/src/counters.rs | 24 +++- tests/guest/osbench/src/around.rs | 77 +++++++++++ tests/guest/osbench/src/crunch.rs | 17 ++- tests/guest/osbench/src/main.rs | 1 + tests/guest/osbench/src/mem.rs | 8 +- tests/guest/osbench/src/sys/toyos.rs | 22 ++++ tests/guest/osbench/src/tail.rs | 35 +---- 19 files changed, 400 insertions(+), 118 deletions(-) create mode 100644 tests/guest/osbench/src/around.rs diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 420d0782731..202a03a2cfd 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -34,6 +34,24 @@ faster than a `hlt` control. Its `work.build_shaped.j1` p50 read 1176 ms against the control's 967, with its own 5 runs spread from 817 to 1594 ms, the control on a different `main`: unresolved. +**The exit cost of deep states, accepted** (orchestrator, 2026-10-28, under +the owner's direction of 2026-10-27), as #1073's first boots read it at +`59dcd1f03`, p50 against main's `ea6414e53` and against Linux 6.8 on the same +bench (Linux's 50 µs timer slack included): + +| row | T14 main | T14 deep idle | T14 Linux | AMD main | AMD deep idle | AMD Linux | +|---|---|---|---|---|---|---| +| `time.sleep_late.1ms` | 17.7 µs | 22.9 µs | 71.5-71.8 µs | 32.5 µs | 5.4 µs | 69.7-70.8 µs | +| `time.sleep_late.10ms` | 17.8 µs | 29.5 µs | 213 µs | 32.6 µs | 32.3 µs | 342-344 µs | +| `wake.storm.drain.w4` | 37 µs | 127 µs | 121-123 µs | 105 µs | 307 µs | 77-78 µs | +| `wake.storm.drain.w16` | 77 µs | 144 µs | 370-1041 µs | 333 µs | 329 µs | 113-200 µs | + +A sleep long enough for a deep state is woken early by its exit and still +lands later the deeper the state was (none at 100 µs, C2 at 1 ms, C10 at +10 ms), which an early wake that fell short would make hundreds of µs: +consistent with the cost of the caches a deep state powers off, not measured +as that. A burst of wakes to CPUs in deep states pays their exits. + **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, timer lateness no worse than `hlt`'s by more than the chosen state's exit latency, and osbench's `cpu.clock` maximum at ratio diff --git a/kernel/pure/cpu.rs b/kernel/pure/cpu.rs index c636a03f82b..71d6cc7dfee 100644 --- a/kernel/pure/cpu.rs +++ b/kernel/pure/cpu.rs @@ -190,6 +190,8 @@ pub struct Counters { pub aperf_mperf: bool, /// `MSR_SMI_COUNT` (34H). pub smi: bool, + /// `MSR_CORE_C6_RESIDENCY` and `MSR_CORE_C7_RESIDENCY` (3FDH, 3FEH). + pub core_residency: bool, } pub fn counters(facts: &CounterFacts) -> Counters { @@ -201,9 +203,26 @@ pub fn counters(facts: &CounterFacts) -> Counters { // firmware's count it is read for. smi: counts_smis(&Ident::new(facts.vendor, facts.signature)) && facts.cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR == 0, + // The same rule: a hypervisor's emulation is not the core's count. + core_residency: counts_core_residency(&Ident::new(facts.vendor, facts.signature)) + && facts.cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR == 0, } } +/// The family 6 DisplayModels SDM Vol. 4 names as supporting Table 2-20 +/// (§2.11-§2.17), which carries `MSR_CORE_C6_RESIDENCY` and +/// `MSR_CORE_C7_RESIDENCY`; [`SMI_COUNT_MODELS`]' Table 2-20 run. AMD +/// documents no core residency counter in the PPRs in hand. +const CORE_RESIDENCY_MODELS: [u32; 37] = [ + 0x2a, 0x2d, 0x3a, 0x3e, 0x3c, 0x45, 0x46, 0x3f, 0x3d, 0x47, 0x4f, 0x56, + 0x4e, 0x5e, 0x55, 0x8e, 0x9e, 0x66, 0xa5, 0xa6, 0x7e, 0x8c, 0x8d, 0x6a, + 0x6c, 0x97, 0x9a, 0xba, 0xb7, 0xbf, 0x8f, 0xcf, 0xaa, 0xad, 0xae, 0xaf, 0xbd, +]; + +fn counts_core_residency(id: &Ident) -> bool { + id.vendor == Vendor::Intel && id.family == 6 && CORE_RESIDENCY_MODELS.contains(&id.model) +} + /// The Intel family 6 DisplayModels a section of SDM Vol. 4 names as /// supporting a table that carries `MSR_SMI_COUNT`: Table 2-6 (§2.4-§2.7), /// 2-15 (§2.8-§2.10), 2-20 (§2.11-§2.17) and 2-61 (§2.18). A model Table 2-1 @@ -348,7 +367,7 @@ mod tests { /// 06_8CH, which §2.17 names as supporting Table 2-20. #[test] fn the_t14_counts_smis_and_has_aperf_and_mperf() { - assert_eq!(counters(&intel(T14_SIGNATURE)), Counters { aperf_mperf: true, smi: true }); + assert_eq!(counters(&intel(T14_SIGNATURE)), Counters { aperf_mperf: true, smi: true, core_residency: true }); } /// The AMD bench, Family 17h Model 60h: PPR 55922 gives it `EffFreq` and @@ -356,19 +375,19 @@ mod tests { #[test] fn the_amd_bench_has_aperf_and_mperf_and_no_smi_count() { let renoir = CounterFacts { vendor: Vendor::Amd, signature: 0x0086_0f01, cpuid_1_ecx: 0, cpuid_6_ecx: 1 }; - assert_eq!(counters(&renoir), Counters { aperf_mperf: true, smi: false }); + assert_eq!(counters(&renoir), Counters { aperf_mperf: true, smi: false, core_residency: false }); } #[test] fn a_vendor_neither_intel_nor_amd_has_neither() { let other = CounterFacts { vendor: Vendor::Other(*b"CentaurHauls"), ..intel(T14_SIGNATURE) }; - assert_eq!(counters(&other), Counters { aperf_mperf: false, smi: false }); + assert_eq!(counters(&other), Counters { aperf_mperf: false, smi: false, core_residency: false }); } #[test] fn a_guest_counts_no_smis() { let guest = CounterFacts { cpuid_1_ecx: CPUID_1_ECX_HYPERVISOR, ..intel(T14_SIGNATURE) }; - assert_eq!(counters(&guest), Counters { aperf_mperf: true, smi: false }); + assert_eq!(counters(&guest), Counters { aperf_mperf: true, smi: false, core_residency: false }); } /// Family 6 signatures with the given DisplayModel, stepping 1. diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index 8b49317772e..8589eebf71c 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -89,12 +89,14 @@ impl RunToken { #[must_use] pub struct SleepToken { armed: Option, + idle: crate::sched::idle::Plan, } impl SleepToken { - fn new(_quiesced: Quiesced, timer: TimerApplied) -> Self { + fn new(_quiesced: Quiesced, timer: TimerApplied, idle: crate::sched::idle::Plan) -> Self { Self { armed: timer.armed(), + idle, } } @@ -102,6 +104,11 @@ impl SleepToken { pub fn armed(&self) -> Option { self.armed } + + /// The idle state the pass chose, whose wake its timer plan includes. + pub fn idle(&self) -> &crate::sched::idle::Plan { + &self.idle + } } /// What the driver must do when the pass ends. @@ -2116,7 +2123,14 @@ impl SchedPass<'_, '_, H, P, Disposed> { fn try_sleep(&mut self) -> Result::Payload>, ()> { self.post_steal_probe(); let rearm = self.rearm_deadline(); - let timer = self.apply_timer_no_later_than(rearm); + // The idle choice is made against the timer the CPU would sleep + // with, and its own wake joins the same plan: one write of the timer. + let deadline = TimerPlan::compute(None, self.cpu.earliest_deadline()).no_later_than(rearm).armed(); + let idle = self.env.hw.idle_plan(deadline, self.now); + let timer = self.apply_timer_no_later_than(match (rearm, idle.wake_at().map(Nanos)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }); let arm: SleepArm<'_> = self.env.cpus.get(self.cpu.id).doorbell().arm_sleep(); // [`Balance::PushOnSurplus`]'s other half, and the half that makes the // push an ordering rather than a hope: SLEEPING is published above, so @@ -2131,7 +2145,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { return Err(()); } match arm.confirm(&self.cpu.mailbox) { - Ok(quiesced) => Ok(Action::Idle(SleepToken::new(quiesced, timer))), + Ok(quiesced) => Ok(Action::Idle(SleepToken::new(quiesced, timer, idle))), Err(_awake) => { self.env.cpus.get(self.cpu.id).doorbell().begin_pass(); self.cpu.drain(self.env, self.now); @@ -2670,6 +2684,10 @@ mod tests { need_resched: Vec, kicks: Vec, switches: Vec>, + /// Each timer write, `None` a stop. + timers: Vec>, + /// The revisit an idle plan asks, where a test sets one. + revisit: Option, } #[derive(Default)] @@ -2694,8 +2712,17 @@ mod tests { fn now(&self) -> Nanos { Nanos::ZERO } - fn set_timer(&self, _deadline: Nanos) {} - fn stop_timer(&self) {} + fn set_timer(&self, deadline: Nanos) { + self.state().timers.push(Some(deadline)); + } + fn stop_timer(&self) { + self.state().timers.push(None); + } + fn idle_plan(&self, deadline: Option, now: Nanos) -> crate::sched::idle::Plan { + let mut plan = crate::sched::idle::Plan::c1(deadline.map(|d| d.0), now.0); + plan.choice.revisit_after = self.state().revisit; + plan + } fn halt(&self) {} fn need_resched(&self, cpu: CpuId) { self.state().need_resched.push(cpu); @@ -4304,6 +4331,44 @@ mod tests { } } + /// **A spawn's placement reads no idle exit**: the shallowest-sleeper + /// preference is a wake forward's, so a new thread, the crunch row's or a + /// churn worker's, lands where it would with every CPU in C1, and a + /// thread that never blocks is never moved by it. + #[test] + fn a_spawn_lands_where_it_would_whatever_the_sleepers_exits() { + let (handles, _consumers) = bare_handles(two_llcs()); + sleep(&handles, &[0, 1, 2, 3, 4, 5, 6, 7]); + let plain: Vec = (0..8).map(|start| handles.place(CpuId(start), NOW)).collect(); + for (cpu, exit) in [(0, 1_048_000), (2, 253_000), (5, 1_048_000), (7, 253_000)] { + handles.get(CpuId(cpu)).publish_idle_exit(exit); + } + let deep: Vec = (0..8).map(|start| handles.place(CpuId(start), NOW)).collect(); + assert_eq!(plain, deep); + } + + /// **An idle CPU's plan joins its pass's one timer write**: the revisit + /// the plan asks is the instant the timer is armed for, written once, and + /// the sleep token carries the plan to the wait. + #[test] + fn an_idle_pass_arms_its_plans_wake_in_its_one_timer_write() { + let mut w = World::new(1); + w.hw.state().revisit = Some(500); + let (cpus, env) = w.split(); + let pass = SchedPass::begin(&mut cpus[0], env, NOW); + let Action::Idle(token) = pass.dispose_none().finish() else { panic!("an empty CPU sleeps") }; + assert_eq!(token.armed(), Some(Nanos(NOW.0 + 500))); + assert_eq!(token.idle().choice.revisit_after, Some(500)); + assert_eq!(w.hw.state().timers, [Some(Nanos(NOW.0 + 500))], "one write, at the revisit"); + + let mut w = World::new(1); + let (cpus, env) = w.split(); + let pass = SchedPass::begin(&mut cpus[0], env, NOW); + let Action::Idle(token) = pass.dispose_none().finish() else { panic!("an empty CPU sleeps") }; + assert_eq!(token.armed(), None); + assert_eq!(w.hw.state().timers, [None], "a plan with no wake leaves the plan main makes: stopped"); + } + /// **Locality first, then a free physical core, then rotation**: a forward /// or push from cpu0 takes a core whose threads all sleep in its own LLC, /// shares a busy core there before crossing to the other LLC's free ones, diff --git a/kernel/pure/sched/hw.rs b/kernel/pure/sched/hw.rs index 1c48b6ff473..50a1b0bf280 100644 --- a/kernel/pure/sched/hw.rs +++ b/kernel/pure/sched/hw.rs @@ -9,6 +9,7 @@ //! transition or ordering-sensitive code may live behind them. use crate::sched::cpu::{RunToken, SleepToken}; +use crate::sched::idle; use crate::sched::task::{SchedPayload, TaskAccounting, TaskKey}; /// CPU identity. Always a field or a parameter, never an ambient query — @@ -97,6 +98,15 @@ pub trait Machine: Kicker + 'static { fn stop_timer(&self); + /// The idle state this CPU waits in once the pass that asks ends, chosen + /// with the scheduler's timer due at `deadline` (`crate::sched::idle`). + /// The pass folds [`idle::Plan::wake_at`] into its one timer plan, and the + /// [`SleepToken`] carries the plan to the wait. A machine with no idle + /// states plans C1. + fn idle_plan(&self, deadline: Option, now: Nanos) -> idle::Plan { + idle::Plan::c1(deadline.map(|d| d.0), now.0) + } + /// Enable interrupts and halt, atomically — on x86 the `sti;hlt` pair and /// its STI shadow, which is why this is one operation and not an /// IRQ-guard drop followed by a halt. A wake that lands in diff --git a/kernel/pure/sched/idle.rs b/kernel/pure/sched/idle.rs index 5ac6cded7a7..f0e661150ad 100644 --- a/kernel/pure/sched/idle.rs +++ b/kernel/pure/sched/idle.rs @@ -96,7 +96,7 @@ impl History { } /// What [`choose`] picked. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] pub struct Choice { /// A declared state, or `None` for C1. pub state: Option, @@ -109,6 +109,42 @@ pub struct Choice { pub revisit_after: Option, } +/// What a CPU decided in its last pass before it sleeps, so the timer its +/// choice needs is part of the pass's one timer plan ([`wake_at`]) and the +/// wait enters what the pass chose. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub struct Plan { + pub choice: Choice, + /// The chosen state's exit, 0 for C1. + pub exit: u64, + /// Which declaration the choice indexes, the machine's own token for it: + /// a wait that finds another declared since enters C1. + pub declaration: u64, + /// The pass's instant, at which the choice was made. + pub chosen_at: u64, + /// When the scheduler's own timer was due before the choice moved it. + pub deadline: Option, +} + +impl Plan { + /// C1, with nothing declared or nothing chosen: what a machine with no + /// declaration plans. + pub const fn c1(deadline: Option, chosen_at: u64) -> Self { + Self { choice: Choice { state: None, wake_early_by: 0, revisit_after: None }, exit: 0, declaration: 0, chosen_at, deadline } + } + + /// The instant the pass's timer must fire by for this plan, sooner than + /// its deadline ([`wake_at`]). + pub fn wake_at(&self) -> Option { + wake_at(&self.choice, self.deadline, self.chosen_at) + } + + /// What ended the period that began with this plan and ended at `woke`. + pub fn ended(&self, woke: u64) -> Ended { + ended(&self.choice, self.deadline, self.chosen_at, woke) + } +} + /// The state to wait in. `horizon` is how long until the CPU's timer fires, /// `None` where none is armed; `limit` the held wake bound, `None` where /// nothing is held. diff --git a/kernel/src/arch/aarch64/counters.rs b/kernel/src/arch/aarch64/counters.rs index 5a123549238..7fb20966ec2 100644 --- a/kernel/src/arch/aarch64/counters.rs +++ b/kernel/src/arch/aarch64/counters.rs @@ -7,5 +7,5 @@ use crate::counters::Hardware; pub fn bring_up() {} pub fn read() -> Hardware { - Hardware { smi: None, aperf: None, mperf: None, envelope: None, firmware: None } + Hardware { smi: None, aperf: None, mperf: None, envelope: None, firmware: None, core_c6: None, core_c7: None } } diff --git a/kernel/src/arch/x86_64/counters.rs b/kernel/src/arch/x86_64/counters.rs index d1208eafe70..47908c331ab 100644 --- a/kernel/src/arch/x86_64/counters.rs +++ b/kernel/src/arch/x86_64/counters.rs @@ -10,11 +10,14 @@ use super::cpu; use crate::counters::{Firmware, Hardware}; const MSR_SMI_COUNT: u32 = 0x34; +const MSR_CORE_C6_RESIDENCY: u32 = 0x3FD; +const MSR_CORE_C7_RESIDENCY: u32 = 0x3FE; const IA32_MPERF: u32 = 0xE7; const IA32_APERF: u32 = 0xE8; const SMI: u8 = 1 << 0; const APERF_MPERF: u8 = 1 << 1; +const CORE_RESIDENCY: u8 = 1 << 2; /// A CPU's verdict, written by that CPU before it reads a counter. pub struct Local { @@ -40,7 +43,9 @@ pub fn bring_up() { cpuid_1_ecx, cpuid_6_ecx, }); - let bits = if verdict.smi { SMI } else { 0 } | if verdict.aperf_mperf { APERF_MPERF } else { 0 }; + let bits = if verdict.smi { SMI } else { 0 } + | if verdict.aperf_mperf { APERF_MPERF } else { 0 } + | if verdict.core_residency { CORE_RESIDENCY } else { 0 }; admitted().store(bits, Relaxed); } @@ -55,5 +60,7 @@ pub fn read() -> Hardware { mperf: msr(APERF_MPERF, IA32_MPERF), envelope: super::control_regs::envelope(), firmware: super::smi_cmd::counted().map(|(calls, nanos)| Firmware { calls, nanos }), + core_c6: msr(CORE_RESIDENCY, MSR_CORE_C6_RESIDENCY), + core_c7: msr(CORE_RESIDENCY, MSR_CORE_C7_RESIDENCY), } } diff --git a/kernel/src/counters.rs b/kernel/src/counters.rs index 25945279c71..393c057cefe 100644 --- a/kernel/src/counters.rs +++ b/kernel/src/counters.rs @@ -51,6 +51,9 @@ pub struct Hardware { pub mperf: Option, pub envelope: Option, pub firmware: Option, + /// `MSR_CORE_C6_RESIDENCY` and `MSR_CORE_C7_RESIDENCY`, where admitted. + pub core_c6: Option, + pub core_c7: Option, } /// The commands this CPU wrote to the firmware, where it is the one that @@ -103,18 +106,20 @@ pub fn bring_up() { let me = percpu::cpu_id() as usize; let words = sample(me); block(me).publish(words); - let [smi, aperf, mperf, request, pkg, epb] = [ + let [smi, aperf, mperf, request, pkg, epb, c6, c7] = [ Counter::Smi, Counter::Aperf, Counter::Mperf, Counter::HwpRequest, Counter::HwpRequestPkg, Counter::EnergyPerfBias, + Counter::CoreC6, + Counter::CoreC7, ] .map(|c| held(&words, c)); log!( "counters: cpu{me} reads smi={smi} aperf={aperf} mperf={mperf} hwp_request={request} \ - hwp_request_pkg={pkg} energy_perf_bias={epb}" + hwp_request_pkg={pkg} energy_perf_bias={epb} core_c6={c6} core_c7={c7}" ); } @@ -142,6 +147,8 @@ fn sample(me: usize) -> [u64; WORDS] { Counter::FirmwareNanos => hardware.firmware.as_ref().map(|f| f.nanos), Counter::DeepIdles => Some(crate::cpu_local::of(me as u32).idle.deep().0), Counter::DeepIdleNanos => Some(crate::cpu_local::of(me as u32).idle.deep().1), + Counter::CoreC6 => hardware.core_c6, + Counter::CoreC7 => hardware.core_c7, }; if let Some(value) = value { words[1] |= 1 << counter as usize; diff --git a/kernel/src/hw.rs b/kernel/src/hw.rs index 4510b519d83..80d0cc040db 100644 --- a/kernel/src/hw.rs +++ b/kernel/src/hw.rs @@ -62,6 +62,10 @@ impl Machine for KernelHw { irqchip::stop_timer(); } + fn idle_plan(&self, deadline: Option, now: Nanos) -> kernel::sched::idle::Plan { + crate::idle::plan(deadline, now) + } + fn halt(&self) { crate::arch::hw::halt(); } @@ -78,14 +82,13 @@ impl Machine for KernelHw { /// Waits in the state `crate::idle` chooses. Diagnostic builds arm a /// periodic wake first, so a quiescent CPU still reports. fn idle_wait(&self, token: SleepToken) { - let armed = token.armed(); #[cfg(feature = "boot-actuators")] if crate::actuator::diag_tick() { irqchip::arm_within(DIAG_TICK_NS); } #[cfg(feature = "mask-windows")] crate::windows::halting(); - crate::idle::wait(armed); + crate::idle::wait(token.idle()); #[cfg(feature = "mask-windows")] crate::windows::woken(); // **A CPU that is executing has a one-shot armed, and this is where diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index e8148dd3a03..72d81e7187d 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -117,73 +117,83 @@ fn limit() -> Option { Some(LIMIT.load(Ordering::Relaxed)).filter(|&l| l != u64::MAX) } -/// Wait, with interrupts masked and the scheduler's timer armed for `armed`, -/// in the state this idle period chooses; returns once the CPU has left it, -/// for an interrupt or, from `MWAIT`, for an event of the processor's own -/// (SDM Vol. 2B, MWAIT), which is a short unforeseen period. The kernel's +/// This CPU's declared states, read whole, and the declaration they came +/// from; `None` with nothing declared or a declaration mid-publication, which +/// this idle period treats as nothing declared and the next reads. +struct Declared { + generation: u64, + len: usize, + entries: [Entry; MAX_STATES], + costs: [Cost; MAX_STATES], +} + +fn declared(local: &Local) -> Option { + let read = toyos_userbound::idle::read(&local.table.snapshot()?); + let mut d = Declared { generation: read.generation, len: 0, entries: [Entry::Mwait { hint: 0 }; MAX_STATES], costs: [Cost { exit: 0, residency: 0 }; MAX_STATES] }; + for (i, state) in read.states.iter().flatten().enumerate() { + d.entries[i] = state.entry; + d.costs[i] = Cost { exit: state.exit_ns, residency: state.residency_ns }; + d.len = i + 1; + } + Some(d) +} + +/// What this CPU learned of its wakes, forgotten where its declaration is +/// not the one it learned under. +fn history(local: &Local, generation: u64) -> History { + let learned = History::from_bits(local.history.load(Ordering::Relaxed)); + if generation == local.learned_under.load(Ordering::Relaxed) { learned } else { History::default() } +} + +/// The idle state this CPU waits in once its pass ends, chosen with the +/// scheduler's timer due at `deadline`: the kernel's `Machine::idle_plan`. +/// The pass folds the plan's own wake into its one timer write. +pub fn plan(deadline: Option, now: Nanos) -> idle::Plan { + let (deadline, now) = (deadline.map(|d| d.0), now.0); + let local = &crate::cpu_local::here().idle; + let Some(d) = declared(local).filter(|d| d.len > 0) else { return idle::Plan::c1(deadline, now) }; + let costs = &d.costs[..d.len]; + let choice = idle::choose(costs, &history(local, d.generation), deadline.map(|at| at.saturating_sub(now)), limit()); + idle::Plan { choice, exit: choice.state.map_or(0, |state| costs[state].exit), declaration: d.generation, chosen_at: now, deadline } +} + +/// Wait, with interrupts masked and the timer armed by the pass, in the +/// state its `plan` chose; returns once the CPU has left it, for an +/// interrupt or, from `MWAIT`, for an event of the processor's own (SDM Vol. +/// 2B, MWAIT), which is a short unforeseen period. The kernel's /// `Machine::idle_wait`. -pub fn wait(armed: Option) { +/// +/// The plan's exit is published, fenced, and the bound read again: a hold +/// made since the pass either is read here, and the CPU waits in C1 under +/// the timer the plan armed, or sees the exit and wakes it. +pub fn wait(plan: &idle::Plan) { let local = &crate::cpu_local::here().idle; - let mut history = History::from_bits(local.history.load(Ordering::Relaxed)); - let now = crate::clock::nanos_since_boot(); - // A declaration mid-publication is this period in C1, and the next reads it. - let declared = local.table.snapshot().map(|words| toyos_userbound::idle::read(&words)); - let mut entries = [Entry::Mwait { hint: 0 }; MAX_STATES]; - let mut costs = [Cost { exit: 0, residency: 0 }; MAX_STATES]; - let mut len = 0; - if let Some(d) = &declared { - for (i, state) in d.states.iter().flatten().enumerate() { - entries[i] = state.entry; - costs[i] = Cost { exit: state.exit_ns, residency: state.residency_ns }; - len = i + 1; - } - if d.generation != local.learned_under.load(Ordering::Relaxed) { - history.forget(); - local.learned_under.store(d.generation, Ordering::Relaxed); - } - } - let costs = &costs[..len]; - let horizon = armed.map(|at| at.0.saturating_sub(now)); - // Chosen under the bound read after the exit is published: a hold made - // meanwhile either is read here or sees this exit and wakes this CPU. - let mut bound = limit(); - let choice = loop { - let choice = idle::choose(costs, &history, horizon, bound); - let exit = choice.state.map_or(0, |state| costs[state].exit); - crate::sched::driver::publish_idle_exit(exit); - fence(Ordering::SeqCst); - let now_bound = limit(); - if now_bound.is_none_or(|b| exit <= b) { - break choice; - } - bound = now_bound; - }; - let armed = armed.map(|at| at.0); - // The pass just armed the timer for `armed`, or stopped it (`SleepToken`), - // and `wake_at` is never later: one write moves it earlier, with no read - // of the timer, which on an xAPIC is an uncached access on every idle. - if let Some(at) = idle::wake_at(&choice, armed, now) { - crate::arch::irqchip::arm_one_shot(at.saturating_sub(now)); - } - match choice.state { - Some(state) => crate::arch::idle::enter(entries[state]), - None => crate::arch::hw::halt(), + let d = declared(local).filter(|d| d.len > 0 && d.generation == plan.declaration); + crate::sched::driver::publish_idle_exit(plan.exit); + fence(Ordering::SeqCst); + let state = plan.choice.state.filter(|_| d.is_some() && limit().is_none_or(|bound| plan.exit <= bound)); + let entered = crate::clock::nanos_since_boot(); + match (state, &d) { + (Some(state), Some(d)) => crate::arch::idle::enter(d.entries[state]), + _ => crate::arch::hw::halt(), } // Awake: no hold has a reason to wake this CPU until it waits again. crate::sched::driver::publish_idle_exit(0); let woke = crate::clock::nanos_since_boot(); - let slept = woke.saturating_sub(now); - if choice.state.is_some() { + let slept = woke.saturating_sub(entered); + if state.is_some() { local.deep.fetch_add(1, Ordering::Relaxed); local.deep_nanos.fetch_add(slept, Ordering::Relaxed); } - if len > 0 { - match idle::ended(&choice, armed, now, woke) { - idle::Ended::Deadline => {} - idle::Ended::Revisit => history.quiet(costs, slept), - idle::Ended::Unforeseen => history.unforeseen(costs, slept), - } + let Some(d) = d else { return }; + let costs = &d.costs[..d.len]; + let mut history = history(local, d.generation); + match plan.ended(woke) { + idle::Ended::Deadline => {} + idle::Ended::Revisit => history.quiet(costs, woke.saturating_sub(plan.chosen_at)), + idle::Ended::Unforeseen => history.unforeseen(costs, woke.saturating_sub(plan.chosen_at)), } + local.learned_under.store(d.generation, Ordering::Relaxed); local.history.store(history.bits(), Ordering::Relaxed); } diff --git a/lib/inspect/src/kernel.rs b/lib/inspect/src/kernel.rs index 0b1f02937ee..bdcb28e246b 100644 --- a/lib/inspect/src/kernel.rs +++ b/lib/inspect/src/kernel.rs @@ -67,7 +67,7 @@ mod tests { /// calls on the boot processor, which makes them. fn whole(cpu: u32) -> Record { let (calls, nanos) = if cpu == 0 { (Some(1), Some(15_968)) } else { (None, None) }; - record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3), Some(0x8000_2a04), Some(0x8000_ff01), Some(6), calls, nanos, Some(40), Some(2_000_000)]) + record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3), Some(0x8000_2a04), Some(0x8000_ff01), Some(6), calls, nanos, Some(40), Some(2_000_000), Some(1_000), Some(3_000)]) } #[test] @@ -88,7 +88,7 @@ mod tests { /// `TRACE`: only what the record carries, and no zero in place of the rest. #[test] fn an_absent_counter_has_no_path() { - let got = render(&[record(0, false, [Some(10), None, None, None, None, None, None, None, None, None, None, None])]).unwrap(); + let got = render(&[record(0, false, [Some(10), None, None, None, None, None, None, None, None, None, None, None, None, None])]).unwrap(); let paths: Vec<&str> = got.keys().map(String::as_str).collect(); assert_eq!(paths, vec!["kernel.cpu.0.hardware_id", "kernel.cpu.0.stale", "kernel.cpu.0.stamp"]); } diff --git a/lib/userbound/src/idle.rs b/lib/userbound/src/idle.rs index 6b513413872..cc2ac04d8de 100644 --- a/lib/userbound/src/idle.rs +++ b/lib/userbound/src/idle.rs @@ -11,8 +11,13 @@ //! MONITOR/MWAIT and its extensions (Intel SDM Vol. 2B, MWAIT, Table 1-3); //! - a port is one the holder could read itself through its claim: no port //! the kernel keeps, protects or lent to another claim -//! (`crate::firmware::port`, asked by the caller), since a read the kernel -//! makes for the holder is one the holder's AML could make; +//! (`crate::firmware::port`, asked by the caller). The port comes from the +//! firmware's AML, which is trusted platform code, and a read the kernel +//! makes for the holder is one the holder's AML could make, so this fence +//! keeps the read off what the kernel owns and needs no list of which parts +//! trap which ports. On a part where the port is no trapped C-state request +//! the read returns at once and the CPU passes back through selection: a +//! short unforeseen period the history learns, never a hang; //! - the CPU keeps time in any state: its APIC timer always runs (CPUID.06H: //! EAX[2]) and its TSC is invariant (CPUID.80000007H:EDX[8]). Without either a //! deep state stops the clock or the one-shot, so a CPU lacking one keeps diff --git a/sdk/abi/src/counters.rs b/sdk/abi/src/counters.rs index 7e2b5ee5788..8a883d5edca 100644 --- a/sdk/abi/src/counters.rs +++ b/sdk/abi/src/counters.rs @@ -55,10 +55,16 @@ pub enum Counter { DeepIdles, /// Nanoseconds those periods lasted, each from the choice to the wake. DeepIdleNanos, + /// Time the CPU's core was in its processor-specific C6 states, in TSC + /// ticks (`MSR_CORE_C6_RESIDENCY`): the hardware's own count, beside the + /// kernel's choices. + CoreC6, + /// The same for C7 (`MSR_CORE_C7_RESIDENCY`). + CoreC7, } impl Counter { - pub const COUNT: usize = 12; + pub const COUNT: usize = 14; pub const ALL: [Counter; Self::COUNT] = [ Self::Stamp, Self::Smi, @@ -72,6 +78,8 @@ impl Counter { Self::FirmwareNanos, Self::DeepIdles, Self::DeepIdleNanos, + Self::CoreC6, + Self::CoreC7, ]; pub const fn name(self) -> &'static str { @@ -88,6 +96,8 @@ impl Counter { Self::FirmwareNanos => "firmware_nanos", Self::DeepIdles => "deep_idles", Self::DeepIdleNanos => "deep_idle_nanos", + Self::CoreC6 => "core_c6", + Self::CoreC7 => "core_c7", } } @@ -102,7 +112,9 @@ impl Counter { | Self::HwpRequestPkg | Self::EnergyPerfBias | Self::DeepIdles - | Self::DeepIdleNanos => Rights::COUNTERS.union(Rights::TRACE), + | Self::DeepIdleNanos + | Self::CoreC6 + | Self::CoreC7 => Rights::COUNTERS.union(Rights::TRACE), } } } @@ -204,8 +216,8 @@ mod tests { #[test] fn every_shape_of_record_reads_back_as_written() { for r in [ - record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0), Some(0x8000_2a04), Some(6), Some(0), Some(2), Some(31_936), Some(0), Some(7_000_000)]), - record(true, [Some(9), None, None, None, Some(4), None, Some(0x8000_ff01), None, Some(0), None, Some(12), None]), + record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0), Some(0x8000_2a04), Some(6), Some(0), Some(2), Some(31_936), Some(0), Some(7_000_000), Some(5), None]), + record(true, [Some(9), None, None, None, Some(4), None, Some(0x8000_ff01), None, Some(0), None, Some(12), None, None, Some(9)]), record(true, [None; Counter::COUNT]), ] { assert_eq!(Record::decode(&r.encode()), Ok(r)); @@ -215,7 +227,7 @@ mod tests { /// Absent is not zero: a zero that is there reads back there. #[test] fn a_present_zero_is_not_an_absent_counter() { - let r = record(false, [Some(5), Some(0), None, None, None, None, None, None, None, None, None, None]); + let r = record(false, [Some(5), Some(0), None, None, None, None, None, None, None, None, None, None, None, None]); assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Smi), Some(0)); assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Aperf), None); } @@ -232,7 +244,7 @@ mod tests { #[test] fn a_word_under_an_absent_counter_is_refused() { - let mut raw = record(false, [Some(1), None, None, None, None, None, None, None, None, None, None, None]).encode(); + let mut raw = record(false, [Some(1), None, None, None, None, None, None, None, None, None, None, None, None, None]).encode(); raw.0[16 + 8 * Counter::Mperf as usize] = 1; assert_eq!(Record::decode(&raw), Err(Undecodable::Absent(Counter::Mperf))); } diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs new file mode 100644 index 00000000000..7440aafb1b9 --- /dev/null +++ b/tests/guest/osbench/src/around.rs @@ -0,0 +1,77 @@ +//! What the machine's CPUs did across one measured row, printed beside it: +//! the CPUs the row's threads ran on, on both systems; and on ToyOS, where +//! the boot grants the counters, each CPU's busy fraction, the clock it ran +//! at, its core's C6 and C7 residency where the CPU counts them, and the +//! idle periods the kernel chose a deep state for. A row that moves with the +//! machine's idle states, or with where its threads ran, says so here. + +use std::time::Instant; + +use crate::report::note; + +/// The counters as a row began. +pub struct Around { + began: Instant, + #[cfg(target_os = "toyos")] + before: Option>, +} + +/// Read the counters before a row. +pub fn begin() -> Around { + Around { + began: Instant::now(), + #[cfg(target_os = "toyos")] + before: crate::sys::diag::shared().and_then(|d| d.counters().ok()), + } +} + +impl Around { + /// Note what the CPUs did since [`begin`], for `bench`, whose threads ran + /// on `placed`: each thread's CPU as it began and as it ended, by + /// [`crate::sys::current_cpu`]'s naming. + pub fn end(self, bench: &str, placed: &[(u32, u32)]) { + let placed: Vec = placed.iter().map(|(from, to)| format!("{from}>{to}")).collect(); + note(&format!("{bench}.placement"), &[(crate::sys::CPU_NAMING, placed.join(","))]); + #[cfg(target_os = "toyos")] + if let (Some(before), Some(d)) = (self.before, crate::sys::diag::shared()) { + match d.counters() { + Ok(after) => cpus(bench, &before, &after, crate::ns_since(self.began)), + Err(why) => note(&format!("{bench}.cpus"), &[("failed", why)]), + } + } + #[cfg(not(target_os = "toyos"))] + let _ = self.began; + } +} + +/// One line a CPU: what it did between `before` and `after`, `ns` apart. +#[cfg(target_os = "toyos")] +pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate::sys::diag::CpuCounts], ns: u64) { + let delta = |x: Option, y: Option| x.zip(y).map(|(x, y)| y.wrapping_sub(x)); + let shown = |v: Option| v.map_or("-".to_string(), |v| v.to_string()); + let permille = |part: Option, whole: Option| shown(part.zip(whole).map(|(p, w)| p * 1000 / w.max(1))); + for b in after { + let Some(a) = before.iter().find(|a| a.cpu == b.cpu) else { continue }; + let stamp = delta(a.stamp, b.stamp); + let (aperf, mperf) = (delta(a.aperf, b.aperf), delta(a.mperf, b.mperf)); + // MPERF counts at the TSC's rate while the CPU runs, so APERF/MPERF + // of the TSC's rate is the clock it ran at. + let tsc_mhz = stamp.map(|s| s * 1000 / ns.max(1)); + let run_mhz = aperf.zip(mperf).zip(tsc_mhz).map(|((a, m), t)| a * t / m.max(1)); + note( + &format!("{bench}.cpus"), + &[ + ("cpu", b.cpu.to_string()), + ("apic", b.apic.to_string()), + ("busy_permille", permille(mperf, stamp)), + ("run_mhz", shown(run_mhz)), + ("core_c6_permille", permille(delta(a.core_c6, b.core_c6), stamp)), + ("core_c7_permille", permille(delta(a.core_c7, b.core_c7), stamp)), + ("deep_idles", shown(delta(a.deep_idles, b.deep_idles))), + ("deep_permille", permille(delta(a.deep_idle_nanos, b.deep_idle_nanos), Some(ns))), + ("kicks", shown(delta(a.kicks, b.kicks))), + ("smi", shown(delta(a.smi, b.smi))), + ], + ); + } +} diff --git a/tests/guest/osbench/src/crunch.rs b/tests/guest/osbench/src/crunch.rs index 2d3115fdf9a..0fdffa4ed27 100644 --- a/tests/guest/osbench/src/crunch.rs +++ b/tests/guest/osbench/src/crunch.rs @@ -218,7 +218,9 @@ fn run_phase(phase: &str, threads: usize, inputs: &Arc, heat: Option<&He continue; } } - let (mut ns, elapsed) = slice(workload, threads, inputs, &stopped); + let around = crate::around::begin(); + let (mut ns, elapsed, placed) = slice(workload, threads, inputs, &stopped); + around.end(&bench, &placed); let units = ns.len() as u64; let mut fields = vec![ ("work_unit", workload.unit.to_string()), @@ -244,8 +246,9 @@ fn stop_fields(heat: Option<&Heat>, hottest: &AtomicI64) -> Vec<(&'static str, S } /// `workload`'s units on `threads` threads until [`SLICE`] has passed or the -/// phase is stopped: every unit's ns, and the slice's own. -fn slice(workload: &Workload, threads: usize, inputs: &Arc, stopped: &AtomicU8) -> (Vec, u64) { +/// phase is stopped: every unit's ns, the slice's own, and each thread's CPU +/// as it began and as it ended. +fn slice(workload: &Workload, threads: usize, inputs: &Arc, stopped: &AtomicU8) -> (Vec, u64, Vec<(u32, u32)>) { let go = Barrier::new(threads + 1); let start = std::sync::OnceLock::new(); let samples = thread::scope(|scope| { @@ -255,20 +258,22 @@ fn slice(workload: &Workload, threads: usize, inputs: &Arc, stopped: &At scope.spawn(move || { go.wait(); let start: &Instant = start.get().expect("the slice's start is set before it goes"); + let began_on = crate::sys::current_cpu(); let mut ns = Vec::new(); while start.elapsed() < SLICE && stopped.load(Ordering::Relaxed) == RUNNING { let t = Instant::now(); std::hint::black_box((workload.run)(std::hint::black_box(inputs))); ns.push(ns_since(t)); } - ns + (ns, (began_on, crate::sys::current_cpu())) }) }) .collect(); start.set(Instant::now()).expect("one start"); go.wait(); - workers.into_iter().flat_map(|w| w.join().expect("a crunch worker panicked")).collect::>() + workers.into_iter().map(|w| w.join().expect("a crunch worker panicked")).collect::>() }); let elapsed = ns_since(*start.get().expect("set")); - (samples, elapsed) + let placed = samples.iter().map(|(_, placed)| *placed).collect(); + (samples.into_iter().flat_map(|(ns, _)| ns).collect(), elapsed, placed) } diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index d9e64686c28..4d127c6efff 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -37,6 +37,7 @@ //! beneath it (musl's on Linux) are each system's own. mod arch; +mod around; mod boot; mod clock; mod crunch; diff --git a/tests/guest/osbench/src/mem.rs b/tests/guest/osbench/src/mem.rs index 3fc9bd177f1..fc2d7e9eab0 100644 --- a/tests/guest/osbench/src/mem.rs +++ b/tests/guest/osbench/src/mem.rs @@ -135,18 +135,20 @@ fn churn(size: usize, threads: usize) { let bench = format!("mem.malloc_churn.{size}.t{threads}"); let ring = if size >= MIB { 4 } else { 64 }; let layout = Layout::from_size_align(size, 16).expect("a small layout"); + let placed = Arc::new(std::sync::Mutex::new(Vec::new())); let round = || -> u64 { let start = Arc::new(Barrier::new(threads + 1)); let stop = Arc::new(AtomicBool::new(false)); let done = Arc::new(AtomicU64::new(0)); let workers: Vec<_> = (0..threads) .map(|_| { - let (start, stop, done) = (start.clone(), stop.clone(), done.clone()); + let (start, stop, done, placed) = (start.clone(), stop.clone(), done.clone(), placed.clone()); thread::spawn(move || { // SAFETY: `layout` has a non-zero size; every block is // freed with it once, below. let mut live: Vec<*mut u8> = (0..ring).map(|_| unsafe { alloc(layout) }).collect(); start.wait(); + let began_on = crate::sys::current_cpu(); let mut ops = 0u64; while !stop.load(Ordering::Relaxed) { for slot in live.iter_mut() { @@ -166,6 +168,7 @@ fn churn(size: usize, threads: usize) { unsafe { dealloc(block, layout) }; } done.fetch_add(ops, Ordering::Relaxed); + placed.lock().expect("a churn worker panicked holding the placements").push((began_on, crate::sys::current_cpu())); }) }) .collect(); @@ -181,7 +184,10 @@ fn churn(size: usize, threads: usize) { }; round(); crate::clock::settle(); + placed.lock().expect("no worker runs now").clear(); + let around = crate::around::begin(); let mut rates: Vec = (0..5).map(|_| round()).collect(); + around.end(&bench, &placed.lock().expect("no worker runs now")); result(&bench, "kops/s", &mut rates, &[("block_bytes", size.to_string()), ("threads", threads.to_string()), ("ring", ring.to_string())]); } diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index d4a50d33247..9ad12a66c3a 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -223,10 +223,27 @@ pub mod diag { pub struct CpuCounts { pub cpu: u32, pub apic: u32, + /// The CPU's own counter when it read the rest (the TSC). + pub stamp: Option, pub smi: Option, pub aperf: Option, pub mperf: Option, pub kicks: Option, + /// Core C6 and C7 residency in TSC ticks, where the CPU counts it. + pub core_c6: Option, + pub core_c7: Option, + /// Idle periods the kernel chose a declared deep state for, and + /// their nanoseconds. + pub deep_idles: Option, + pub deep_idle_nanos: Option, + } + + /// The capability's diagnostics for the rows that read the counters + /// around them ([`crate::around`]): opened once a process, and `None` + /// where the boot endows no capability. + pub fn shared() -> Option<&'static Diag> { + static SHARED: std::sync::OnceLock> = std::sync::OnceLock::new(); + SHARED.get_or_init(open).as_ref() } pub fn open() -> Option { @@ -247,10 +264,15 @@ pub mod diag { Ok(CpuCounts { cpu: r.cpu, apic: r.hardware_id, + stamp: r.get(Counter::Stamp), smi: r.get(Counter::Smi), aperf: r.get(Counter::Aperf), mperf: r.get(Counter::Mperf), kicks: r.get(Counter::Kicks), + core_c6: r.get(Counter::CoreC6), + core_c7: r.get(Counter::CoreC7), + deep_idles: r.get(Counter::DeepIdles), + deep_idle_nanos: r.get(Counter::DeepIdleNanos), }) }) .collect() diff --git a/tests/guest/osbench/src/tail.rs b/tests/guest/osbench/src/tail.rs index d29e18fbb4a..d24182e158a 100644 --- a/tests/guest/osbench/src/tail.rs +++ b/tests/guest/osbench/src/tail.rs @@ -48,14 +48,14 @@ pub fn suite() { #[cfg(target_os = "toyos")] let diag = sys::diag::open(); #[cfg(target_os = "toyos")] - let before = diag.as_ref().map(|d| d.counters()); + let (before, began) = (diag.as_ref().map(|d| d.counters()), Instant::now()); let late = periodic("tail.wake.periodic.60s", Duration::from_secs(60), &mut |_| {}); print_outliers("tail.wake.periodic.60s", &late); #[cfg(target_os = "toyos")] if let (Some(d), Some(before)) = (diag.as_ref(), before) { - counters_over("tail.wake.periodic.60s", before, d); + counters_over("tail.wake.periodic.60s", before, d, crate::ns_since(began)); } stolen(Duration::from_secs(10)); @@ -204,33 +204,12 @@ fn stolen(span: Duration) { } } -/// Each CPU's SMIs, its average clock and its kicks between `before` and now, -/// by the kernel's index of the CPU and its APIC id. +/// What each CPU did between `before`, `ns` ago, and now ([`crate::around`]). #[cfg(target_os = "toyos")] -fn counters_over(bench: &str, before: Result, String>, d: &sys::diag::Diag) { - let (before, after) = match (before, d.counters()) { - (Ok(before), Ok(after)) => (before, after), - (Err(why), _) | (_, Err(why)) => return note(&format!("{bench}.counters"), &[("failed", why)]), - }; - for b in &after { - let Some(a) = before.iter().find(|a| a.cpu == b.cpu) else { continue }; - let delta = |x: Option, y: Option| match (x, y) { - (Some(x), Some(y)) => Some(y.wrapping_sub(x)), - _ => None, - }; - let shown = |v: Option| v.map_or("-".to_string(), |v| v.to_string()); - let (aperf, mperf) = (delta(a.aperf, b.aperf), delta(a.mperf, b.mperf)); - note( - &format!("{bench}.counters"), - &[ - ("cpu", b.cpu.to_string()), - ("apic", b.apic.to_string()), - ("smi", shown(delta(a.smi, b.smi))), - ("kicks", shown(delta(a.kicks, b.kicks))), - ("aperf_per_mperf_permille", shown(aperf.zip(mperf).map(|(a, m)| a * 1000 / m.max(1)))), - ("mperf", shown(mperf)), - ], - ); +fn counters_over(bench: &str, before: Result, String>, d: &sys::diag::Diag, ns: u64) { + match (before, d.counters()) { + (Ok(before), Ok(after)) => crate::around::cpus(&format!("{bench}.counters"), &before, &after, ns), + (Err(why), _) | (_, Err(why)) => note(&format!("{bench}.counters"), &[("failed", why)]), } } From f8749dc6972f8cd764bd514d52f48a6e2af390eb Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 02:31:28 +0100 Subject: [PATCH 10/24] osbench: the tail diary times its counters too Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/src/tail.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/guest/osbench/src/tail.rs b/tests/guest/osbench/src/tail.rs index d24182e158a..533e25a46db 100644 --- a/tests/guest/osbench/src/tail.rs +++ b/tests/guest/osbench/src/tail.rs @@ -222,7 +222,7 @@ fn counters_over(bench: &str, before: Result, String>, fn diary_run(mut d: sys::diag::Diag) { const BENCH: &str = "tail.diag.periodic.30s"; const SHOWN: usize = 16; - let before = d.counters(); + let (before, began) = (d.counters(), Instant::now()); // The diary names a CPU by the kernel's index, and the sleeper's CPUs // are APIC ids: the counters records carry both. let index_of: std::collections::BTreeMap = @@ -295,5 +295,5 @@ fn diary_run(mut d: sys::diag::Diag) { for ((pid, name), ns) in by_process.into_iter().filter(|(_, ns)| *ns > 0) { note(&format!("{BENCH}.cpu_time"), &[("pid", pid.to_string()), ("name", name), ("us", (ns / 1000).to_string())]); } - counters_over(BENCH, before, &d); + counters_over(BENCH, before, &d, crate::ns_since(began)); } From e14605abc6e0a6f580928d2acd56b50fdbe7fcc9 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 03:15:37 +0100 Subject: [PATCH 11/24] sched: a CPU beside a running SMT sibling waits in C1, and the sibling's sleep pokes it A core enters a core C-state only when every thread of it asks for one, so a thread waiting deep beside a running sibling saves nothing and pays the deeper exit; on the AMD bench it also cost the running sibling. Round 2's instruments read it: the one slow deflate (66.77 ms against 63.0) ran beside a sibling 99 % in port 0x414, every fast one beside a sibling in C1, at the same clock. On the T14 two runs beside deep siblings were as fast as without. A CPU whose plan is deeper than C1 marks itself held, fences and reads its siblings; a running one keeps it in C1. The sibling that sleeps publishes SLEEPING, fences and pokes each held sibling, which chooses again: an event, so a held CPU is not left in C1 when its core could go deep. Nothing is held where nothing deeper than C1 is declared. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/sched/cpu.rs | 105 +++++++++++++++++++++++++++++- tests/guest/osbench/src/around.rs | 1 + 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index 8589eebf71c..29464e57190 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -30,7 +30,7 @@ use crate::sched::mailbox::{ }; use crate::sched::msg::Msg; use crate::sched::queue::{RunQueue, MIN_SLICE_NS}; -use crate::sched::sync::{fence, Arc, AtomicU32, AtomicU64, Ordering}; +use crate::sched::sync::{fence, Arc, AtomicBool, AtomicU32, AtomicU64, Ordering}; use crate::sched::task::{ BlockedTask, Claim, DeadTask, ReadyTask, RunningTask, SchedPayload, TaskKey, TaskShared, TaskState, TransitTask, WaitClass, WakeCause, WakeReason, QUANTUM_NS, @@ -1753,6 +1753,8 @@ impl SchedPass<'_, '_, H, P, Disposed> { // The re-arm's allowance is per idle period, so a CPU that is // given work starts its next one with a full one. self.cpu.idle_probes_spent = 0; + // Running, it is held by nothing: a sibling going to sleep owes it no poke. + handle.held_by_sibling.store(false, Ordering::Relaxed); return self.switch_to_current(); } if !self.cpu.is_idle() { @@ -2126,7 +2128,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { // The idle choice is made against the timer the CPU would sleep // with, and its own wake joins the same plan: one write of the timer. let deadline = TimerPlan::compute(None, self.cpu.earliest_deadline()).no_later_than(rearm).armed(); - let idle = self.env.hw.idle_plan(deadline, self.now); + let idle = self.held_shallow_by_a_running_sibling(self.env.hw.idle_plan(deadline, self.now)); let timer = self.apply_timer_no_later_than(match (rearm, idle.wake_at().map(Nanos)) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), @@ -2145,7 +2147,10 @@ impl SchedPass<'_, '_, H, P, Disposed> { return Err(()); } match arm.confirm(&self.cpu.mailbox) { - Ok(quiesced) => Ok(Action::Idle(SleepToken::new(quiesced, timer, idle))), + Ok(quiesced) => { + self.poke_held_siblings(); + Ok(Action::Idle(SleepToken::new(quiesced, timer, idle))) + } Err(_awake) => { self.env.cpus.get(self.cpu.id).doorbell().begin_pass(); self.cpu.drain(self.env, self.now); @@ -2155,6 +2160,41 @@ impl SchedPass<'_, '_, H, P, Disposed> { } } + /// **A deep state beside a running SMT sibling saves nothing**: a core + /// enters a core C-state only when every thread of it asks for one, and + /// the thread waiting deep still pays the deeper exit (and on Zen 2 slows + /// the sibling it shares the core with, #1073's readings). So a CPU whose + /// plan is deeper than C1 waits in C1 while a sibling runs, marked held, + /// and the sibling that sleeps next pokes it to choose again + /// ([`Self::poke_held_siblings`]). The mark is stored and fenced before + /// the siblings are read, and a sleeper's SLEEPING before it reads the + /// marks, so one of the two sees the other. + fn held_shallow_by_a_running_sibling(&self, plan: crate::sched::idle::Plan) -> crate::sched::idle::Plan { + if plan.choice.state.is_none() { + return plan; + } + let handle = self.env.cpus.get(self.cpu.id); + handle.held_by_sibling.store(true, Ordering::Relaxed); + fence(Ordering::SeqCst); + if self.env.cpus.busy_siblings(self.cpu.id) == 0 { + handle.held_by_sibling.store(false, Ordering::Relaxed); + return plan; + } + crate::sched::idle::Plan { declaration: plan.declaration, ..crate::sched::idle::Plan::c1(plan.deadline, plan.chosen_at) } + } + + /// Poke each SMT sibling held in C1 for this CPU's running: SLEEPING is + /// published, so it now chooses again with this CPU asleep. + fn poke_held_siblings(&self) { + fence(Ordering::SeqCst); + for &sibling in self.env.cpus.topology().core(self.cpu.id) { + let handle = self.env.cpus.get(sibling); + if sibling != self.cpu.id && handle.held_by_sibling() && handle.poke() == Kick::Send { + self.env.hw.kick(sibling); + } + } + } + /// When [`Balance::PullWithRearm`] wants this CPU back, and the allowance it /// spends to ask. /// @@ -2324,6 +2364,9 @@ pub struct CpuHandle { /// waits, after it published SLEEPING, so a reader may see the previous /// idle period's: a tiebreak, never a correctness input. idle_exit: AtomicU64, + /// **This CPU waits in C1 only because an SMT sibling runs**: the sibling + /// that sleeps next pokes it to choose again ([`SchedPass::try_sleep`]). + held_by_sibling: AtomicBool, /// The on-target counterpart to the simulator's invariants: the sim asserts /// what a pass *does*, this measures what a pass *costs*. /// @@ -2351,6 +2394,7 @@ impl CpuHandle { weight: AtomicU64::new(0), last_pass: AtomicU64::new(0), idle_exit: AtomicU64::new(0), + held_by_sibling: AtomicBool::new(false), #[cfg(feature = "sched-check")] pass_costs: PassCosts::new(), } @@ -2397,6 +2441,10 @@ impl CpuHandle { self.idle_exit.store(nanos, Ordering::Relaxed); } + pub fn held_by_sibling(&self) -> bool { + self.held_by_sibling.load(Ordering::Relaxed) + } + /// Stamp the numbers above with the pass that published them. pub fn publish_pass(&self, now: Nanos) { self.last_pass.store(now.0, Ordering::Relaxed); @@ -2688,6 +2736,8 @@ mod tests { timers: Vec>, /// The revisit an idle plan asks, where a test sets one. revisit: Option, + /// Plans a declared state, 0, where a test sets it. + deep: bool, } #[derive(Default)] @@ -2721,6 +2771,10 @@ mod tests { fn idle_plan(&self, deadline: Option, now: Nanos) -> crate::sched::idle::Plan { let mut plan = crate::sched::idle::Plan::c1(deadline.map(|d| d.0), now.0); plan.choice.revisit_after = self.state().revisit; + if self.state().deep { + plan.choice.state = Some(0); + plan.exit = 253_000; + } plan } fn halt(&self) {} @@ -4331,6 +4385,51 @@ mod tests { } } + /// One core of two threads, cpu0 and cpu1. + fn one_smt_core() -> Topology { + use crate::sched::topology::CpuKeys; + Topology::new([0, 0].iter().map(|&core| CpuKeys { core, llc: 0, package: 0 }).collect()).expect("a tree") + } + + fn idle_pass(w: &mut World, cpu: usize) -> SleepToken { + let (cpus, env) = w.split(); + let pass = SchedPass::begin(&mut cpus[cpu], env, NOW); + let Action::Idle(token) = pass.dispose_none().finish() else { panic!("an empty CPU sleeps") }; + token + } + + /// **A CPU beside a running sibling waits in C1, and the sibling that + /// sleeps wakes it to choose again**: the core saves nothing from one + /// thread deep, and the held CPU is not left in C1 once the core could go + /// deep, since a sibling's sleep is the event that says so. + #[test] + fn a_cpu_beside_a_running_sibling_waits_in_c1_until_the_sibling_sleeps() { + let mut w = World::on(one_smt_core()); + w.hw.state().deep = true; + let held = idle_pass(&mut w, 0); + assert_eq!(held.idle().choice.state, None, "cpu1 runs, so cpu0 waits in C1"); + assert!(w.handles.get(CpuId(0)).held_by_sibling()); + let kicks = w.hw.state().kicks.len(); + let sleeper = idle_pass(&mut w, 1); + assert_eq!(sleeper.idle().choice.state, Some(0), "cpu0 sleeps, so cpu1 may wait deep"); + assert_eq!(w.hw.state().kicks[kicks..], [CpuId(0)], "cpu1's sleep pokes the cpu it held"); + assert!(w.handles.get(CpuId(0)).doorbell().kick_pending()); + let again = idle_pass(&mut w, 0); + assert_eq!(again.idle().choice.state, Some(0), "poked, cpu0 chooses again with the core asleep"); + assert!(!w.handles.get(CpuId(0)).held_by_sibling()); + } + + /// With nothing declared nothing is held, so a sibling's sleep pokes no + /// one: a machine with C1 alone pays nothing for the rule. + #[test] + fn with_no_deep_plan_no_cpu_is_held_and_no_sleep_pokes() { + let mut w = World::on(one_smt_core()); + let _ = idle_pass(&mut w, 0); + assert!(!w.handles.get(CpuId(0)).held_by_sibling()); + let _ = idle_pass(&mut w, 1); + assert!(w.hw.state().kicks.is_empty()); + } + /// **A spawn's placement reads no idle exit**: the shallowest-sleeper /// preference is a wake forward's, so a new thread, the crunch row's or a /// churn worker's, lands where it would with every CPU in C1, and a diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 7440aafb1b9..4a6a7f34ca6 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -67,6 +67,7 @@ pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate: ("run_mhz", shown(run_mhz)), ("core_c6_permille", permille(delta(a.core_c6, b.core_c6), stamp)), ("core_c7_permille", permille(delta(a.core_c7, b.core_c7), stamp)), + // Counted when a period ends, so one begun before the row counts whole: past 1000 is that. ("deep_idles", shown(delta(a.deep_idles, b.deep_idles))), ("deep_permille", permille(delta(a.deep_idle_nanos, b.deep_idle_nanos), Some(ns))), ("kicks", shown(delta(a.kicks, b.kicks))), From 6a0fa3f29d5adabbff85f881f63ee13b5f0dd9a8 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 03:21:53 +0100 Subject: [PATCH 12/24] osbench: each group's cpu.clock names what ran beside it, from the roster A clock held below the one-core bin at a group's start means three or more cores ran then; the roster, read on the measuring CPU before and after the clock's few milliseconds, names the processes that did, and wakes no other CPU. The counters, whose read kicks every CPU, stay around rows of seconds. Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/src/around.rs | 28 ++++++++++++++++++++++++++++ tests/guest/osbench/src/main.rs | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 4a6a7f34ca6..3b98fb38321 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -4,6 +4,11 @@ //! at, its core's C6 and C7 residency where the CPU counts them, and the //! idle periods the kernel chose a deep state for. A row that moves with the //! machine's idle states, or with where its threads ran, says so here. +//! +//! **Reading the counters kicks every CPU** (`toyos_abi::counters`), so they +//! are read around rows of seconds, where one wake is nothing, and never +//! around `cpu.clock`'s few milliseconds, which [`ran_during`] names from the +//! roster instead. use std::time::Instant; @@ -76,3 +81,26 @@ pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate: ); } } + +/// Run `measure`, and note each process that ran on any CPU meanwhile, with +/// its CPU time: on ToyOS from the roster, read on this CPU before and after, +/// which wakes no other CPU and so leaves a short measurement as it was. A +/// clock held below its one-core bin is then named by what ran beside it. +pub fn ran_during(bench: &str, measure: impl FnOnce()) { + #[cfg(target_os = "toyos")] + let before = crate::sys::diag::shared().map(|d| d.roster()); + measure(); + #[cfg(target_os = "toyos")] + if let (Some(before), Some(d)) = (before, crate::sys::diag::shared()) { + let mut by_process: std::collections::BTreeMap<(u32, String), u64> = std::collections::BTreeMap::new(); + for (&(pid, tid), (name, after)) in &d.roster() { + let was = before.get(&(pid, tid)).map_or(0, |(_, ns)| *ns); + *by_process.entry((pid, name.clone())).or_default() += after.saturating_sub(was); + } + for ((pid, name), ns) in by_process.into_iter().filter(|(_, ns)| *ns > 0) { + note(&format!("{bench}.ran"), &[("pid", pid.to_string()), ("name", name), ("us", (ns / 1000).to_string())]); + } + } + #[cfg(not(target_os = "toyos"))] + let _ = bench; +} diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index 4d127c6efff..bd1beaa1fad 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -177,7 +177,7 @@ const BY_NAME: [(&str, fn()); 2] = [("lan", lan::suite), ("internet", internet:: fn group(name: &str, run: fn()) { report::group(name, "start"); conditions(); - clock::distribution(); + around::ran_during(&format!("{name}.cpu.clock"), clock::distribution); let t = Instant::now(); run(); report::group(name, &format!("done secs={}", t.elapsed().as_secs())); From f6510defd3ae6a01470a4a75761fbb5f01ab1538 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 03:32:57 +0100 Subject: [PATCH 13/24] issues: the idle track records where its exit stands after the second boots Co-Authored-By: Claude Opus 5.5 --- issues/idle-cpus-enter-the-c-states-cst-names.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 202a03a2cfd..fecf4c52147 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -52,6 +52,22 @@ lands later the deeper the state was (none at 100 µs, C2 at 1 ms, C10 at consistent with the cost of the caches a deep state powers off, not measured as that. A burst of wakes to CPUs in deep states pays their exits. +**Where the exit stands** (#1073's second boots at `f8749dc69`, two boots +each against its control with the deep states off, all else equal): + +- T14 `crunch.single` 1.10 to 1.11 times faster (deflate 60.3 to 54.5 ms, + Linux's 54.6 to 54.7), each row's thread at 4.21 to 4.23 GHz against + 3.83 by APERF/MPERF, its idle cores 40 to 80 % of the row in core C7. +- T14 `cpu.clock` at 4189 at 8 and 9 group starts of 17, the control at + none; Linux reaches 4190 at 13. Boot, identity, energy, system and mem + stay at 3789 even when every idle CPU waits deepest (#1073's first + always-deepest control), so three or more cores ran at those starts; + osbench now names what ran beside each group's clock (`.cpu.clock.ran`). +- AMD: no clock gap and no throughput change; the IPC cost of the first + boots is gone with the idle choice folded into the pass's timer. A + thread whose SMT sibling waited in port 0x414 ran deflate 6 % slower at + the same clock, so a CPU beside a running sibling now waits in C1. + **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, timer lateness no worse than `hlt`'s by more than the chosen state's exit latency, and osbench's `cpu.clock` maximum at ratio From d286dddf3512e79f4b09947b2ea3642806c0f979 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 04:20:15 +0100 Subject: [PATCH 14/24] idle: review r2; the SMT-sibling rule goes, its premise refuted by its control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 put a CPU beside a running SMT sibling in C1 because one AMD deflate row ran 6 % slow beside a sibling in port 0x414. The third boots read the same slow deflate with the deep states off, every idle CPU in C1: both slow boots placed deflate on CPU 4, the six fast ones elsewhere, at the same clock. The rule answered nothing, and review r2 found three defects in it (a stale held flag, the history learning the kernel's own poke, and two idle siblings holding each other), so it is removed whole. The spread is recorded as a property of the bench. - The hold's kick, `CpuHandles::wake_slower_than`, moves into the pure scheduler beside the tiebreak that reads the same word, with a host test: it kicks each other CPU above the bound once, not the holder and not one at or below it. - `idle_exit`'s contract says what it now is: published before the wait and fenced against the bound, 0 while awake, the hold's input and the wake forward's tiebreak. - The history's update is the plan's own (`Plan::learn`), and a revisit clears the residency it was armed for even when its timer fired up to `FORESEEN_SLACK` early. - `History::forget` is gone: a new declaration already starts from a fresh history. `Choice` and `Plan` lose their unused `Default`, and the helpers only `Plan` calls are private. - The core C6/C7 residency models are cited section by section to SDM Vol. 4 325462-093: each §2.11-§2.16 opening and §2.17's name the model as supporting Table 2-20, which defines 3FDH and 3FEH, and of the tables that supersede it only Table 2-57 (06_AFH) names either, defining both. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 18 ++- ...ompares-toyos-with-linux-on-one-machine.md | 6 + ...percent-slow-on-cpu-4-at-the-same-clock.md | 29 ++++ kernel/pure/cpu.rs | 8 +- kernel/pure/sched/cpu.rs | 146 +++++------------- kernel/pure/sched/idle.rs | 53 ++++--- kernel/src/idle.rs | 6 +- kernel/src/sched/driver.rs | 8 +- 8 files changed, 124 insertions(+), 150 deletions(-) create mode 100644 issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index fecf4c52147..5b78e2c111a 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -61,12 +61,22 @@ each against its control with the deep states off, all else equal): - T14 `cpu.clock` at 4189 at 8 and 9 group starts of 17, the control at none; Linux reaches 4190 at 13. Boot, identity, energy, system and mem stay at 3789 even when every idle CPU waits deepest (#1073's first - always-deepest control), so three or more cores ran at those starts; + always-deepest control), so idle depth does not hold them there; osbench now names what ran beside each group's clock (`.cpu.clock.ran`). - AMD: no clock gap and no throughput change; the IPC cost of the first - boots is gone with the idle choice folded into the pass's timer. A - thread whose SMT sibling waited in port 0x414 ran deflate 6 % slower at - the same clock, so a CPU beside a running sibling now waits in C1. + boots is gone with the idle choice folded into the pass's timer. The one + slow deflate is a property of the bench, not of the idle states: it came + back with every idle CPU in C1 + (`issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md`). +- Third boots at `f6510defd`: the T14 at 4189 at 8 and 10 group starts. + Boot, identity, energy and system run while acpiserver is still loading + its tables, about 12.2 s after power-on, and acpiserver ran for the whole + of each of those clock windows. At the starts of net, procs, mem, tail + and nvme still at 3789, the roster names nothing beside osbench for more + than 0.9 ms of the 5.5 ms window. Mem stays unexplained. Net, procs, tail + and nvme reached 4189 under the always-deepest control, so the history + holding idle CPUs shallow after the group before is the open hypothesis + there; a per-CPU state read inside the window would test it. **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, timer lateness no worse than `hlt`'s by more than diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index fb51fb9df22..2b8167966f7 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -48,6 +48,12 @@ OpenSSL and AWS-LC by the crypto oracle probe, and that table is the evidence: `ToyOSOrg/evidence` at `bed0b00efac82546b995f0841da54aea11c45694`, `r2/table.md`. +The AMD bench's `crunch.single.deflate` reads 6 % slower in boots that place +it on CPU 4, at the same clock, with or without the deep idle states +(`issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md`). +An AMD single-thread row compared across boots must therefore hold the +thread's placement equal. + What ToyOS cannot measure is reported `unsupported` by name, and is not a comparison until it can: diff --git a/issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md b/issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md new file mode 100644 index 00000000000..76a3a913e9b --- /dev/null +++ b/issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md @@ -0,0 +1,29 @@ +--- +status: open +kind: finding +opened: 2026-10-28 +--- + +# The AMD bench's deflate runs 6 % slow on CPU 4 at the same clock + +osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has two +values across eight boots. In the two boots whose deflate thread ran on +CPU 4, its p50 was 66.6 and 66.8 ms. In the six boots that placed it on +CPUs 0, 2, 10 or 12, its p50 was 62.9 to 63.6 ms. APERF/MPERF put the +thread's clock at 4214 to 4250 MHz in every boot. The two slow boots were +one with the deep idle states and one without them, with every idle CPU in +C1 (`ToyOSOrg/evidence` branch `pr-1073`, `r3/`). Idle states do not explain +it. + +`sha256`, `matmul` and `parse` show no such split, including the one boot +that ran `sha256` on CPU 4. Two boots are too few to tell whether CPU 4 +itself is slower, or whether that placement shares the core or cache with +other work. + +Every AMD single-thread timing row can carry this spread. A comparison +between two AMD boots, ToyOS's or Linux's, that does not hold the placement +equal can read 6 % that neither system caused. + +**Exit**: in one boot, deflate pinned to each CPU in turn, with each +sibling's and each CCX's load noted, either names the cause or shows that +the spread follows the core. diff --git a/kernel/pure/cpu.rs b/kernel/pure/cpu.rs index 71d6cc7dfee..09a59d7a14f 100644 --- a/kernel/pure/cpu.rs +++ b/kernel/pure/cpu.rs @@ -209,9 +209,11 @@ pub fn counters(facts: &CounterFacts) -> Counters { } } -/// The family 6 DisplayModels SDM Vol. 4 names as supporting Table 2-20 -/// (§2.11-§2.17), which carries `MSR_CORE_C6_RESIDENCY` and -/// `MSR_CORE_C7_RESIDENCY`; [`SMI_COUNT_MODELS`]' Table 2-20 run. AMD +/// The family 6 DisplayModels SDM Vol. 4 (325462-093) names as supporting +/// Table 2-20, each in its section's opening (§2.11-§2.16 and §2.17), which +/// defines `MSR_CORE_C6_RESIDENCY` (3FDH) and `MSR_CORE_C7_RESIDENCY` (3FEH) +/// for them all. Of the tables that supersede it for these models, only +/// Table 2-57 (06_AFH) names either address, and it defines both. AMD /// documents no core residency counter in the PPRs in hand. const CORE_RESIDENCY_MODELS: [u32; 37] = [ 0x2a, 0x2d, 0x3a, 0x3e, 0x3c, 0x45, 0x46, 0x3f, 0x3d, 0x47, 0x4f, 0x56, diff --git a/kernel/pure/sched/cpu.rs b/kernel/pure/sched/cpu.rs index 29464e57190..9273993293b 100644 --- a/kernel/pure/sched/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -23,14 +23,14 @@ use alloc::collections::{BTreeMap, BTreeSet, VecDeque}; use alloc::vec::Vec; use core::marker::PhantomData; -use crate::sched::hw::{CpuId, Hw, Nanos, TraceEvent, TraceKind}; +use crate::sched::hw::{CpuId, Hw, Kicker, Nanos, TraceEvent, TraceKind}; use crate::sched::mailbox::{ Doorbell, Kick, MailboxConsumer, MailboxNode, MailboxProducer, PostSlot, PreemptGuard, Quiesced, SchedMsg, SleepArm, Urgency, }; use crate::sched::msg::Msg; use crate::sched::queue::{RunQueue, MIN_SLICE_NS}; -use crate::sched::sync::{fence, Arc, AtomicBool, AtomicU32, AtomicU64, Ordering}; +use crate::sched::sync::{fence, Arc, AtomicU32, AtomicU64, Ordering}; use crate::sched::task::{ BlockedTask, Claim, DeadTask, ReadyTask, RunningTask, SchedPayload, TaskKey, TaskShared, TaskState, TransitTask, WaitClass, WakeCause, WakeReason, QUANTUM_NS, @@ -1753,8 +1753,6 @@ impl SchedPass<'_, '_, H, P, Disposed> { // The re-arm's allowance is per idle period, so a CPU that is // given work starts its next one with a full one. self.cpu.idle_probes_spent = 0; - // Running, it is held by nothing: a sibling going to sleep owes it no poke. - handle.held_by_sibling.store(false, Ordering::Relaxed); return self.switch_to_current(); } if !self.cpu.is_idle() { @@ -2128,7 +2126,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { // The idle choice is made against the timer the CPU would sleep // with, and its own wake joins the same plan: one write of the timer. let deadline = TimerPlan::compute(None, self.cpu.earliest_deadline()).no_later_than(rearm).armed(); - let idle = self.held_shallow_by_a_running_sibling(self.env.hw.idle_plan(deadline, self.now)); + let idle = self.env.hw.idle_plan(deadline, self.now); let timer = self.apply_timer_no_later_than(match (rearm, idle.wake_at().map(Nanos)) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), @@ -2147,10 +2145,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { return Err(()); } match arm.confirm(&self.cpu.mailbox) { - Ok(quiesced) => { - self.poke_held_siblings(); - Ok(Action::Idle(SleepToken::new(quiesced, timer, idle))) - } + Ok(quiesced) => Ok(Action::Idle(SleepToken::new(quiesced, timer, idle))), Err(_awake) => { self.env.cpus.get(self.cpu.id).doorbell().begin_pass(); self.cpu.drain(self.env, self.now); @@ -2160,41 +2155,6 @@ impl SchedPass<'_, '_, H, P, Disposed> { } } - /// **A deep state beside a running SMT sibling saves nothing**: a core - /// enters a core C-state only when every thread of it asks for one, and - /// the thread waiting deep still pays the deeper exit (and on Zen 2 slows - /// the sibling it shares the core with, #1073's readings). So a CPU whose - /// plan is deeper than C1 waits in C1 while a sibling runs, marked held, - /// and the sibling that sleeps next pokes it to choose again - /// ([`Self::poke_held_siblings`]). The mark is stored and fenced before - /// the siblings are read, and a sleeper's SLEEPING before it reads the - /// marks, so one of the two sees the other. - fn held_shallow_by_a_running_sibling(&self, plan: crate::sched::idle::Plan) -> crate::sched::idle::Plan { - if plan.choice.state.is_none() { - return plan; - } - let handle = self.env.cpus.get(self.cpu.id); - handle.held_by_sibling.store(true, Ordering::Relaxed); - fence(Ordering::SeqCst); - if self.env.cpus.busy_siblings(self.cpu.id) == 0 { - handle.held_by_sibling.store(false, Ordering::Relaxed); - return plan; - } - crate::sched::idle::Plan { declaration: plan.declaration, ..crate::sched::idle::Plan::c1(plan.deadline, plan.chosen_at) } - } - - /// Poke each SMT sibling held in C1 for this CPU's running: SLEEPING is - /// published, so it now chooses again with this CPU asleep. - fn poke_held_siblings(&self) { - fence(Ordering::SeqCst); - for &sibling in self.env.cpus.topology().core(self.cpu.id) { - let handle = self.env.cpus.get(sibling); - if sibling != self.cpu.id && handle.held_by_sibling() && handle.poke() == Kick::Send { - self.env.hw.kick(sibling); - } - } - } - /// When [`Balance::PullWithRearm`] wants this CPU back, and the allowance it /// spends to ask. /// @@ -2358,15 +2318,13 @@ pub struct CpuHandle { /// can tell a claim about the present from one a stopped CPU left behind. /// See [`CpuHandle::answering`]. last_pass: AtomicU64, - /// **How long this CPU takes to wake from the state it last chose to wait - /// in**, in nanoseconds, 0 for C1: what a wake forward weighs between - /// sleeping CPUs ([`CpuHandles::idle_near`]). Written before the CPU - /// waits, after it published SLEEPING, so a reader may see the previous - /// idle period's: a tiebreak, never a correctness input. + /// **How long this CPU takes to wake from the state it is waiting in**, + /// in nanoseconds, 0 for C1 and while awake. The wait publishes it and + /// then fences before it reads the wake bound, which is what lets a hold + /// find every CPU slower than its bound ([`CpuHandles::wake_slower_than`], + /// `kernel::idle`). A wake forward weighs it between sleepers as a + /// tiebreak ([`CpuHandles::idle_near`]). idle_exit: AtomicU64, - /// **This CPU waits in C1 only because an SMT sibling runs**: the sibling - /// that sleeps next pokes it to choose again ([`SchedPass::try_sleep`]). - held_by_sibling: AtomicBool, /// The on-target counterpart to the simulator's invariants: the sim asserts /// what a pass *does*, this measures what a pass *costs*. /// @@ -2394,7 +2352,6 @@ impl CpuHandle { weight: AtomicU64::new(0), last_pass: AtomicU64::new(0), idle_exit: AtomicU64::new(0), - held_by_sibling: AtomicBool::new(false), #[cfg(feature = "sched-check")] pass_costs: PassCosts::new(), } @@ -2441,10 +2398,6 @@ impl CpuHandle { self.idle_exit.store(nanos, Ordering::Relaxed); } - pub fn held_by_sibling(&self) -> bool { - self.held_by_sibling.load(Ordering::Relaxed) - } - /// Stamp the numbers above with the pass that published them. pub fn publish_pass(&self, now: Nanos) { self.last_pass.store(now.0, Ordering::Relaxed); @@ -2612,6 +2565,18 @@ impl CpuHandles { &self.topology } + /// Ring every CPU but `me` whose published exit exceeds `bound`, and + /// kick each whose ring is the one that sends: it leaves its state and + /// chooses again under the bound. The caller has stored the bound and + /// fenced. + pub fn wake_slower_than(&self, me: CpuId, bound: u64, kicker: &impl Kicker) { + for handle in self.handles.iter().filter(|handle| handle.id != me) { + if handle.idle_exit() > bound && handle.poke() == Kick::Send { + kicker.kick(handle.id); + } + } + } + /// How many of `cpu`'s SMT siblings, `cpu` excluded, read other than SLEEPING. fn busy_siblings(&self, cpu: CpuId) -> usize { self.topology @@ -2709,7 +2674,7 @@ mod tests { extern crate std; use super::*; - use crate::sched::hw::{Kicker, Machine}; + use crate::sched::hw::Machine; use crate::sched::job::{Job, JobKey}; use crate::sched::mailbox::{mailbox, NoPreempt}; use crate::sched::queue::SLICE_NS; @@ -2736,8 +2701,6 @@ mod tests { timers: Vec>, /// The revisit an idle plan asks, where a test sets one. revisit: Option, - /// Plans a declared state, 0, where a test sets it. - deep: bool, } #[derive(Default)] @@ -2771,10 +2734,6 @@ mod tests { fn idle_plan(&self, deadline: Option, now: Nanos) -> crate::sched::idle::Plan { let mut plan = crate::sched::idle::Plan::c1(deadline.map(|d| d.0), now.0); plan.choice.revisit_after = self.state().revisit; - if self.state().deep { - plan.choice.state = Some(0); - plan.exit = 253_000; - } plan } fn halt(&self) {} @@ -4385,51 +4344,6 @@ mod tests { } } - /// One core of two threads, cpu0 and cpu1. - fn one_smt_core() -> Topology { - use crate::sched::topology::CpuKeys; - Topology::new([0, 0].iter().map(|&core| CpuKeys { core, llc: 0, package: 0 }).collect()).expect("a tree") - } - - fn idle_pass(w: &mut World, cpu: usize) -> SleepToken { - let (cpus, env) = w.split(); - let pass = SchedPass::begin(&mut cpus[cpu], env, NOW); - let Action::Idle(token) = pass.dispose_none().finish() else { panic!("an empty CPU sleeps") }; - token - } - - /// **A CPU beside a running sibling waits in C1, and the sibling that - /// sleeps wakes it to choose again**: the core saves nothing from one - /// thread deep, and the held CPU is not left in C1 once the core could go - /// deep, since a sibling's sleep is the event that says so. - #[test] - fn a_cpu_beside_a_running_sibling_waits_in_c1_until_the_sibling_sleeps() { - let mut w = World::on(one_smt_core()); - w.hw.state().deep = true; - let held = idle_pass(&mut w, 0); - assert_eq!(held.idle().choice.state, None, "cpu1 runs, so cpu0 waits in C1"); - assert!(w.handles.get(CpuId(0)).held_by_sibling()); - let kicks = w.hw.state().kicks.len(); - let sleeper = idle_pass(&mut w, 1); - assert_eq!(sleeper.idle().choice.state, Some(0), "cpu0 sleeps, so cpu1 may wait deep"); - assert_eq!(w.hw.state().kicks[kicks..], [CpuId(0)], "cpu1's sleep pokes the cpu it held"); - assert!(w.handles.get(CpuId(0)).doorbell().kick_pending()); - let again = idle_pass(&mut w, 0); - assert_eq!(again.idle().choice.state, Some(0), "poked, cpu0 chooses again with the core asleep"); - assert!(!w.handles.get(CpuId(0)).held_by_sibling()); - } - - /// With nothing declared nothing is held, so a sibling's sleep pokes no - /// one: a machine with C1 alone pays nothing for the rule. - #[test] - fn with_no_deep_plan_no_cpu_is_held_and_no_sleep_pokes() { - let mut w = World::on(one_smt_core()); - let _ = idle_pass(&mut w, 0); - assert!(!w.handles.get(CpuId(0)).held_by_sibling()); - let _ = idle_pass(&mut w, 1); - assert!(w.hw.state().kicks.is_empty()); - } - /// **A spawn's placement reads no idle exit**: the shallowest-sleeper /// preference is a wake forward's, so a new thread, the crunch row's or a /// churn worker's, lands where it would with every CPU in C1, and a @@ -4516,6 +4430,20 @@ mod tests { assert_eq!(handles.idle_near(me, 0, asleep), Some(CpuId(4)), "cpu4 shares a busy core, but is in this LLC: locality before wake cost"); } + #[test] + fn a_hold_kicks_each_other_cpu_slower_to_wake_than_its_bound_once() { + let (handles, _consumers) = bare_handles(two_llcs()); + sleep(&handles, &[0, 1, 2, 3, 4, 5, 6, 7]); + for (cpu, exit) in [(0, 1_048_000), (1, 1_048_000), (2, 253_000), (3, 253_001), (4, 0)] { + handles.get(CpuId(cpu)).publish_idle_exit(exit); + } + let hw = TestHw::default(); + handles.wake_slower_than(CpuId(0), 253_000, &hw); + assert_eq!(hw.state().kicks, [CpuId(1), CpuId(3)], "not the holder, not at or below the bound"); + handles.wake_slower_than(CpuId(0), 253_000, &hw); + assert_eq!(hw.state().kicks, [CpuId(1), CpuId(3)], "a kick already coming is not sent twice"); + } + /// **A spawn takes the least queue, then a sleeping CPU, then the CPU /// whose SMT siblings run least**, scanned from its rotating start. #[test] diff --git a/kernel/pure/sched/idle.rs b/kernel/pure/sched/idle.rs index f0e661150ad..2d580fbe578 100644 --- a/kernel/pure/sched/idle.rs +++ b/kernel/pure/sched/idle.rs @@ -75,12 +75,6 @@ impl History { } } - /// The states a new declaration replaced are not its states: what was - /// learned of them says nothing about these. - pub fn forget(&mut self) { - *self = Self::default(); - } - /// The whole history as one word, for a CPU to keep in an atomic of its own. pub const fn bits(&self) -> u64 { u64::from_le_bytes(self.early) @@ -96,7 +90,7 @@ impl History { } /// What [`choose`] picked. -#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Choice { /// A declared state, or `None` for C1. pub state: Option, @@ -112,7 +106,7 @@ pub struct Choice { /// What a CPU decided in its last pass before it sleeps, so the timer its /// choice needs is part of the pass's one timer plan ([`wake_at`]) and the /// wait enters what the pass chose. -#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Plan { pub choice: Choice, /// The chosen state's exit, 0 for C1. @@ -143,6 +137,18 @@ impl Plan { pub fn ended(&self, woke: u64) -> Ended { ended(&self.choice, self.deadline, self.chosen_at, woke) } + + /// What `history` learns from the period this plan began, ended at + /// `woke`. A revisit covers the residency it was armed for, however + /// early within [`FORESEEN_SLACK`] it fired. + pub fn learn(&self, history: &mut History, costs: &[Cost], woke: u64) { + let slept = woke.saturating_sub(self.chosen_at); + match self.ended(woke) { + Ended::Deadline => {} + Ended::Revisit => history.quiet(costs, slept.max(self.choice.revisit_after.unwrap_or(0))), + Ended::Unforeseen => history.unforeseen(costs, slept), + } + } } /// The state to wait in. `horizon` is how long until the CPU's timer fires, @@ -174,7 +180,7 @@ pub const FORESEEN_SLACK: u64 = 1_000; /// When the scheduler's own timer fires after this choice, where one is armed /// for `armed`: its deadline, pulled in by the chosen state's exit. -pub fn deadline_fires(choice: &Choice, armed: Option) -> Option { +fn deadline_fires(choice: &Choice, armed: Option) -> Option { armed.map(|at| at.saturating_sub(choice.wake_early_by)) } @@ -182,7 +188,7 @@ pub fn deadline_fires(choice: &Choice, armed: Option) -> Option { /// it, for `choice` made at `now`: the deadline pulled in by the exit, or the /// revisit, whichever is sooner. Never at or after `armed`, so writing it can /// only move the timer earlier; `None` leaves it as the scheduler armed it. -pub fn wake_at(choice: &Choice, armed: Option, now: u64) -> Option { +fn wake_at(choice: &Choice, armed: Option, now: u64) -> Option { let early = deadline_fires(choice, armed).filter(|_| choice.wake_early_by > 0); let revisit = choice.revisit_after.map(|after| now.saturating_add(after)); early.into_iter().chain(revisit).min().filter(|&at| armed.is_none_or(|armed| at < armed)) @@ -191,7 +197,7 @@ pub fn wake_at(choice: &Choice, armed: Option, now: u64) -> Option { /// Whether a period that ended at `woke` was ended by the scheduler's own /// timer, firing for `fires`: what the history must not learn from, since the /// horizon already said it. -pub fn foreseen(fires: Option, woke: u64) -> bool { +fn foreseen(fires: Option, woke: u64) -> bool { fires.is_some_and(|fires| woke.saturating_add(FORESEEN_SLACK) >= fires) } @@ -208,7 +214,7 @@ pub enum Ended { /// What ended an idle period chosen as `choice` at `now`, with the /// scheduler's timer armed for `armed`, that ended at `woke`. -pub fn ended(choice: &Choice, armed: Option, now: u64, woke: u64) -> Ended { +fn ended(choice: &Choice, armed: Option, now: u64, woke: u64) -> Ended { let revisit = choice.revisit_after.map(|after| now.saturating_add(after)); if foreseen(deadline_fires(choice, armed), woke) { Ended::Deadline @@ -336,16 +342,6 @@ mod tests { assert_eq!(waited, WINDOW - EARLY_ALLOWED); } - #[test] - fn a_new_declaration_starts_with_nothing_learned() { - let mut h = History::default(); - for _ in 0..WINDOW { - h.unforeseen(&T14, 10 * US); - } - h.forget(); - assert_eq!(pick(&h, None, None), Some(1)); - } - #[test] fn no_declared_state_is_c1() { assert_eq!(choose(&[], &History::default(), None, None), Choice { state: None, wake_early_by: 0, revisit_after: None }); @@ -411,6 +407,19 @@ mod tests { assert_eq!(ended(&deep, None, 0, 9_000 * US), Ended::Unforeseen); } + #[test] + fn a_revisit_a_tick_early_clears_the_state_it_was_armed_for() { + let mut h = History::default(); + for _ in 0..WINDOW { + h.unforeseen(&T14, 10 * US); + } + let choice = choose(&T14, &h, None, None); + assert_eq!((choice.state, choice.revisit_after), (None, Some(506 * US))); + let plan = Plan { choice, exit: 0, declaration: 1, chosen_at: 0, deadline: None }; + plan.learn(&mut h, &T14, 506 * US - FORESEEN_SLACK); + assert_eq!(pick(&h, None, None), Some(0), "C2 is admitted again, C3 still learns"); + } + #[test] fn the_least_bound_held_is_the_machines_and_goes_with_its_last_hold() { let mut b = Bounds::default(); diff --git a/kernel/src/idle.rs b/kernel/src/idle.rs index 72d81e7187d..429fa7aeaf5 100644 --- a/kernel/src/idle.rs +++ b/kernel/src/idle.rs @@ -188,11 +188,7 @@ pub fn wait(plan: &idle::Plan) { let Some(d) = d else { return }; let costs = &d.costs[..d.len]; let mut history = history(local, d.generation); - match plan.ended(woke) { - idle::Ended::Deadline => {} - idle::Ended::Revisit => history.quiet(costs, woke.saturating_sub(plan.chosen_at)), - idle::Ended::Unforeseen => history.unforeseen(costs, woke.saturating_sub(plan.chosen_at)), - } + plan.learn(&mut history, costs, woke); local.learned_under.store(d.generation, Ordering::Relaxed); local.history.store(history.bits(), Ordering::Relaxed); } diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 95d25f666f7..18a482ce350 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -772,13 +772,7 @@ pub fn publish_idle_exit(nanos: u64) { /// Ring and kick every other CPU whose published exit exceeds `bound`, so it /// leaves its state and chooses again under the bound. pub fn wake_slower_than(bound: u64) { - let me = current_cpu(); - for cpu in (0..cpus().len() as u32).map(CpuId).filter(|&cpu| cpu != me) { - let handle = cpus().get(cpu); - if handle.idle_exit() > bound && handle.poke() == Kick::Send { - HW.kick(cpu); - } - } + cpus().wake_slower_than(current_cpu(), bound, &HW); } pub fn current_cpu() -> CpuId { From 49a98e31e77c9423452eeb7ea2355ffb1b409733 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 04:37:59 +0100 Subject: [PATCH 15/24] osbench: the runner's capability has one owner; identity and fair exit 0 again The group-start roster instrument (6a0fa3f29) opened the counters' Diag before each group's first row, and Diag took the process's one SysCap endowment. `osbench identity` and `osbench fair` each take it again for a row of their own, found nothing and refused, so both groups exited 1 on every bench in every boot since. The metal judge says so when it is run over the readbacks; it was not run. The endowment is now taken once, by `sys::syscap()`, and every user borrows it: the counters' Diag, identity's inventory read and fair's job builder. `osbench_capability_groups_exit_0` boots benchcase.toml under QEMU with those two groups alone and holds each to exit 0, so a capability taken twice fails a PR's own QEMU run rather than a nightly bench. Review r3's suggestions: - A wait the bound or a new declaration sends to C1 publishes exit 0, so `idle_exit` is what the CPU waits in. - `Plan::learn`'s Unforeseen and Deadline arms have a test. - The AMD deflate finding records only what was measured. B4 ran the slow mode on CPU 10, which earlier read fast, so the record is renamed: the mode varies per boot at the same clock and with no running neighbour. - The track names #1066's exit as unmet, 8 to 10 of 17 group starts against Linux's 13, with acpiserver's start-up a measured busy core at four. - A stray blank line goes. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 23 ++++++++----- ...ompares-toyos-with-linux-on-one-machine.md | 10 +++--- ...nt-slow-in-some-boots-at-the-same-clock.md | 29 ++++++++++++++++ ...percent-slow-on-cpu-4-at-the-same-clock.md | 29 ---------------- kernel/pure/cpu.rs | 1 - kernel/pure/sched/idle.rs | 13 +++++++ kernel/src/idle.rs | 4 +++ tests/guest/osbench/src/fair.rs | 7 +--- tests/guest/osbench/src/identity.rs | 4 +-- tests/guest/osbench/src/sys/toyos.rs | 13 +++++-- tests/toyos.rs | 34 +++++++++++++++++++ 11 files changed, 112 insertions(+), 55 deletions(-) create mode 100644 issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md delete mode 100644 issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 5b78e2c111a..54924a8e40d 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -1,5 +1,5 @@ --- -status: assigned +status: open kind: track opened: 2026-09-29 --- @@ -64,19 +64,26 @@ each against its control with the deep states off, all else equal): always-deepest control), so idle depth does not hold them there; osbench now names what ran beside each group's clock (`.cpu.clock.ran`). - AMD: no clock gap and no throughput change; the IPC cost of the first - boots is gone with the idle choice folded into the pass's timer. The one - slow deflate is a property of the bench, not of the idle states: it came - back with every idle CPU in C1 - (`issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md`). + boots is gone with the idle choice folded into the pass's timer. The slow + deflate is not the idle states: it came back with every idle CPU in C1, + at the same clock + (`issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md`). - Third boots at `f6510defd`: the T14 at 4189 at 8 and 10 group starts. Boot, identity, energy and system run while acpiserver is still loading - its tables, about 12.2 s after power-on, and acpiserver ran for the whole - of each of those clock windows. At the starts of net, procs, mem, tail - and nvme still at 3789, the roster names nothing beside osbench for more + its tables (12.2 s by the kernel log's clock in the first B3 boot, + `r4/` in the evidence), and acpiserver ran for the whole of each of those + clock windows: a measured busy core beside the clock's thread. At the + starts of net, procs, mem, tail and nvme still at 3789, the roster names nothing beside osbench for more than 0.9 ms of the 5.5 ms window. Mem stays unexplained. Net, procs, tail and nvme reached 4189 under the always-deepest control, so the history holding idle CPUs shallow after the group before is the open hypothesis there; a per-CPU state read inside the window would test it. +- The B4 boots at `d286dddf3` reached 4189 at 10 of 17 group starts. + +**#1066's exit is unmet** where this track stands: the T14's `cpu.clock` reaches +ratio 42 at 8 to 10 of 17 group starts against Linux's 13. Four of those +groups are held by acpiserver's measured start-up work. The rest are the +open question above. **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, timer lateness no worse than `hlt`'s by more than diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index 2b8167966f7..836ab30aae5 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -48,11 +48,11 @@ OpenSSL and AWS-LC by the crypto oracle probe, and that table is the evidence: `ToyOSOrg/evidence` at `bed0b00efac82546b995f0841da54aea11c45694`, `r2/table.md`. -The AMD bench's `crunch.single.deflate` reads 6 % slower in boots that place -it on CPU 4, at the same clock, with or without the deep idle states -(`issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md`). -An AMD single-thread row compared across boots must therefore hold the -thread's placement equal. +The AMD bench's `crunch.single.deflate` read 6 % slower in three boots of +nine, at the same clock, with and without the deep idle states +(`issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md`). +An AMD single-thread row compared across boots needs repeats enough to +see which mode each boot read. What ToyOS cannot measure is reported `unsupported` by name, and is not a comparison until it can: diff --git a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md new file mode 100644 index 00000000000..12071feae08 --- /dev/null +++ b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md @@ -0,0 +1,29 @@ +--- +status: open +kind: finding +opened: 2026-10-28 +--- + +# The AMD bench's deflate runs 6 % slow in some boots at the same clock + +osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has two +values across nine boots (`ToyOSOrg/evidence` branch `pr-1073`, `r3/` and +`r4/`): + +- In three boots its p50 was 66.6 to 66.8 ms. Two ran the thread on CPU 4 + and one on CPU 10. +- In the six others it was 62.9 to 63.6 ms, on CPUs 0, 2, 10 and 12. + +APERF/MPERF put the thread's clock at 4214 to 4250 MHz in every boot. The +slow boots include one with every idle CPU in C1 and two with the deep +states. In the slow A\* boot and the slow B4 boot, every other CPU, the +sibling included, read `busy_permille=0` across the row. So it is not the +idle states, the clock, the placement, or a running neighbour. + +`sha256`, `matmul` and `parse` show no such split. Every AMD single-thread +timing row may carry this spread. A comparison between two AMD boots, +ToyOS's or Linux's, can therefore read 6 % that neither system caused. + +**Exit**: one boot running deflate several times, at intervals, on one +pinned CPU, which says whether the mode is per boot or per run, and with the +package's power and temperature beside it once a counter carries them. diff --git a/issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md b/issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md deleted file mode 100644 index 76a3a913e9b..00000000000 --- a/issues/the-amd-benchs-deflate-runs-6-percent-slow-on-cpu-4-at-the-same-clock.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: finding -opened: 2026-10-28 ---- - -# The AMD bench's deflate runs 6 % slow on CPU 4 at the same clock - -osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has two -values across eight boots. In the two boots whose deflate thread ran on -CPU 4, its p50 was 66.6 and 66.8 ms. In the six boots that placed it on -CPUs 0, 2, 10 or 12, its p50 was 62.9 to 63.6 ms. APERF/MPERF put the -thread's clock at 4214 to 4250 MHz in every boot. The two slow boots were -one with the deep idle states and one without them, with every idle CPU in -C1 (`ToyOSOrg/evidence` branch `pr-1073`, `r3/`). Idle states do not explain -it. - -`sha256`, `matmul` and `parse` show no such split, including the one boot -that ran `sha256` on CPU 4. Two boots are too few to tell whether CPU 4 -itself is slower, or whether that placement shares the core or cache with -other work. - -Every AMD single-thread timing row can carry this spread. A comparison -between two AMD boots, ToyOS's or Linux's, that does not hold the placement -equal can read 6 % that neither system caused. - -**Exit**: in one boot, deflate pinned to each CPU in turn, with each -sibling's and each CCX's load noted, either names the cause or shows that -the spread follows the core. diff --git a/kernel/pure/cpu.rs b/kernel/pure/cpu.rs index 09a59d7a14f..7f3ba5a4e24 100644 --- a/kernel/pure/cpu.rs +++ b/kernel/pure/cpu.rs @@ -300,7 +300,6 @@ pub fn topology_keys(levels: &[[u32; 4]], caches: &[[u32; 4]]) -> Result crate::arch::idle::enter(d.entries[state]), diff --git a/tests/guest/osbench/src/fair.rs b/tests/guest/osbench/src/fair.rs index cf3af89326f..d022791d86d 100644 --- a/tests/guest/osbench/src/fair.rs +++ b/tests/guest/osbench/src/fair.rs @@ -246,12 +246,7 @@ impl Grouping { #[cfg(target_os = "toyos")] Grouping::Job => { use std::os::toyos::process::CommandExt; - use toyos::syscap::SysCap; - static CAP: std::sync::OnceLock> = std::sync::OnceLock::new(); - let cap = CAP - .get_or_init(|| toyos::endow::Endowments::get().take(toyos::endow::SYSCAP_LABEL)) - .as_ref() - .ok_or("no capability was endowed, so no job can be made")?; + let cap = crate::sys::syscap().ok_or("no capability was endowed, so no job can be made")?; let job = cap.new_job().map_err(|e| format!("a new job was refused: {e:?}"))?; let mut command = Command::new(exe); command.endow(toyos::job::JOB_LABEL, job.into_raw().0); diff --git a/tests/guest/osbench/src/identity.rs b/tests/guest/osbench/src/identity.rs index 843b1d1432f..d49178a39c5 100644 --- a/tests/guest/osbench/src/identity.rs +++ b/tests/guest/osbench/src/identity.rs @@ -109,8 +109,6 @@ mod imp { #[cfg(target_os = "toyos")] mod imp { - use toyos::endow::{Endowments, SYSCAP_LABEL}; - use toyos::syscap::SysCap; use toyos_abi::inventory::{Driven, RawRecord, Record}; use crate::report::{note, refuse, unsupported}; @@ -125,7 +123,7 @@ mod imp { /// Every PCI function off the kernel's inventory, which the runner's /// capability carries `inventory` for. fn pci() { - let Some(cap) = Endowments::get().take::(SYSCAP_LABEL) else { + let Some(cap) = crate::sys::syscap() else { return refuse("identity.pci", "this program was endowed no SysCap"); }; let records: Vec = match cap.records(|n| vec![RawRecord::EMPTY; n]) { diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index 9ad12a66c3a..23b3d8c5043 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -197,6 +197,14 @@ pub mod ipc { } } +/// The capability the runner endowed this process, `None` where the boot +/// endows none. The endowment is taken once per process, so every row that +/// uses the capability borrows it here. +pub fn syscap() -> Option<&'static toyos::syscap::SysCap> { + static CAP: std::sync::OnceLock> = std::sync::OnceLock::new(); + CAP.get_or_init(|| toyos::endow::Endowments::get().take(toyos::endow::SYSCAP_LABEL)).as_ref() +} + /// What the tail group reads to say where a late wake's time went, through /// the runner's capability: each CPU's firmware interrupts and clock, the /// scheduler's diary and the roster's names. Read-only observation rights; @@ -204,7 +212,6 @@ pub mod ipc { pub mod diag { use std::collections::BTreeMap; - use toyos::endow::{Endowments, SYSCAP_LABEL}; use toyos::syscap::{SysCap, TraceCursor}; use toyos_abi::counters::{Counter, RawRecord, Record}; use toyos_abi::syscall::{SYSINFO_ENTRY_SIZE, SYSINFO_HEADER_SIZE}; @@ -212,7 +219,7 @@ pub mod diag { use toyos_trace::{Entry, Event}; pub struct Diag { - cap: SysCap, + cap: &'static SysCap, cursor: Option>>, } @@ -247,7 +254,7 @@ pub mod diag { } pub fn open() -> Option { - let cap: SysCap = Endowments::get().take(SYSCAP_LABEL)?; + let cap = super::syscap()?; let cursor = cap.trace_cursor_words().ok().map(|n| TraceCursor::new(vec![0u64; n])); Some(Diag { cap, cursor }) } diff --git a/tests/toyos.rs b/tests/toyos.rs index 8b936da6736..fb4ddbc359a 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -499,6 +499,13 @@ const MACHINE_TESTS: &[&str] = &[ // and its mask have no host build, QEMU raises the event on demand, and // nothing presses the T14's button but a hand. "acpi_press_is_one_sci", + // The osbench groups that use the runner's capability for a row of their + // own, on the image they are measured on: which endowments a job gets is + // the image's and the supervisor's, which no host build has, and the + // benches run osbench only nightly. `osbench_identity` and `osbench_fair` + // once exited 1 on every bench because an instrument took the one + // capability first. + "osbench_capability_groups_exit_0", // What the kernel reads and writes for the `acpi` claim's holder and what // it refuses. A red here is a write the kernel made — to RAM, to the // firmware's tables, to COM1, to `PM1a_CNT`, to a function's configuration @@ -2688,6 +2695,32 @@ fn acpi_press_is_one_sci() -> Result<(), String> { Ok(()) } +/// `osbench identity` and `osbench fair` on tests/images/benchcase.toml with +/// only those two jobs: each exits 0, which it does only once every row it +/// ran measured or said why not, its capability rows among them. +fn osbench_capability_groups_exit_0() -> Result<(), String> { + const JOBS: [&str; 2] = ["osbench_identity", "osbench_fair"]; + let committed = compile::repo_root().join("tests/images/benchcase.toml"); + let text = fs::read_to_string(&committed).map_err(|e| format!("{}: {e}", committed.display()))?; + let derived = toyos_metal::image::derive(&text, 300_000, &JOBS, &[]).map_err(|why| why.to_string())?; + let config = common::lane::dir().join("osbench-capability-groups.toml"); + fs::write(&config, derived).map_err(|e| format!("{}: {e}", config.display()))?; + let mut qemu = QemuInstance::boot_with_options( + &config, + &[], + &[], + BootOptions { ready_marker: bootlog::COMPLETE, ..Default::default() }, + ); + let mut console = format!("{}\n", qemu.boot_log()); + let last = format!("===TEST_END {} ", JOBS[JOBS.len() - 1]); + await_guest(&mut qemu, &mut console, "osbench's two groups to end", |said| said.contains(&last))?; + let said = serial::Serial::named("the osbench boot", console); + for job in JOBS { + said.must_say(&format!("===TEST_END {job} exit=0==="))?; + } + Ok(()) +} + /// Boot the image `--console-boot` builds, on the machine shape with a panel /// for `/system/bin/console` to claim, and wait for what that image owes at /// boot: the console up on its panel, having read the log and the keyboard, @@ -4398,6 +4431,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "nested_nmi_is_loud" => faults::nested_nmi_is_loud(test_config), "kernel_wx_leaf_is_refused" => faults::kernel_wx_leaf_is_refused(test_config, qemu::Profile::Metal), "acpi_press_is_one_sci" => acpi_press_is_one_sci(), + "osbench_capability_groups_exit_0" => osbench_capability_groups_exit_0(), "acpi_mediated_access" => acpi_mediated_access(2), "acpi_lock_given_back_on_one_cpu" => acpi_mediated_access(1), "acpi_supply_outlives_holder" => acpi_supply_outlives_holder(), From d60d727aa82a49984b7ec6995716a566b02a56c7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 04:39:45 +0100 Subject: [PATCH 16/24] tests: the osbench boot names itself once Co-Authored-By: Claude Opus 5.5 --- tests/toyos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index fb4ddbc359a..ec8f8e7d8a8 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2714,7 +2714,7 @@ fn osbench_capability_groups_exit_0() -> Result<(), String> { let mut console = format!("{}\n", qemu.boot_log()); let last = format!("===TEST_END {} ", JOBS[JOBS.len() - 1]); await_guest(&mut qemu, &mut console, "osbench's two groups to end", |said| said.contains(&last))?; - let said = serial::Serial::named("the osbench boot", console); + let said = serial::Serial::named("osbench boot", console); for job in JOBS { said.must_say(&format!("===TEST_END {job} exit=0==="))?; } From 10fdff43959afbb2b5fd38e96a438424d811cc0a Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 05:21:54 +0100 Subject: [PATCH 17/24] issues: the B5 and A* testcases readings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The T14's group starts at ratio 42 across six boots with the deep states read 8, 8, 8, 10, 10 and 5: only sched, files, ipc and work in all six, so a spread from boot to boot. The A* testcases control puts soundserver's worst IRQ lateness where the deep states put it on both benches, the AMD's 2661 µs window included, so the deep states move neither. The AMD deflate finding counts B5's fast run, three slow of ten. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 24 +++++++++++++------ ...ompares-toyos-with-linux-on-one-machine.md | 2 +- ...nt-slow-in-some-boots-at-the-same-clock.md | 6 ++--- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 54924a8e40d..eb80ffa4026 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -58,7 +58,7 @@ each against its control with the deep states off, all else equal): - T14 `crunch.single` 1.10 to 1.11 times faster (deflate 60.3 to 54.5 ms, Linux's 54.6 to 54.7), each row's thread at 4.21 to 4.23 GHz against 3.83 by APERF/MPERF, its idle cores 40 to 80 % of the row in core C7. -- T14 `cpu.clock` at 4189 at 8 and 9 group starts of 17, the control at +- T14 `cpu.clock` at 4189 at 8 and 8 group starts of 17, the control at none; Linux reaches 4190 at 13. Boot, identity, energy, system and mem stay at 3789 even when every idle CPU waits deepest (#1073's first always-deepest control), so idle depth does not hold them there; @@ -73,17 +73,27 @@ each against its control with the deep states off, all else equal): its tables (12.2 s by the kernel log's clock in the first B3 boot, `r4/` in the evidence), and acpiserver ran for the whole of each of those clock windows: a measured busy core beside the clock's thread. At the - starts of net, procs, mem, tail and nvme still at 3789, the roster names nothing beside osbench for more - than 0.9 ms of the 5.5 ms window. Mem stays unexplained. Net, procs, tail + starts of net, procs, mem, tail and nvme still at 3789, the roster names + nothing beside osbench for more than 0.9 ms of the 5.5 ms window. Mem stays unexplained. Net, procs, tail and nvme reached 4189 under the always-deepest control, so the history holding idle CPUs shallow after the group before is the open hypothesis there; a per-CPU state read inside the window would test it. -- The B4 boots at `d286dddf3` reached 4189 at 10 of 17 group starts. +- Group starts at 4189 across the six T14 boots with the deep states: + 8 and 8 (r2), 8 and 10 (r3), 10 (B4) and 5 (B5 at `d60d727aa`), counted + as a maximum above 4100 MHz. Only sched, files, ipc and work reached it + in all six; B5 lost time, which every earlier boot had. It is a spread + from boot to boot, not a fixed set of groups. +- Soundserver's worst IRQ lateness in the testcases boots, with every idle + CPU in C1 against with the deep states: T14 457 to 572 µs against 166 to + 564; AMD 0 to 2661 µs against 0 to 2661, the 2661 µs window in both. The + deep states move neither, and no boot underran. **#1066's exit is unmet** where this track stands: the T14's `cpu.clock` reaches -ratio 42 at 8 to 10 of 17 group starts against Linux's 13. Four of those -groups are held by acpiserver's measured start-up work. The rest are the -open question above. +ratio 42 at 5 to 10 of 17 group starts against Linux's 13. Of the groups +below it, boot, identity, energy and system are held by acpiserver's +measured start-up work; the rest are the open question above, which two +diagnostic boots take up next: the counters read around each group's clock, +and the history switched off. **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, timer lateness no worse than `hlt`'s by more than diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index 836ab30aae5..5065a66b57f 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -49,7 +49,7 @@ evidence: `ToyOSOrg/evidence` at `bed0b00efac82546b995f0841da54aea11c45694`, `r2/table.md`. The AMD bench's `crunch.single.deflate` read 6 % slower in three boots of -nine, at the same clock, with and without the deep idle states +ten, at the same clock, with and without the deep idle states (`issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md`). An AMD single-thread row compared across boots needs repeats enough to see which mode each boot read. diff --git a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md index 12071feae08..c1583825eab 100644 --- a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md +++ b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md @@ -7,12 +7,12 @@ opened: 2026-10-28 # The AMD bench's deflate runs 6 % slow in some boots at the same clock osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has two -values across nine boots (`ToyOSOrg/evidence` branch `pr-1073`, `r3/` and -`r4/`): +values across ten boots (`ToyOSOrg/evidence` branch `pr-1073`, `r3/` to +`r5/`): - In three boots its p50 was 66.6 to 66.8 ms. Two ran the thread on CPU 4 and one on CPU 10. -- In the six others it was 62.9 to 63.6 ms, on CPUs 0, 2, 10 and 12. +- In the seven others it was 62.9 to 63.7 ms, on CPUs 0, 2, 10 and 12. APERF/MPERF put the thread's clock at 4214 to 4250 MHz in every boot. The slow boots include one with every idle CPU in C1 and two with the deep From c9d5ee4c0e69c81c7b2d71a42a5b62a7cd591835 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 05:24:45 +0100 Subject: [PATCH 18/24] review r4: the lateness control over every window, and the QEMU row names its machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The track states soundserver's worst IRQ lateness over every window line of each testcases boot: T14 181 to 572 µs with every idle CPU in C1, 0 to 564 with the deep states; the AMD 0 to 2661 both ways. The registration of `osbench_capability_groups_exit_0` names QEMU x86-64's q35 and drops its history, which 49a98e31e's message carries. An assertion says the deadline is the scheduler's own, pulled in by the exit. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 18 ++++++++++-------- kernel/pure/sched/idle.rs | 2 +- tests/toyos.rs | 8 +++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index eb80ffa4026..e465ebd7f81 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -74,19 +74,21 @@ each against its control with the deep states off, all else equal): `r4/` in the evidence), and acpiserver ran for the whole of each of those clock windows: a measured busy core beside the clock's thread. At the starts of net, procs, mem, tail and nvme still at 3789, the roster names - nothing beside osbench for more than 0.9 ms of the 5.5 ms window. Mem stays unexplained. Net, procs, tail - and nvme reached 4189 under the always-deepest control, so the history - holding idle CPUs shallow after the group before is the open hypothesis - there; a per-CPU state read inside the window would test it. + nothing beside osbench for more than 0.9 ms of the 5.5 ms window. Mem + stays unexplained. Net, procs, tail and nvme reached 4189 under the + always-deepest control, so the history holding idle CPUs shallow after + the group before is the open hypothesis there; a per-CPU state read + inside the window would test it. - Group starts at 4189 across the six T14 boots with the deep states: 8 and 8 (r2), 8 and 10 (r3), 10 (B4) and 5 (B5 at `d60d727aa`), counted as a maximum above 4100 MHz. Only sched, files, ipc and work reached it in all six; B5 lost time, which every earlier boot had. It is a spread from boot to boot, not a fixed set of groups. -- Soundserver's worst IRQ lateness in the testcases boots, with every idle - CPU in C1 against with the deep states: T14 457 to 572 µs against 166 to - 564; AMD 0 to 2661 µs against 0 to 2661, the 2661 µs window in both. The - deep states move neither, and no boot underran. +- Soundserver's worst IRQ lateness, over every window line of the + testcases boots: with every idle CPU in C1 (one boot a bench), T14 181 to + 572 µs and AMD 0 to 2661; with the deep states (two boots a bench), T14 0 + to 564 µs and AMD 0 to 2661. The deep states move neither, and no window + underran. **#1066's exit is unmet** where this track stands: the T14's `cpu.clock` reaches ratio 42 at 5 to 10 of 17 group starts against Linux's 13. Of the groups diff --git a/kernel/pure/sched/idle.rs b/kernel/pure/sched/idle.rs index 46aeaab6f2f..ac75cf8cde7 100644 --- a/kernel/pure/sched/idle.rs +++ b/kernel/pure/sched/idle.rs @@ -430,7 +430,7 @@ mod tests { assert_eq!(h, History::from_bits(0x0101), "a 10 µs wake came sooner than both residencies"); let mut h = fresh; plan.learn(&mut h, &T14, 10_000 * US - 1048 * US); - assert_eq!(h, fresh, "the early wake before the deadline is the scheduler's own"); + assert_eq!(h, fresh, "the deadline, pulled in by the exit, is the scheduler's own"); } #[test] diff --git a/tests/toyos.rs b/tests/toyos.rs index ec8f8e7d8a8..d5173fa98a1 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -500,11 +500,9 @@ const MACHINE_TESTS: &[&str] = &[ // nothing presses the T14's button but a hand. "acpi_press_is_one_sci", // The osbench groups that use the runner's capability for a row of their - // own, on the image they are measured on: which endowments a job gets is - // the image's and the supervisor's, which no host build has, and the - // benches run osbench only nightly. `osbench_identity` and `osbench_fair` - // once exited 1 on every bench because an instrument took the one - // capability first. + // own, on the image they are measured on, on QEMU x86-64's q35: which + // endowments a job gets is the image's and the supervisor's, which no + // host build has, and the benches run osbench only nightly. "osbench_capability_groups_exit_0", // What the kernel reads and writes for the `acpi` claim's holder and what // it refuses. A red here is a write the kernel made — to RAM, to the From b1ae80c02611c8b5caaccca21bdd9e3978f26a65 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 06:12:48 +0100 Subject: [PATCH 19/24] osbench: the counters' figures are of the readings' own window, and a CPU whose readings are not the row's is refused The instrument around crunch, churn and tail divided by the row's wall time what the counters measured over theirs. The two differ: a counters read within 10 ms of another is served that read's values, so a reading can predate its row or be the same reading twice. Deep idle was also credited per period at its end, so a period begun before the row counted whole. The shipped rows printed deep fractions up to 2278 permille. Each CPU's window is now the difference of the counter stamps its two readings carry, converted by the clock page, and `osbench_rules::window` refuses it by name when the readings are one or span more than 10 % off the row. Busy and C6/C7 fractions and the clock are of that window, and both `window_us` and `row_us` are printed. Deep idle is printed as the periods that ended in the window and their microseconds, which is what the kernel counts. The rule is host-tested: a shared reading, a reading taken before a short row and a missing stamp are refused, and the fractions and the clock are of the readings' window. The kernel serving an old reading as new is recorded with an owner and an exit, for the track's follow-up. The track's C7 and clock claims are re-derived from rows whose readings matched them: the deflate thread busy 989 permille across its row, at 4.23 GHz against 3.83, every other core 99.5 to 99.9 % in C7. Co-Authored-By: Claude Opus 5.5 --- ...nother-returns-the-older-reading-as-new.md | 31 +++++ .../idle-cpus-enter-the-c-states-cst-names.md | 8 +- tests/guest/osbench/rules/Cargo.toml | 2 +- tests/guest/osbench/rules/src/lib.rs | 121 +++++++++++++++++- tests/guest/osbench/src/around.rs | 45 ++++--- 5 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 issues/a-counters-read-within-10-ms-of-another-returns-the-older-reading-as-new.md diff --git a/issues/a-counters-read-within-10-ms-of-another-returns-the-older-reading-as-new.md b/issues/a-counters-read-within-10-ms-of-another-returns-the-older-reading-as-new.md new file mode 100644 index 00000000000..08f4f672f1c --- /dev/null +++ b/issues/a-counters-read-within-10-ms-of-another-returns-the-older-reading-as-new.md @@ -0,0 +1,31 @@ +--- +status: assigned +kind: defect +opened: 2026-10-28 +--- + +# A counters read within 10 ms of another returns the older reading as new + +`SYS_COUNTERS` reads every CPU in a round: the reader kicks each CPU and +waits for its answer (`kernel/src/counters.rs`). A read issued within +`JOIN` (10 ms) of the last round's issue waits on that round and is served +its values. Nothing in the reply says so. The reader gets a reading taken up +to 10 ms before it asked, and two reads 5 ms apart can get one reading twice. + +#1073's group-start diagnostic read the counters around each group's 5 ms +`cpu.clock` window. 13 of its 17 windows came back with before equal to +after on every CPU (`ToyOSOrg/evidence` branch `pr-1073`, `r6/`). osbench's +shipped instruments now take each window from the readings' own counter +stamps and refuse a CPU whose readings are not the row's +(`osbench_rules::window`). That makes the defect visible, not absent: a +caller that does not compare stamps still reads an old reading as current. + +Owner: the idle track's follow-up +(`issues/idle-cpus-enter-the-c-states-cst-names.md`), in +`kernel/src/counters.rs` and the `toyos_abi::counters` record. + +**Exit**: a read never returns a reading older than its own call as +current. Either it waits for a round issued after it, or its record says it +was served an earlier round and when that round was issued; the ABI test +and a kernel host test hold whichever is chosen. osbench's window rule +stays as the reader's check. diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index e465ebd7f81..7ddbdaebc49 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -56,8 +56,12 @@ as that. A burst of wakes to CPUs in deep states pays their exits. each against its control with the deep states off, all else equal): - T14 `crunch.single` 1.10 to 1.11 times faster (deflate 60.3 to 54.5 ms, - Linux's 54.6 to 54.7), each row's thread at 4.21 to 4.23 GHz against - 3.83 by APERF/MPERF, its idle cores 40 to 80 % of the row in core C7. + Linux's 54.6 to 54.7). By APERF/MPERF over windows that matched their + rows (the thread busy 989 ‰ across deflate's), deflate's thread ran at + 4.23 GHz against 3.83, every other core 99.5 to 99.9 % of the row in core + C7. `malloc_churn.1048576.t1`'s threads ran at 4.19 to 4.21 GHz against + 3.79, with the cores 40 to 99 % in C7, the lower ones having run part of + the row. - T14 `cpu.clock` at 4189 at 8 and 8 group starts of 17, the control at none; Linux reaches 4190 at 13. Boot, identity, energy, system and mem stay at 3789 even when every idle CPU waits deepest (#1073's first diff --git a/tests/guest/osbench/rules/Cargo.toml b/tests/guest/osbench/rules/Cargo.toml index 6f8008706fe..017682bdf1a 100644 --- a/tests/guest/osbench/rules/Cargo.toml +++ b/tests/guest/osbench/rules/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "osbench-rules" -description = "osbench's pure rules, host-tested: when a munmap pair's x2APIC readings admit its figure, where the CPU crunch stops for heat, and an energy counter's span across its wrap." +description = "osbench's pure rules, host-tested: when a munmap pair's x2APIC readings admit its figure, where the CPU crunch stops for heat, an energy counter's span across its wrap, and whether two counters readings describe their row." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/tests/guest/osbench/rules/src/lib.rs b/tests/guest/osbench/rules/src/lib.rs index cb55320caaf..bee7699ff44 100644 --- a/tests/guest/osbench/rules/src/lib.rs +++ b/tests/guest/osbench/rules/src/lib.rs @@ -1,7 +1,8 @@ //! The judgements osbench makes over what it read, apart from the reading, so //! a host tests them: whether a munmap pair's figure stands ([`pair`]), where -//! the CPU crunch stops for heat ([`heat`]), and how far an energy counter -//! advanced across its wrap ([`energy`]). +//! the CPU crunch stops for heat ([`heat`]), how far an energy counter +//! advanced across its wrap ([`energy`]), and whether two counters readings +//! describe the row they were read around ([`window`]). pub mod pair { //! A munmap pair is an initiator that unmaps and a sibling that touched @@ -243,3 +244,119 @@ pub mod energy { } } } + +pub mod window { + //! Whether one CPU's two counters readings describe the row they were + //! read around. Each reading carries the CPU's own counter as it was + //! taken, so the readings' window is the difference of the two, whatever + //! the kernel served: a read within its join interval of another is + //! served that read's values, which is one reading twice, or one taken + //! before the row began. + + /// How far the readings' window may differ from the row's: 10 %. + pub const SLACK_PERMILLE: u64 = 100; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Refusal { + /// A reading without the CPU's counter: no window can be read off it. + NoStamp, + /// Both readings are one: nothing was measured. + OneReading, + /// The readings span `window_ns` against the row's `row_ns`. + NotTheRow { window_ns: u64, row_ns: u64 }, + } + + impl std::fmt::Display for Refusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Refusal::NoStamp => write!(f, "a reading carries no counter"), + Refusal::OneReading => write!(f, "both reads were served one reading"), + Refusal::NotTheRow { window_ns, row_ns } => write!( + f, + "the readings span {} us against the row's {} us", + window_ns / 1000, + row_ns / 1000 + ), + } + } + } + + /// The readings' window in counter ticks and nanoseconds, if it is the + /// row's within [`SLACK_PERMILLE`]. `to_ns` converts ticks of the CPU's + /// counter to nanoseconds. + pub fn of( + before: Option, + after: Option, + row_ns: u64, + to_ns: impl Fn(u64) -> u64, + ) -> Result<(u64, u64), Refusal> { + let (Some(before), Some(after)) = (before, after) else { return Err(Refusal::NoStamp) }; + let ticks = after.wrapping_sub(before); + if ticks == 0 { + return Err(Refusal::OneReading); + } + let window_ns = to_ns(ticks); + let slack = row_ns * SLACK_PERMILLE / 1000; + if window_ns.abs_diff(row_ns) > slack { + return Err(Refusal::NotTheRow { window_ns, row_ns }); + } + Ok((ticks, window_ns)) + } + + /// `part` of the window's `ticks`, in thousandths. + pub fn permille(part: u64, ticks: u64) -> u64 { + part.saturating_mul(1000) / ticks.max(1) + } + + /// The clock a CPU ran at while it ran, in MHz: APERF over MPERF of the + /// counter's rate, MPERF counting at that rate. + pub fn run_mhz(aperf: u64, mperf: u64, ticks: u64, window_ns: u64) -> Option { + let counter_mhz = ticks.saturating_mul(1000) / window_ns.max(1); + (mperf > 0).then(|| aperf.saturating_mul(counter_mhz) / mperf) + } + + #[cfg(test)] + mod tests { + use super::*; + + /// A 2 GHz counter: two ticks a nanosecond. + fn ns(ticks: u64) -> u64 { + ticks / 2 + } + + #[test] + fn a_window_that_is_the_row_stands() { + assert_eq!(of(Some(1_000), Some(1_000 + 2_000_000_000), 1_000_000_000, ns), Ok((2_000_000_000, 1_000_000_000))); + assert_eq!(of(Some(0), Some(2_180_000_000), 1_000_000_000, ns), Ok((2_180_000_000, 1_090_000_000)), "within 10 %"); + } + + #[test] + fn one_reading_served_twice_is_refused() { + assert_eq!(of(Some(77), Some(77), 5_000_000, ns), Err(Refusal::OneReading)); + } + + #[test] + fn a_reading_taken_before_a_short_row_is_refused() { + // A 2.5 ms row whose first read joined a reading 10 ms older. + assert_eq!( + of(Some(0), Some(25_000_000), 2_500_000, ns), + Err(Refusal::NotTheRow { window_ns: 12_500_000, row_ns: 2_500_000 }) + ); + } + + #[test] + fn a_missing_counter_is_refused() { + assert_eq!(of(None, Some(5), 5, ns), Err(Refusal::NoStamp)); + } + + #[test] + fn fractions_and_the_clock_are_of_the_readings_window() { + let (ticks, window_ns) = of(Some(0), Some(2_000_000_000), 1_000_000_000, ns).unwrap(); + assert_eq!(permille(2_000_000_000, ticks), 1000, "running throughout"); + assert_eq!(permille(500_000_000, ticks), 250); + // MPERF at the counter's 2000 MHz, APERF at 4200. + assert_eq!(run_mhz(4_200_000_000, 2_000_000_000, ticks, window_ns), Some(4200)); + assert_eq!(run_mhz(5, 0, ticks, window_ns), None, "a CPU that never ran has no clock"); + } + } +} diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 3b98fb38321..3f37ae9c854 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -2,7 +2,7 @@ //! the CPUs the row's threads ran on, on both systems; and on ToyOS, where //! the boot grants the counters, each CPU's busy fraction, the clock it ran //! at, its core's C6 and C7 residency where the CPU counts them, and the -//! idle periods the kernel chose a deep state for. A row that moves with the +//! deep idle periods that ended, all over the readings' own window. A row that moves with the //! machine's idle states, or with where its threads ran, says so here. //! //! **Reading the counters kicks every CPU** (`toyos_abi::counters`), so they @@ -49,32 +49,47 @@ impl Around { } } -/// One line a CPU: what it did between `before` and `after`, `ns` apart. +/// One line a CPU: what it did between `before` and `after`, read around a +/// row of `row_ns`. Every figure is of the readings' own window, off the +/// CPU's counter in each; a CPU whose readings are not the row's is refused +/// by name ([`osbench_rules::window`]). Deep idle is counted as periods that +/// ended in the window and their microseconds, as the kernel credits a +/// period when it ends. #[cfg(target_os = "toyos")] -pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate::sys::diag::CpuCounts], ns: u64) { +pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate::sys::diag::CpuCounts], row_ns: u64) { + use osbench_rules::window; + let page = toyos_abi::clock::page(); + let to_ns = |ticks: u64| toyos_abi::clock::ticks_to_nanos(ticks, page.period_fs); let delta = |x: Option, y: Option| x.zip(y).map(|(x, y)| y.wrapping_sub(x)); let shown = |v: Option| v.map_or("-".to_string(), |v| v.to_string()); - let permille = |part: Option, whole: Option| shown(part.zip(whole).map(|(p, w)| p * 1000 / w.max(1))); for b in after { let Some(a) = before.iter().find(|a| a.cpu == b.cpu) else { continue }; - let stamp = delta(a.stamp, b.stamp); + let (ticks, window_ns) = match window::of(a.stamp, b.stamp, row_ns, to_ns) { + Ok(window) => window, + Err(why) => { + note( + &format!("{bench}.cpus"), + &[("cpu", b.cpu.to_string()), ("refused", why.to_string()), ("row_us", (row_ns / 1000).to_string())], + ); + continue; + } + }; + let permille = |part: Option| shown(part.map(|p| window::permille(p, ticks))); let (aperf, mperf) = (delta(a.aperf, b.aperf), delta(a.mperf, b.mperf)); - // MPERF counts at the TSC's rate while the CPU runs, so APERF/MPERF - // of the TSC's rate is the clock it ran at. - let tsc_mhz = stamp.map(|s| s * 1000 / ns.max(1)); - let run_mhz = aperf.zip(mperf).zip(tsc_mhz).map(|((a, m), t)| a * t / m.max(1)); + let run_mhz = aperf.zip(mperf).and_then(|(a, m)| window::run_mhz(a, m, ticks, window_ns)); note( &format!("{bench}.cpus"), &[ ("cpu", b.cpu.to_string()), ("apic", b.apic.to_string()), - ("busy_permille", permille(mperf, stamp)), + ("window_us", (window_ns / 1000).to_string()), + ("row_us", (row_ns / 1000).to_string()), + ("busy_permille", permille(mperf)), ("run_mhz", shown(run_mhz)), - ("core_c6_permille", permille(delta(a.core_c6, b.core_c6), stamp)), - ("core_c7_permille", permille(delta(a.core_c7, b.core_c7), stamp)), - // Counted when a period ends, so one begun before the row counts whole: past 1000 is that. - ("deep_idles", shown(delta(a.deep_idles, b.deep_idles))), - ("deep_permille", permille(delta(a.deep_idle_nanos, b.deep_idle_nanos), Some(ns))), + ("core_c6_permille", permille(delta(a.core_c6, b.core_c6))), + ("core_c7_permille", permille(delta(a.core_c7, b.core_c7))), + ("deep_ended", shown(delta(a.deep_idles, b.deep_idles))), + ("deep_ended_us", shown(delta(a.deep_idle_nanos, b.deep_idle_nanos).map(|ns| ns / 1000))), ("kicks", shown(delta(a.kicks, b.kicks))), ("smi", shown(delta(a.smi, b.smi))), ], From 1d46a3ccc728666a7624e7c78f9a514082c785f4 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 06:45:39 +0100 Subject: [PATCH 20/24] osbench: deep idle is noted as the periods that ended, without their time The kernel credits a deep period's nanoseconds when it ends, so the time of the periods that ended in a window includes what they spent before it. B6 printed it above `window_us` on 13 T14 and 31 AMD lines. A count of ended periods is true of the window; their time is no figure of it, so it goes. Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/src/around.rs | 7 +++---- tests/guest/osbench/src/sys/toyos.rs | 6 ++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 3f37ae9c854..0ec07322904 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -52,9 +52,9 @@ impl Around { /// One line a CPU: what it did between `before` and `after`, read around a /// row of `row_ns`. Every figure is of the readings' own window, off the /// CPU's counter in each; a CPU whose readings are not the row's is refused -/// by name ([`osbench_rules::window`]). Deep idle is counted as periods that -/// ended in the window and their microseconds, as the kernel credits a -/// period when it ends. +/// by name ([`osbench_rules::window`]). Deep idle is counted as the periods +/// that ended in the window: the kernel credits a period's time when it +/// ends, so its time is no fraction of the window. #[cfg(target_os = "toyos")] pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate::sys::diag::CpuCounts], row_ns: u64) { use osbench_rules::window; @@ -89,7 +89,6 @@ pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate: ("core_c6_permille", permille(delta(a.core_c6, b.core_c6))), ("core_c7_permille", permille(delta(a.core_c7, b.core_c7))), ("deep_ended", shown(delta(a.deep_idles, b.deep_idles))), - ("deep_ended_us", shown(delta(a.deep_idle_nanos, b.deep_idle_nanos).map(|ns| ns / 1000))), ("kicks", shown(delta(a.kicks, b.kicks))), ("smi", shown(delta(a.smi, b.smi))), ], diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index 10e87066b7b..5e4edd77163 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -239,10 +239,9 @@ pub mod diag { /// Core C6 and C7 residency in TSC ticks, where the CPU counts it. pub core_c6: Option, pub core_c7: Option, - /// Idle periods the kernel chose a declared deep state for, and - /// their nanoseconds. + /// Idle periods the kernel chose a declared deep state for, counted + /// as each ends. pub deep_idles: Option, - pub deep_idle_nanos: Option, } /// The capability's diagnostics for the rows that read the counters @@ -279,7 +278,6 @@ pub mod diag { core_c6: r.get(Counter::CoreC6), core_c7: r.get(Counter::CoreC7), deep_idles: r.get(Counter::DeepIdles), - deep_idle_nanos: r.get(Counter::DeepIdleNanos), }) }) .collect() From 862f68cb64a718cb859e4d427c753279f8090d91 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 06:52:02 +0100 Subject: [PATCH 21/24] osbench: APERF/MPERF as the ratio its vendors define, and a window bounded by the join and the reads' cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither SDM Vol. 3B §17.2 nor AMD's APM Vol. 2 §17.3 defines the rate MPERF counts at: only the APERF/MPERF ratio is architectural. The instrument divided MPERF by the TSC for a busy fraction and multiplied the ratio by the TSC's rate for a clock, so a fully busy T14 CPU read 989 permille and its clock 1 % above `cpu.clock`'s, 4230 against 4189. The ratio is now printed as `aperf_mperf_permille` and neither figure is derived from it; the clock stays `cpu.clock`'s. Core C6/C7 residency counts at the TSC's rate (SDM Vol. 4 Table 2-20) and stays a fraction of the window. The kernel leaves Sierra Forest (06_AFH) out of the residency counters, as Table 2-57 counts its residency at the P1 clock, not the TSC's rate that `Counter::CoreC6` promises. A CPU's window now stands only between the row less its two reads' cost and the row plus the kernel's 10 ms join, the two ways a reading can differ from the row, and a CPU the kernel read stale is refused by name. The row is the span from the first read's call to the second's return, and `reads_us` is printed beside it. The track takes the clock step from each row's `mhz`, `cpu.clock`'s reading beside it: 4187 to 4189 MHz in the seven boots with the deep states against 3788 to 3789 in the four without. It links the stale-reading issue and says what the three diagnostic boots read. Co-Authored-By: Claude Opus 5.5 --- .../idle-cpus-enter-the-c-states-cst-names.md | 25 ++-- kernel/pure/cpu.rs | 21 ++- tests/guest/osbench/rules/src/lib.rs | 122 +++++++++++------- tests/guest/osbench/src/around.rs | 71 ++++++---- tests/guest/osbench/src/sys/toyos.rs | 4 + tests/guest/osbench/src/tail.rs | 18 +-- 6 files changed, 166 insertions(+), 95 deletions(-) diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 7ddbdaebc49..97124312d29 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -56,17 +56,18 @@ as that. A burst of wakes to CPUs in deep states pays their exits. each against its control with the deep states off, all else equal): - T14 `crunch.single` 1.10 to 1.11 times faster (deflate 60.3 to 54.5 ms, - Linux's 54.6 to 54.7). By APERF/MPERF over windows that matched their - rows (the thread busy 989 ‰ across deflate's), deflate's thread ran at - 4.23 GHz against 3.83, every other core 99.5 to 99.9 % of the row in core - C7. `malloc_churn.1048576.t1`'s threads ran at 4.19 to 4.21 GHz against - 3.79, with the cores 40 to 99 % in C7, the lower ones having run part of - the row. + Linux's 54.6 to 54.7). The clock beside both deflate and + `malloc_churn.1048576.t1` read 4187 to 4189 MHz in all seven boots with + the deep states, against 3788 to 3789 in the four A\* boots (each row's + `mhz`, `cpu.clock`'s reading beside it). Across deflate, every core but + the thread's was 99.5 to 99.9 % of the row in core C7, which counts at the + TSC's rate. - T14 `cpu.clock` at 4189 at 8 and 8 group starts of 17, the control at none; Linux reaches 4190 at 13. Boot, identity, energy, system and mem stay at 3789 even when every idle CPU waits deepest (#1073's first always-deepest control), so idle depth does not hold them there; - osbench now names what ran beside each group's clock (`.cpu.clock.ran`). + osbench now names what ran beside each group's clock + (`.cpu.clock.ran`). - AMD: no clock gap and no throughput change; the IPC cost of the first boots is gone with the idle choice folded into the pass's timer. The slow deflate is not the idle states: it came back with every idle CPU in C1, @@ -97,9 +98,13 @@ each against its control with the deep states off, all else equal): **#1066's exit is unmet** where this track stands: the T14's `cpu.clock` reaches ratio 42 at 5 to 10 of 17 group starts against Linux's 13. Of the groups below it, boot, identity, energy and system are held by acpiserver's -measured start-up work; the rest are the open question above, which two -diagnostic boots take up next: the counters read around each group's clock, -and the history switched off. +measured start-up work; the rest are the open question above. Three +diagnostic boots (#1073's `r6/`) read the counters around each group's +clock, the same with the history switched off, and the scheduler's diary +across it. The counters read came back as one reading twice in 13 of 17 +windows (`issues/a-counters-read-within-10-ms-of-another-returns-the-older-reading-as-new.md`), +and switching the history off left the count inside the spread. The +finding goes into this record with the track's next change. **Exit**: on the T14, since a KVM guest's idle is its host's: the idle minute's package energy, timer lateness no worse than `hlt`'s by more than diff --git a/kernel/pure/cpu.rs b/kernel/pure/cpu.rs index 7f3ba5a4e24..10b61f5377f 100644 --- a/kernel/pure/cpu.rs +++ b/kernel/pure/cpu.rs @@ -212,13 +212,17 @@ pub fn counters(facts: &CounterFacts) -> Counters { /// The family 6 DisplayModels SDM Vol. 4 (325462-093) names as supporting /// Table 2-20, each in its section's opening (§2.11-§2.16 and §2.17), which /// defines `MSR_CORE_C6_RESIDENCY` (3FDH) and `MSR_CORE_C7_RESIDENCY` (3FEH) -/// for them all. Of the tables that supersede it for these models, only -/// Table 2-57 (06_AFH) names either address, and it defines both. AMD -/// documents no core residency counter in the PPRs in hand. -const CORE_RESIDENCY_MODELS: [u32; 37] = [ +/// as counting at the TSC's rate, the unit [`Counter::CoreC6`] promises. +/// Of the tables that supersede it for these models, only Table 2-57 names +/// either address, for 06_AFH, and counts both in units of the P1 clock, +/// so 06_AFH is left out. AMD documents no core residency counter in the +/// PPRs in hand. +/// +/// [`Counter::CoreC6`]: toyos_abi::counters::Counter::CoreC6 +const CORE_RESIDENCY_MODELS: [u32; 36] = [ 0x2a, 0x2d, 0x3a, 0x3e, 0x3c, 0x45, 0x46, 0x3f, 0x3d, 0x47, 0x4f, 0x56, 0x4e, 0x5e, 0x55, 0x8e, 0x9e, 0x66, 0xa5, 0xa6, 0x7e, 0x8c, 0x8d, 0x6a, - 0x6c, 0x97, 0x9a, 0xba, 0xb7, 0xbf, 0x8f, 0xcf, 0xaa, 0xad, 0xae, 0xaf, 0xbd, + 0x6c, 0x97, 0x9a, 0xba, 0xb7, 0xbf, 0x8f, 0xcf, 0xaa, 0xad, 0xae, 0xbd, ]; fn counts_core_residency(id: &Ident) -> bool { @@ -371,6 +375,13 @@ mod tests { assert_eq!(counters(&intel(T14_SIGNATURE)), Counters { aperf_mperf: true, smi: true, core_residency: true }); } + /// 06_AFH: Table 2-57 counts its core residency at P1, not the TSC. + #[test] + fn sierra_forest_counts_no_core_residency_in_tsc_ticks() { + assert!(!counters(&intel(0x000a_06f0)).core_residency); + assert!(counters(&intel(0x000a_06f0)).smi, "its 34H stands"); + } + /// The AMD bench, Family 17h Model 60h: PPR 55922 gives it `EffFreq` and /// no `MSR0000_0034`. #[test] diff --git a/tests/guest/osbench/rules/src/lib.rs b/tests/guest/osbench/rules/src/lib.rs index bee7699ff44..1b94183a951 100644 --- a/tests/guest/osbench/rules/src/lib.rs +++ b/tests/guest/osbench/rules/src/lib.rs @@ -247,72 +247,96 @@ pub mod energy { pub mod window { //! Whether one CPU's two counters readings describe the row they were - //! read around. Each reading carries the CPU's own counter as it was - //! taken, so the readings' window is the difference of the two, whatever - //! the kernel served: a read within its join interval of another is - //! served that read's values, which is one reading twice, or one taken - //! before the row began. - - /// How far the readings' window may differ from the row's: 10 %. - pub const SLACK_PERMILLE: u64 = 100; + //! read around. Each reading carries the CPU's own counter as it + //! answered, so the readings' window is the difference of the two, + //! whatever the kernel served. + //! + //! The row is the span from the first read's call to the second read's + //! return. A CPU answers each read inside its call, so its window lies + //! within the row and is shorter by at most the two calls' cost, except + //! that a read issued within [`JOIN_NS`] of another round is served that + //! round, issued up to [`JOIN_NS`] before the call. A window outside + //! those bounds is not the row's. + + /// The kernel's counters join interval (`kernel/src/counters.rs`'s + /// `JOIN`): how long before its call a read may be served a round. + pub const JOIN_NS: u64 = 10_000_000; + + /// The row's span and what its two reads cost, in nanoseconds. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct Span { + pub row_ns: u64, + pub reads_ns: u64, + } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Refusal { + /// The kernel read the CPU stale: it did not answer within the + /// read's bound, and its values are the last it published. + Stale, /// A reading without the CPU's counter: no window can be read off it. NoStamp, /// Both readings are one: nothing was measured. OneReading, - /// The readings span `window_ns` against the row's `row_ns`. - NotTheRow { window_ns: u64, row_ns: u64 }, + /// The readings span `window_ns`, outside what a row of `row_ns` + /// read at a cost of `reads_ns` allows. + NotTheRow { window_ns: u64, row_ns: u64, reads_ns: u64 }, } impl std::fmt::Display for Refusal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Refusal::Stale => write!(f, "the kernel read this CPU stale"), Refusal::NoStamp => write!(f, "a reading carries no counter"), Refusal::OneReading => write!(f, "both reads were served one reading"), - Refusal::NotTheRow { window_ns, row_ns } => write!( + Refusal::NotTheRow { window_ns, row_ns, reads_ns } => write!( f, - "the readings span {} us against the row's {} us", + "the readings span {} us against a row of {} us read in {} us", window_ns / 1000, - row_ns / 1000 + row_ns / 1000, + reads_ns / 1000 ), } } } - /// The readings' window in counter ticks and nanoseconds, if it is the - /// row's within [`SLACK_PERMILLE`]. `to_ns` converts ticks of the CPU's - /// counter to nanoseconds. + /// The readings' window in counter ticks and nanoseconds, where it is + /// the row's. `to_ns` converts ticks of the CPU's counter to nanoseconds. pub fn of( + stale: bool, before: Option, after: Option, - row_ns: u64, + span: Span, to_ns: impl Fn(u64) -> u64, ) -> Result<(u64, u64), Refusal> { + if stale { + return Err(Refusal::Stale); + } let (Some(before), Some(after)) = (before, after) else { return Err(Refusal::NoStamp) }; let ticks = after.wrapping_sub(before); if ticks == 0 { return Err(Refusal::OneReading); } let window_ns = to_ns(ticks); - let slack = row_ns * SLACK_PERMILLE / 1000; - if window_ns.abs_diff(row_ns) > slack { - return Err(Refusal::NotTheRow { window_ns, row_ns }); + let shortest = span.row_ns.saturating_sub(span.reads_ns); + let longest = span.row_ns.saturating_add(JOIN_NS); + if !(shortest..=longest).contains(&window_ns) { + return Err(Refusal::NotTheRow { window_ns, row_ns: span.row_ns, reads_ns: span.reads_ns }); } Ok((ticks, window_ns)) } - /// `part` of the window's `ticks`, in thousandths. + /// `part` of the window's `ticks`, in thousandths: for a counter that + /// counts at the window counter's rate. pub fn permille(part: u64, ticks: u64) -> u64 { part.saturating_mul(1000) / ticks.max(1) } - /// The clock a CPU ran at while it ran, in MHz: APERF over MPERF of the - /// counter's rate, MPERF counting at that rate. - pub fn run_mhz(aperf: u64, mperf: u64, ticks: u64, window_ns: u64) -> Option { - let counter_mhz = ticks.saturating_mul(1000) / window_ns.max(1); - (mperf > 0).then(|| aperf.saturating_mul(counter_mhz) / mperf) + /// APERF over MPERF, in thousandths: the one figure SDM Vol. 3B §17.2 + /// and AMD's APM Vol. 2 §17.3 define of the pair. Neither defines + /// MPERF's rate, so neither a clock nor a busy fraction is read off it. + pub fn aperf_mperf_permille(aperf: u64, mperf: u64) -> Option { + (mperf > 0).then(|| aperf.saturating_mul(1000) / mperf) } #[cfg(test)] @@ -324,39 +348,49 @@ pub mod window { ticks / 2 } + const ROW: Span = Span { row_ns: 1_000_000_000, reads_ns: 200_000 }; + #[test] - fn a_window_that_is_the_row_stands() { - assert_eq!(of(Some(1_000), Some(1_000 + 2_000_000_000), 1_000_000_000, ns), Ok((2_000_000_000, 1_000_000_000))); - assert_eq!(of(Some(0), Some(2_180_000_000), 1_000_000_000, ns), Ok((2_180_000_000, 1_090_000_000)), "within 10 %"); + fn a_window_inside_the_row_by_less_than_its_reads_stands() { + assert_eq!(of(false, Some(1_000), Some(1_000 + 1_999_800_000), ROW, ns), Ok((1_999_800_000, 999_900_000))); + assert_eq!(of(false, Some(0), Some(1_999_600_000), ROW, ns), Ok((1_999_600_000, 999_800_000)), "short by the reads exactly"); } #[test] - fn one_reading_served_twice_is_refused() { - assert_eq!(of(Some(77), Some(77), 5_000_000, ns), Err(Refusal::OneReading)); + fn a_before_served_from_a_joined_round_stands_within_the_join() { + assert_eq!(of(false, Some(0), Some(2_020_000_000), ROW, ns), Ok((2_020_000_000, 1_010_000_000))); + assert_eq!( + of(false, Some(0), Some(2_020_000_002), ROW, ns), + Err(Refusal::NotTheRow { window_ns: 1_010_000_001, row_ns: 1_000_000_000, reads_ns: 200_000 }) + ); } #[test] - fn a_reading_taken_before_a_short_row_is_refused() { - // A 2.5 ms row whose first read joined a reading 10 ms older. + fn a_window_shorter_than_the_row_less_its_reads_is_refused() { + // An `after` served a round issued inside the row. assert_eq!( - of(Some(0), Some(25_000_000), 2_500_000, ns), - Err(Refusal::NotTheRow { window_ns: 12_500_000, row_ns: 2_500_000 }) + of(false, Some(0), Some(1_900_000_000), ROW, ns), + Err(Refusal::NotTheRow { window_ns: 950_000_000, row_ns: 1_000_000_000, reads_ns: 200_000 }) ); } #[test] - fn a_missing_counter_is_refused() { - assert_eq!(of(None, Some(5), 5, ns), Err(Refusal::NoStamp)); + fn one_reading_served_twice_is_refused() { + assert_eq!(of(false, Some(77), Some(77), ROW, ns), Err(Refusal::OneReading)); + } + + #[test] + fn a_stale_or_unstamped_reading_is_refused() { + assert_eq!(of(true, Some(0), Some(2_000_000_000), ROW, ns), Err(Refusal::Stale)); + assert_eq!(of(false, None, Some(5), ROW, ns), Err(Refusal::NoStamp)); } #[test] - fn fractions_and_the_clock_are_of_the_readings_window() { - let (ticks, window_ns) = of(Some(0), Some(2_000_000_000), 1_000_000_000, ns).unwrap(); - assert_eq!(permille(2_000_000_000, ticks), 1000, "running throughout"); - assert_eq!(permille(500_000_000, ticks), 250); - // MPERF at the counter's 2000 MHz, APERF at 4200. - assert_eq!(run_mhz(4_200_000_000, 2_000_000_000, ticks, window_ns), Some(4200)); - assert_eq!(run_mhz(5, 0, ticks, window_ns), None, "a CPU that never ran has no clock"); + fn residency_is_a_fraction_of_the_window_and_the_pair_a_ratio() { + let (ticks, _) = of(false, Some(0), Some(2_000_000_000), ROW, ns).unwrap(); + assert_eq!(permille(1_995_000_000, ticks), 997); + assert_eq!(aperf_mperf_permille(4_200, 2_400), Some(1750)); + assert_eq!(aperf_mperf_permille(5, 0), None, "a CPU that never ran has no ratio"); } } } diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 0ec07322904..23e39e4fe9e 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -1,8 +1,8 @@ //! What the machine's CPUs did across one measured row, printed beside it: //! the CPUs the row's threads ran on, on both systems; and on ToyOS, where -//! the boot grants the counters, each CPU's busy fraction, the clock it ran -//! at, its core's C6 and C7 residency where the CPU counts them, and the -//! deep idle periods that ended, all over the readings' own window. A row that moves with the +//! the boot grants the counters, each CPU's APERF/MPERF ratio, its core's +//! C6 and C7 residency where the CPU counts them, and the deep idle periods +//! that ended, over the readings' own window. A row that moves with the //! machine's idle states, or with where its threads ran, says so here. //! //! **Reading the counters kicks every CPU** (`toyos_abi::counters`), so they @@ -16,17 +16,15 @@ use crate::report::note; /// The counters as a row began. pub struct Around { - began: Instant, #[cfg(target_os = "toyos")] - before: Option>, + before: Option, } /// Read the counters before a row. pub fn begin() -> Around { Around { - began: Instant::now(), #[cfg(target_os = "toyos")] - before: crate::sys::diag::shared().and_then(|d| d.counters().ok()), + before: crate::sys::diag::shared().and_then(|d| read(d).ok()), } } @@ -39,53 +37,72 @@ impl Around { note(&format!("{bench}.placement"), &[(crate::sys::CPU_NAMING, placed.join(","))]); #[cfg(target_os = "toyos")] if let (Some(before), Some(d)) = (self.before, crate::sys::diag::shared()) { - match d.counters() { - Ok(after) => cpus(bench, &before, &after, crate::ns_since(self.began)), + match read(d) { + Ok(after) => cpus(bench, &before, &after), Err(why) => note(&format!("{bench}.cpus"), &[("failed", why)]), } } - #[cfg(not(target_os = "toyos"))] - let _ = self.began; } } -/// One line a CPU: what it did between `before` and `after`, read around a -/// row of `row_ns`. Every figure is of the readings' own window, off the -/// CPU's counter in each; a CPU whose readings are not the row's is refused -/// by name ([`osbench_rules::window`]). Deep idle is counted as the periods -/// that ended in the window: the kernel credits a period's time when it -/// ends, so its time is no fraction of the window. +/// One counters read, with when its call began and what it cost. #[cfg(target_os = "toyos")] -pub fn cpus(bench: &str, before: &[crate::sys::diag::CpuCounts], after: &[crate::sys::diag::CpuCounts], row_ns: u64) { +pub struct Reading { + pub cpus: Vec, + called: Instant, + cost_ns: u64, +} + +#[cfg(target_os = "toyos")] +pub fn read(d: &crate::sys::diag::Diag) -> Result { + let called = Instant::now(); + let cpus = d.counters()?; + Ok(Reading { cpus, called, cost_ns: crate::ns_since(called) }) +} + +/// One line a CPU: what it did between `before` and `after`. Every figure +/// is of the readings' own window, off the CPU's counter in each, and a CPU +/// whose readings are not the row's is refused by name +/// ([`osbench_rules::window`]). Core residency counts at the TSC's rate (SDM +/// Vol. 4 Table 2-20), the window's own counter; APERF/MPERF is printed as +/// the ratio, the one figure of the pair either vendor defines. Deep idle is +/// counted as the periods that ended in the window: the kernel credits a +/// period's time when it ends, so its time is no fraction of the window. +#[cfg(target_os = "toyos")] +pub fn cpus(bench: &str, before: &Reading, after: &Reading) { use osbench_rules::window; let page = toyos_abi::clock::page(); let to_ns = |ticks: u64| toyos_abi::clock::ticks_to_nanos(ticks, page.period_fs); + let span = window::Span { + row_ns: u64::try_from(after.called.duration_since(before.called).as_nanos()).expect("a row under 584 years") + + after.cost_ns, + reads_ns: before.cost_ns + after.cost_ns, + }; let delta = |x: Option, y: Option| x.zip(y).map(|(x, y)| y.wrapping_sub(x)); let shown = |v: Option| v.map_or("-".to_string(), |v| v.to_string()); - for b in after { - let Some(a) = before.iter().find(|a| a.cpu == b.cpu) else { continue }; - let (ticks, window_ns) = match window::of(a.stamp, b.stamp, row_ns, to_ns) { + for b in &after.cpus { + let Some(a) = before.cpus.iter().find(|a| a.cpu == b.cpu) else { continue }; + let (ticks, window_ns) = match window::of(a.stale || b.stale, a.stamp, b.stamp, span, to_ns) { Ok(window) => window, Err(why) => { note( &format!("{bench}.cpus"), - &[("cpu", b.cpu.to_string()), ("refused", why.to_string()), ("row_us", (row_ns / 1000).to_string())], + &[("cpu", b.cpu.to_string()), ("refused", why.to_string()), ("row_us", (span.row_ns / 1000).to_string())], ); continue; } }; let permille = |part: Option| shown(part.map(|p| window::permille(p, ticks))); - let (aperf, mperf) = (delta(a.aperf, b.aperf), delta(a.mperf, b.mperf)); - let run_mhz = aperf.zip(mperf).and_then(|(a, m)| window::run_mhz(a, m, ticks, window_ns)); + let ratio = delta(a.aperf, b.aperf).zip(delta(a.mperf, b.mperf)).and_then(|(a, m)| window::aperf_mperf_permille(a, m)); note( &format!("{bench}.cpus"), &[ ("cpu", b.cpu.to_string()), ("apic", b.apic.to_string()), ("window_us", (window_ns / 1000).to_string()), - ("row_us", (row_ns / 1000).to_string()), - ("busy_permille", permille(mperf)), - ("run_mhz", shown(run_mhz)), + ("row_us", (span.row_ns / 1000).to_string()), + ("reads_us", (span.reads_ns / 1000).to_string()), + ("aperf_mperf_permille", shown(ratio)), ("core_c6_permille", permille(delta(a.core_c6, b.core_c6))), ("core_c7_permille", permille(delta(a.core_c7, b.core_c7))), ("deep_ended", shown(delta(a.deep_idles, b.deep_idles))), diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index 5e4edd77163..45a995986ce 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -228,6 +228,9 @@ pub mod diag { /// names it by. #[derive(Clone)] pub struct CpuCounts { + /// The CPU did not answer within the read's bound: its values are the + /// last it published. + pub stale: bool, pub cpu: u32, pub apic: u32, /// The CPU's own counter when it read the rest (the TSC). @@ -268,6 +271,7 @@ pub mod diag { .map(|r| { let r = Record::decode(r).map_err(|e| format!("a counters record did not decode: {e:?}"))?; Ok(CpuCounts { + stale: r.stale, cpu: r.cpu, apic: r.hardware_id, stamp: r.get(Counter::Stamp), diff --git a/tests/guest/osbench/src/tail.rs b/tests/guest/osbench/src/tail.rs index 533e25a46db..4df8fa5faae 100644 --- a/tests/guest/osbench/src/tail.rs +++ b/tests/guest/osbench/src/tail.rs @@ -48,14 +48,14 @@ pub fn suite() { #[cfg(target_os = "toyos")] let diag = sys::diag::open(); #[cfg(target_os = "toyos")] - let (before, began) = (diag.as_ref().map(|d| d.counters()), Instant::now()); + let before = diag.as_ref().map(crate::around::read); let late = periodic("tail.wake.periodic.60s", Duration::from_secs(60), &mut |_| {}); print_outliers("tail.wake.periodic.60s", &late); #[cfg(target_os = "toyos")] if let (Some(d), Some(before)) = (diag.as_ref(), before) { - counters_over("tail.wake.periodic.60s", before, d, crate::ns_since(began)); + counters_over("tail.wake.periodic.60s", before, d); } stolen(Duration::from_secs(10)); @@ -204,11 +204,11 @@ fn stolen(span: Duration) { } } -/// What each CPU did between `before`, `ns` ago, and now ([`crate::around`]). +/// What each CPU did between `before` and now ([`crate::around`]). #[cfg(target_os = "toyos")] -fn counters_over(bench: &str, before: Result, String>, d: &sys::diag::Diag, ns: u64) { - match (before, d.counters()) { - (Ok(before), Ok(after)) => crate::around::cpus(&format!("{bench}.counters"), &before, &after, ns), +fn counters_over(bench: &str, before: Result, d: &sys::diag::Diag) { + match (before, crate::around::read(d)) { + (Ok(before), Ok(after)) => crate::around::cpus(&format!("{bench}.counters"), &before, &after), (Err(why), _) | (_, Err(why)) => note(&format!("{bench}.counters"), &[("failed", why)]), } } @@ -222,11 +222,11 @@ fn counters_over(bench: &str, before: Result, String>, fn diary_run(mut d: sys::diag::Diag) { const BENCH: &str = "tail.diag.periodic.30s"; const SHOWN: usize = 16; - let (before, began) = (d.counters(), Instant::now()); + let before = crate::around::read(&d); // The diary names a CPU by the kernel's index, and the sleeper's CPUs // are APIC ids: the counters records carry both. let index_of: std::collections::BTreeMap = - before.iter().flatten().map(|c| (c.apic, c.cpu)).collect(); + before.iter().flat_map(|r| &r.cpus).map(|c| (c.apic, c.cpu)).collect(); let names_before = d.roster(); d.diary(); let mut shown = 0usize; @@ -295,5 +295,5 @@ fn diary_run(mut d: sys::diag::Diag) { for ((pid, name), ns) in by_process.into_iter().filter(|(_, ns)| *ns > 0) { note(&format!("{BENCH}.cpu_time"), &[("pid", pid.to_string()), ("name", name), ("us", (ns / 1000).to_string())]); } - counters_over(BENCH, before, &d, crate::ns_since(began)); + counters_over(BENCH, before, &d); } From 18fe4e1504b3eb59ae3b31660c322b554a489d43 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 07:03:24 +0100 Subject: [PATCH 22/24] osbench: the Linux build has no use for Instant at the top; the deflate finding names B6 and no MPERF-rate clock `Instant` is used only by the ToyOS half of the instrument now, so the Linux build's clippy refused the import. The AMD deflate finding counts B6, 65.7 ms on CPU 0, and compares the boots by the APERF/MPERF ratio rather than a clock that assumed MPERF's rate. Co-Authored-By: Claude Opus 5.5 --- ...ompares-toyos-with-linux-on-one-machine.md | 4 ++-- ...nt-slow-in-some-boots-at-the-same-clock.md | 22 +++++++++++-------- tests/guest/osbench/src/around.rs | 6 ++--- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index 5065a66b57f..aa2f3735570 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -48,8 +48,8 @@ OpenSSL and AWS-LC by the crypto oracle probe, and that table is the evidence: `ToyOSOrg/evidence` at `bed0b00efac82546b995f0841da54aea11c45694`, `r2/table.md`. -The AMD bench's `crunch.single.deflate` read 6 % slower in three boots of -ten, at the same clock, with and without the deep idle states +The AMD bench's `crunch.single.deflate` read 4 to 6 % slower in four boots +of eleven, at the same clock, with and without the deep idle states (`issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md`). An AMD single-thread row compared across boots needs repeats enough to see which mode each boot read. diff --git a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md index c1583825eab..5645ea01131 100644 --- a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md +++ b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md @@ -6,19 +6,23 @@ opened: 2026-10-28 # The AMD bench's deflate runs 6 % slow in some boots at the same clock -osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has two -values across ten boots (`ToyOSOrg/evidence` branch `pr-1073`, `r3/` to -`r5/`): +osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has more +than one value across eleven boots (`ToyOSOrg/evidence` branch `pr-1073`, +`r3/` to `r7/`): - In three boots its p50 was 66.6 to 66.8 ms. Two ran the thread on CPU 4 and one on CPU 10. -- In the seven others it was 62.9 to 63.7 ms, on CPUs 0, 2, 10 and 12. +- In one, B6's, it was 65.7 ms, on CPU 0. +- In the seven others it was 62.9 to 63.7 ms, on CPUs 0, 2, 6, 10 and 12. -APERF/MPERF put the thread's clock at 4214 to 4250 MHz in every boot. The -slow boots include one with every idle CPU in C1 and two with the deep -states. In the slow A\* boot and the slow B4 boot, every other CPU, the -sibling included, read `busy_permille=0` across the row. So it is not the -idle states, the clock, the placement, or a running neighbour. +The thread's APERF/MPERF ratio, the one figure of the pair AMD defines +(APM Vol. 2 §17.3), differed by under 1 % across the boots. The slow boots +include one with every idle CPU in C1 and the rest with the deep states. +In the slow A\* boot and the slow B4 boot, no other CPU's MPERF advanced by +a thousandth of its TSC's count across the row, and MPERF counts only in +C0 (§17.3). +So it is not the idle states, the clock, the placement, or a running +neighbour. `sha256`, `matmul` and `parse` show no such split. Every AMD single-thread timing row may carry this spread. A comparison between two AMD boots, diff --git a/tests/guest/osbench/src/around.rs b/tests/guest/osbench/src/around.rs index 23e39e4fe9e..431d6c4f65f 100644 --- a/tests/guest/osbench/src/around.rs +++ b/tests/guest/osbench/src/around.rs @@ -10,8 +10,6 @@ //! around `cpu.clock`'s few milliseconds, which [`ran_during`] names from the //! roster instead. -use std::time::Instant; - use crate::report::note; /// The counters as a row began. @@ -49,13 +47,13 @@ impl Around { #[cfg(target_os = "toyos")] pub struct Reading { pub cpus: Vec, - called: Instant, + called: std::time::Instant, cost_ns: u64, } #[cfg(target_os = "toyos")] pub fn read(d: &crate::sys::diag::Diag) -> Result { - let called = Instant::now(); + let called = std::time::Instant::now(); let cpus = d.counters()?; Ok(Reading { cpus, called, cost_ns: crate::ns_since(called) }) } From efe09aa0c7316dc829eb81621509ddab7a3bbd2c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 07:59:16 +0100 Subject: [PATCH 23/24] issues: B7 is a fast AMD deflate boot, four slow of twelve Co-Authored-By: Claude Opus 5.5 --- ...ublished-run-compares-toyos-with-linux-on-one-machine.md | 2 +- ...e-runs-6-percent-slow-in-some-boots-at-the-same-clock.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index aa2f3735570..24e214b8c1c 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -49,7 +49,7 @@ evidence: `ToyOSOrg/evidence` at `bed0b00efac82546b995f0841da54aea11c45694`, `r2/table.md`. The AMD bench's `crunch.single.deflate` read 4 to 6 % slower in four boots -of eleven, at the same clock, with and without the deep idle states +of twelve, at the same clock, with and without the deep idle states (`issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md`). An AMD single-thread row compared across boots needs repeats enough to see which mode each boot read. diff --git a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md index 5645ea01131..079e7d34f96 100644 --- a/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md +++ b/issues/the-amd-benchs-deflate-runs-6-percent-slow-in-some-boots-at-the-same-clock.md @@ -7,13 +7,13 @@ opened: 2026-10-28 # The AMD bench's deflate runs 6 % slow in some boots at the same clock osbench's `crunch.single.deflate` on the AMD bench (Ryzen 7 4800U) has more -than one value across eleven boots (`ToyOSOrg/evidence` branch `pr-1073`, -`r3/` to `r7/`): +than one value across twelve boots (`ToyOSOrg/evidence` branch `pr-1073`, +`r3/` to `r8/`): - In three boots its p50 was 66.6 to 66.8 ms. Two ran the thread on CPU 4 and one on CPU 10. - In one, B6's, it was 65.7 ms, on CPU 0. -- In the seven others it was 62.9 to 63.7 ms, on CPUs 0, 2, 6, 10 and 12. +- In the eight others it was 62.9 to 63.7 ms, on CPUs 0, 2, 6, 10 and 12. The thread's APERF/MPERF ratio, the one figure of the pair AMD defines (APM Vol. 2 §17.3), differed by under 1 % across the boots. The slow boots From 5c812f1b39601c8d1840edf526660c6653f72fa7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 08:00:16 +0100 Subject: [PATCH 24/24] issues: the track counts B6 and B7 in the clock spread and the clock step Co-Authored-By: Claude Opus 5.5 --- issues/idle-cpus-enter-the-c-states-cst-names.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/issues/idle-cpus-enter-the-c-states-cst-names.md b/issues/idle-cpus-enter-the-c-states-cst-names.md index 97124312d29..6cad4707afd 100644 --- a/issues/idle-cpus-enter-the-c-states-cst-names.md +++ b/issues/idle-cpus-enter-the-c-states-cst-names.md @@ -57,7 +57,7 @@ each against its control with the deep states off, all else equal): - T14 `crunch.single` 1.10 to 1.11 times faster (deflate 60.3 to 54.5 ms, Linux's 54.6 to 54.7). The clock beside both deflate and - `malloc_churn.1048576.t1` read 4187 to 4189 MHz in all seven boots with + `malloc_churn.1048576.t1` read 4187 to 4189 MHz in all eight boots with the deep states, against 3788 to 3789 in the four A\* boots (each row's `mhz`, `cpu.clock`'s reading beside it). Across deflate, every core but the thread's was 99.5 to 99.9 % of the row in core C7, which counts at the @@ -84,10 +84,10 @@ each against its control with the deep states off, all else equal): always-deepest control, so the history holding idle CPUs shallow after the group before is the open hypothesis there; a per-CPU state read inside the window would test it. -- Group starts at 4189 across the six T14 boots with the deep states: - 8 and 8 (r2), 8 and 10 (r3), 10 (B4) and 5 (B5 at `d60d727aa`), counted +- Group starts at 4189 across the eight T14 boots with the deep states: + 8 and 8 (r2), 8 and 10 (r3), 10 (B4), 5 (B5), 10 (B6) and 10 (B7), counted as a maximum above 4100 MHz. Only sched, files, ipc and work reached it - in all six; B5 lost time, which every earlier boot had. It is a spread + in all eight; B5 and B6 lost time, which the others had. It is a spread from boot to boot, not a fixed set of groups. - Soundserver's worst IRQ lateness, over every window line of the testcases boots: with every idle CPU in C1 (one boot a bench), T14 181 to