From 545e9cd565dda13c46fab133128faaeb33522f11 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 28 Sep 2026 21:50:42 +0200 Subject: [PATCH 01/12] The kernel declares the CPU's HWP request and a perf-state claim reads it back per CPU The self-hosting bar is measured under a power envelope that must be read back for a whole build span, and ToyOS left every register of it where firmware put it. This is the first stage of the track that makes the kernel own it (issues/kernel/the-kernel-owns-cpu-performance-state.md). The declaration. control_regs.rs, the one CPU-state declaration, now also writes IA32_PM_ENABLE, IA32_HWP_REQUEST, IA32_HWP_REQUEST_PKG and IA32_ENERGY_PERF_BIAS whole on the BSP and every AP, and asserts each on each. The values are the bar's: min is the package's maximum-efficiency ratio (MSR_PLATFORM_INFO[47:40]), max the CPU's HWP highest performance, desired 0, EPP 128, window 0, no package control, which on the T14's inputs is 0x80002a04, what Linux's intel_pstate held there; the package request is 0x8000ff01 and EPB 6. The declaration is all or nothing. A CPU missing any register it names (HWP, EPP, package request, EPB, package thermal status, or not Intel, or hybrid) gets no request, and the BSP says why once: "control_regs: no performance request is declared: no HWP ...". Whether the machine declared one is the BSP's verdict and every AP must reach it; the request itself is per CPU, since a CPU's highest performance is its own. IA32_MISC_ENABLE's turbo bit is read back and not written, because its other bits are model-specific and firmware's. The arithmetic is toyos-perfstate, a pure host-tested crate, because no QEMU CPU has HWP. Its oracle is the T14's Linux MSR readings. The read-back needs no new syscall. It is a device class, perf-state (class 9, the manifest's `devices = ["perf-state"]`), whose read answers toyos_abi::perf's records: the package's registers (HWP package request, PLATFORM_INFO, RAPL unit, PKG_POWER_LIMIT, PKG_ENERGY_STATUS, package thermal status, TEMPERATURE_TARGET), then each CPU's (PM_ENABLE, HWP capabilities, HWP request, EPB, MISC_ENABLE). A per-CPU MSR is readable only on its CPU, so a read asks every CPU. It issues a generation on a second shootdown::Shootdown (the loom-modelled TLB ack protocol), answers for its own CPU, and kicks the rest. Each answers from its next scheduler pass in drain_irqs, which costs two relaxed loads when nothing is owed, and posts perf_state::WATCH. The read parks there, bounded by 250 ms, past which it is refused Io and the silent CPUs are named. A claim exists only with the control_regs::HwpDeclared proof, so no rdmsr of these registers is reachable on a CPU without them. On AArch64 that proof is an uninhabited type and the claim is refused by name. /system/bin/perfstate holds the row and prints one read, checked against the declaration. The guest binary perf_state runs the same code, and perf_request drives it: in QEMU it asserts the named refusal, no request line, and the claim refused NotFound; on the T14 it asserts every CPU logged the bar's literal values and the binary read them all back. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6rME2DoqwjcYFStYHHY4j --- Cargo.lock | 4 + Cargo.toml | 1 + .../the-kernel-owns-cpu-performance-state.md | 51 +++ kernel/Cargo.lock | 5 + kernel/Cargo.toml | 1 + kernel/src/arch/aarch64/mod.rs | 1 + kernel/src/arch/aarch64/perf_state.rs | 20 ++ kernel/src/arch/x86_64/control_regs.rs | 145 +++++++- kernel/src/arch/x86_64/mod.rs | 1 + kernel/src/arch/x86_64/perf_state.rs | 38 +++ kernel/src/device.rs | 8 + kernel/src/main.rs | 1 + kernel/src/object/device.rs | 12 +- kernel/src/object/ops.rs | 12 +- kernel/src/perf_state.rs | 148 +++++++++ kernel/src/sched/driver.rs | 2 + kernel/src/shootdown.rs | 3 +- kernel/src/syscall/device.rs | 6 +- kernel/src/syscall/io.rs | 18 + system.toml | 5 + tests/toyos-rust-tests/Cargo.lock | 14 + tests/toyos-rust-tests/Cargo.toml | 1 + tests/toyos-rust-tests/src/bin/perf_state.rs | 33 ++ tests/toyos.rs | 64 ++++ toyos-abi/src/lib.rs | 1 + toyos-abi/src/perf.rs | 109 ++++++ toyos-abi/src/syscall.rs | 5 + toyos-perfstate/Cargo.toml | 14 + toyos-perfstate/src/lib.rs | 313 ++++++++++++++++++ userland/Cargo.lock | 13 + userland/Cargo.toml | 1 + userland/perfstate/Cargo.toml | 20 ++ userland/perfstate/src/lib.rs | 62 ++++ userland/perfstate/src/main.rs | 26 ++ 34 files changed, 1141 insertions(+), 17 deletions(-) create mode 100644 issues/kernel/the-kernel-owns-cpu-performance-state.md create mode 100644 kernel/src/arch/aarch64/perf_state.rs create mode 100644 kernel/src/arch/x86_64/perf_state.rs create mode 100644 kernel/src/perf_state.rs create mode 100644 tests/toyos-rust-tests/src/bin/perf_state.rs create mode 100644 toyos-abi/src/perf.rs create mode 100644 toyos-perfstate/Cargo.toml create mode 100644 toyos-perfstate/src/lib.rs create mode 100644 userland/perfstate/Cargo.toml create mode 100644 userland/perfstate/src/lib.rs create mode 100644 userland/perfstate/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 1e18b1d023b..d10c19d40c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,6 +1143,10 @@ dependencies = [ name = "toyos-pcid" version = "0.1.0" +[[package]] +name = "toyos-perfstate" +version = "0.1.0" + [[package]] name = "toyos-proclife" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index aff7f48c5ef..98bc31bdcc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "toyos-net-wire", "toyos-pci", "toyos-pcid", + "toyos-perfstate", "toyos-proclife", "toyos-ps2", "toyos-quiesce", diff --git a/issues/kernel/the-kernel-owns-cpu-performance-state.md b/issues/kernel/the-kernel-owns-cpu-performance-state.md new file mode 100644 index 00000000000..c090fff7ac9 --- /dev/null +++ b/issues/kernel/the-kernel-owns-cpu-performance-state.md @@ -0,0 +1,51 @@ +--- +status: open +kind: track +opened: 2026-09-28 +--- + +# The kernel owns CPU performance state + +The self-hosting bar (`issues/build/toyos-builds-itself.md`) is measured +under a fixed power envelope that must be read back for a whole build span. +Until this track, ToyOS left every register of it where firmware put it. The +kernel declares the envelope from the one CPU-state declaration +(`kernel/src/arch/x86_64/control_regs.rs`), and a `perf-state` claim reads it +back per CPU. The values the kernel programs are the bar's, not its own. + +- **1 — the HWP request, declared and read back.** `IA32_PM_ENABLE`, + `IA32_HWP_REQUEST` (min: the package's maximum-efficiency ratio; max: the + CPU's highest performance; EPP 128), `IA32_HWP_REQUEST_PKG` and EPB 6, + written whole on every CPU and asserted on each, refused by name on a CPU + that lacks any of them. Read back through the `perf-state` claim, together + with the turbo bit, `MSR_PKG_POWER_LIMIT`, the energy counter and package + thermal status (`/system/bin/perfstate`). *Exit*: `perf_request` green in + QEMU, which proves the refusal, and on the T14, which proves every CPU holds + `0x80002a04` and reads it back. +- **2 — RAPL, declared.** PL1, PL2 and their windows through + `MSR_PKG_POWER_LIMIT`, and the MMIO mirror in the host bridge's MCHBAR, at + the bar's values; the peak limit beside them. A limit firmware locked (bit + 63) is refused by name, never worked around. *Exit*: the T14 reads both + back at the bar's values. +- **3 — turbo, declared.** `IA32_MISC_ENABLE` bit 38 is read back and not + written: its other bits are model-specific and firmware's, so declaring one + bit needs the owner's ruling on writing that register whole. Linux's + `platform_profile` has no ToyOS counterpart, and it is how the bar's firmware + limits were chosen. *Exit*: the ruling, and the bit declared or recorded + as firmware's. +- **4 — the sampler.** A program that reads the envelope every 60 s and at a + span's start and end, and turns `MSR_PKG_ENERGY_STATUS` into the first + 60 s's package power and `IA32_PACKAGE_THERM_STATUS` into a temperature, + which are the bar's validity conditions. *Exit*: one valid span on the T14. + +**Not covered.** A hybrid CPU is refused: its HWP scale is not its ratio scale +and the declared minimum is a ratio. AMD's CPPC and AArch64 declare nothing; +the AArch64 kernel refuses the claim by name. + +**What only the T14 proves.** No QEMU CPU enumerates HWP (TCG's `qemu64`, and +KVM, which reduces leaf 6 to `ARAT`), so every write and every read of these +registers runs only there. Under Linux on the T14 every CPU held +`IA32_HWP_REQUEST` `0x80002a04` with `IA32_HWP_CAPABILITIES` `0x010d182a` or +`0x010e182a`, and the package `IA32_HWP_REQUEST_PKG` `0x8000ff01`. +`MSR_PLATFORM_INFO` was not read there; its ratio 4 is inferred from Linux's +`cpuinfo_min_freq` of 400000 kHz, and ToyOS's boot line prints the register. diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index be62dcc5ee5..431c8bbd4e3 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -50,6 +50,7 @@ dependencies = [ "toyos-hda", "toyos-pci", "toyos-pcid", + "toyos-perfstate", "toyos-proclife", "toyos-ps2", "toyos-quiesce", @@ -140,6 +141,10 @@ dependencies = [ name = "toyos-pcid" version = "0.1.0" +[[package]] +name = "toyos-perfstate" +version = "0.1.0" + [[package]] name = "toyos-proclife" version = "0.1.0" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index f8d2ea7f322..3ca18cd1f6d 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -404,6 +404,7 @@ toyos-gpt = { path = "../toyos-gpt" } toyos-hda = { path = "../toyos-hda" } toyos-pci = { path = "../toyos-pci" } toyos-pcid = { path = "../toyos-pcid" } +toyos-perfstate = { path = "../toyos-perfstate" } toyos-tco = { path = "../toyos-tco" } toyos-proclife = { path = "../toyos-proclife" } toyos-ps2 = { path = "../toyos-ps2" } diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index 1ac0e061e99..37d7a9e5adc 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -35,6 +35,7 @@ pub mod irqchip; pub mod keyboard_controller; pub mod paging; pub mod percpu; +pub mod perf_state; pub mod pio; pub mod pmu; pub mod rtc; diff --git a/kernel/src/arch/aarch64/perf_state.rs b/kernel/src/arch/aarch64/perf_state.rs new file mode 100644 index 00000000000..5ed6b330467 --- /dev/null +++ b/kernel/src/arch/aarch64/perf_state.rs @@ -0,0 +1,20 @@ +//! The performance envelope's registers. AArch64 declares no performance +//! request, so there is no proof to read one with and every claim is refused. + +use toyos_abi::perf::{CpuRegisters, PackageRegisters}; + +/// Uninhabited: nothing on this architecture can hold one. +pub enum Declared {} + +pub fn declared() -> Result { + Err("AArch64 declares no performance request: this kernel programs none of its CPUs' \ + performance controls") +} + +pub fn read_cpu(declared: &Declared) -> CpuRegisters { + match *declared {} +} + +pub fn read_package(declared: &Declared) -> PackageRegisters { + match *declared {} +} diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 3ad0eb2ff98..529de667cc9 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -1,11 +1,20 @@ -//! What `CR0`, `CR4` and `IA32_EFER` hold on every CPU in this machine. One -//! declaration, applied by the BSP and every AP and checked on each; nothing -//! else may write any of the three. Each register is written whole: `CR0` -//! and `EFER` are constants, `CR4` is required bits plus whatever optional -//! bits this CPU offers. `EFER.NXE` lets bit 63 of a paging entry mean *not -//! executable* ([`Prot`](crate::mm::policy::Prot)). - -use core::sync::atomic::{AtomicU64, Ordering}; +//! What `CR0`, `CR4`, `IA32_EFER` and the performance request hold on every +//! CPU in this machine. One declaration, applied by the BSP and every AP and +//! checked on each; nothing else may write any of them. Each register is +//! written whole: `CR0` and `EFER` are constants, `CR4` is required bits plus +//! whatever optional bits this CPU offers. `EFER.NXE` lets bit 63 of a paging +//! entry mean *not executable* ([`Prot`](crate::mm::policy::Prot)). +//! +//! The performance request is HWP's (`toyos_perfstate`): `IA32_PM_ENABLE`, +//! `IA32_HWP_REQUEST`, `IA32_HWP_REQUEST_PKG` and `IA32_ENERGY_PERF_BIAS`, on +//! a machine whose CPUs have every register it names, and none of them on one +//! that does not — refused by name once, and firmware's values stand. +//! `IA32_MISC_ENABLE`'s turbo bit is not declared: that register's other bits +//! are model-specific and firmware's, and writing it whole would decide them. + +use core::sync::atomic::{AtomicU64, AtomicU8, Ordering}; + +use toyos_perfstate::msr; use super::cpu; use crate::log; @@ -112,11 +121,91 @@ pub fn init_cr0(cpu_id: u32) { bench::report(cpu_id, before); } -/// Puts this CPU's `CR4` and `EFER` into the declaration and checks all -/// three against it. Must run after [`init_cr0`] and before `arch::syscall::init`, which needs `SCE` set. +/// Whether the machine's declaration carries the performance request: the +/// BSP's verdict, which every AP must reach too. +static HWP: AtomicU8 = AtomicU8::new(HWP_UNDECIDED); +const HWP_UNDECIDED: u8 = 0; +const HWP_DECLARED: u8 = 1; +const HWP_REFUSED: u8 = 2; + +/// Proof that the machine's declaration carries the performance request, so +/// every register [`toyos_perfstate::msr`] names exists on every CPU and +/// reading one is no `#GP`. +#[derive(Clone, Copy)] +pub struct HwpDeclared(()); + +impl HwpDeclared { + /// The proof, or the reason this machine has none. + pub fn ask() -> Result { + match HWP.load(Ordering::Acquire) { + HWP_DECLARED => Ok(Self(())), + HWP_REFUSED => Err(toyos_perfstate::refusal(&perf_cpuid()) + .expect("every CPU reached the BSP's refusal, this one included")), + _ => panic!("control_regs: the performance request is asked about before the BSP declared it"), + } + } +} + +/// This CPU's `IA32_HWP_REQUEST`, or `None` where the machine has no request. +/// Recomputed per CPU, as `CR4` is, and not required to match the BSP's: a +/// CPU's highest performance is its own. Whether there is one at all is the +/// machine's, and a CPU that disagrees is named. +fn hwp_declaration(cpu_id: u32) -> Option { + let refusal = toyos_perfstate::refusal(&perf_cpuid()); + let mine = if refusal.is_none() { HWP_DECLARED } else { HWP_REFUSED }; + match HWP.compare_exchange(HWP_UNDECIDED, mine, Ordering::Release, Ordering::Acquire) { + Ok(_) => { + if let Some(refusal) = refusal { + log!("control_regs: no performance request is declared: {}", refusal.reason()); + } + } + Err(machine) => assert!( + machine == mine, + "control_regs: cpu{cpu_id} {} a performance request and the BSP {} one", + if mine == HWP_DECLARED { "can hold" } else { "cannot hold" }, + if machine == HWP_DECLARED { "declared" } else { "refused" }, + ), + } + refusal.is_none().then(|| { + toyos_perfstate::hwp_request( + cpu::rdmsr(msr::HWP_CAPABILITIES), + cpu::rdmsr(msr::PLATFORM_INFO), + ) + }) +} + +fn perf_cpuid() -> toyos_perfstate::Cpuid { + let (max_leaf, ebx, ecx, edx) = cpu::cpuid(0, 0); + let mut vendor = [0u8; 12]; + for (at, word) in [ebx, edx, ecx].into_iter().enumerate() { + vendor[at * 4..at * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + // A leaf above the maximum answers with the highest basic leaf's data. + let (leaf6_eax, _, leaf6_ecx, _) = if max_leaf >= 6 { cpu::cpuid(6, 0) } else { (0, 0, 0, 0) }; + let leaf7_edx = if max_leaf >= 7 { cpu::cpuid(7, 0).3 } else { 0 }; + toyos_perfstate::Cpuid { vendor, max_leaf, leaf6_eax, leaf6_ecx, leaf7_edx } +} + +/// Puts this CPU's `CR4`, `EFER` and performance request into the declaration +/// and checks every register against it. Must run after [`init_cr0`] and +/// before `arch::syscall::init`, which needs `SCE` set. pub fn init(cpu_id: u32) { let declared = declaration(cpu_id); + let hwp = hwp_declaration(cpu_id); if !skipped(cpu_id) { + if let Some(request) = hwp { + // SAFETY: `hwp_declaration` answered `Some` only where CPUID + // enumerates HWP with EPP and the package request and EPB, so each + // MSR exists; `PM_ENABLE` goes first because a request written + // before it is `#GP` (SDM Vol. 3B, HWP's enabling), and every + // value fits the register's defined bits. + unsafe { + cpu::wrmsr(msr::PM_ENABLE, toyos_perfstate::PM_ENABLE); + cpu::wrmsr(msr::HWP_REQUEST, request); + cpu::wrmsr(msr::HWP_REQUEST_PKG, toyos_perfstate::HWP_REQUEST_PKG); + cpu::wrmsr(msr::ENERGY_PERF_BIAS, toyos_perfstate::ENERGY_PERF_BIAS); + } + } // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked // the first two and both callers use PCID 0; `wrmsr` writes [`EFER`], whose @@ -132,6 +221,9 @@ pub fn init(cpu_id: u32) { } } self_check(cpu_id, declared); + if let Some(request) = hwp { + hwp_check(cpu_id, request); + } } /// Whether the declaration carries `PCIDE`, and therefore whether `INVPCID` is this machine's flush. @@ -278,6 +370,39 @@ fn self_check(cpu_id: u32, declared_cr4: u64) { CHECKED.fetch_add(1, Ordering::Relaxed); } +/// [`self_check`] for the performance request, logged first for the same +/// reason; the line carries the request's two inputs, so a reader can +/// recompute it. +fn hwp_check(cpu_id: u32, request: u64) { + let pm_enable = cpu::rdmsr(msr::PM_ENABLE); + let live = cpu::rdmsr(msr::HWP_REQUEST); + let pkg = cpu::rdmsr(msr::HWP_REQUEST_PKG); + let epb = cpu::rdmsr(msr::ENERGY_PERF_BIAS); + log!( + "control_regs: cpu{} pm_enable={} hwp_request={:#010x} hwp_request_pkg={:#010x} epb={} \ + hwp_capabilities={:#010x} platform_info={:#018x}", + cpu_id, + pm_enable, + live, + pkg, + epb, + cpu::rdmsr(msr::HWP_CAPABILITIES), + cpu::rdmsr(msr::PLATFORM_INFO), + ); + let want = [ + ("pm_enable", pm_enable, toyos_perfstate::PM_ENABLE), + ("hwp_request", live, request), + ("hwp_request_pkg", pkg, toyos_perfstate::HWP_REQUEST_PKG), + ("epb", epb, toyos_perfstate::ENERGY_PERF_BIAS), + ]; + for (name, holds, declared) in want { + assert!( + holds == declared, + "control_regs: cpu{cpu_id} holds {name}={holds:#x}, the declaration is {declared:#x}", + ); + } +} + /// How many CPUs hold the declaration, said once after the last of them has /// been checked. A divergent CPU panics inside [`self_check`], so what this /// line adds is the *count*: a CPU that never reached [`init`] at all is diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index e44661ba2fb..e54f51d1625 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -30,6 +30,7 @@ pub mod nmi_gate; pub mod paging; pub mod pat; pub mod percpu; +pub mod perf_state; pub mod pio; pub mod pmu; pub mod rtc; diff --git a/kernel/src/arch/x86_64/perf_state.rs b/kernel/src/arch/x86_64/perf_state.rs new file mode 100644 index 00000000000..e4f5f134c5d --- /dev/null +++ b/kernel/src/arch/x86_64/perf_state.rs @@ -0,0 +1,38 @@ +//! The performance envelope's registers, read on the CPU that runs this: the +//! read-back half of [`super::control_regs`]'s performance request. + +use toyos_abi::perf::{CpuRegisters, PackageRegisters}; +use toyos_perfstate::msr; + +use super::cpu; + +pub use super::control_regs::HwpDeclared as Declared; + +/// The proof every read below needs, or why this machine has none. +pub fn declared() -> Result { + Declared::ask().map_err(toyos_perfstate::Refusal::reason) +} + +/// This CPU's own registers. +pub fn read_cpu(_: &Declared) -> CpuRegisters { + CpuRegisters { + pm_enable: cpu::rdmsr(msr::PM_ENABLE), + hwp_capabilities: cpu::rdmsr(msr::HWP_CAPABILITIES), + hwp_request: cpu::rdmsr(msr::HWP_REQUEST), + energy_perf_bias: cpu::rdmsr(msr::ENERGY_PERF_BIAS), + misc_enable: cpu::rdmsr(msr::MISC_ENABLE), + } +} + +/// The registers of the package this CPU is in. +pub fn read_package(_: &Declared) -> PackageRegisters { + PackageRegisters { + hwp_request_pkg: cpu::rdmsr(msr::HWP_REQUEST_PKG), + platform_info: cpu::rdmsr(msr::PLATFORM_INFO), + rapl_power_unit: cpu::rdmsr(msr::RAPL_POWER_UNIT), + pkg_power_limit: cpu::rdmsr(msr::PKG_POWER_LIMIT), + pkg_energy_status: cpu::rdmsr(msr::PKG_ENERGY_STATUS), + package_therm_status: cpu::rdmsr(msr::PACKAGE_THERM_STATUS), + temperature_target: cpu::rdmsr(msr::TEMPERATURE_TARGET), + } +} diff --git a/kernel/src/device.rs b/kernel/src/device.rs index 8fe19f0fea8..03039bfde6b 100644 --- a/kernel/src/device.rs +++ b/kernel/src/device.rs @@ -184,6 +184,14 @@ pub fn try_claim(class: DeviceType, selector: [u64; 2]) -> Result { + let declared = crate::arch::perf_state::declared().map_err(|why| { + log!("perf-state: no claim: {why}"); + ClaimError::Absent + })?; + let claim = Claim::acquire(class)?; + Ok(DeviceClaim::new(class, DeviceInfo::PerfState(crate::perf_state::Reader::new(declared)), claim)) + } } } diff --git a/kernel/src/main.rs b/kernel/src/main.rs index ef64ca60a95..463e9864dc2 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -31,6 +31,7 @@ mod params; mod blackbox; mod deadline; mod quiesce; +mod perf_state; mod hardlockup; mod mm; mod panic; diff --git a/kernel/src/object/device.rs b/kernel/src/object/device.rs index 2694474e484..6b6967e9816 100644 --- a/kernel/src/object/device.rs +++ b/kernel/src/object/device.rs @@ -28,6 +28,8 @@ pub enum DeviceInfo { /// Which partition, how long, and both its GUIDs; the view it moves blocks /// through is the claim's own (`device::Claim::partition`). Partition(toyos_abi::part::PartitionInfo), + /// Answers the performance envelope's registers, never a description. + PerfState(crate::perf_state::Reader), } /// The two scanout buffers and the cursor plane. @@ -56,7 +58,7 @@ impl DeviceInfo { // `described.bytes` unset, so the next read re-mints instead of binding stranded handles. fn mint(&self, table: &mut HandleTable) -> Result, SyscallError> { Ok(match self { - Self::Events => Box::new([]), + Self::Events | Self::PerfState(_) => Box::new([]), Self::Framebuffer(info, buffers) => { let mut info = *info; let h = install_buffers( @@ -156,6 +158,14 @@ impl DeviceClaim { Some((device, unique)) } + /// A performance-state claim's read, which [`crate::perf_state::Reader`] answers. + pub fn read_perf_state(&self, buf: &mut crate::user_ptr::UserBytesMut) -> Option { + match &self.described.lock().info { + DeviceInfo::PerfState(reader) => reader.read(buf), + _ => unreachable!("a {:?} claim is not read as a performance-state one", self.class), + } + } + pub fn info_read(&self) -> bool { self.info_read.load(Ordering::Relaxed) } diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 3808154fb29..8fafc18e97a 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -263,6 +263,7 @@ pub fn read_watch(object: &KObjectRef) -> Option { device_registry::DeviceType::Framebuffer => None, // A partition answers its description and has nothing to wait for. device_registry::DeviceType::Partition => None, + device_registry::DeviceType::PerfState => Some(WatchRef::Static(&crate::perf_state::WATCH)), }, // Named unconditionally: the watch alone cannot enforce rights. KObjectRef::SysCap(_) => Some(WatchRef::Static(&crate::log::user::WATCH)), @@ -304,7 +305,8 @@ fn close_ends_polls(object: &KObjectRef) -> bool { | device_registry::DeviceType::HdaAudio | device_registry::DeviceType::VirtioSound | device_registry::DeviceType::Framebuffer - | device_registry::DeviceType::Partition => true, + | device_registry::DeviceType::Partition + | device_registry::DeviceType::PerfState => true, }, KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) @@ -389,6 +391,7 @@ pub fn read_device( // Every read is the description: a partition's bytes move through // `SYS_PARTITION_READ`, never through a read of the claim. device_registry::DeviceType::Partition => Some(claim.describe(table, buf)), + device_registry::DeviceType::PerfState => claim.read_perf_state(buf), // The description first and interrupts after, the shape the HDA stub // has: a driver reads what it is driving once, and everything it reads // afterwards is what its device has been doing. @@ -601,7 +604,8 @@ pub fn fstat(object: &KObjectRef) -> Stat { device_registry::DeviceType::PciFunction => FileType::Unknown, device_registry::DeviceType::HdaAudio | device_registry::DeviceType::VirtioSound => FileType::Unknown, - device_registry::DeviceType::Partition => FileType::Unknown, + device_registry::DeviceType::Partition + | device_registry::DeviceType::PerfState => FileType::Unknown, }), } } @@ -767,7 +771,8 @@ fn partition_fsync(claim: &DeviceClaim) -> u64 { | device_registry::DeviceType::Framebuffer | device_registry::DeviceType::HdaAudio | device_registry::DeviceType::VirtioSound - | device_registry::DeviceType::PciFunction => { + | device_registry::DeviceType::PciFunction + | device_registry::DeviceType::PerfState => { return SyscallError::PermissionDenied.to_u64(); } } @@ -838,6 +843,7 @@ pub fn has_data(object: &KObjectRef) -> bool { } device_registry::DeviceType::Framebuffer => true, device_registry::DeviceType::Partition => true, + device_registry::DeviceType::PerfState => crate::perf_state::answered(), device_registry::DeviceType::HdaAudio => { !d.info_read() || crate::drivers::hda::has_pending() } diff --git a/kernel/src/perf_state.rs b/kernel/src/perf_state.rs new file mode 100644 index 00000000000..f77664580f5 --- /dev/null +++ b/kernel/src/perf_state.rs @@ -0,0 +1,148 @@ +//! A `perf-state` claim's read. A per-CPU register is readable only on its own +//! CPU, so the reading CPU answers for itself and asks the rest: it issues a +//! generation and kicks every other CPU, each answers from its next scheduler +//! pass ([`serve_if_owed`], from `drain_irqs`) by reading its registers into +//! its slot and posting [`WATCH`], and the read blocks on that watch until +//! every CPU has answered — bounded by [`ANSWER`], past which it is refused +//! `Io` and the silent CPUs are named. +//! +//! The ask and its answers are [`Shootdown`]'s protocol: an answer published +//! for a generation was read after that generation was issued. A slot that a +//! later ask overwrites mid-copy mixes two answers, each read after this one +//! asked. + +use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +use toyos_abi::perf::{answer_len, CpuRegisters}; +use toyos_abi::syscall::SyscallError; + +use crate::arch::perf_state::{self, Declared}; +use crate::shootdown::{Generation, Shootdown, MAX_CPUS}; +use crate::sync::Lock; +use crate::time::{Budget, Deadline, Duration, Instant}; +use crate::user_ptr::UserBytesMut; +use crate::watch::Watch; + +static ASKS: Shootdown = Shootdown::new(); +static SLOTS: [Slot; MAX_CPUS] = [const { Slot::new() }; MAX_CPUS]; + +/// Posted by every answer; a blocked read waits here. +pub static WATCH: Watch = Watch::new(); + +/// A kicked CPU reaches a pass within one timer interrupt; this is the +/// kernel's choice, the blocked-task dump's for the same question. +const ANSWER: Budget = Budget::of( + Duration::from_millis(250), + "the read is refused `Io`, and the CPUs that did not answer are named", +); + +struct Slot([AtomicU64; 5]); + +impl Slot { + const fn new() -> Self { + Self([const { AtomicU64::new(0) }; 5]) + } + + fn store(&self, r: CpuRegisters) { + let words = [r.pm_enable, r.hwp_capabilities, r.hwp_request, r.energy_perf_bias, r.misc_enable]; + for (slot, word) in self.0.iter().zip(words) { + slot.store(word, Relaxed); + } + } + + fn load(&self) -> CpuRegisters { + let [pm_enable, hwp_capabilities, hwp_request, energy_perf_bias, misc_enable] = + self.0.each_ref().map(|word| word.load(Relaxed)); + CpuRegisters { pm_enable, hwp_capabilities, hwp_request, energy_perf_bias, misc_enable } + } +} + +/// One claim's side of the protocol: the proof its reads need, and the ask +/// its reads wait on — joined by every read of the claim until it is answered. +pub struct Reader { + declared: Declared, + pending: Lock>, +} + +#[derive(Clone, Copy)] +struct Ask { + generation: Generation, + deadline: Deadline, +} + +impl Reader { + pub fn new(declared: Declared) -> Self { + Self { declared, pending: Lock::new(None) } + } + + /// The answer's bytes into `buf`, or `None` while a CPU has not answered — + /// the caller then waits on [`WATCH`] until [`answered`]. + pub fn read(&self, buf: &mut UserBytesMut) -> Option { + let cpus = crate::arch::smp::cpu_count() as usize; + let len = answer_len(cpus); + if buf.len() < len { + return Some(SyscallError::ResourceExhausted.to_u64()); + } + let now = crate::clock::now(); + let mut pending = self.pending.lock(); + let ask = *pending.get_or_insert_with(|| self.ask(cpus, now)); + let answered = (0..cpus).all(|cpu| ASKS.served(cpu, ask.generation)); + if !answered && !ask.deadline.reached(now) { + return None; + } + *pending = None; + drop(pending); + if !answered { + for cpu in (0..cpus).filter(|&cpu| !ASKS.served(cpu, ask.generation)) { + crate::log!("perf_state: cpu{cpu} did not answer a read within {ANSWER}"); + } + return Some(SyscallError::Io.to_u64()); + } + buf.write_at(0, perf_state::read_package(&self.declared).as_bytes()); + // `answer_len(cpu)` is where CPU `cpu`'s record starts. + for (cpu, slot) in SLOTS[..cpus].iter().enumerate() { + buf.write_at(answer_len(cpu), slot.load().as_bytes()); + } + Some(len as u64) + } + + /// Issued, this CPU's answer given, and every other CPU kicked — in that + /// order, so no kicked CPU can look before the generation it owes exists. + fn ask(&self, cpus: usize, now: Instant) -> Ask { + let generation = ASKS.issue(); + let me = crate::arch::percpu::cpu_id() as usize; + ASKS.serve(me, || SLOTS[me].store(perf_state::read_cpu(&self.declared))); + for cpu in (0..cpus).filter(|&cpu| cpu != me) { + crate::arch::irqchip::kick_cpu(cpu as u32); + } + Ask { generation, deadline: Deadline::at(now + ANSWER.duration()) } + } +} + +/// Whether every CPU has answered the latest ask: a blocked read's wake +/// condition, and a hint — [`Reader::read`] decides. +pub fn answered() -> bool { + (0..crate::arch::smp::cpu_count() as usize).all(|cpu| !ASKS.owes(cpu)) +} + +/// How long a blocked read parks before it looks again; its own ask's +/// deadline is what refuses it. +pub fn park_deadline() -> Deadline { + Deadline::at(crate::clock::now() + ANSWER.duration()) +} + +/// This CPU's answer, if one is owed. Called from `drain_irqs` every pass, so +/// what it costs when nothing is owed is two relaxed loads. +pub fn serve_if_owed() { + let me = crate::arch::percpu::cpu_id() as usize; + if !ASKS.owes(me) { + return; + } + // The proof is used inside the closure only: on an architecture where it + // is uninhabited, binding one here would make the rest unreachable. + let read = perf_state::declared() + .map(|declared| move || perf_state::read_cpu(&declared)) + .expect("an ask is made only through a claim, and a claim only where the request is declared"); + ASKS.serve(me, || SLOTS[me].store(read())); + WATCH.post(); +} diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 43233c12614..f75b0932e1d 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -669,6 +669,8 @@ fn drain_irqs(entered: super::dump::Entered) { // A CPU cannot read a sibling's `CpuSched`, so the dump reaches every CPU // by asking, and this is where each one answers. super::dump::serve_if_owed(); + // Per-CPU registers are readable only on their own CPU, so a performance-state read asks here too. + crate::perf_state::serve_if_owed(); // Repaints the panel if whoever owns the screen has drawn over the report. crate::drivers::panic_console::hold_report(); diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 6c453a99c17..7ccf2daa03e 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -1,4 +1,5 @@ -//! The acknowledgement half of a TLB shootdown, with no hardware in it. +//! The acknowledgement half of a machine-wide ask — a TLB shootdown, or a +//! performance-state read — with no hardware in it. //! Compiled a second time into `kernel-loom/` against loom's atomics, so this file must hold no `crate::` references. //! The read must happen before the flush, or a target could publish a generation its flush has not yet completed. diff --git a/kernel/src/syscall/device.rs b/kernel/src/syscall/device.rs index 8472b219b3d..1afa87e338e 100644 --- a/kernel/src/syscall/device.rs +++ b/kernel/src/syscall/device.rs @@ -129,7 +129,8 @@ pub(super) fn sys_device_claim(syscap: RawHandle, class: u64, selector: [u64; 2] | device::DeviceType::Mouse | device::DeviceType::Framebuffer | device::DeviceType::HdaAudio - | device::DeviceType::VirtioSound => 0, + | device::DeviceType::VirtioSound + | device::DeviceType::PerfState => 0, device::DeviceType::PciFunction => 1, device::DeviceType::Partition => 2, }; @@ -397,7 +398,8 @@ pub(super) fn sys_partition_transfer( | device::DeviceType::Framebuffer | device::DeviceType::HdaAudio | device::DeviceType::VirtioSound - | device::DeviceType::PciFunction => { + | device::DeviceType::PciFunction + | device::DeviceType::PerfState => { drop(claim); return crate::object::HandleError::WrongType { held: class.class_name(), diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index 20560cc0cdc..fa809d5ffc9 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -39,6 +39,8 @@ enum ReadBlock { /// the serial line, never on the keyboard's queue, which only a claim /// drains and which would answer it at once for as long as a key sits there. Console(Deadline), + /// A performance-state read, until every CPU has answered its ask. + PerfState(Deadline), /// Nothing to wait for: the answer is this word. Refused(u64), /// Carried out of the process's lock: `HandleError::refuse` may take the @@ -97,6 +99,7 @@ fn read_block_device(claim: &crate::object::device::DeviceClaim) -> ReadBlock { device::DeviceType::Keyboard => ReadBlock::Keyboard(Deadline::never()), device::DeviceType::VirtioSound if claim.info_read() => ReadBlock::VirtioSound, device::DeviceType::HdaAudio if claim.info_read() => ReadBlock::Hda, + device::DeviceType::PerfState => ReadBlock::PerfState(crate::perf_state::park_deadline()), _ => ReadBlock::Refused(SyscallError::NotFound.to_u64()), } } @@ -213,6 +216,21 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { return cancelled(); } } + Err(ReadBlock::PerfState(deadline)) => { + let parkable = crate::scheduler::Parkable::at_entry(); + if watch::wait_until( + &parkable, + &crate::perf_state::WATCH, + 0, + WaitClass::Io, + deadline, + crate::perf_state::answered, + ) + .is_err() + { + return cancelled(); + } + } Err(ReadBlock::Console(deadline)) => { let parkable = crate::scheduler::Parkable::at_entry(); if watch::wait_until( diff --git a/system.toml b/system.toml index 4d398193186..239e93621e2 100644 --- a/system.toml +++ b/system.toml @@ -106,6 +106,11 @@ receives = ["netd"] receives = ["netd", "soundd", "log", "compositor"] syscap = ["inventory"] +# The CPU performance envelope read back once: `perf-state` is the whole of +# what it holds, and a machine that declared no request has none to give it. +[programs.perfstate] +devices = ["perf-state"] + [programs.filepicker] service = true serves = ["filepicker"] diff --git a/tests/toyos-rust-tests/Cargo.lock b/tests/toyos-rust-tests/Cargo.lock index 5879429cd2b..738c90b7dd8 100644 --- a/tests/toyos-rust-tests/Cargo.lock +++ b/tests/toyos-rust-tests/Cargo.lock @@ -1444,6 +1444,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perfstate" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", + "toyos-perfstate", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -2139,6 +2148,10 @@ dependencies = [ name = "toyos-keymap" version = "0.1.0" +[[package]] +name = "toyos-perfstate" +version = "0.1.0" + [[package]] name = "toyos-quiesce" version = "0.1.0" @@ -2151,6 +2164,7 @@ dependencies = [ "cpal", "libloading 0.8.9 (git+https://github.com/ToyOSOrg/rust_libloading?branch=toyos-sdk-0.12)", "memmap2", + "perfstate", "rustls", "rustls-pki-types", "rustls-rustcrypto", diff --git a/tests/toyos-rust-tests/Cargo.toml b/tests/toyos-rust-tests/Cargo.toml index cffc67a0f95..97b34af0afa 100644 --- a/tests/toyos-rust-tests/Cargo.toml +++ b/tests/toyos-rust-tests/Cargo.toml @@ -9,6 +9,7 @@ toyos-abi = { path = "../../toyos-abi" } toyos = { path = "../../toyos" } toyos-window = { path = "../../userland/toyos-window" } toyos-tco = { path = "../../toyos-tco" } +perfstate = { path = "../../userland/perfstate" } toyos-quiesce = { path = "../../toyos-quiesce" } toyos-i219 = { path = "../../toyos-i219" } toyos-inspect = { path = "../../toyos-inspect" } diff --git a/tests/toyos-rust-tests/src/bin/perf_state.rs b/tests/toyos-rust-tests/src/bin/perf_state.rs new file mode 100644 index 00000000000..4328731477f --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/perf_state.rs @@ -0,0 +1,33 @@ +//! The performance-state claim. Where the machine declared no request the +//! claim is refused `NotFound`; where it declared one, `/system/bin/perfstate`'s +//! own read answers every CPU holding its declaration, twice. Which branch is +//! right is the machine's to say, so `perf_request` drives it and reads which +//! one it took. + +use toyos::endow::Endowments; +use toyos::syscap::SysCap; +use toyos::Device; +use toyos_abi::perf::answer_len; +use toyos_abi::syscall::{self, DeviceType, SyscallError, SYSCAP_LABEL}; + +fn main() { + let cap: SysCap = Endowments::get() + .take(SYSCAP_LABEL) + .expect("the test estate is endowed a device-minting capability"); + match cap.claim::(DeviceType::PerfState) { + Err(SyscallError::NotFound) => println!("perf-state: refused NotFound"), + Err(e) => panic!("perf-state claim: {e:?}, want NotFound or a claim"), + Ok(claim) => { + let mut short = vec![0u8; answer_len(syscall::cpu_count() as usize) - 1]; + assert_eq!(claim.read(&mut short), Err(SyscallError::ResourceExhausted)); + // Twice: an answered read leaves nothing outstanding for the next. + for _ in 0..2 { + if let Err(why) = perfstate::read_back(&claim) { + panic!("{why}"); + } + } + println!("perf-state: declared"); + } + } + println!("===PERF_STATE_OK==="); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index ff90ef5aae8..09a11c9f883 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -247,6 +247,9 @@ const RUST_SKIP: &[&str] = &[ // Meaningful only on `MetalNoUsb`, where no input source exists; on every // other machine both claims succeed. `input_claim_absent` runs it. "input_absent", + // Which of its two branches is right is the machine's to say: QEMU's CPUs + // have no HWP and the T14's do. `perf_request` runs it and reads which. + "perf_state", // Needs a display whose mode can change, which is `Profile::VirtioGpu` // alone; the shared boot has no display at all. `gpu_set_resolution` runs // it there, and `iommu_gpu_scanout_swap` the second. @@ -609,6 +612,7 @@ const MACHINE_TESTS: &[(&str, Sched, Tier)] = &[ ("irq_census_conservation", Sched::Parallel, Tier::Weekly), ("control_regs", Sched::Parallel, Tier::Fast), ("control_regs_negative", Sched::Parallel, Tier::Fast), + ("perf_request", Sched::Parallel, Tier::Fast), // The boot facts the metal suite reads off a machine's own records: every // CPU the firmware named came up and none of their timestamp counters // trails the BSP's; the physical memory manager's accounting against the @@ -1518,6 +1522,7 @@ const CARRIES: &[(&str, &[&str])] = &[ ("launcher_refusals", &["test_rs_launcher_refusals"]), ("spawn_cwd", &["test_rs_spawn_cwd"]), ("input_claim_absent", &["test_rs_input_absent"]), + ("perf_request", &["test_rs_perf_state"]), ("gpu_set_resolution", &["test_rs_gpu_set_resolution"]), ("iommu_gpu_scanout_swap", &["test_rs_gpu_scanout_swap"]), ("userdev_dma_fault", &["test_rs_log_origin"]), @@ -1696,6 +1701,10 @@ const METAL: &[(&str, metal::Metal)] = &[ judge: |b| control_regs(b[0].kernel().text(), b[0].cpus()?), }, ), + ( + "perf_request", + metal::Metal::Runs { arms: TESTCASES, judge: |b| perf_request_on_metal(b[0]) }, + ), ( "ioapic_topology", metal::Metal::Runs { arms: TESTCASES, judge: |b| ioapic_topology(b[0].kernel().text()) }, @@ -13695,6 +13704,7 @@ fn run_machine_test( control_regs(qemu.boot_log(), CPUS) } "control_regs_negative" => control_regs_negative(test_config, c_bins, rust_bins), + "perf_request" => perf_request(test_config, c_bins, rust_bins), "smp_roster_and_tsc_trail" => { // Eight, which is the T14's own count and this suite's ceiling. const CPUS: u32 = 8; @@ -19653,3 +19663,57 @@ fn root_withheld_refused(log: &str) -> Result<(), String> { eprintln!(" [root] a handoff with no ROOT image refused the boot by name"); Ok(()) } + +/// The kernel's performance request, on a machine that cannot hold one: every +/// QEMU CPU this repository launches has no HWP (`Arch::cpu`'s `qemu64` and +/// KVM's `host`, whose leaf 6 KVM reduces to `ARAT`), so the kernel refuses it +/// by name, programs none of it, and a `perf-state` claim is `NotFound`. +fn perf_request( + test_config: &Path, + c_bins: &[(String, Vec)], + rust_bins: &[(String, Vec)], +) -> Result<(), String> { + const REFUSED: &str = "control_regs: no performance request is declared: no HWP"; + let mut qemu = QemuInstance::boot(test_config, c_bins, rust_bins); + let boot = qemu.boot_log().to_string(); + if !boot.contains(REFUSED) { + return Err(format!("the kernel never said {REFUSED:?}:\n{boot}")); + } + if let Some(line) = boot.lines().find(|l| l.contains(" hwp_request=")) { + return Err(format!("a CPU with no HWP was given a request: {line}")); + } + let result = qemu.run_test("test_rs_perf_state", Duration::from_secs(30)); + if let Some(err) = &result.error { + return Err(format!("{err}\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("perf_state exited {:?}:\n{}", result.exit_code, result.stdout)); + } + if !result.stdout.contains("perf-state: refused NotFound") { + return Err(format!("the claim was not refused NotFound:\n{}", result.stdout)); + } + eprintln!(" [perf_request] no HWP: refused by name, and the claim refused NotFound"); + Ok(()) +} + +/// The same on the T14, whose CPUs have every register the request names: +/// each CPU holds the bar's power envelope — the `IA32_HWP_REQUEST`, +/// `IA32_HWP_REQUEST_PKG` and EPB the Linux run it is held against held — and +/// the guest binary read every CPU back holding its declaration. +fn perf_request_on_metal(boot: &metal::Readback) -> Result<(), String> { + const BAR: &str = "pm_enable=1 hwp_request=0x80002a04 hwp_request_pkg=0x8000ff01 epb=6 "; + let cpus = boot.cpus()?; + let log = boot.kernel(); + for cpu in 0..cpus { + let head = format!("control_regs: cpu{cpu} pm_enable="); + let Some(line) = log.text().lines().find(|l| l.contains(&head)) else { + return Err(format!("cpu{cpu} logged no performance request:\n{}", log.text())); + }; + if !line.contains(&format!("control_regs: cpu{cpu} {BAR}")) { + return Err(format!("cpu{cpu} does not hold the bar's envelope {BAR:?}: {line}")); + } + } + boot.job_passed("test_rs_perf_state")?; + eprintln!(" [perf_request] {cpus} CPUs hold the bar's request and read it back"); + Ok(()) +} diff --git a/toyos-abi/src/lib.rs b/toyos-abi/src/lib.rs index 276eb4dbf1b..bd1e323edfb 100644 --- a/toyos-abi/src/lib.rs +++ b/toyos-abi/src/lib.rs @@ -26,6 +26,7 @@ pub mod inventory; pub mod log; pub mod part; pub mod pci; +pub mod perf; pub mod ring; pub mod syscall; pub mod virtio_sound; diff --git a/toyos-abi/src/perf.rs b/toyos-abi/src/perf.rs new file mode 100644 index 00000000000..f317bf45c9f --- /dev/null +++ b/toyos-abi/src/perf.rs @@ -0,0 +1,109 @@ +//! What a read of a `perf-state` claim answers: one [`PackageRegisters`], then +//! one [`CpuRegisters`] per CPU in CPU order — the machine's whole CPU count, +//! or the read is refused whole with `ResourceExhausted`, and with `Io` when a +//! CPU did not answer within the kernel's bound. Every field is the +//! register's raw value, named by its x86-64 MSR; decoding is the reader's. + +/// The package-wide registers, read on whichever CPU answered the read. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PackageRegisters { + /// `IA32_HWP_REQUEST_PKG`, 0x772. + pub hwp_request_pkg: u64, + /// `MSR_PLATFORM_INFO`, 0xCE. + pub platform_info: u64, + /// `MSR_RAPL_POWER_UNIT`, 0x606. + pub rapl_power_unit: u64, + /// `MSR_PKG_POWER_LIMIT`, 0x610. + pub pkg_power_limit: u64, + /// `MSR_PKG_ENERGY_STATUS`, 0x611. + pub pkg_energy_status: u64, + /// `IA32_PACKAGE_THERM_STATUS`, 0x1B1. + pub package_therm_status: u64, + /// `MSR_TEMPERATURE_TARGET`, 0x1A2. + pub temperature_target: u64, +} + +/// One CPU's registers, read on that CPU. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CpuRegisters { + /// `IA32_PM_ENABLE`, 0x770. + pub pm_enable: u64, + /// `IA32_HWP_CAPABILITIES`, 0x771. + pub hwp_capabilities: u64, + /// `IA32_HWP_REQUEST`, 0x774. + pub hwp_request: u64, + /// `IA32_ENERGY_PERF_BIAS`, 0x1B0. + pub energy_perf_bias: u64, + /// `IA32_MISC_ENABLE`, 0x1A0. + pub misc_enable: u64, +} + +// Every byte belongs to a field: both cross the boundary as bytes, so a gap +// would publish whatever the kernel stack held. +const _: () = assert!(core::mem::size_of::() == 7 * 8); +const _: () = assert!(core::mem::size_of::() == 5 * 8); + +/// The bytes a read answers on a machine of `cpus` CPUs. +pub const fn answer_len(cpus: usize) -> usize { + core::mem::size_of::() + cpus * core::mem::size_of::() +} + +macro_rules! plain_bytes { + ($($ty:ty),+) => {$( + impl $ty { + pub fn as_bytes(&self) -> &[u8] { + // SAFETY: `self` is a valid `&Self`, readable for + // `size_of::()` bytes, and the const asserts above prove + // the `repr(C)` layout is all `u64` fields with no padding. + unsafe { + core::slice::from_raw_parts( + self as *const Self as *const u8, + core::mem::size_of::(), + ) + } + } + + /// The record at the start of `bytes`, or `None` if it is shorter. + pub fn read_from(bytes: &[u8]) -> Option { + let bytes = bytes.get(..core::mem::size_of::())?; + let mut out = Self::default(); + // SAFETY: `out` is `size_of::()` bytes of `u64` fields, + // every bit pattern of which is a valid value, and `bytes` is + // exactly that long. + unsafe { + core::ptr::copy_nonoverlapping( + bytes.as_ptr(), + &mut out as *mut Self as *mut u8, + bytes.len(), + ); + } + Some(out) + } + } + )+}; +} + +plain_bytes!(PackageRegisters, CpuRegisters); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_record_round_trips_through_its_bytes() { + let cpu = CpuRegisters { + pm_enable: 1, + hwp_capabilities: 0x010d_182a, + hwp_request: 0x8000_2a04, + energy_perf_bias: 6, + misc_enable: 0x0085_0089, + }; + assert_eq!(CpuRegisters::read_from(cpu.as_bytes()), Some(cpu)); + assert_eq!(CpuRegisters::read_from(&cpu.as_bytes()[1..]), None); + let pkg = PackageRegisters { pkg_power_limit: 0x0042_8200_00dd_8200, ..Default::default() }; + assert_eq!(PackageRegisters::read_from(pkg.as_bytes()), Some(pkg)); + assert_eq!(answer_len(8), 56 + 8 * 40); + } +} diff --git a/toyos-abi/src/syscall.rs b/toyos-abi/src/syscall.rs index f471ce741fe..29f3d7e88ac 100644 --- a/toyos-abi/src/syscall.rs +++ b/toyos-abi/src/syscall.rs @@ -1264,6 +1264,11 @@ device_classes! { /// Like `pci`, a class whose name is not the whole of the entry — /// `part:` — and [`DeviceRequest`] is the one parser. Partition = 8 => "part", + /// The CPU performance envelope's registers, read back: every read answers + /// [`crate::perf`]'s records, each CPU's taken on that CPU after the read + /// asked. Read-only — the kernel writes the declaration and nothing else + /// does. `NotFound` on a machine whose CPUs got no declared request. + PerfState = 9 => "perf-state", } /// A PCI function named by what identifies the *card*, not the slot firmware diff --git a/toyos-perfstate/Cargo.toml b/toyos-perfstate/Cargo.toml new file mode 100644 index 00000000000..9c157af13ee --- /dev/null +++ b/toyos-perfstate/Cargo.toml @@ -0,0 +1,14 @@ +# A member of the host workspace (root `Cargo.toml`), like toyos-tco: the +# kernel depends on it by path and its tests run on the host, because no QEMU +# CPU has HWP and the arithmetic that decides what a real laptop's CPUs are +# asked for is exactly what a guest cannot exercise. + +[package] +name = "toyos-perfstate" +description = "The CPU performance request the kernel declares on an x86-64 CPU with HWP, and the register layouts a reader checks it against." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] diff --git a/toyos-perfstate/src/lib.rs b/toyos-perfstate/src/lib.rs new file mode 100644 index 00000000000..acee94134e6 --- /dev/null +++ b/toyos-perfstate/src/lib.rs @@ -0,0 +1,313 @@ +//! The CPU performance request this kernel declares on an x86-64 CPU with +//! hardware-controlled performance states (HWP), and the register layouts a +//! reader checks it against. Pure: the kernel supplies CPUID and the MSR reads +//! and does the writes. +//! +//! Layouts are the Intel SDM's — Vol. 3B, *Power and Thermal Management*, for +//! HWP, the energy/performance bias, RAPL and package thermal status; Vol. 4 +//! for the addresses. The declared values are the power envelope the +//! self-hosting bar is measured under (`issues/build/toyos-builds-itself.md`), +//! so a ToyOS run and the Linux run it is held against ask the CPU for the +//! same thing. + +#![no_std] +#![forbid(unsafe_code)] + +/// MSR addresses, SDM Vol. 4. +pub mod msr { + /// `IA32_PM_ENABLE`: bit 0 enables HWP, and only a reset clears it. + pub const PM_ENABLE: u32 = 0x770; + pub const HWP_CAPABILITIES: u32 = 0x771; + pub const HWP_REQUEST_PKG: u32 = 0x772; + pub const HWP_REQUEST: u32 = 0x774; + pub const MISC_ENABLE: u32 = 0x1A0; + pub const ENERGY_PERF_BIAS: u32 = 0x1B0; + pub const PACKAGE_THERM_STATUS: u32 = 0x1B1; + // Model-specific, and named by no CPUID bit: every Intel core since Sandy + // Bridge has them, and HWP is younger than all four. + pub const PLATFORM_INFO: u32 = 0xCE; + pub const TEMPERATURE_TARGET: u32 = 0x1A2; + pub const RAPL_POWER_UNIT: u32 = 0x606; + pub const PKG_POWER_LIMIT: u32 = 0x610; + pub const PKG_ENERGY_STATUS: u32 = 0x611; +} + +/// `IA32_MISC_ENABLE` bit 38: set, turbo is off. +pub const TURBO_DISABLE: u64 = 1 << 38; + +/// `IA32_PM_ENABLE` on every CPU. +pub const PM_ENABLE: u64 = 1; + +/// `IA32_ENERGY_PERF_BIAS` on every CPU: the bar's 6, on the scale where 0 is +/// performance and 15 is energy saving. +pub const ENERGY_PERF_BIAS: u64 = 6; + +/// The energy/performance preference every CPU's request carries: the bar's +/// 128, which is Linux's `balance_performance` on the bar's machine. +pub const EPP: u8 = 128; + +/// `IA32_HWP_REQUEST_PKG`: the bar's value. Inert — no CPU's request sets +/// package control — and declared so that firmware does not decide it either. +pub const HWP_REQUEST_PKG: u64 = + HwpRequest { min: 1, max: 255, desired: 0, epp: EPP, window: 0, package_control: false }.raw(); + +/// What CPUID says about the registers the envelope names. +#[derive(Clone, Copy, Debug)] +pub struct Cpuid { + /// Leaf 0's `EBX`, `EDX`, `ECX`, in that order: the vendor string. + pub vendor: [u8; 12], + pub max_leaf: u32, + pub leaf6_eax: u32, + pub leaf6_ecx: u32, + pub leaf7_edx: u32, +} + +/// Why a CPU gets no declared performance request, and its performance state +/// stays whatever firmware left. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + NoLeaf6, + NoHwp, + NotIntel, + NoEpp, + NoPackageRequest, + NoEnergyPerfBias, + NoPackageThermal, + Hybrid, +} + +impl Refusal { + pub const fn reason(self) -> &'static str { + match self { + Self::NoLeaf6 => "CPUID has no leaf 6, so no power management is enumerated", + Self::NoHwp => "no HWP (CPUID.06H:EAX[7] clear)", + Self::NotIntel => { + "HWP on a CPU that is not Intel, whose RAPL and thermal registers are Intel's \ + model-specific ones" + } + Self::NoEpp => "HWP without an energy/performance preference (CPUID.06H:EAX[10] clear)", + Self::NoPackageRequest => "HWP without a package-level request (CPUID.06H:EAX[11] clear)", + Self::NoEnergyPerfBias => "no energy/performance bias (CPUID.06H:ECX[3] clear)", + Self::NoPackageThermal => "no package thermal status (CPUID.06H:EAX[6] clear)", + Self::Hybrid => { + "a hybrid CPU (CPUID.07H:EDX[15]), whose HWP scale is not its ratio scale, \ + and the declared minimum is a ratio" + } + } + } +} + +/// `None` where every register the envelope names exists on this CPU. +/// **All or nothing**: a CPU missing one gets no request at all, so no reader +/// ever takes firmware's value for one register as the declaration's. +pub const fn refusal(cpuid: &Cpuid) -> Option { + const PTM: u32 = 1 << 6; + const HWP: u32 = 1 << 7; + const HWP_EPP: u32 = 1 << 10; + const HWP_PKG: u32 = 1 << 11; + const EPB: u32 = 1 << 3; + const HYBRID: u32 = 1 << 15; + let eax = cpuid.leaf6_eax; + if cpuid.max_leaf < 6 { + Some(Refusal::NoLeaf6) + } else if eax & HWP == 0 { + Some(Refusal::NoHwp) + } else if !is_intel(&cpuid.vendor) { + Some(Refusal::NotIntel) + } else if eax & HWP_EPP == 0 { + Some(Refusal::NoEpp) + } else if eax & HWP_PKG == 0 { + Some(Refusal::NoPackageRequest) + } else if cpuid.leaf6_ecx & EPB == 0 { + Some(Refusal::NoEnergyPerfBias) + } else if eax & PTM == 0 { + Some(Refusal::NoPackageThermal) + } else if cpuid.max_leaf >= 7 && cpuid.leaf7_edx & HYBRID != 0 { + Some(Refusal::Hybrid) + } else { + None + } +} + +const fn is_intel(vendor: &[u8; 12]) -> bool { + let want = b"GenuineIntel"; + let mut i = 0; + while i < want.len() { + if vendor[i] != want[i] { + return false; + } + i += 1; + } + true +} + +/// `IA32_HWP_REQUEST` for a CPU whose `IA32_HWP_CAPABILITIES` reads +/// `capabilities`, in a package whose `MSR_PLATFORM_INFO` reads +/// `platform_info`. The bar names every field: the minimum is the package's +/// maximum-efficiency ratio, the maximum is the CPU's highest performance — +/// turbo included — and the CPU chooses between them, unwindowed, on its own. +pub const fn hwp_request(capabilities: u64, platform_info: u64) -> u64 { + HwpRequest { + min: ((platform_info >> 40) & 0xff) as u8, + max: HwpCapabilities::of(capabilities).highest, + desired: 0, + epp: EPP, + window: 0, + package_control: false, + } + .raw() +} + +/// `IA32_HWP_REQUEST`'s fields; `IA32_HWP_REQUEST_PKG` is the same layout +/// with bit 42 reserved. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HwpRequest { + pub min: u8, + pub max: u8, + /// 0 leaves the choice to the CPU. + pub desired: u8, + pub epp: u8, + /// Bits 41:32, ten bits wide; 0 leaves the window to the CPU. + pub window: u16, + /// Set, this CPU follows the package request instead. + pub package_control: bool, +} + +impl HwpRequest { + pub const fn raw(self) -> u64 { + self.min as u64 + | (self.max as u64) << 8 + | (self.desired as u64) << 16 + | (self.epp as u64) << 24 + | ((self.window & 0x3ff) as u64) << 32 + | (self.package_control as u64) << 42 + } + + /// The fields; bits above 42 are not one of them. + pub const fn of(raw: u64) -> Self { + Self { + min: raw as u8, + max: (raw >> 8) as u8, + desired: (raw >> 16) as u8, + epp: (raw >> 24) as u8, + window: ((raw >> 32) & 0x3ff) as u16, + package_control: raw & 1 << 42 != 0, + } + } +} + +/// `IA32_HWP_CAPABILITIES`' four performance levels. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HwpCapabilities { + pub highest: u8, + pub guaranteed: u8, + pub efficient: u8, + pub lowest: u8, +} + +impl HwpCapabilities { + pub const fn of(raw: u64) -> Self { + Self { + highest: raw as u8, + guaranteed: (raw >> 8) as u8, + efficient: (raw >> 16) as u8, + lowest: (raw >> 24) as u8, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two `IA32_HWP_CAPABILITIES` the T14's i5-1135G7 reads under Linux: its + /// cores differ in the most efficient level and agree in the rest. + const T14_CAPABILITIES: [u64; 2] = [0x010d_182a, 0x010e_182a]; + /// `MSR_PLATFORM_INFO` bits 47:40 on the T14: Linux's `cpuinfo_min_freq` + /// there is 400000 kHz, which intel_pstate computes as that ratio times + /// 100 MHz. The register's other bits are not the declaration's business. + const T14_PLATFORM_INFO: u64 = 4 << 40; + /// What every one of the T14's eight CPUs held under Linux, intel_pstate + /// active with EPP `balance_performance`. + const T14_REQUEST: u64 = 0x8000_2a04; + + fn intel(leaf6_eax: u32, leaf6_ecx: u32, leaf7_edx: u32) -> Cpuid { + Cpuid { vendor: *b"GenuineIntel", max_leaf: 0x1b, leaf6_eax, leaf6_ecx, leaf7_edx } + } + + /// The bits a CPU needs: PTM, HWP, EPP and the package request in `EAX`, + /// EPB in `ECX`. + const EAX: u32 = 1 << 6 | 1 << 7 | 1 << 10 | 1 << 11; + const ECX: u32 = 1 << 3; + + #[test] + fn the_t14_is_asked_for_what_linux_asked_for() { + for caps in T14_CAPABILITIES { + assert_eq!(hwp_request(caps, T14_PLATFORM_INFO), T14_REQUEST, "{caps:#x}"); + } + } + + #[test] + fn the_package_request_is_the_bars() { + assert_eq!(HWP_REQUEST_PKG, 0x8000_ff01); + } + + #[test] + fn the_t14s_registers_decode_to_the_bars_fields() { + assert_eq!( + HwpRequest::of(T14_REQUEST), + HwpRequest { min: 4, max: 42, desired: 0, epp: 128, window: 0, package_control: false }, + ); + assert_eq!( + HwpCapabilities::of(T14_CAPABILITIES[0]), + HwpCapabilities { highest: 42, guaranteed: 24, efficient: 13, lowest: 1 }, + ); + } + + /// Each field alone, at the bit the SDM gives it — so two fields swapped + /// with equal values in the T14's case cannot hide. + #[test] + fn every_field_is_where_the_sdm_puts_it() { + let zero = HwpRequest { min: 0, max: 0, desired: 0, epp: 0, window: 0, package_control: false }; + assert_eq!(HwpRequest { min: 0xff, ..zero }.raw(), 0xff); + assert_eq!(HwpRequest { max: 0xff, ..zero }.raw(), 0xff << 8); + assert_eq!(HwpRequest { desired: 0xff, ..zero }.raw(), 0xff << 16); + assert_eq!(HwpRequest { epp: 0xff, ..zero }.raw(), 0xff << 24); + assert_eq!(HwpRequest { window: 0x3ff, ..zero }.raw(), 0x3ff << 32); + assert_eq!(HwpRequest { window: 0xffff, ..zero }.raw(), 0x3ff << 32); + assert_eq!(HwpRequest { package_control: true, ..zero }.raw(), 1 << 42); + let all = HwpRequest { min: 1, max: 2, desired: 3, epp: 4, window: 5, package_control: true }; + assert_eq!(HwpRequest::of(all.raw()), all); + } + + #[test] + fn a_cpu_with_every_register_is_declared() { + assert_eq!(refusal(&intel(EAX, ECX, 0)), None); + } + + /// QEMU's two x86-64 CPUs: TCG's `qemu64` says AMD and leaf 6 has only + /// `ARAT` (bit 2); KVM's `host` passes the vendor and the same leaf 6. + #[test] + fn no_qemu_cpu_is_declared() { + let tcg = Cpuid { vendor: *b"AuthenticAMD", max_leaf: 0xd, leaf6_eax: 1 << 2, leaf6_ecx: 0, leaf7_edx: 0 }; + assert_eq!(refusal(&tcg), Some(Refusal::NoHwp)); + assert_eq!(refusal(&intel(1 << 2, 0, 0)), Some(Refusal::NoHwp)); + } + + #[test] + fn a_missing_register_refuses_the_whole_request_by_name() { + let without = |eax: u32| refusal(&intel(EAX & !eax, ECX, 0)); + assert_eq!(without(1 << 10), Some(Refusal::NoEpp)); + assert_eq!(without(1 << 11), Some(Refusal::NoPackageRequest)); + assert_eq!(without(1 << 6), Some(Refusal::NoPackageThermal)); + assert_eq!(refusal(&intel(EAX, 0, 0)), Some(Refusal::NoEnergyPerfBias)); + assert_eq!(refusal(&intel(EAX, ECX, 1 << 15)), Some(Refusal::Hybrid)); + let old = Cpuid { max_leaf: 5, ..intel(EAX, ECX, 0) }; + assert_eq!(refusal(&old), Some(Refusal::NoLeaf6)); + let amd = Cpuid { vendor: *b"AuthenticAMD", ..intel(EAX, ECX, 0) }; + assert_eq!(refusal(&amd), Some(Refusal::NotIntel)); + // Leaf 7 above the maximum is not read: a stale `EDX` there says nothing. + let six = Cpuid { max_leaf: 6, ..intel(EAX, ECX, 1 << 15) }; + assert_eq!(refusal(&six), None); + } +} diff --git a/userland/Cargo.lock b/userland/Cargo.lock index 30245ea0bae..1c5a40f4a72 100644 --- a/userland/Cargo.lock +++ b/userland/Cargo.lock @@ -2642,6 +2642,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perfstate" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", + "toyos-perfstate", +] + [[package]] name = "pico-args" version = "0.5.0" @@ -4135,6 +4144,10 @@ dependencies = [ name = "toyos-mixer" version = "0.1.0" +[[package]] +name = "toyos-perfstate" +version = "0.1.0" + [[package]] name = "toyos-swap" version = "0.1.0" diff --git a/userland/Cargo.toml b/userland/Cargo.toml index ab9a3cf6828..e1e2dc3fd9b 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -18,6 +18,7 @@ members = [ "metalprobe", "netd", "paint", + "perfstate", "pkg", "proctest", "shell", diff --git a/userland/perfstate/Cargo.toml b/userland/perfstate/Cargo.toml new file mode 100644 index 00000000000..88bbcfc3a28 --- /dev/null +++ b/userland/perfstate/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "perfstate" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +# The library is the read and its check, so the guest test that mints its own +# claim runs exactly what `/system/bin/perfstate` runs. +[lib] +path = "src/lib.rs" +doctest = false + +[[bin]] +name = "perfstate" +path = "src/main.rs" + +[dependencies] +toyos-abi = { path = "../../toyos-abi" } +toyos = { path = "../../toyos" } +toyos-perfstate = { path = "../../toyos-perfstate" } diff --git a/userland/perfstate/src/lib.rs b/userland/perfstate/src/lib.rs new file mode 100644 index 00000000000..4898ee9ea4c --- /dev/null +++ b/userland/perfstate/src/lib.rs @@ -0,0 +1,62 @@ +//! One read of a `perf-state` claim: every register it answers, one line for +//! the package and one per CPU, checked against the request the kernel +//! declares for that CPU (`toyos_perfstate`). + +use toyos::Device; +use toyos_abi::perf::{answer_len, CpuRegisters, PackageRegisters}; +use toyos_abi::syscall::{self, SyscallError}; +use toyos_perfstate::{HwpRequest, TURBO_DISABLE}; + +/// Prints the answer, and names the first register that is not the +/// declaration's. The turbo bit is printed and not checked: the kernel +/// does not declare it. +pub fn read_back(claim: &Device) -> Result<(), String> { + let cpus = syscall::cpu_count() as usize; + let mut buf = vec![0u8; answer_len(cpus)]; + let n = claim.read(&mut buf).map_err(|e: SyscallError| format!("the read was refused: {e:?}"))?; + if n != buf.len() { + return Err(format!("the read answered {n} bytes, and {cpus} CPUs are {}", buf.len())); + } + let pkg = PackageRegisters::read_from(&buf).expect("the buffer holds the package record"); + println!( + "pkg hwp_request_pkg={:#010x} platform_info={:#018x} rapl_power_unit={:#x} \ + pkg_power_limit={:#018x} pkg_energy_status={:#x} package_therm_status={:#x} \ + temperature_target={:#x}", + pkg.hwp_request_pkg, + pkg.platform_info, + pkg.rapl_power_unit, + pkg.pkg_power_limit, + pkg.pkg_energy_status, + pkg.package_therm_status, + pkg.temperature_target, + ); + check("pkg", "hwp_request_pkg", pkg.hwp_request_pkg, toyos_perfstate::HWP_REQUEST_PKG)?; + for cpu in 0..cpus { + let regs = CpuRegisters::read_from(&buf[answer_len(cpu)..]) + .expect("the buffer holds every CPU's record"); + println!( + "cpu{cpu} pm_enable={} hwp_request={:#010x} {:?} epb={} turbo={} \ + hwp_capabilities={:#010x}", + regs.pm_enable, + regs.hwp_request, + HwpRequest::of(regs.hwp_request), + regs.energy_perf_bias, + if regs.misc_enable & TURBO_DISABLE == 0 { "on" } else { "off" }, + regs.hwp_capabilities, + ); + let who = format!("cpu{cpu}"); + let declared = toyos_perfstate::hwp_request(regs.hwp_capabilities, pkg.platform_info); + check(&who, "pm_enable", regs.pm_enable, toyos_perfstate::PM_ENABLE)?; + check(&who, "hwp_request", regs.hwp_request, declared)?; + check(&who, "epb", regs.energy_perf_bias, toyos_perfstate::ENERGY_PERF_BIAS)?; + } + Ok(()) +} + +fn check(who: &str, name: &str, holds: u64, declared: u64) -> Result<(), String> { + if holds == declared { + Ok(()) + } else { + Err(format!("{who} holds {name}={holds:#x}, and the declaration is {declared:#x}")) + } +} diff --git a/userland/perfstate/src/main.rs b/userland/perfstate/src/main.rs new file mode 100644 index 00000000000..2274128402f --- /dev/null +++ b/userland/perfstate/src/main.rs @@ -0,0 +1,26 @@ +//! `perfstate`: the CPU performance envelope's registers, read back once +//! through the `perf-state` claim its row names. Exits 0 when every CPU holds +//! the kernel's declaration, 1 when one does not or there is no claim. + +use std::process::ExitCode; + +use toyos::endow; +use toyos::Device; +use toyos_abi::syscall::DeviceType; + +fn main() -> ExitCode { + let Some(claim) = endow::device::(DeviceType::PerfState) else { + eprintln!( + "perfstate: no perf-state claim was endowed; init's log says why, and a machine \ + that declared no performance request has none to give" + ); + return ExitCode::FAILURE; + }; + match perfstate::read_back(&claim) { + Ok(()) => ExitCode::SUCCESS, + Err(why) => { + eprintln!("perfstate: {why}"); + ExitCode::FAILURE + } + } +} From c157f95b20c6e4147d7d2328ac56c5c4f072965b Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 00:13:12 +0200 Subject: [PATCH 02/12] Round 1 review fixes: HWP enabled before its registers are read, and no claim reaches an unproven register The review sent #590 back with five blockers. Each is answered here, except the reading from the T14, which only that machine can give. - control_regs: CPUID alone now decides whether a CPU gets a request. IA32_HWP_INTERRUPT is set to 0 where CPUID.06H:EAX[8] enumerates it, and IA32_PM_ENABLE is written next. Only after that are IA32_HWP_CAPABILITIES and MSR_PLATFORM_INFO read, and the request computed and written. This is intel_pstate's order (intel_pstate_hwp_enable, then intel_pstate_get_hwp_cap). The HWP writes moved out of the no-ap-control-regs skip, into hwp_init after self_check. - toyos-perfstate: no CPUID bit enumerates MSR_PLATFORM_INFO (0xCE). SDM Vol. 4 documents it in the model tables of DisplayFamily 06H, so a CPU of any other family is refused by name (Refusal::NotFamily6) before the register is read. - The read-back drops MSR_RAPL_POWER_UNIT, MSR_PKG_POWER_LIMIT, MSR_PKG_ENERGY_STATUS and MSR_TEMPERATURE_TARGET. No CPUID bit enumerates them, and nothing at boot read them, so a userland read was the first to touch them. The track's stages 2 and 4 now require each one to be proven present at boot first, and keep the energy counter (CVE-2020-8694) off any row a session can launch. - A cancelled perf-state read takes its ask back (Reader::cancel, reached from sys_read's cancelled wait), so no later read is answered from registers sampled before it began. The ABI doc now says that overlapping reads of one claim share one ask. - HwpCapabilities is deleted. The request's maximum is `capabilities as u8`. - perf-state-deaf-cpu grants the claim on a machine with no declaration and answers zeros. The last CPU answers no ask. perf_state_silent_cpu (Fast) asserts two reads refused Io, each naming that CPU alone. - perf-request-diverges has cpu1 move its request one ratio off the declaration when it answers a read, then run hwp_check. perf_request's metal row gains a second boot, perfdiverge, which must carry that panic on the page after the reset. It is priced in the metal profile and ruled flashable. - TESTCASES now runs test_rs_perf_state, before null_sink_client_exits. The metal row's job_passed("test_rs_perf_state") named a job that no boot ran. - Deleted per the review's REMOVEs: the narration in toyos-perfstate's Cargo.toml; "since Sandy Bridge"; "two relaxed loads"; the cross-reference to the dump's bound; shootdown.rs's list of callers; the track's chronology and the claim that its values are the bar's. The stale "four" in io.rs's device-class count goes too. Co-Authored-By: Claude Opus 5.5 --- .../the-kernel-owns-cpu-performance-state.md | 49 ++++--- kernel/src/actuator.rs | 10 ++ kernel/src/arch/aarch64/perf_state.rs | 4 + kernel/src/arch/x86_64/control_regs.rs | 125 ++++++++++++------ kernel/src/arch/x86_64/perf_state.rs | 5 +- kernel/src/device.rs | 4 +- kernel/src/object/device.rs | 8 ++ kernel/src/perf_state.rs | 71 +++++++--- kernel/src/shootdown.rs | 3 +- kernel/src/syscall/io.rs | 18 ++- src/metal.rs | 6 + tests/metal-profile.toml | 35 +++++ .../src/bin/perf_state_silent.rs | 25 ++++ tests/toyos.rs | 100 +++++++++++++- toyos-abi/src/perf.rs | 14 +- toyos-abi/src/syscall.rs | 5 +- toyos-perfstate/Cargo.toml | 5 - toyos-perfstate/src/lib.rs | 121 +++++++++++------ userland/perfstate/src/lib.rs | 12 +- 19 files changed, 464 insertions(+), 156 deletions(-) create mode 100644 tests/toyos-rust-tests/src/bin/perf_state_silent.rs diff --git a/issues/kernel/the-kernel-owns-cpu-performance-state.md b/issues/kernel/the-kernel-owns-cpu-performance-state.md index c090fff7ac9..1e9d64a1d07 100644 --- a/issues/kernel/the-kernel-owns-cpu-performance-state.md +++ b/issues/kernel/the-kernel-owns-cpu-performance-state.md @@ -8,25 +8,37 @@ opened: 2026-09-28 The self-hosting bar (`issues/build/toyos-builds-itself.md`) is measured under a fixed power envelope that must be read back for a whole build span. -Until this track, ToyOS left every register of it where firmware put it. The -kernel declares the envelope from the one CPU-state declaration +The kernel declares the envelope from the one CPU-state declaration (`kernel/src/arch/x86_64/control_regs.rs`), and a `perf-state` claim reads it -back per CPU. The values the kernel programs are the bar's, not its own. +back per CPU. -- **1 — the HWP request, declared and read back.** `IA32_PM_ENABLE`, - `IA32_HWP_REQUEST` (min: the package's maximum-efficiency ratio; max: the - CPU's highest performance; EPP 128), `IA32_HWP_REQUEST_PKG` and EPB 6, - written whole on every CPU and asserted on each, refused by name on a CPU - that lacks any of them. Read back through the `perf-state` claim, together - with the turbo bit, `MSR_PKG_POWER_LIMIT`, the energy counter and package - thermal status (`/system/bin/perfstate`). *Exit*: `perf_request` green in - QEMU, which proves the refusal, and on the T14, which proves every CPU holds - `0x80002a04` and reads it back. +**A register reaches a claim only once boot has proven it.** The kernel has no +`rdmsr` fault fixup, so a register a userland read is the first to touch is a +kernel `#GP` any holder of the claim can cause. Each register a stage adds is +enumerated by CPUID or read at boot under the declaration's proof +(`control_regs::HwpDeclared`) before any read of the claim can reach it. + +- **1 — the HWP request, declared and read back.** `IA32_HWP_INTERRUPT` 0 + where CPUID enumerates it, `IA32_PM_ENABLE`, `IA32_HWP_REQUEST` (min: the + package's maximum-efficiency ratio; max: the CPU's highest performance; EPP + 128), `IA32_HWP_REQUEST_PKG` and EPB 6, written whole on every CPU and + asserted on each, refused by name on a CPU that lacks any of them or is not + DisplayFamily 06H (`MSR_PLATFORM_INFO`'s family). Read back through the + `perf-state` claim, together with the turbo bit and package thermal status + (`/system/bin/perfstate`). *Exit*: in QEMU, `perf_request` (the refusal) and + `perf_state_silent_cpu` (a CPU that never answers is refused `Io` by name); + on the T14, **owed**, `perf_request`'s metal row: on `testcases` every CPU + logs `control_regs: cpuN pm_enable=1 hwp_request=0x80002a04 + hwp_request_pkg=0x8000ff01 epb=6` and `test_rs_perf_state` exits 0; on + `perfdiverge` the page after the reset carries the panic `control_regs: + cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04`. - **2 — RAPL, declared.** PL1, PL2 and their windows through `MSR_PKG_POWER_LIMIT`, and the MMIO mirror in the host bridge's MCHBAR, at the bar's values; the peak limit beside them. A limit firmware locked (bit - 63) is refused by name, never worked around. *Exit*: the T14 reads both - back at the bar's values. + 63) is refused by name, never worked around. `MSR_RAPL_POWER_UNIT` and + `MSR_PKG_POWER_LIMIT` are enumerated by no CPUID bit, so each is read at + boot before the claim answers it. *Exit*: the T14 reads both back at the + bar's values. - **3 — turbo, declared.** `IA32_MISC_ENABLE` bit 38 is read back and not written: its other bits are model-specific and firmware's, so declaring one bit needs the owner's ruling on writing that register whole. Linux's @@ -35,8 +47,13 @@ back per CPU. The values the kernel programs are the bar's, not its own. as firmware's. - **4 — the sampler.** A program that reads the envelope every 60 s and at a span's start and end, and turns `MSR_PKG_ENERGY_STATUS` into the first - 60 s's package power and `IA32_PACKAGE_THERM_STATUS` into a temperature, - which are the bar's validity conditions. *Exit*: one valid span on the T14. + 60 s's package power and `IA32_PACKAGE_THERM_STATUS` with + `MSR_TEMPERATURE_TARGET` into a temperature, which are the bar's validity + conditions. The energy counter and the temperature target are enumerated by + no CPUID bit, so each is read at boot before the claim answers it; and the + energy counter is a side channel (CVE-2020-8694), so it reaches only a row + the owner rules on, never the `perfstate` row any session can launch. + *Exit*: one valid span on the T14. **Not covered.** A hybrid CPU is refused: its HWP scale is not its ratio scale and the declared minimum is a ratio. AMD's CPPC and AArch64 declare nothing; diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 1c79d5b12f0..fd9e231e663 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -291,6 +291,16 @@ actuators! { /// Make one CPU ignore a kick. dump_deaf_cpu = "dump-deaf-cpu"; + /// Grant a `perf-state` claim where no performance request was declared, + /// answering zeros, and have the last CPU never answer its ask: what the + /// read's bound refuses, on a machine QEMU can stage. + perf_state_deaf_cpu = "perf-state-deaf-cpu"; + + /// Have cpu1 move its HWP request off the declaration when it answers a + /// `perf-state` read, then check it as boot does: the negative control on + /// that check, which only a CPU with HWP reaches. + perf_request_diverges = "perf-request-diverges"; + /// On one CPU, file Ctrl+Alt+D's request inside each kind of pass that may not serve it and inside a report, and count the Ring 3 returns each is left pending across. dump_in_blocking_pass = "dump-in-blocking-pass"; diff --git a/kernel/src/arch/aarch64/perf_state.rs b/kernel/src/arch/aarch64/perf_state.rs index 5ed6b330467..64f947eae10 100644 --- a/kernel/src/arch/aarch64/perf_state.rs +++ b/kernel/src/arch/aarch64/perf_state.rs @@ -18,3 +18,7 @@ pub fn read_cpu(declared: &Declared) -> CpuRegisters { pub fn read_package(declared: &Declared) -> PackageRegisters { match *declared {} } + +pub fn diverge(declared: &Declared, _: u32) { + match *declared {} +} diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 529de667cc9..e8ac4965c2d 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -5,10 +5,11 @@ //! whatever optional bits this CPU offers. `EFER.NXE` lets bit 63 of a paging //! entry mean *not executable* ([`Prot`](crate::mm::policy::Prot)). //! -//! The performance request is HWP's (`toyos_perfstate`): `IA32_PM_ENABLE`, -//! `IA32_HWP_REQUEST`, `IA32_HWP_REQUEST_PKG` and `IA32_ENERGY_PERF_BIAS`, on -//! a machine whose CPUs have every register it names, and none of them on one -//! that does not — refused by name once, and firmware's values stand. +//! The performance request is HWP's (`toyos_perfstate`): `IA32_HWP_INTERRUPT` +//! where it exists, `IA32_PM_ENABLE`, `IA32_HWP_REQUEST`, `IA32_HWP_REQUEST_PKG` +//! and `IA32_ENERGY_PERF_BIAS`, on a machine whose CPUs have every register it +//! names, and none of them on one that does not — refused by name once, and +//! firmware's values stand. //! `IA32_MISC_ENABLE`'s turbo bit is not declared: that register's other bits //! are model-specific and firmware's, and writing it whole would decide them. @@ -129,8 +130,8 @@ const HWP_DECLARED: u8 = 1; const HWP_REFUSED: u8 = 2; /// Proof that the machine's declaration carries the performance request, so -/// every register [`toyos_perfstate::msr`] names exists on every CPU and -/// reading one is no `#GP`. +/// every register [`toyos_perfstate::msr`] names but `HWP_INTERRUPT` exists on +/// every CPU, HWP is enabled there, and reading one is no `#GP`. #[derive(Clone, Copy)] pub struct HwpDeclared(()); @@ -146,11 +147,9 @@ impl HwpDeclared { } } -/// This CPU's `IA32_HWP_REQUEST`, or `None` where the machine has no request. -/// Recomputed per CPU, as `CR4` is, and not required to match the BSP's: a -/// CPU's highest performance is its own. Whether there is one at all is the -/// machine's, and a CPU that disagrees is named. -fn hwp_declaration(cpu_id: u32) -> Option { +/// Whether this CPU gets a performance request, from CPUID alone: the +/// machine's answer, and a CPU that disagrees is named. +fn hwp_declared(cpu_id: u32) -> bool { let refusal = toyos_perfstate::refusal(&perf_cpuid()); let mine = if refusal.is_none() { HWP_DECLARED } else { HWP_REFUSED }; match HWP.compare_exchange(HWP_UNDECIDED, mine, Ordering::Release, Ordering::Acquire) { @@ -166,16 +165,12 @@ fn hwp_declaration(cpu_id: u32) -> Option { if machine == HWP_DECLARED { "declared" } else { "refused" }, ), } - refusal.is_none().then(|| { - toyos_perfstate::hwp_request( - cpu::rdmsr(msr::HWP_CAPABILITIES), - cpu::rdmsr(msr::PLATFORM_INFO), - ) - }) + refusal.is_none() } fn perf_cpuid() -> toyos_perfstate::Cpuid { let (max_leaf, ebx, ecx, edx) = cpu::cpuid(0, 0); + let leaf1_eax = cpu::cpuid(1, 0).0; let mut vendor = [0u8; 12]; for (at, word) in [ebx, edx, ecx].into_iter().enumerate() { vendor[at * 4..at * 4 + 4].copy_from_slice(&word.to_le_bytes()); @@ -183,7 +178,7 @@ fn perf_cpuid() -> toyos_perfstate::Cpuid { // A leaf above the maximum answers with the highest basic leaf's data. let (leaf6_eax, _, leaf6_ecx, _) = if max_leaf >= 6 { cpu::cpuid(6, 0) } else { (0, 0, 0, 0) }; let leaf7_edx = if max_leaf >= 7 { cpu::cpuid(7, 0).3 } else { 0 }; - toyos_perfstate::Cpuid { vendor, max_leaf, leaf6_eax, leaf6_ecx, leaf7_edx } + toyos_perfstate::Cpuid { vendor, max_leaf, leaf1_eax, leaf6_eax, leaf6_ecx, leaf7_edx } } /// Puts this CPU's `CR4`, `EFER` and performance request into the declaration @@ -191,21 +186,8 @@ fn perf_cpuid() -> toyos_perfstate::Cpuid { /// before `arch::syscall::init`, which needs `SCE` set. pub fn init(cpu_id: u32) { let declared = declaration(cpu_id); - let hwp = hwp_declaration(cpu_id); + let hwp = hwp_declared(cpu_id); if !skipped(cpu_id) { - if let Some(request) = hwp { - // SAFETY: `hwp_declaration` answered `Some` only where CPUID - // enumerates HWP with EPP and the package request and EPB, so each - // MSR exists; `PM_ENABLE` goes first because a request written - // before it is `#GP` (SDM Vol. 3B, HWP's enabling), and every - // value fits the register's defined bits. - unsafe { - cpu::wrmsr(msr::PM_ENABLE, toyos_perfstate::PM_ENABLE); - cpu::wrmsr(msr::HWP_REQUEST, request); - cpu::wrmsr(msr::HWP_REQUEST_PKG, toyos_perfstate::HWP_REQUEST_PKG); - cpu::wrmsr(msr::ENERGY_PERF_BIAS, toyos_perfstate::ENERGY_PERF_BIAS); - } - } // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked // the first two and both callers use PCID 0; `wrmsr` writes [`EFER`], whose @@ -221,11 +203,58 @@ pub fn init(cpu_id: u32) { } } self_check(cpu_id, declared); - if let Some(request) = hwp { - hwp_check(cpu_id, request); + if hwp { + hwp_init(cpu_id); } } +/// Puts this CPU's performance request into the declaration and checks it. +/// Every HWP register but `IA32_HWP_INTERRUPT` is touched only once +/// `IA32_PM_ENABLE` is set, the request's inputs included, and the interrupt +/// is cleared before it: intel_pstate's order (`intel_pstate_hwp_enable`, +/// then `intel_pstate_get_hwp_cap`). +fn hwp_init(cpu_id: u32) { + let notifies = toyos_perfstate::hwp_notification(&perf_cpuid()); + // SAFETY: `hwp_declared` answered `true` only where CPUID enumerates HWP, + // and `IA32_HWP_INTERRUPT` is written only where CPUID enumerates it too; + // both values fit their registers' defined bits. + unsafe { + if notifies { + cpu::wrmsr(msr::HWP_INTERRUPT, toyos_perfstate::HWP_INTERRUPT); + } + cpu::wrmsr(msr::PM_ENABLE, toyos_perfstate::PM_ENABLE); + } + let request = hwp_request(); + // SAFETY: HWP is enabled, so its registers are live; CPUID enumerated EPP, + // the package request and EPB, and every value fits its register's + // defined bits. + unsafe { + cpu::wrmsr(msr::HWP_REQUEST, request); + cpu::wrmsr(msr::HWP_REQUEST_PKG, toyos_perfstate::HWP_REQUEST_PKG); + cpu::wrmsr(msr::ENERGY_PERF_BIAS, toyos_perfstate::ENERGY_PERF_BIAS); + } + hwp_check(cpu_id, request, notifies); +} + +/// This CPU's `IA32_HWP_REQUEST`, once `IA32_PM_ENABLE` is set. Recomputed per +/// CPU, as `CR4` is, and not required to match the BSP's: a CPU's highest +/// performance is its own. +fn hwp_request() -> u64 { + toyos_perfstate::hwp_request(cpu::rdmsr(msr::HWP_CAPABILITIES), cpu::rdmsr(msr::PLATFORM_INFO)) +} + +/// `perf-request-diverges`: this CPU's request moved one ratio off the +/// declaration once the machine is up, then checked as boot checks it — the +/// negative control on [`hwp_check`]'s asserts, which only a CPU with HWP +/// reaches. Returns only where that check is broken. +pub fn diverge(_: &HwpDeclared, cpu_id: u32) { + let request = hwp_request(); + // SAFETY: the proof says HWP is enabled on every CPU; the minimum moves + // from the package's most efficient ratio by one, still a defined value. + unsafe { cpu::wrmsr(msr::HWP_REQUEST, request ^ 1) }; + hwp_check(cpu_id, request, toyos_perfstate::hwp_notification(&perf_cpuid())); +} + /// Whether the declaration carries `PCIDE`, and therefore whether `INVPCID` is this machine's flush. pub fn pcid_active() -> bool { DECLARED_CR4.load(Ordering::Acquire) & cr4::PCIDE != 0 @@ -373,29 +402,33 @@ fn self_check(cpu_id: u32, declared_cr4: u64) { /// [`self_check`] for the performance request, logged first for the same /// reason; the line carries the request's two inputs, so a reader can /// recompute it. -fn hwp_check(cpu_id: u32, request: u64) { +fn hwp_check(cpu_id: u32, request: u64, notifies: bool) { let pm_enable = cpu::rdmsr(msr::PM_ENABLE); let live = cpu::rdmsr(msr::HWP_REQUEST); let pkg = cpu::rdmsr(msr::HWP_REQUEST_PKG); let epb = cpu::rdmsr(msr::ENERGY_PERF_BIAS); + let interrupt = notifies.then(|| cpu::rdmsr(msr::HWP_INTERRUPT)); log!( "control_regs: cpu{} pm_enable={} hwp_request={:#010x} hwp_request_pkg={:#010x} epb={} \ - hwp_capabilities={:#010x} platform_info={:#018x}", + hwp_interrupt={} hwp_capabilities={:#010x} platform_info={:#018x}", cpu_id, pm_enable, live, pkg, epb, + Enumerated(interrupt), cpu::rdmsr(msr::HWP_CAPABILITIES), cpu::rdmsr(msr::PLATFORM_INFO), ); let want = [ - ("pm_enable", pm_enable, toyos_perfstate::PM_ENABLE), - ("hwp_request", live, request), - ("hwp_request_pkg", pkg, toyos_perfstate::HWP_REQUEST_PKG), - ("epb", epb, toyos_perfstate::ENERGY_PERF_BIAS), + ("pm_enable", Some(pm_enable), toyos_perfstate::PM_ENABLE), + ("hwp_request", Some(live), request), + ("hwp_request_pkg", Some(pkg), toyos_perfstate::HWP_REQUEST_PKG), + ("epb", Some(epb), toyos_perfstate::ENERGY_PERF_BIAS), + ("hwp_interrupt", interrupt, toyos_perfstate::HWP_INTERRUPT), ]; for (name, holds, declared) in want { + let Some(holds) = holds else { continue }; assert!( holds == declared, "control_regs: cpu{cpu_id} holds {name}={holds:#x}, the declaration is {declared:#x}", @@ -403,6 +436,18 @@ fn hwp_check(cpu_id: u32, request: u64) { } } +/// A register's value, or `absent` where CPUID enumerates no such register. +struct Enumerated(Option); + +impl core::fmt::Display for Enumerated { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0 { + Some(value) => write!(f, "{value:#x}"), + None => f.write_str("absent"), + } + } +} + /// How many CPUs hold the declaration, said once after the last of them has /// been checked. A divergent CPU panics inside [`self_check`], so what this /// line adds is the *count*: a CPU that never reached [`init`] at all is diff --git a/kernel/src/arch/x86_64/perf_state.rs b/kernel/src/arch/x86_64/perf_state.rs index e4f5f134c5d..7da5cb70747 100644 --- a/kernel/src/arch/x86_64/perf_state.rs +++ b/kernel/src/arch/x86_64/perf_state.rs @@ -7,6 +7,7 @@ use toyos_perfstate::msr; use super::cpu; pub use super::control_regs::HwpDeclared as Declared; +pub use super::control_regs::diverge; /// The proof every read below needs, or why this machine has none. pub fn declared() -> Result { @@ -29,10 +30,6 @@ pub fn read_package(_: &Declared) -> PackageRegisters { PackageRegisters { hwp_request_pkg: cpu::rdmsr(msr::HWP_REQUEST_PKG), platform_info: cpu::rdmsr(msr::PLATFORM_INFO), - rapl_power_unit: cpu::rdmsr(msr::RAPL_POWER_UNIT), - pkg_power_limit: cpu::rdmsr(msr::PKG_POWER_LIMIT), - pkg_energy_status: cpu::rdmsr(msr::PKG_ENERGY_STATUS), package_therm_status: cpu::rdmsr(msr::PACKAGE_THERM_STATUS), - temperature_target: cpu::rdmsr(msr::TEMPERATURE_TARGET), } } diff --git a/kernel/src/device.rs b/kernel/src/device.rs index 03039bfde6b..e5806bd0851 100644 --- a/kernel/src/device.rs +++ b/kernel/src/device.rs @@ -185,12 +185,12 @@ pub fn try_claim(class: DeviceType, selector: [u64; 2]) -> Result { - let declared = crate::arch::perf_state::declared().map_err(|why| { + let reader = crate::perf_state::Reader::claim().map_err(|why| { log!("perf-state: no claim: {why}"); ClaimError::Absent })?; let claim = Claim::acquire(class)?; - Ok(DeviceClaim::new(class, DeviceInfo::PerfState(crate::perf_state::Reader::new(declared)), claim)) + Ok(DeviceClaim::new(class, DeviceInfo::PerfState(reader), claim)) } } } diff --git a/kernel/src/object/device.rs b/kernel/src/object/device.rs index 6b6967e9816..141c90236cb 100644 --- a/kernel/src/object/device.rs +++ b/kernel/src/object/device.rs @@ -166,6 +166,14 @@ impl DeviceClaim { } } + /// A performance-state read that was cancelled while it waited. + pub fn cancel_perf_state(&self) { + match &self.described.lock().info { + DeviceInfo::PerfState(reader) => reader.cancel(), + _ => unreachable!("a {:?} claim has no performance-state read to cancel", self.class), + } + } + pub fn info_read(&self) -> bool { self.info_read.load(Ordering::Relaxed) } diff --git a/kernel/src/perf_state.rs b/kernel/src/perf_state.rs index f77664580f5..c5077e78939 100644 --- a/kernel/src/perf_state.rs +++ b/kernel/src/perf_state.rs @@ -30,7 +30,7 @@ static SLOTS: [Slot; MAX_CPUS] = [const { Slot::new() }; MAX_CPUS]; pub static WATCH: Watch = Watch::new(); /// A kicked CPU reaches a pass within one timer interrupt; this is the -/// kernel's choice, the blocked-task dump's for the same question. +/// kernel's choice. const ANSWER: Budget = Budget::of( Duration::from_millis(250), "the read is refused `Io`, and the CPUs that did not answer are named", @@ -58,9 +58,12 @@ impl Slot { } /// One claim's side of the protocol: the proof its reads need, and the ask -/// its reads wait on — joined by every read of the claim until it is answered. +/// its reads wait on — joined by every read of the claim until it is answered +/// or a reader waiting on it is cancelled. pub struct Reader { - declared: Declared, + /// `None` only under `perf-state-deaf-cpu`, whose claim has no + /// declaration behind it and answers zeros. + declared: Option, pending: Lock>, } @@ -71,8 +74,18 @@ struct Ask { } impl Reader { - pub fn new(declared: Declared) -> Self { - Self { declared, pending: Lock::new(None) } + /// A claim's reader, or why this machine has none. + pub fn claim() -> Result { + let declared = perf_state::declared().map(Some).or_else(|why| { + if crate::actuator::perf_state_deaf_cpu() { Ok(None) } else { Err(why) } + })?; + Ok(Self { declared, pending: Lock::new(None) }) + } + + /// A read that was cancelled takes its ask back, so the next read asks + /// afresh and is never answered with registers read before it began. + pub fn cancel(&self) { + *self.pending.lock() = None; } /// The answer's bytes into `buf`, or `None` while a CPU has not answered — @@ -98,7 +111,8 @@ impl Reader { } return Some(SyscallError::Io.to_u64()); } - buf.write_at(0, perf_state::read_package(&self.declared).as_bytes()); + let package = self.declared.as_ref().map_or_else(Default::default, perf_state::read_package); + buf.write_at(0, package.as_bytes()); // `answer_len(cpu)` is where CPU `cpu`'s record starts. for (cpu, slot) in SLOTS[..cpus].iter().enumerate() { buf.write_at(answer_len(cpu), slot.load().as_bytes()); @@ -111,7 +125,9 @@ impl Reader { fn ask(&self, cpus: usize, now: Instant) -> Ask { let generation = ASKS.issue(); let me = crate::arch::percpu::cpu_id() as usize; - ASKS.serve(me, || SLOTS[me].store(perf_state::read_cpu(&self.declared))); + if !deaf(me, cpus) { + ASKS.serve(me, || SLOTS[me].store(sample(me))); + } for cpu in (0..cpus).filter(|&cpu| cpu != me) { crate::arch::irqchip::kick_cpu(cpu as u32); } @@ -131,18 +147,43 @@ pub fn park_deadline() -> Deadline { Deadline::at(crate::clock::now() + ANSWER.duration()) } -/// This CPU's answer, if one is owed. Called from `drain_irqs` every pass, so -/// what it costs when nothing is owed is two relaxed loads. +/// This CPU's answer, if one is owed. Called from `drain_irqs` every pass. pub fn serve_if_owed() { let me = crate::arch::percpu::cpu_id() as usize; - if !ASKS.owes(me) { + if !ASKS.owes(me) || deaf(me, crate::arch::smp::cpu_count() as usize) { return; } + ASKS.serve(me, || SLOTS[me].store(sample(me))); + WATCH.post(); +} + +/// This CPU's registers, or zeros where the claim has no declaration behind it. +fn sample(me: usize) -> CpuRegisters { // The proof is used inside the closure only: on an architecture where it // is uninhabited, binding one here would make the rest unreachable. - let read = perf_state::declared() - .map(|declared| move || perf_state::read_cpu(&declared)) - .expect("an ask is made only through a claim, and a claim only where the request is declared"); - ASKS.serve(me, || SLOTS[me].store(read())); - WATCH.post(); + let read = perf_state::declared().map(|declared| { + move || { + if crate::actuator::perf_request_diverges() && me == 1 { + perf_state::diverge(&declared, 1); + } + perf_state::read_cpu(&declared) + } + }); + match read { + Ok(read) => read(), + Err(why) => { + assert!( + crate::actuator::perf_state_deaf_cpu(), + "an ask is made only through a claim, and a claim only where the request is \ + declared: {why}", + ); + CpuRegisters::default() + } + } +} + +/// `perf-state-deaf-cpu`'s silent CPU, the last: it answers no ask, its own +/// included. +fn deaf(cpu: usize, cpus: usize) -> bool { + crate::actuator::perf_state_deaf_cpu() && cpu + 1 == cpus } diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 7ccf2daa03e..190b97f0783 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -1,5 +1,4 @@ -//! The acknowledgement half of a machine-wide ask — a TLB shootdown, or a -//! performance-state read — with no hardware in it. +//! The acknowledgement half of a machine-wide ask, with no hardware in it. //! Compiled a second time into `kernel-loom/` against loom's atomics, so this file must hold no `crate::` references. //! The read must happen before the flush, or a target could publish a generation its flush has not yet completed. diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index fa809d5ffc9..3ff37f26727 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -39,8 +39,9 @@ enum ReadBlock { /// the serial line, never on the keyboard's queue, which only a claim /// drains and which would answer it at once for as long as a key sits there. Console(Deadline), - /// A performance-state read, until every CPU has answered its ask. - PerfState(Deadline), + /// A performance-state read, until every CPU has answered its ask; the + /// claim is carried so a cancelled wait can take the ask back. + PerfState(alloc::sync::Arc, Deadline), /// Nothing to wait for: the answer is this word. Refused(u64), /// Carried out of the process's lock: `HandleError::refuse` may take the @@ -92,14 +93,16 @@ pub(super) fn sys_write(h: RawHandle, buf: &UserBytes) -> u64 { } } -/// Only these four device classes block; the rest answer `NotFound` on an -/// empty blocking read. -fn read_block_device(claim: &crate::object::device::DeviceClaim) -> ReadBlock { +/// Only these device classes block; the rest answer `NotFound` on an empty +/// blocking read. +fn read_block_device(claim: &alloc::sync::Arc) -> ReadBlock { match claim.class() { device::DeviceType::Keyboard => ReadBlock::Keyboard(Deadline::never()), device::DeviceType::VirtioSound if claim.info_read() => ReadBlock::VirtioSound, device::DeviceType::HdaAudio if claim.info_read() => ReadBlock::Hda, - device::DeviceType::PerfState => ReadBlock::PerfState(crate::perf_state::park_deadline()), + device::DeviceType::PerfState => { + ReadBlock::PerfState(claim.clone(), crate::perf_state::park_deadline()) + } _ => ReadBlock::Refused(SyscallError::NotFound.to_u64()), } } @@ -216,7 +219,7 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { return cancelled(); } } - Err(ReadBlock::PerfState(deadline)) => { + Err(ReadBlock::PerfState(claim, deadline)) => { let parkable = crate::scheduler::Parkable::at_entry(); if watch::wait_until( &parkable, @@ -228,6 +231,7 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { ) .is_err() { + claim.cancel_perf_state(); return cancelled(); } } diff --git a/src/metal.rs b/src/metal.rs index e1f7d6d4bd7..f86ddd02031 100644 --- a/src/metal.rs +++ b/src/metal.rs @@ -803,6 +803,12 @@ pub const FLASHABLE: &[(&str, Flash)] = &[ // hold, reaches no firmware state, and the worst // it leaves is a stick a replug clears — the defect the arm exists to stage. ("usb-transport-break", Flash::Ok), + // It moves cpu1's HWP request one ratio off the declaration once the + // machine is up, to a value the CPU accepts, and the kernel's own check + // then panics the boot. It reaches no device register and no firmware + // state, and the reset that ends the boot clears `IA32_PM_ENABLE`, which + // only a reset does. + ("perf-request-diverges", Flash::Ok), ( "quiesce-late-word", Flash::Never( diff --git a/tests/metal-profile.toml b/tests/metal-profile.toml index c7636091b61..743f319d4ba 100644 --- a/tests/metal-profile.toml +++ b/tests/metal-profile.toml @@ -457,6 +457,29 @@ ceiling = 2000 ceiling_from = "twice the sample period `kernel/src/hardlockup`'s SAMPLE_NS states, 1000 ms: a CPU is found stuck at the first sample after its bound, so one period is the lateness and a second is the widest a delayed one costs" measured = 0 +# --- the boot a performance check ends --- +# `perfdiverge` moves cpu1's HWP request off the declaration when its one job +# reads the claim, and the kernel's own check panics; the panic, not a +# shutdown, resets the machine. + +[[number]] +name = "boot.perfdiverge.complete_ms" +unit = "ms" +ceiling = 60000 +ceiling_from = "toyos_tco::JOB_BOUND_MS — as boot.testcases.complete_ms: the panic comes after `Boot: complete`, from the job list" + +[[number]] +name = "boot.perfdiverge.back_secs" +unit = "s" +ceiling = 420 +ceiling_from = "toyos_build::metal::return_secs" + +[[number]] +name = "boot.perfdiverge.stick_secs" +unit = "s" +ceiling = 30 +ceiling_from = "as boot.testcases.stick_secs" + # --- the boot that is never idle --- # `usbload` stops no CPU. It sweeps the last eighth of the stick once, reading # each run and writing it back, until the deadline takes the machine out from @@ -876,6 +899,12 @@ ceiling = 16667 ceiling_from = "as boot.deadlinewedge.panel_max_us" measured = 3890 +[[number]] +name = "boot.perfdiverge.panel_max_us" +unit = "us" +ceiling = 16667 +ceiling_from = "as boot.testcases.panel_max_us" + [[number]] name = "boot.usbbreak.panel_max_us" unit = "us" @@ -999,6 +1028,12 @@ ceiling = 100000 ceiling_from = "as boot.testcases.panel_us" measured = 19488 +[[number]] +name = "boot.perfdiverge.panel_us" +unit = "us" +ceiling = 100000 +ceiling_from = "as boot.testcases.panel_us" + # --- what the block layer had open where the stop ended # How long the stop took is priced nowhere: a stop that gave up spends its # budget and no more, so the record's own shortfall clause is the verdict, and diff --git a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs new file mode 100644 index 00000000000..b4f73dcd81c --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs @@ -0,0 +1,25 @@ +//! A `perf-state` read one CPU never answers (`perf-state-deaf-cpu`, which +//! also grants the claim where no request was declared): refused `Io` once the +//! kernel's bound has passed, never answered and never left waiting. +//! `perf_state_silent_cpu` drives it and reads which CPU the kernel named. + +use toyos::endow::Endowments; +use toyos::syscap::SysCap; +use toyos::Device; +use toyos_abi::perf::answer_len; +use toyos_abi::syscall::{self, DeviceType, SyscallError, SYSCAP_LABEL}; + +fn main() { + let cap: SysCap = Endowments::get() + .take(SYSCAP_LABEL) + .expect("the test estate is endowed a device-minting capability"); + let claim = cap + .claim::(DeviceType::PerfState) + .expect("perf-state-deaf-cpu grants the claim on any machine"); + let mut buf = vec![0u8; answer_len(syscall::cpu_count() as usize)]; + // Twice: a refused read leaves no ask behind that answers the next. + for _ in 0..2 { + assert_eq!(claim.read(&mut buf), Err(SyscallError::Io)); + } + println!("===PERF_STATE_SILENT_OK==="); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 09a11c9f883..40f222b8959 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -250,6 +250,9 @@ const RUST_SKIP: &[&str] = &[ // Which of its two branches is right is the machine's to say: QEMU's CPUs // have no HWP and the T14's do. `perf_request` runs it and reads which. "perf_state", + // Meaningful only under `perf-state-deaf-cpu`, which grants a claim no + // other boot has. `perf_state_silent_cpu` runs it. + "perf_state_silent", // Needs a display whose mode can change, which is `Profile::VirtioGpu` // alone; the shared boot has no display at all. `gpu_set_resolution` runs // it there, and `iommu_gpu_scanout_swap` the second. @@ -613,6 +616,7 @@ const MACHINE_TESTS: &[(&str, Sched, Tier)] = &[ ("control_regs", Sched::Parallel, Tier::Fast), ("control_regs_negative", Sched::Parallel, Tier::Fast), ("perf_request", Sched::Parallel, Tier::Fast), + ("perf_state_silent_cpu", Sched::Parallel, Tier::Fast), // The boot facts the metal suite reads off a machine's own records: every // CPU the firmware named came up and none of their timestamp counters // trails the BSP's; the physical memory manager's accounting against the @@ -1523,6 +1527,7 @@ const CARRIES: &[(&str, &[&str])] = &[ ("spawn_cwd", &["test_rs_spawn_cwd"]), ("input_claim_absent", &["test_rs_input_absent"]), ("perf_request", &["test_rs_perf_state"]), + ("perf_state_silent_cpu", &["test_rs_perf_state_silent"]), ("gpu_set_resolution", &["test_rs_gpu_set_resolution"]), ("iommu_gpu_scanout_swap", &["test_rs_gpu_scanout_swap"]), ("userdev_dma_fault", &["test_rs_log_origin"]), @@ -1702,8 +1707,17 @@ const METAL: &[(&str, metal::Metal)] = &[ }, ), ( + // Two boots: `testcases`, whose CPUs hold the declaration and read it + // back, and `perfdiverge`, where cpu1's request is moved off it once the + // machine is up and the boot's own check must panic naming it. "perf_request", - metal::Metal::Runs { arms: TESTCASES, judge: |b| perf_request_on_metal(b[0]) }, + metal::Metal::Runs { + arms: PERF_REQUEST, + judge: |b| { + perf_request_on_metal(b[0])?; + perf_request_diverged(b[1]) + }, + }, ), ( "ioapic_topology", @@ -2060,9 +2074,22 @@ const TESTCASES: &[metal::Arm] = &[metal::once( "testcases", "tests/testcases", &[], - &["test_rs_abuse_short_sleep", "test_rs_null_sink_client_exits", "log-close"], + &[ + "test_rs_abuse_short_sleep", + "test_rs_perf_state", + "test_rs_null_sink_client_exits", + "log-close", + ], )]; +/// `perf_request`'s two boots. The first is [`TESTCASES`], whose list carries +/// `test_rs_perf_state` because a job this row added would land after +/// `log-close`; the second is its own, since it ends in a panic. +const PERF_REQUEST: &[metal::Arm] = &[ + metal::once("testcases", "tests/testcases", &[], &[]), + metal::once("perfdiverge", "tests/testcases", &["perf-request-diverges"], &["test_rs_perf_state"]), +]; + /// **Two boots of one config, because these two cannot share one.** Each fills /// a machine-wide cap and leaves it filled: `mkdir_cap` fills the directory cap, /// and `readdir_bound`'s own `create_dir("/tmp/empty")` is then refused with @@ -13705,6 +13732,7 @@ fn run_machine_test( } "control_regs_negative" => control_regs_negative(test_config, c_bins, rust_bins), "perf_request" => perf_request(test_config, c_bins, rust_bins), + "perf_state_silent_cpu" => perf_state_silent_cpu(test_config, c_bins, rust_bins), "smp_roster_and_tsc_trail" => { // Eight, which is the T14's own count and this suite's ceiling. const CPUS: u32 = 8; @@ -19717,3 +19745,71 @@ fn perf_request_on_metal(boot: &metal::Readback) -> Result<(), String> { eprintln!(" [perf_request] {cpus} CPUs hold the bar's request and read it back"); Ok(()) } + +/// The second boot: `perf-request-diverges` moves cpu1's request one ratio off +/// the declaration when it answers `test_rs_perf_state`'s read, and runs the +/// check boot runs. That check panics naming what cpu1 holds against what it +/// was declared — the page after the reset carries it — and a boot whose check +/// did not assert reaches no such panic. +fn perf_request_diverged(boot: &metal::Readback) -> Result<(), String> { + const NAMED: &str = "control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04"; + let after = boot.after_the_reset()?; + let said = after.must_say_after(bootlog::PREVIOUS_PANIC, NAMED)?.to_string(); + eprintln!(" [perf_request] a request moved off the declaration panicked: {}", said.trim()); + Ok(()) +} + +/// A read one CPU never answers is refused `Io` once the kernel's bound has +/// passed, naming that CPU and no other — twice, so a refused ask answers no +/// later read. `perf-state-deaf-cpu` grants the claim on QEMU's CPUs, which +/// have no HWP, and silences the last CPU; a read with no bound waits for +/// ever, and this reds at its ceiling. +fn perf_state_silent_cpu( + test_config: &Path, + c_bins: &[(String, Vec)], + rust_bins: &[(String, Vec)], +) -> Result<(), String> { + const CPUS: u32 = 2; + const SILENT: &str = " did not answer a read within "; + const READS: usize = 2; + let mut qemu = QemuInstance::boot_with_options( + test_config, + c_bins, + rust_bins, + BootOptions { smp: CPUS, kernel_params: &["perf-state-deaf-cpu"], ..Default::default() }, + ); + let result = qemu.run_test("test_rs_perf_state_silent", Duration::from_secs(30)); + if let Some(err) = &result.error { + return Err(format!("{err}\n{}", result.serial)); + } + if result.exit_code != Some(0) || !result.stdout.contains("===PERF_STATE_SILENT_OK===") { + return Err(format!("perf_state_silent exited {:?}:\n{}", result.exit_code, result.serial)); + } + // The kernel's records reach the console through `klogd` and the test's + // lines through `logd`, so the refusals may still be on their way. + let named = |text: &str| -> Vec { + text.lines().filter(|l| l.contains(SILENT)).map(str::to_string).collect() + }; + let mut refusals = named(&result.serial); + if refusals.len() < READS { + let owed = READS - refusals.len(); + let seen = std::cell::Cell::new(0); + let more = qemu.drain_until(Duration::from_secs(10), |line| { + seen.set(seen.get() + usize::from(line.contains(SILENT))); + seen.get() >= owed + }); + refusals.extend(named(&more)); + } + let want = format!("perf_state: cpu{}{SILENT}", CPUS - 1); + if refusals.len() != READS || !refusals.iter().all(|l| l.contains(&want)) { + return Err(format!( + "want {READS} refusals each naming cpu{} alone, got {}:\n{}\n{}", + CPUS - 1, + refusals.len(), + refusals.join("\n"), + result.serial, + )); + } + eprintln!(" [perf_state_silent_cpu] {READS} reads refused Io, each naming cpu{}", CPUS - 1); + Ok(()) +} diff --git a/toyos-abi/src/perf.rs b/toyos-abi/src/perf.rs index f317bf45c9f..9cbd0b4be6a 100644 --- a/toyos-abi/src/perf.rs +++ b/toyos-abi/src/perf.rs @@ -12,16 +12,8 @@ pub struct PackageRegisters { pub hwp_request_pkg: u64, /// `MSR_PLATFORM_INFO`, 0xCE. pub platform_info: u64, - /// `MSR_RAPL_POWER_UNIT`, 0x606. - pub rapl_power_unit: u64, - /// `MSR_PKG_POWER_LIMIT`, 0x610. - pub pkg_power_limit: u64, - /// `MSR_PKG_ENERGY_STATUS`, 0x611. - pub pkg_energy_status: u64, /// `IA32_PACKAGE_THERM_STATUS`, 0x1B1. pub package_therm_status: u64, - /// `MSR_TEMPERATURE_TARGET`, 0x1A2. - pub temperature_target: u64, } /// One CPU's registers, read on that CPU. @@ -42,7 +34,7 @@ pub struct CpuRegisters { // Every byte belongs to a field: both cross the boundary as bytes, so a gap // would publish whatever the kernel stack held. -const _: () = assert!(core::mem::size_of::() == 7 * 8); +const _: () = assert!(core::mem::size_of::() == 3 * 8); const _: () = assert!(core::mem::size_of::() == 5 * 8); /// The bytes a read answers on a machine of `cpus` CPUs. @@ -102,8 +94,8 @@ mod tests { }; assert_eq!(CpuRegisters::read_from(cpu.as_bytes()), Some(cpu)); assert_eq!(CpuRegisters::read_from(&cpu.as_bytes()[1..]), None); - let pkg = PackageRegisters { pkg_power_limit: 0x0042_8200_00dd_8200, ..Default::default() }; + let pkg = PackageRegisters { package_therm_status: 0x8830_0000, ..Default::default() }; assert_eq!(PackageRegisters::read_from(pkg.as_bytes()), Some(pkg)); - assert_eq!(answer_len(8), 56 + 8 * 40); + assert_eq!(answer_len(8), 24 + 8 * 40); } } diff --git a/toyos-abi/src/syscall.rs b/toyos-abi/src/syscall.rs index 29f3d7e88ac..272c0c689d4 100644 --- a/toyos-abi/src/syscall.rs +++ b/toyos-abi/src/syscall.rs @@ -1266,8 +1266,9 @@ device_classes! { Partition = 8 => "part", /// The CPU performance envelope's registers, read back: every read answers /// [`crate::perf`]'s records, each CPU's taken on that CPU after the read - /// asked. Read-only — the kernel writes the declaration and nothing else - /// does. `NotFound` on a machine whose CPUs got no declared request. + /// asked; reads of one claim that overlap share one ask. Read-only — the + /// kernel writes the declaration and nothing else does. `NotFound` on a + /// machine whose CPUs got no declared request. PerfState = 9 => "perf-state", } diff --git a/toyos-perfstate/Cargo.toml b/toyos-perfstate/Cargo.toml index 9c157af13ee..87b3050bbc1 100644 --- a/toyos-perfstate/Cargo.toml +++ b/toyos-perfstate/Cargo.toml @@ -1,8 +1,3 @@ -# A member of the host workspace (root `Cargo.toml`), like toyos-tco: the -# kernel depends on it by path and its tests run on the host, because no QEMU -# CPU has HWP and the arithmetic that decides what a real laptop's CPUs are -# asked for is exactly what a guest cannot exercise. - [package] name = "toyos-perfstate" description = "The CPU performance request the kernel declares on an x86-64 CPU with HWP, and the register layouts a reader checks it against." diff --git a/toyos-perfstate/src/lib.rs b/toyos-perfstate/src/lib.rs index acee94134e6..c279650f1d2 100644 --- a/toyos-perfstate/src/lib.rs +++ b/toyos-perfstate/src/lib.rs @@ -4,11 +4,10 @@ //! and does the writes. //! //! Layouts are the Intel SDM's — Vol. 3B, *Power and Thermal Management*, for -//! HWP, the energy/performance bias, RAPL and package thermal status; Vol. 4 -//! for the addresses. The declared values are the power envelope the -//! self-hosting bar is measured under (`issues/build/toyos-builds-itself.md`), -//! so a ToyOS run and the Linux run it is held against ask the CPU for the -//! same thing. +//! HWP, the energy/performance bias and package thermal status; Vol. 4 for the +//! addresses. The declared values are the power envelope the self-hosting bar +//! is measured under (`issues/build/toyos-builds-itself.md`), so a ToyOS run +//! and the Linux run it is held against ask the CPU for the same thing. #![no_std] #![forbid(unsafe_code)] @@ -19,17 +18,15 @@ pub mod msr { pub const PM_ENABLE: u32 = 0x770; pub const HWP_CAPABILITIES: u32 = 0x771; pub const HWP_REQUEST_PKG: u32 = 0x772; + /// Exists where CPUID.06H:EAX[8] says so ([`super::hwp_notification`]). + pub const HWP_INTERRUPT: u32 = 0x773; pub const HWP_REQUEST: u32 = 0x774; pub const MISC_ENABLE: u32 = 0x1A0; pub const ENERGY_PERF_BIAS: u32 = 0x1B0; pub const PACKAGE_THERM_STATUS: u32 = 0x1B1; - // Model-specific, and named by no CPUID bit: every Intel core since Sandy - // Bridge has them, and HWP is younger than all four. + /// `MSR_PLATFORM_INFO`: enumerated by no CPUID bit. Vol. 4 documents it in + /// the model tables of DisplayFamily 06H, which [`super::refusal`] requires. pub const PLATFORM_INFO: u32 = 0xCE; - pub const TEMPERATURE_TARGET: u32 = 0x1A2; - pub const RAPL_POWER_UNIT: u32 = 0x606; - pub const PKG_POWER_LIMIT: u32 = 0x610; - pub const PKG_ENERGY_STATUS: u32 = 0x611; } /// `IA32_MISC_ENABLE` bit 38: set, turbo is off. @@ -38,6 +35,10 @@ pub const TURBO_DISABLE: u64 = 1 << 38; /// `IA32_PM_ENABLE` on every CPU. pub const PM_ENABLE: u64 = 1; +/// `IA32_HWP_INTERRUPT` on every CPU that has it: no HWP notification is +/// enabled, as intel_pstate leaves it — nothing in this kernel takes one. +pub const HWP_INTERRUPT: u64 = 0; + /// `IA32_ENERGY_PERF_BIAS` on every CPU: the bar's 6, on the scale where 0 is /// performance and 15 is energy saving. pub const ENERGY_PERF_BIAS: u64 = 6; @@ -57,6 +58,7 @@ pub struct Cpuid { /// Leaf 0's `EBX`, `EDX`, `ECX`, in that order: the vendor string. pub vendor: [u8; 12], pub max_leaf: u32, + pub leaf1_eax: u32, pub leaf6_eax: u32, pub leaf6_ecx: u32, pub leaf7_edx: u32, @@ -69,6 +71,7 @@ pub enum Refusal { NoLeaf6, NoHwp, NotIntel, + NotFamily6, NoEpp, NoPackageRequest, NoEnergyPerfBias, @@ -85,6 +88,10 @@ impl Refusal { "HWP on a CPU that is not Intel, whose RAPL and thermal registers are Intel's \ model-specific ones" } + Self::NotFamily6 => { + "an Intel CPU outside DisplayFamily 06H, for which the SDM documents no \ + MSR_PLATFORM_INFO (0xCE), and no CPUID bit enumerates it" + } Self::NoEpp => "HWP without an energy/performance preference (CPUID.06H:EAX[10] clear)", Self::NoPackageRequest => "HWP without a package-level request (CPUID.06H:EAX[11] clear)", Self::NoEnergyPerfBias => "no energy/performance bias (CPUID.06H:ECX[3] clear)", @@ -114,6 +121,8 @@ pub const fn refusal(cpuid: &Cpuid) -> Option { Some(Refusal::NoHwp) } else if !is_intel(&cpuid.vendor) { Some(Refusal::NotIntel) + } else if display_family(cpuid.leaf1_eax) != 6 { + Some(Refusal::NotFamily6) } else if eax & HWP_EPP == 0 { Some(Refusal::NoEpp) } else if eax & HWP_PKG == 0 { @@ -129,6 +138,22 @@ pub const fn refusal(cpuid: &Cpuid) -> Option { } } +/// Whether this CPU has `IA32_HWP_INTERRUPT` (CPUID.06H:EAX[8]). +pub const fn hwp_notification(cpuid: &Cpuid) -> bool { + cpuid.leaf6_eax & 1 << 8 != 0 +} + +/// CPUID.01H:EAX's DisplayFamily (SDM Vol. 2A, CPUID): the extended family +/// counts only where the family field is 0FH. +const fn display_family(leaf1_eax: u32) -> u32 { + let family = (leaf1_eax >> 8) & 0xf; + if family == 0xf { + family + ((leaf1_eax >> 20) & 0xff) + } else { + family + } +} + const fn is_intel(vendor: &[u8; 12]) -> bool { let want = b"GenuineIntel"; let mut i = 0; @@ -145,11 +170,12 @@ const fn is_intel(vendor: &[u8; 12]) -> bool { /// `capabilities`, in a package whose `MSR_PLATFORM_INFO` reads /// `platform_info`. The bar names every field: the minimum is the package's /// maximum-efficiency ratio, the maximum is the CPU's highest performance — -/// turbo included — and the CPU chooses between them, unwindowed, on its own. +/// turbo included, the capabilities' bits 7:0 — and the CPU chooses between +/// them, unwindowed, on its own. pub const fn hwp_request(capabilities: u64, platform_info: u64) -> u64 { HwpRequest { min: ((platform_info >> 40) & 0xff) as u8, - max: HwpCapabilities::of(capabilities).highest, + max: capabilities as u8, desired: 0, epp: EPP, window: 0, @@ -196,26 +222,6 @@ impl HwpRequest { } } -/// `IA32_HWP_CAPABILITIES`' four performance levels. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct HwpCapabilities { - pub highest: u8, - pub guaranteed: u8, - pub efficient: u8, - pub lowest: u8, -} - -impl HwpCapabilities { - pub const fn of(raw: u64) -> Self { - Self { - highest: raw as u8, - guaranteed: (raw >> 8) as u8, - efficient: (raw >> 16) as u8, - lowest: (raw >> 24) as u8, - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -230,9 +236,18 @@ mod tests { /// What every one of the T14's eight CPUs held under Linux, intel_pstate /// active with EPP `balance_performance`. const T14_REQUEST: u64 = 0x8000_2a04; + /// CPUID.01H:EAX's family field at 6, every other field 0. + const FAMILY_6: u32 = 6 << 8; fn intel(leaf6_eax: u32, leaf6_ecx: u32, leaf7_edx: u32) -> Cpuid { - Cpuid { vendor: *b"GenuineIntel", max_leaf: 0x1b, leaf6_eax, leaf6_ecx, leaf7_edx } + Cpuid { + vendor: *b"GenuineIntel", + max_leaf: 0x1b, + leaf1_eax: FAMILY_6, + leaf6_eax, + leaf6_ecx, + leaf7_edx, + } } /// The bits a CPU needs: PTM, HWP, EPP and the package request in `EAX`, @@ -247,21 +262,25 @@ mod tests { } } + /// The maximum is the capabilities' highest level alone: the three levels + /// above it in the register are not read into the request. + #[test] + fn the_maximum_is_the_highest_level_alone() { + assert_eq!(HwpRequest::of(hwp_request(0xffff_ff2a, 0)).max, 42); + assert_eq!(HwpRequest::of(hwp_request(0x0000_0000, 0)).max, 0); + } + #[test] fn the_package_request_is_the_bars() { assert_eq!(HWP_REQUEST_PKG, 0x8000_ff01); } #[test] - fn the_t14s_registers_decode_to_the_bars_fields() { + fn the_t14s_request_decodes_to_the_bars_fields() { assert_eq!( HwpRequest::of(T14_REQUEST), HwpRequest { min: 4, max: 42, desired: 0, epp: 128, window: 0, package_control: false }, ); - assert_eq!( - HwpCapabilities::of(T14_CAPABILITIES[0]), - HwpCapabilities { highest: 42, guaranteed: 24, efficient: 13, lowest: 1 }, - ); } /// Each field alone, at the bit the SDM gives it — so two fields swapped @@ -289,7 +308,7 @@ mod tests { /// `ARAT` (bit 2); KVM's `host` passes the vendor and the same leaf 6. #[test] fn no_qemu_cpu_is_declared() { - let tcg = Cpuid { vendor: *b"AuthenticAMD", max_leaf: 0xd, leaf6_eax: 1 << 2, leaf6_ecx: 0, leaf7_edx: 0 }; + let tcg = Cpuid { vendor: *b"AuthenticAMD", max_leaf: 0xd, ..intel(1 << 2, 0, 0) }; assert_eq!(refusal(&tcg), Some(Refusal::NoHwp)); assert_eq!(refusal(&intel(1 << 2, 0, 0)), Some(Refusal::NoHwp)); } @@ -310,4 +329,26 @@ mod tests { let six = Cpuid { max_leaf: 6, ..intel(EAX, ECX, 1 << 15) }; assert_eq!(refusal(&six), None); } + + /// `MSR_PLATFORM_INFO` is the family's, so a CPU of any other family is + /// refused before the register is read — the extended family included, + /// which is 0FH plus its field and never its field alone. + #[test] + fn a_cpu_outside_family_6_is_refused_by_name() { + let of = |leaf1_eax: u32| refusal(&Cpuid { leaf1_eax, ..intel(EAX, ECX, 0) }); + assert_eq!(of(FAMILY_6), None); + // Family 0FH, extended family 0: a Pentium 4. + assert_eq!(of(0x0000_0f41), Some(Refusal::NotFamily6)); + // Family 0FH, extended family 3: DisplayFamily 12H. + assert_eq!(of(0x0030_0f00), Some(Refusal::NotFamily6)); + // An extended family beside family 6 does not count. + assert_eq!(of(0x0ff0_0600), None); + } + + #[test] + fn notification_is_bit_8_alone() { + assert!(hwp_notification(&intel(EAX | 1 << 8, ECX, 0))); + assert!(!hwp_notification(&intel(EAX, ECX, 0))); + assert!(!hwp_notification(&intel(!(1 << 8), ECX, 0))); + } } diff --git a/userland/perfstate/src/lib.rs b/userland/perfstate/src/lib.rs index 4898ee9ea4c..3bddc6af84b 100644 --- a/userland/perfstate/src/lib.rs +++ b/userland/perfstate/src/lib.rs @@ -19,16 +19,8 @@ pub fn read_back(claim: &Device) -> Result<(), String> { } let pkg = PackageRegisters::read_from(&buf).expect("the buffer holds the package record"); println!( - "pkg hwp_request_pkg={:#010x} platform_info={:#018x} rapl_power_unit={:#x} \ - pkg_power_limit={:#018x} pkg_energy_status={:#x} package_therm_status={:#x} \ - temperature_target={:#x}", - pkg.hwp_request_pkg, - pkg.platform_info, - pkg.rapl_power_unit, - pkg.pkg_power_limit, - pkg.pkg_energy_status, - pkg.package_therm_status, - pkg.temperature_target, + "pkg hwp_request_pkg={:#010x} platform_info={:#018x} package_therm_status={:#x}", + pkg.hwp_request_pkg, pkg.platform_info, pkg.package_therm_status, ); check("pkg", "hwp_request_pkg", pkg.hwp_request_pkg, toyos_perfstate::HWP_REQUEST_PKG)?; for cpu in 0..cpus { From ee20f53d8f2dd593b476443e9f8e98f1ae9dfe89 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 00:14:39 +0200 Subject: [PATCH 03/12] perf_state_silent_cpu claims only what two refusals show A second Io read cannot tell a cleared ask from a stale one, since a stale ask past its deadline is refused Io too. So the second read shows that the claim still answers after a refusal, and nothing more. The comments now say only that. Co-Authored-By: Claude Opus 5.5 --- tests/toyos-rust-tests/src/bin/perf_state_silent.rs | 2 +- tests/toyos.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs index b4f73dcd81c..9d36c412ab6 100644 --- a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs +++ b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs @@ -17,7 +17,7 @@ fn main() { .claim::(DeviceType::PerfState) .expect("perf-state-deaf-cpu grants the claim on any machine"); let mut buf = vec![0u8; answer_len(syscall::cpu_count() as usize)]; - // Twice: a refused read leaves no ask behind that answers the next. + // Twice: the claim still answers after a refusal. for _ in 0..2 { assert_eq!(claim.read(&mut buf), Err(SyscallError::Io)); } diff --git a/tests/toyos.rs b/tests/toyos.rs index 40f222b8959..c5c3daae472 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -19760,8 +19760,8 @@ fn perf_request_diverged(boot: &metal::Readback) -> Result<(), String> { } /// A read one CPU never answers is refused `Io` once the kernel's bound has -/// passed, naming that CPU and no other — twice, so a refused ask answers no -/// later read. `perf-state-deaf-cpu` grants the claim on QEMU's CPUs, which +/// passed, naming that CPU and no other — twice, so the claim still answers +/// after a refusal. `perf-state-deaf-cpu` grants the claim on QEMU's CPUs, which /// have no HWP, and silences the last CPU; a read with no bound waits for /// ever, and this reds at its ceiling. fn perf_state_silent_cpu( From 148c20786a47ba690a13dd66cc43cd96d5dd3f31 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 03:27:46 +0200 Subject: [PATCH 04/12] Round 2 review fixes: a perf-state read owns its ask, and no QEMU verdict rests on time A read's ask now lives on `sys_read`'s stack (`perf_state::Ask`), made by the read's first look and gone with the read. The claim holds no ask, so a read that ends answered, refused, cancelled or as a nonblocking `WouldBlock` leaves nothing a later read is answered from or refused by, which is what toyos-abi/src/syscall.rs's `PerfState` doc promises. `Reader::cancel` and `DeviceClaim::cancel_perf_state` are deleted: there is nothing left to take back. The doc's "reads of one claim that overlap share one ask" is deleted, since every read now asks; a CPU's one answer still serves every ask issued before it. With no standing ask there is no readiness before a read: `has_data` is false and `read_watch` is `None` for the class, so a poll on a claim is refused `NotSupported` rather than reporting a readiness the next nonblocking read would not honour. `perf-state-deaf-cpu` now has no CPU answer a kick: the asker answers itself inline in `Ask::issue`, so every read on two CPUs names exactly the one other CPU whichever CPU the test runs on, and no verdict waits on a kicked vCPU being scheduled inside 250 ms. The refusal names the ask (`did not answer Generation(N)`). `test_rs_perf_state_silent` makes a nonblocking read first (the boot's first ask, `WouldBlock`), then two blocking reads, and `perf_state_silent_cpu` requires the refusals to name Generation(2) and Generation(3), once each, one CPU each: a read answered or refused from an ask not its own reds. kernel-loom/tests/shootdown_answer.rs models the target-to-initiator edge the slots use: `serve`'s closure writes a loom cell and the initiator reads it once `served` answers. `HwpDeclared::ask` asserts `control_regs::report` has run, which is after every committed CPU ran `init`, so the proof's "enabled on every CPU" is enforced rather than true by boot order alone. Also: `Enumerated` inlined as `{:x?}`; the `NotIntel` reason names MSR_PLATFORM_INFO rather than RAPL; the removed comments (toyos-perfstate lib.rs's envelope citation, perf_state.rs's "within one timer interrupt", shootdown.rs's "nothing reads through this edge yet", system.toml's row comment) are deleted; the track issue drops the same false citation, says the Linux readings are #568's unmerged samples, and records that no test launches /system/bin/perfstate. Co-Authored-By: Claude Opus 5.5 --- .../the-kernel-owns-cpu-performance-state.md | 9 +- kernel-loom/tests/shootdown_answer.rs | 57 +++++++++++ kernel/src/actuator.rs | 5 +- kernel/src/arch/x86_64/control_regs.rs | 29 +++--- kernel/src/object/device.rs | 19 ++-- kernel/src/object/ops.rs | 8 +- kernel/src/perf_state.rs | 99 ++++++++----------- kernel/src/shootdown.rs | 2 +- kernel/src/syscall/io.rs | 28 +++--- system.toml | 2 - .../src/bin/perf_state_silent.rs | 13 ++- tests/toyos.rs | 29 +++--- toyos-abi/src/syscall.rs | 5 +- toyos-perfstate/src/lib.rs | 8 +- 14 files changed, 179 insertions(+), 134 deletions(-) create mode 100644 kernel-loom/tests/shootdown_answer.rs diff --git a/issues/kernel/the-kernel-owns-cpu-performance-state.md b/issues/kernel/the-kernel-owns-cpu-performance-state.md index 1e9d64a1d07..57edd6b2c74 100644 --- a/issues/kernel/the-kernel-owns-cpu-performance-state.md +++ b/issues/kernel/the-kernel-owns-cpu-performance-state.md @@ -6,8 +6,8 @@ opened: 2026-09-28 # The kernel owns CPU performance state -The self-hosting bar (`issues/build/toyos-builds-itself.md`) is measured -under a fixed power envelope that must be read back for a whole build span. +The self-hosting bar is measured under a fixed power envelope that must be +read back for a whole build span. The kernel declares the envelope from the one CPU-state declaration (`kernel/src/arch/x86_64/control_regs.rs`), and a `perf-state` claim reads it back per CPU. @@ -32,6 +32,8 @@ enumerated by CPUID or read at boot under the declaration's proof hwp_request_pkg=0x8000ff01 epb=6` and `test_rs_perf_state` exits 0; on `perfdiverge` the page after the reset carries the panic `control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04`. + No test launches `/system/bin/perfstate`, so its row's `devices` is + unmeasured. - **2 — RAPL, declared.** PL1, PL2 and their windows through `MSR_PKG_POWER_LIMIT`, and the MMIO mirror in the host bridge's MCHBAR, at the bar's values; the peak limit beside them. A limit firmware locked (bit @@ -61,7 +63,8 @@ the AArch64 kernel refuses the claim by name. **What only the T14 proves.** No QEMU CPU enumerates HWP (TCG's `qemu64`, and KVM, which reduces leaf 6 to `ARAT`), so every write and every read of these -registers runs only there. Under Linux on the T14 every CPU held +registers runs only there. Under Linux on the T14 (#568's samples, not merged; +the metal row re-measures them) every CPU held `IA32_HWP_REQUEST` `0x80002a04` with `IA32_HWP_CAPABILITIES` `0x010d182a` or `0x010e182a`, and the package `IA32_HWP_REQUEST_PKG` `0x8000ff01`. `MSR_PLATFORM_INFO` was not read there; its ratio 4 is inferred from Linux's diff --git a/kernel-loom/tests/shootdown_answer.rs b/kernel-loom/tests/shootdown_answer.rs new file mode 100644 index 00000000000..f3c9090d82b --- /dev/null +++ b/kernel-loom/tests/shootdown_answer.rs @@ -0,0 +1,57 @@ +//! Loom: what a target wrote while serving a generation is what an initiator +//! that saw it served reads — the edge a `perf-state` read copies each CPU's +//! registers through. The slot is a loom cell rather than an atomic, so a read +//! unordered against the write is loom's own `Causality violation`. + +#![cfg(feature = "loom")] + +use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; + +use kernel_loom::shootdown::Shootdown; +use loom::cell::UnsafeCell; +use loom::sync::Arc; + +/// What the target's serve stores. +const SAMPLE: u64 = 0x8000_2a04; + +/// Set by any execution in which the initiator saw the answer, so the +/// assertion is shown to have run. Outside the model because loom re-runs the +/// closure once per interleaving. +static SEEN: AtomicBool = AtomicBool::new(false); + +struct Machine { + shootdown: Shootdown, + /// cpu 1's answer slot. + slot: UnsafeCell, +} + +// SAFETY: `slot` is written only inside cpu 1's `serve` and read only after +// `served` says that serve finished; that ordering is what the model checks. +unsafe impl Sync for Machine {} + +#[test] +fn what_a_serve_wrote_is_read_once_it_is_served() { + SEEN.store(false, SeqCst); + loom::model(|| { + let m = Arc::new(Machine { shootdown: Shootdown::new(), slot: UnsafeCell::new(0) }); + let generation = m.shootdown.issue(); + + let target = { + let m = m.clone(); + loom::thread::spawn(move || { + // SAFETY: the only write, before the serve's publication. + m.shootdown.serve(1, || m.slot.with_mut(|slot| unsafe { *slot = SAMPLE })); + }) + }; + + if m.shootdown.served(1, generation) { + SEEN.store(true, SeqCst); + // SAFETY: `served` answered, so the write happened before this. + let read = m.slot.with(|slot| unsafe { *slot }); + assert_eq!(read, SAMPLE, "the initiator saw cpu 1 served and read its slot unwritten"); + } + + target.join().unwrap(); + }); + assert!(SEEN.load(SeqCst), "no interleaving saw the answer, so the assertion never ran"); +} diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 275c0c85bf5..1324f21a775 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -286,8 +286,9 @@ actuators! { dump_deaf_cpu = "dump-deaf-cpu"; /// Grant a `perf-state` claim where no performance request was declared, - /// answering zeros, and have the last CPU never answer its ask: what the - /// read's bound refuses, on a machine QEMU can stage. + /// answering zeros, and have no CPU answer a kick, so every CPU but a + /// read's asker is silent: what the read's bound refuses, on a machine + /// QEMU can stage. perf_state_deaf_cpu = "perf-state-deaf-cpu"; /// Have cpu1 move its HWP request off the declaration when it answers a diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index e8ac4965c2d..4781e36fa9f 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -13,7 +13,7 @@ //! `IA32_MISC_ENABLE`'s turbo bit is not declared: that register's other bits //! are model-specific and firmware's, and writing it whole would decide them. -use core::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; use toyos_perfstate::msr; @@ -129,6 +129,10 @@ const HWP_UNDECIDED: u8 = 0; const HWP_DECLARED: u8 = 1; const HWP_REFUSED: u8 = 2; +/// Set by [`report`], once every CPU the roster committed has run [`init`]: +/// before it, an AP may not have enabled HWP yet. +static APPLIED: AtomicBool = AtomicBool::new(false); + /// Proof that the machine's declaration carries the performance request, so /// every register [`toyos_perfstate::msr`] names but `HWP_INTERRUPT` exists on /// every CPU, HWP is enabled there, and reading one is no `#GP`. @@ -138,11 +142,15 @@ pub struct HwpDeclared(()); impl HwpDeclared { /// The proof, or the reason this machine has none. pub fn ask() -> Result { + assert!( + APPLIED.load(Ordering::Acquire), + "control_regs: the performance request is asked about before every CPU applied it", + ); match HWP.load(Ordering::Acquire) { HWP_DECLARED => Ok(Self(())), HWP_REFUSED => Err(toyos_perfstate::refusal(&perf_cpuid()) .expect("every CPU reached the BSP's refusal, this one included")), - _ => panic!("control_regs: the performance request is asked about before the BSP declared it"), + _ => unreachable!("control_regs: every CPU that ran `init` decided"), } } } @@ -410,13 +418,13 @@ fn hwp_check(cpu_id: u32, request: u64, notifies: bool) { let interrupt = notifies.then(|| cpu::rdmsr(msr::HWP_INTERRUPT)); log!( "control_regs: cpu{} pm_enable={} hwp_request={:#010x} hwp_request_pkg={:#010x} epb={} \ - hwp_interrupt={} hwp_capabilities={:#010x} platform_info={:#018x}", + hwp_interrupt={:x?} hwp_capabilities={:#010x} platform_info={:#018x}", cpu_id, pm_enable, live, pkg, epb, - Enumerated(interrupt), + interrupt, cpu::rdmsr(msr::HWP_CAPABILITIES), cpu::rdmsr(msr::PLATFORM_INFO), ); @@ -436,18 +444,6 @@ fn hwp_check(cpu_id: u32, request: u64, notifies: bool) { } } -/// A register's value, or `absent` where CPUID enumerates no such register. -struct Enumerated(Option); - -impl core::fmt::Display for Enumerated { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self.0 { - Some(value) => write!(f, "{value:#x}"), - None => f.write_str("absent"), - } - } -} - /// How many CPUs hold the declaration, said once after the last of them has /// been checked. A divergent CPU panics inside [`self_check`], so what this /// line adds is the *count*: a CPU that never reached [`init`] at all is @@ -464,6 +460,7 @@ pub fn report(cpus: u32) { DECLARED_CR4.load(Ordering::Acquire), EFER, ); + APPLIED.store(true, Ordering::Release); } fn opt(value: u64, bit: u64, name: &'static str) -> &'static str { diff --git a/kernel/src/object/device.rs b/kernel/src/object/device.rs index 141c90236cb..2aa237e9244 100644 --- a/kernel/src/object/device.rs +++ b/kernel/src/object/device.rs @@ -158,22 +158,19 @@ impl DeviceClaim { Some((device, unique)) } - /// A performance-state claim's read, which [`crate::perf_state::Reader`] answers. - pub fn read_perf_state(&self, buf: &mut crate::user_ptr::UserBytesMut) -> Option { + /// A performance-state claim's read, which [`crate::perf_state::Reader`] answers + /// against the read's own `ask`. + pub fn read_perf_state( + &self, + ask: &mut Option, + buf: &mut crate::user_ptr::UserBytesMut, + ) -> Option { match &self.described.lock().info { - DeviceInfo::PerfState(reader) => reader.read(buf), + DeviceInfo::PerfState(reader) => reader.read(ask, buf), _ => unreachable!("a {:?} claim is not read as a performance-state one", self.class), } } - /// A performance-state read that was cancelled while it waited. - pub fn cancel_perf_state(&self) { - match &self.described.lock().info { - DeviceInfo::PerfState(reader) => reader.cancel(), - _ => unreachable!("a {:?} claim has no performance-state read to cancel", self.class), - } - } - pub fn info_read(&self) -> bool { self.info_read.load(Ordering::Relaxed) } diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 8fafc18e97a..1c450335fe8 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -263,7 +263,8 @@ pub fn read_watch(object: &KObjectRef) -> Option { device_registry::DeviceType::Framebuffer => None, // A partition answers its description and has nothing to wait for. device_registry::DeviceType::Partition => None, - device_registry::DeviceType::PerfState => Some(WatchRef::Static(&crate::perf_state::WATCH)), + // A read asks when it runs, so nothing is ready before one. + device_registry::DeviceType::PerfState => None, }, // Named unconditionally: the watch alone cannot enforce rights. KObjectRef::SysCap(_) => Some(WatchRef::Static(&crate::log::user::WATCH)), @@ -391,7 +392,8 @@ pub fn read_device( // Every read is the description: a partition's bytes move through // `SYS_PARTITION_READ`, never through a read of the claim. device_registry::DeviceType::Partition => Some(claim.describe(table, buf)), - device_registry::DeviceType::PerfState => claim.read_perf_state(buf), + // A read that does not wait: its ask ends with it. + device_registry::DeviceType::PerfState => claim.read_perf_state(&mut None, buf), // The description first and interrupts after, the shape the HDA stub // has: a driver reads what it is driving once, and everything it reads // afterwards is what its device has been doing. @@ -843,7 +845,7 @@ pub fn has_data(object: &KObjectRef) -> bool { } device_registry::DeviceType::Framebuffer => true, device_registry::DeviceType::Partition => true, - device_registry::DeviceType::PerfState => crate::perf_state::answered(), + device_registry::DeviceType::PerfState => false, device_registry::DeviceType::HdaAudio => { !d.info_read() || crate::drivers::hda::has_pending() } diff --git a/kernel/src/perf_state.rs b/kernel/src/perf_state.rs index c5077e78939..07ace0573b7 100644 --- a/kernel/src/perf_state.rs +++ b/kernel/src/perf_state.rs @@ -10,6 +10,10 @@ //! for a generation was read after that generation was issued. A slot that a //! later ask overwrites mid-copy mixes two answers, each read after this one //! asked. +//! +//! **An ask belongs to the one read that made it** and lives on that read's +//! stack, so a read that ends — answered, refused, cancelled or not waiting — +//! leaves nothing a later read could be answered from or refused by. use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; @@ -18,7 +22,6 @@ use toyos_abi::syscall::SyscallError; use crate::arch::perf_state::{self, Declared}; use crate::shootdown::{Generation, Shootdown, MAX_CPUS}; -use crate::sync::Lock; use crate::time::{Budget, Deadline, Duration, Instant}; use crate::user_ptr::UserBytesMut; use crate::watch::Watch; @@ -29,8 +32,6 @@ static SLOTS: [Slot; MAX_CPUS] = [const { Slot::new() }; MAX_CPUS]; /// Posted by every answer; a blocked read waits here. pub static WATCH: Watch = Watch::new(); -/// A kicked CPU reaches a pass within one timer interrupt; this is the -/// kernel's choice. const ANSWER: Budget = Budget::of( Duration::from_millis(250), "the read is refused `Io`, and the CPUs that did not answer are named", @@ -57,57 +58,71 @@ impl Slot { } } -/// One claim's side of the protocol: the proof its reads need, and the ask -/// its reads wait on — joined by every read of the claim until it is answered -/// or a reader waiting on it is cancelled. +/// One claim's side of the protocol: the proof its reads need. pub struct Reader { /// `None` only under `perf-state-deaf-cpu`, whose claim has no /// declaration behind it and answers zeros. declared: Option, - pending: Lock>, } +/// One read's ask, made by its first look and gone with the read. #[derive(Clone, Copy)] -struct Ask { +pub struct Ask { generation: Generation, deadline: Deadline, } +impl Ask { + /// Issued, this CPU's answer given, and every other CPU kicked — in that + /// order, so no kicked CPU can look before the generation it owes exists. + fn issue(cpus: usize, now: Instant) -> Self { + let generation = ASKS.issue(); + let me = crate::arch::percpu::cpu_id() as usize; + ASKS.serve(me, || SLOTS[me].store(sample(me))); + for cpu in (0..cpus).filter(|&cpu| cpu != me) { + crate::arch::irqchip::kick_cpu(cpu as u32); + } + Self { generation, deadline: Deadline::at(now + ANSWER.duration()) } + } + + /// Whether every CPU has answered this ask: a blocked read's wake condition. + pub fn answered(&self) -> bool { + (0..crate::arch::smp::cpu_count() as usize).all(|cpu| ASKS.served(cpu, self.generation)) + } + + /// Past this the read is refused, so its wait ends here. + pub fn deadline(&self) -> Deadline { + self.deadline + } +} + impl Reader { /// A claim's reader, or why this machine has none. pub fn claim() -> Result { let declared = perf_state::declared().map(Some).or_else(|why| { if crate::actuator::perf_state_deaf_cpu() { Ok(None) } else { Err(why) } })?; - Ok(Self { declared, pending: Lock::new(None) }) - } - - /// A read that was cancelled takes its ask back, so the next read asks - /// afresh and is never answered with registers read before it began. - pub fn cancel(&self) { - *self.pending.lock() = None; + Ok(Self { declared }) } - /// The answer's bytes into `buf`, or `None` while a CPU has not answered — - /// the caller then waits on [`WATCH`] until [`answered`]. - pub fn read(&self, buf: &mut UserBytesMut) -> Option { + /// The answer's bytes into `buf`, or `None` while a CPU has not answered + /// `ask` — the caller then waits on [`WATCH`] until [`Ask::answered`]. + /// `ask` is the read's own, `None` until its first look makes it. + pub fn read(&self, ask: &mut Option, buf: &mut UserBytesMut) -> Option { let cpus = crate::arch::smp::cpu_count() as usize; let len = answer_len(cpus); if buf.len() < len { return Some(SyscallError::ResourceExhausted.to_u64()); } let now = crate::clock::now(); - let mut pending = self.pending.lock(); - let ask = *pending.get_or_insert_with(|| self.ask(cpus, now)); - let answered = (0..cpus).all(|cpu| ASKS.served(cpu, ask.generation)); + let ask = *ask.get_or_insert_with(|| Ask::issue(cpus, now)); + let answered = ask.answered(); if !answered && !ask.deadline.reached(now) { return None; } - *pending = None; - drop(pending); if !answered { for cpu in (0..cpus).filter(|&cpu| !ASKS.served(cpu, ask.generation)) { - crate::log!("perf_state: cpu{cpu} did not answer a read within {ANSWER}"); + crate::log!("perf_state: cpu{cpu} did not answer {:?} within {ANSWER}", ask.generation); } return Some(SyscallError::Io.to_u64()); } @@ -119,38 +134,14 @@ impl Reader { } Some(len as u64) } - - /// Issued, this CPU's answer given, and every other CPU kicked — in that - /// order, so no kicked CPU can look before the generation it owes exists. - fn ask(&self, cpus: usize, now: Instant) -> Ask { - let generation = ASKS.issue(); - let me = crate::arch::percpu::cpu_id() as usize; - if !deaf(me, cpus) { - ASKS.serve(me, || SLOTS[me].store(sample(me))); - } - for cpu in (0..cpus).filter(|&cpu| cpu != me) { - crate::arch::irqchip::kick_cpu(cpu as u32); - } - Ask { generation, deadline: Deadline::at(now + ANSWER.duration()) } - } -} - -/// Whether every CPU has answered the latest ask: a blocked read's wake -/// condition, and a hint — [`Reader::read`] decides. -pub fn answered() -> bool { - (0..crate::arch::smp::cpu_count() as usize).all(|cpu| !ASKS.owes(cpu)) -} - -/// How long a blocked read parks before it looks again; its own ask's -/// deadline is what refuses it. -pub fn park_deadline() -> Deadline { - Deadline::at(crate::clock::now() + ANSWER.duration()) } /// This CPU's answer, if one is owed. Called from `drain_irqs` every pass. +/// Under `perf-state-deaf-cpu` no CPU answers here, so a read is answered by +/// its asker alone. pub fn serve_if_owed() { let me = crate::arch::percpu::cpu_id() as usize; - if !ASKS.owes(me) || deaf(me, crate::arch::smp::cpu_count() as usize) { + if !ASKS.owes(me) || crate::actuator::perf_state_deaf_cpu() { return; } ASKS.serve(me, || SLOTS[me].store(sample(me))); @@ -181,9 +172,3 @@ fn sample(me: usize) -> CpuRegisters { } } } - -/// `perf-state-deaf-cpu`'s silent CPU, the last: it answers no ask, its own -/// included. -fn deaf(cpu: usize, cpus: usize) -> bool { - crate::actuator::perf_state_deaf_cpu() && cpu + 1 == cpus -} diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 190b97f0783..06f2471914d 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -70,7 +70,7 @@ impl Shootdown { /// Has `cpu` flushed since `generation` was issued? pub fn served(&self, cpu: usize, generation: Generation) -> bool { - // Acquire: nothing reads through this edge yet, but `Relaxed` here would be silently unsafe once something does. + // Acquire: pairs with `serve`'s Release, so what its closure wrote is visible once this is true. self.flushed[cpu].load(Ordering::Acquire) >= generation.0 } diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index 3ff37f26727..fae9a2ba5fa 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -39,9 +39,8 @@ enum ReadBlock { /// the serial line, never on the keyboard's queue, which only a claim /// drains and which would answer it at once for as long as a key sits there. Console(Deadline), - /// A performance-state read, until every CPU has answered its ask; the - /// claim is carried so a cancelled wait can take the ask back. - PerfState(alloc::sync::Arc, Deadline), + /// A performance-state read, until every CPU has answered its own ask. + PerfState(crate::perf_state::Ask), /// Nothing to wait for: the answer is this word. Refused(u64), /// Carried out of the process's lock: `HandleError::refuse` may take the @@ -93,16 +92,13 @@ pub(super) fn sys_write(h: RawHandle, buf: &UserBytes) -> u64 { } } -/// Only these device classes block; the rest answer `NotFound` on an empty -/// blocking read. -fn read_block_device(claim: &alloc::sync::Arc) -> ReadBlock { +/// Only these four device classes block; the rest answer `NotFound` on an +/// empty blocking read. +fn read_block_device(claim: &crate::object::device::DeviceClaim) -> ReadBlock { match claim.class() { device::DeviceType::Keyboard => ReadBlock::Keyboard(Deadline::never()), device::DeviceType::VirtioSound if claim.info_read() => ReadBlock::VirtioSound, device::DeviceType::HdaAudio if claim.info_read() => ReadBlock::Hda, - device::DeviceType::PerfState => { - ReadBlock::PerfState(claim.clone(), crate::perf_state::park_deadline()) - } _ => ReadBlock::Refused(SyscallError::NotFound.to_u64()), } } @@ -129,6 +125,7 @@ fn read_block(object: &KObjectRef) -> ReadBlock { } pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { + let mut ask = None; loop { let action = process::with_process_data(|data| { let object = match data.handles.get_ref(h, Rights::READ) { @@ -143,6 +140,12 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { .handles .get::(h, Rights::READ) .expect("a Device resolved a moment ago under this same hold"); + if claim.class() == device::DeviceType::PerfState { + return match claim.read_perf_state(&mut ask, buf) { + Some(n) => Ok((n, None)), + None => Err(ReadBlock::PerfState(ask.expect("a read that waits has asked"))), + }; + } let blocked = read_block_device(&claim); return match ops::read_device(&claim, &mut data.handles, buf) { Some(n) => Ok((n, None)), @@ -219,19 +222,18 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { return cancelled(); } } - Err(ReadBlock::PerfState(claim, deadline)) => { + Err(ReadBlock::PerfState(ask)) => { let parkable = crate::scheduler::Parkable::at_entry(); if watch::wait_until( &parkable, &crate::perf_state::WATCH, 0, WaitClass::Io, - deadline, - crate::perf_state::answered, + ask.deadline(), + || ask.answered(), ) .is_err() { - claim.cancel_perf_state(); return cancelled(); } } diff --git a/system.toml b/system.toml index 239e93621e2..b2fee1bb2d5 100644 --- a/system.toml +++ b/system.toml @@ -106,8 +106,6 @@ receives = ["netd"] receives = ["netd", "soundd", "log", "compositor"] syscap = ["inventory"] -# The CPU performance envelope read back once: `perf-state` is the whole of -# what it holds, and a machine that declared no request has none to give it. [programs.perfstate] devices = ["perf-state"] diff --git a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs index 9d36c412ab6..0114be445aa 100644 --- a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs +++ b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs @@ -1,11 +1,13 @@ -//! A `perf-state` read one CPU never answers (`perf-state-deaf-cpu`, which -//! also grants the claim where no request was declared): refused `Io` once the -//! kernel's bound has passed, never answered and never left waiting. -//! `perf_state_silent_cpu` drives it and reads which CPU the kernel named. +//! A `perf-state` read no CPU but its asker answers (`perf-state-deaf-cpu`, +//! which also grants the claim where no request was declared): refused `Io` +//! once the kernel's bound has passed, never answered and never left waiting. +//! A read that does not wait goes first, and each read after it must make an +//! ask of its own. `perf_state_silent_cpu` drives it and reads which ask and +//! which CPU the kernel named. use toyos::endow::Endowments; use toyos::syscap::SysCap; -use toyos::Device; +use toyos::{AsHandle, Device}; use toyos_abi::perf::answer_len; use toyos_abi::syscall::{self, DeviceType, SyscallError, SYSCAP_LABEL}; @@ -17,6 +19,7 @@ fn main() { .claim::(DeviceType::PerfState) .expect("perf-state-deaf-cpu grants the claim on any machine"); let mut buf = vec![0u8; answer_len(syscall::cpu_count() as usize)]; + assert_eq!(syscall::read_nonblock(claim.as_handle(), &mut buf), Err(SyscallError::WouldBlock)); // Twice: the claim still answers after a refusal. for _ in 0..2 { assert_eq!(claim.read(&mut buf), Err(SyscallError::Io)); diff --git a/tests/toyos.rs b/tests/toyos.rs index 23108643c53..c2420dee250 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -17733,10 +17733,13 @@ fn perf_request_diverged(boot: &metal::Readback) -> Result<(), String> { Ok(()) } -/// A read one CPU never answers is refused `Io` once the kernel's bound has -/// passed, naming that CPU and no other — twice, so the claim still answers -/// after a refusal. `perf-state-deaf-cpu` grants the claim on QEMU's CPUs, which -/// have no HWP, and silences the last CPU; a read with no bound waits for +/// A read only its asker answers is refused `Io` once the kernel's bound has +/// passed, naming the one other CPU — twice, so the claim still answers after +/// a refusal. `perf-state-deaf-cpu` grants the claim on QEMU's CPUs, which have +/// no HWP, and has no CPU answer a kick, so which CPU the test runs on decides +/// nothing. A read that did not wait goes first and makes the boot's first ask; +/// the two refusals must name the second and the third, so a read answered or +/// refused from an ask that is not its own reds. A read with no bound waits for /// ever, and this reds at its ceiling. fn perf_state_silent_cpu( test_config: &Path, @@ -17744,8 +17747,8 @@ fn perf_state_silent_cpu( rust_bins: &[(String, Vec)], ) -> Result<(), String> { const CPUS: u32 = 2; - const SILENT: &str = " did not answer a read within "; - const READS: usize = 2; + const SILENT: &str = " did not answer Generation("; + const ASKS: [&str; 2] = [" did not answer Generation(2) ", " did not answer Generation(3) "]; let mut qemu = QemuInstance::boot_with_options( test_config, c_bins, @@ -17765,8 +17768,8 @@ fn perf_state_silent_cpu( text.lines().filter(|l| l.contains(SILENT)).map(str::to_string).collect() }; let mut refusals = named(&result.serial); - if refusals.len() < READS { - let owed = READS - refusals.len(); + if refusals.len() < ASKS.len() { + let owed = ASKS.len() - refusals.len(); let seen = std::cell::Cell::new(0); let more = qemu.drain_until(Duration::from_secs(10), |line| { seen.set(seen.get() + usize::from(line.contains(SILENT))); @@ -17774,16 +17777,16 @@ fn perf_state_silent_cpu( }); refusals.extend(named(&more)); } - let want = format!("perf_state: cpu{}{SILENT}", CPUS - 1); - if refusals.len() != READS || !refusals.iter().all(|l| l.contains(&want)) { + let one_cpu = |l: &String| (0..CPUS).filter(|c| l.contains(&format!("perf_state: cpu{c} "))).count() == 1; + let each_once = ASKS.iter().all(|ask| refusals.iter().filter(|l| l.contains(ask)).count() == 1); + if refusals.len() != ASKS.len() || !each_once || !refusals.iter().all(one_cpu) { return Err(format!( - "want {READS} refusals each naming cpu{} alone, got {}:\n{}\n{}", - CPUS - 1, + "want one refusal naming one CPU for each of {ASKS:?}, got {}:\n{}\n{}", refusals.len(), refusals.join("\n"), result.serial, )); } - eprintln!(" [perf_state_silent_cpu] {READS} reads refused Io, each naming cpu{}", CPUS - 1); + eprintln!(" [perf_state_silent_cpu] the second and third asks refused Io, each naming one CPU"); Ok(()) } diff --git a/toyos-abi/src/syscall.rs b/toyos-abi/src/syscall.rs index 272c0c689d4..29f3d7e88ac 100644 --- a/toyos-abi/src/syscall.rs +++ b/toyos-abi/src/syscall.rs @@ -1266,9 +1266,8 @@ device_classes! { Partition = 8 => "part", /// The CPU performance envelope's registers, read back: every read answers /// [`crate::perf`]'s records, each CPU's taken on that CPU after the read - /// asked; reads of one claim that overlap share one ask. Read-only — the - /// kernel writes the declaration and nothing else does. `NotFound` on a - /// machine whose CPUs got no declared request. + /// asked. Read-only — the kernel writes the declaration and nothing else + /// does. `NotFound` on a machine whose CPUs got no declared request. PerfState = 9 => "perf-state", } diff --git a/toyos-perfstate/src/lib.rs b/toyos-perfstate/src/lib.rs index c279650f1d2..9608cbdc67f 100644 --- a/toyos-perfstate/src/lib.rs +++ b/toyos-perfstate/src/lib.rs @@ -5,9 +5,7 @@ //! //! Layouts are the Intel SDM's — Vol. 3B, *Power and Thermal Management*, for //! HWP, the energy/performance bias and package thermal status; Vol. 4 for the -//! addresses. The declared values are the power envelope the self-hosting bar -//! is measured under (`issues/build/toyos-builds-itself.md`), so a ToyOS run -//! and the Linux run it is held against ask the CPU for the same thing. +//! addresses. #![no_std] #![forbid(unsafe_code)] @@ -85,8 +83,8 @@ impl Refusal { Self::NoLeaf6 => "CPUID has no leaf 6, so no power management is enumerated", Self::NoHwp => "no HWP (CPUID.06H:EAX[7] clear)", Self::NotIntel => { - "HWP on a CPU that is not Intel, whose RAPL and thermal registers are Intel's \ - model-specific ones" + "HWP on a CPU that is not Intel, and the request's minimum comes from \ + MSR_PLATFORM_INFO (0xCE), which is Intel's" } Self::NotFamily6 => { "an Intel CPU outside DisplayFamily 06H, for which the SDM documents no \ From 09970cc482fed72f742a813b5fb6db398f8d13c2 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 06:21:54 +0200 Subject: [PATCH 05/12] Round 3 review fixes: drop the duplicate loom model and two false claims `shootdown_answer.rs` duplicated `tlb_shootdown.rs`, which already goes red under the same `served` Acquire-to-Relaxed mutation, so its own model was dead weight. `kernel/src/syscall/io.rs`'s device-class count was already wrong against its match arm, and the issue's citation of an unmerged PR's unlogged numbers rotted the moment it was written. The nonblocking-read NOTE is filed rather than fixed, since it stays a NOTE this round. Co-Authored-By: Claude Opus 5.5 --- ...f-state-read-can-only-answer-wouldblock.md | 19 +++++++ .../the-kernel-owns-cpu-performance-state.md | 3 +- kernel-loom/tests/shootdown_answer.rs | 57 ------------------- kernel/src/syscall/io.rs | 2 - 4 files changed, 20 insertions(+), 61 deletions(-) create mode 100644 issues/kernel/a-nonblocking-perf-state-read-can-only-answer-wouldblock.md delete mode 100644 kernel-loom/tests/shootdown_answer.rs diff --git a/issues/kernel/a-nonblocking-perf-state-read-can-only-answer-wouldblock.md b/issues/kernel/a-nonblocking-perf-state-read-can-only-answer-wouldblock.md new file mode 100644 index 00000000000..76c23100804 --- /dev/null +++ b/issues/kernel/a-nonblocking-perf-state-read-can-only-answer-wouldblock.md @@ -0,0 +1,19 @@ +--- +status: open +kind: defect +opened: 2026-09-29 +--- + +# A nonblocking perf-state read can only answer `WouldBlock` + +`kernel/src/object/ops.rs:396` (`DeviceType::PerfState => claim.read_perf_state(&mut None, buf)`) +is the nonblocking arm: on SMP it always issues a fresh ask, sends an IPI to +kick every other CPU, and still returns `WouldBlock`, because the ask cannot +be answered within the call that issued it. No retry ever succeeds, and a +poll on the claim is refused rather than reporting the readiness a +nonblocking read would need. A CPU-wide shootdown that can only ever answer +"try a blocking read instead" is pure cost. + +Exit: `read_block_device`/poll on `PerfState` refuse `NotSupported` by name, +so a caller learns not to retry nonblocking rather than paying the kick to be +told again; `object::ops.rs`'s `&mut None` arm for `PerfState` goes with it. diff --git a/issues/kernel/the-kernel-owns-cpu-performance-state.md b/issues/kernel/the-kernel-owns-cpu-performance-state.md index 57edd6b2c74..033c33889ff 100644 --- a/issues/kernel/the-kernel-owns-cpu-performance-state.md +++ b/issues/kernel/the-kernel-owns-cpu-performance-state.md @@ -63,8 +63,7 @@ the AArch64 kernel refuses the claim by name. **What only the T14 proves.** No QEMU CPU enumerates HWP (TCG's `qemu64`, and KVM, which reduces leaf 6 to `ARAT`), so every write and every read of these -registers runs only there. Under Linux on the T14 (#568's samples, not merged; -the metal row re-measures them) every CPU held +registers runs only there. Under Linux on the T14 every CPU held `IA32_HWP_REQUEST` `0x80002a04` with `IA32_HWP_CAPABILITIES` `0x010d182a` or `0x010e182a`, and the package `IA32_HWP_REQUEST_PKG` `0x8000ff01`. `MSR_PLATFORM_INFO` was not read there; its ratio 4 is inferred from Linux's diff --git a/kernel-loom/tests/shootdown_answer.rs b/kernel-loom/tests/shootdown_answer.rs deleted file mode 100644 index f3c9090d82b..00000000000 --- a/kernel-loom/tests/shootdown_answer.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! Loom: what a target wrote while serving a generation is what an initiator -//! that saw it served reads — the edge a `perf-state` read copies each CPU's -//! registers through. The slot is a loom cell rather than an atomic, so a read -//! unordered against the write is loom's own `Causality violation`. - -#![cfg(feature = "loom")] - -use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; - -use kernel_loom::shootdown::Shootdown; -use loom::cell::UnsafeCell; -use loom::sync::Arc; - -/// What the target's serve stores. -const SAMPLE: u64 = 0x8000_2a04; - -/// Set by any execution in which the initiator saw the answer, so the -/// assertion is shown to have run. Outside the model because loom re-runs the -/// closure once per interleaving. -static SEEN: AtomicBool = AtomicBool::new(false); - -struct Machine { - shootdown: Shootdown, - /// cpu 1's answer slot. - slot: UnsafeCell, -} - -// SAFETY: `slot` is written only inside cpu 1's `serve` and read only after -// `served` says that serve finished; that ordering is what the model checks. -unsafe impl Sync for Machine {} - -#[test] -fn what_a_serve_wrote_is_read_once_it_is_served() { - SEEN.store(false, SeqCst); - loom::model(|| { - let m = Arc::new(Machine { shootdown: Shootdown::new(), slot: UnsafeCell::new(0) }); - let generation = m.shootdown.issue(); - - let target = { - let m = m.clone(); - loom::thread::spawn(move || { - // SAFETY: the only write, before the serve's publication. - m.shootdown.serve(1, || m.slot.with_mut(|slot| unsafe { *slot = SAMPLE })); - }) - }; - - if m.shootdown.served(1, generation) { - SEEN.store(true, SeqCst); - // SAFETY: `served` answered, so the write happened before this. - let read = m.slot.with(|slot| unsafe { *slot }); - assert_eq!(read, SAMPLE, "the initiator saw cpu 1 served and read its slot unwritten"); - } - - target.join().unwrap(); - }); - assert!(SEEN.load(SeqCst), "no interleaving saw the answer, so the assertion never ran"); -} diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index fae9a2ba5fa..2276c8d19eb 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -92,8 +92,6 @@ pub(super) fn sys_write(h: RawHandle, buf: &UserBytes) -> u64 { } } -/// Only these four device classes block; the rest answer `NotFound` on an -/// empty blocking read. fn read_block_device(claim: &crate::object::device::DeviceClaim) -> ReadBlock { match claim.class() { device::DeviceType::Keyboard => ReadBlock::Keyboard(Deadline::never()), From 1daa5ddfd809f663f8f4503ecfbd9c7b38e95fd3 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 09:14:51 +0200 Subject: [PATCH 06/12] Price perfdiverge's one job so its metal list can size to the runner's bound perfdiverge runs test_rs_perf_state, the same binary testcases' own list already prices under list.testcases.job_ms; #590 added the boot.perfdiverge.* rows but left list.perfdiverge.job_ms unpriced, so batches() refused before any boot with "a list nobody has priced cannot be sized to the runner's bound". Give it the same 1400 ms ceiling for the same reason: the same kind of binary spawned off the same stick. Co-Authored-By: Claude Opus 5.5 --- tests/metal-profile.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/metal-profile.toml b/tests/metal-profile.toml index a966607b133..1fd4e206933 100644 --- a/tests/metal-profile.toml +++ b/tests/metal-profile.toml @@ -767,6 +767,12 @@ unit = "ms" ceiling = 1400 ceiling_from = "as list.jobcase.job_ms — the same list" +[[number]] +name = "list.perfdiverge.job_ms" +unit = "ms" +ceiling = 1400 +ceiling_from = "as list.testcases.job_ms — the same one job, test_rs_perf_state, that testcases' own list already prices" + # --- the boots the split above adds --- [[number]] From 00d6966a40a441ead0d375c5b9384f1e31722f9e Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 09:41:46 +0200 Subject: [PATCH 07/12] metal: a boot told to panic is judged by its panic, and a filtered perf_request carries its job The T14 ran perf_request's row at 1daa5ddf and did what the row claims, and the harness scored it red for two reasons of its own. toyos-metal judged every boot that is not a staged wedge by the shutdown's last word, so perfdiverge, which ends in the panic it exists to provoke, exited 1 before perf_request_diverged read the record. An arm now says `panics: Some()`, the harness passes it as `--expect-panic `, and toyos-metal judges such a boot by the kernel panic record the pass after the reset printed: green where that record (the `| ` lines under `Previous boot's panic:`, opening with the kernel's `PANIC (apic `) carries the line, red on a boot that handed the machine back, on a wedge's record, on a panic without the line, and on the line found only in the ring filed after the record. `--expect-panic` beside a wedge arm is refused: a boot ends one way. Every other boot keeps today's rule. A panic seals no panel census (only the stop and seal_wedge write one; the perfdiverge readback carries none), so the per-boot facts would have reded the same boot next: the census joins PATH_TAKEN and perfdiverge's two panel rows go. The per-boot facts are now a function, boot_findings, so a host test can hold the T14's own record to them. perf_request's testcases arm named no job and rode TESTCASES' list, which a run filtered to perf_request does not select, so that boot ran none. The arm now names test_rs_perf_state. The loader marking slot A dead after the expected panic is right and is left: the mark is the loader's A/B policy for any kernel death, it lives in the `attempts` file on the stick's log partition, and every metal boot rewrites the whole stick (wipefs, then dd of the image), which the testcases boot after perfdiverge shows: `this image has had the machine 0 time(s)`, slot A booted. Co-Authored-By: Claude Opus 5.5 --- src/bootlog.rs | 20 ++++ src/metal.rs | 200 ++++++++++++++++++++++++++++--- tests/common/metal.rs | 252 ++++++++++++++++++++++++++++++--------- tests/metal-profile.toml | 12 -- tests/toyos.rs | 21 ++-- 5 files changed, 412 insertions(+), 93 deletions(-) diff --git a/src/bootlog.rs b/src/bootlog.rs index 14aa18ad1e4..ec7d24be733 100644 --- a/src/bootlog.rs +++ b/src/bootlog.rs @@ -132,6 +132,25 @@ pub const LOADER_GOP_LINE: &str = "GOP: mode"; pub const BLACKBOX_HEAD: &str = "Black box:"; pub const PREVIOUS_PANIC: &str = "Previous boot's panic:"; +/// The first words of a record the kernel's panic path sealed, in +/// `kernel/src/panic.rs`'s `first_words`. A wedge, a fault and an unsealed page +/// are reported under [`PREVIOUS_PANIC`] too, and none of them opens with this. +pub const PANIC_RECORD: &str = "PANIC (apic "; + +/// The record a kernel panic sealed, as the pass after the reset printed it: +/// the lines under [`PREVIOUS_PANIC`], the first of them [`PANIC_RECORD`]'s. +/// `None` where that pass reports no panic. +pub fn panic_record(loader: &str) -> Option> { + let after = &loader[loader.find(SEPARATOR)?..]; + let record: Vec<&str> = after + .lines() + .skip_while(|line| !line.starts_with(PREVIOUS_PANIC)) + .skip(1) + .map_while(|line| line.strip_prefix("| ")) + .collect(); + record.first().is_some_and(|head| head.starts_with(PANIC_RECORD)).then_some(record) +} + /// What the loader prints in place of a record's tail, with the count of the /// records it filed instead. pub const TAIL_IN_THE_FILE: &str = @@ -688,6 +707,7 @@ mod tests { "kernel/src/drivers/panic_console/mod.rs", format!("CENSUS: &str = \"{PANEL_CENSUS}\""), ), + ("kernel/src/panic.rs", format!("\"{PANIC_RECORD}{{apic}}): panicked at ")), ("kernel/src/log/mod.rs", format!("TAIL_HEAD: &str = \"{LOG_TAIL_HEAD}\"")), ("kernel/src/log/mod.rs", format!("\"{LOG_TAIL}")), ] { diff --git a/src/metal.rs b/src/metal.rs index d76b7dd6cb7..1676a7451c1 100644 --- a/src/metal.rs +++ b/src/metal.rs @@ -171,9 +171,12 @@ pub enum Refusal { HungWithoutARecord, /// **An image armed to stop itself did not, or was not ended by its own /// bound.** Said only of an image carrying [`WEDGE_ARM`], for which - /// `Rebooting.` is the failure and a sealed `WEDGED` record is the pass — - /// the one boot in this loop whose verdict is not `bootlog::verdict`'s. + /// `Rebooting.` is the failure and a sealed `WEDGED` record is the pass. Wedge { why: &'static str }, + /// **A boot told to end in a panic carrying `want` did not.** Said only of + /// a boot run with `--expect-panic`, for which `Rebooting.` is the failure + /// and the kernel's own panic record carrying that line is the pass. + Panic { want: String, why: String }, /// **What the boot said over its own cable is not what a talking boot /// owes**: the log it serves, a ping, the command's answer and `reboot`, /// each finding by name. Judged after the stick's own verdict, which stays @@ -204,6 +207,7 @@ impl Refusal { | Self::Talk(_) | Self::Swap(_) | Self::Wedge { .. } + | Self::Panic { .. } ) } } @@ -318,6 +322,12 @@ impl fmt::Display for Refusal { "this image is armed to stop itself, so it is judged by the record its own \ deadline sealed and not by the word a shutdown writes — and {why}" ), + Self::Panic { want, why } => write!( + f, + "this boot is to end in a kernel panic whose record carries {want:?}, so it is \ + judged by the record the pass after the reset printed and not by the word a \ + shutdown writes — and {why}" + ), Self::HungWithoutARecord => write!( f, "the last boot of this image was handed the machine and never reported: no panic, \ @@ -1551,17 +1561,19 @@ declare_flags!(METAL = { SWAP = "--swap", Next; BINARY = "--binary", Next; HAND_BACK = "--hand-back", None; + EXPECT_PANIC = "--expect-panic", Next; }); /// The flags a swap of a running machine's service refuses beside it: it /// flashes nothing and reboots nothing, so each of these describes a boot it /// will not make. -const NOT_A_SWAP: &[&Flag] = &[&DRY_RUN, &FAT32_CHECK, &NIC, &INSTALL_SUDOERS, &IMAGE]; +const NOT_A_SWAP: &[&Flag] = + &[&DRY_RUN, &FAT32_CHECK, &NIC, &INSTALL_SUDOERS, &IMAGE, &EXPECT_PANIC]; /// The flags that describe a boot, as against the ones that say which machine /// to reach: [`Args::parse`] refuses an `--install-sudoers` beside any of them. const ABOUT_A_BOOT: &[&Flag] = - &[&DRY_RUN, &IMAGE, &READBACK, &FAT32_CHECK, &NIC, &WAIT_SECS, &TALK]; + &[&DRY_RUN, &IMAGE, &READBACK, &FAT32_CHECK, &NIC, &WAIT_SECS, &TALK, &EXPECT_PANIC]; /// What the binary was asked to do. #[derive(Debug, Clone, PartialEq, Eq)] @@ -1618,6 +1630,9 @@ pub struct Args { /// over ssh: the host saying it is done with a boot held for it. Absent /// leaves the machine running, which is the development loop. hand_back: bool, + /// The line this boot's kernel panic record must carry: the boot is judged + /// by that record, and a boot that hands the machine back is its red. + expect_panic: Option, } impl Args { @@ -1652,6 +1667,7 @@ impl Args { swap: value(&SWAP).map(str::to_string), binary: value(&BINARY).map(PathBuf::from), hand_back: METAL.present(args, &HAND_BACK), + expect_panic: value(&EXPECT_PANIC).map(str::to_string), }; if let Some(host) = value(&HOST) { let (user, machine) = host.split_once('@').ok_or_else(|| { @@ -1992,6 +2008,7 @@ pub fn run(args: &Args) -> Result, Refusal> { // image will arm, judged against the only table that has ruled on any of it. let armed = arms_are_admissible(asked)?; println!("image {}: armed with {armed:?}", image.path.display()); + let ending = Ending::of(&armed, args.expect_panic.as_deref())?; // The client and its key before the machine is asked anything. let cable = match (&args.talk, &args.readback) { (Some(key), Some(dir)) => Some(Talking::prepare(key, dir)?), @@ -2124,20 +2141,7 @@ pub fn run(args: &Args) -> Result, Refusal> { if let Some(said) = reported_and_booted_nothing(&loader, &log) { return Err(Refusal::ReportedAndBootedNothing { said }); } - // **An image armed to stop itself is judged by the record its own bound - // sealed, not by the word a shutdown writes.** Read off what the image - // is armed with rather than off its label or its readback directory: the - // arm comes out of the artifact, so no boot can be judged as something - // it was not flashed as. *Which* bound sealed it is the page's to say - // and not this list's — the arm says a bound was staged, and two of them - // can reach a staged boot. - let ms = if stages_a_wedge(&armed) { - wedged_boot(&loader, &log)? - } else { - let ms = bootlog::verdict(&log).map_err(Refusal::Log)?; - bootlog::handed_back(&loader).map_err(Refusal::Log)?; - ms - }; + let ms = ending.judge(&loader, &log)?; // After the stick's own verdict, which stays the one that names a boot // that never reached its network. if let Some((heard, lines)) = &heard { @@ -2183,6 +2187,74 @@ fn talk_verdict( Ok(()) } +/// How a boot must end to pass, decided before the flash. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ending<'a> { + /// By the shutdown's own last word: every boot not named below. + HandedBack, + /// By the record one of its own bounds sealed: [`wedged_boot`]. + Wedged, + /// By a kernel panic whose record carries this line: [`panicked_boot`]. + Panicked(&'a str), +} + +impl<'a> Ending<'a> { + /// **A wedge is read off what the image is armed with**, not off its label + /// or its readback directory, so no boot is judged as something it was not + /// flashed as. A panic is the caller's to expect, because an arm that + /// panics a machine with the hardware it moves is an ordinary boot on one + /// without it. The two are one boot judged two ways, so they refuse each + /// other. + fn of(armed: &[String], expect_panic: Option<&'a str>) -> Result { + match (stages_a_wedge(armed), expect_panic) { + (false, None) => Ok(Self::HandedBack), + (true, None) => Ok(Self::Wedged), + (false, Some(want)) => Ok(Self::Panicked(want)), + (true, Some(want)) => Err(Refusal::Usage(format!( + "--expect-panic {want:?} beside an image armed with {armed:?}, which is armed to \ + be ended by a bound of its own: a boot ends one way" + ))), + } + } + + /// The boot's millisecond count, where the stick's two files say it ended + /// this way. + fn judge(self, loader: &str, log: &str) -> Result { + match self { + Self::HandedBack => { + let ms = bootlog::verdict(log).map_err(Refusal::Log)?; + bootlog::handed_back(loader).map_err(Refusal::Log)?; + Ok(ms) + } + Self::Wedged => wedged_boot(loader, log), + Self::Panicked(want) => panicked_boot(loader, log, want), + } + } +} + +/// What a boot told to end in a panic owes instead of `Rebooting.`: the pass +/// after the reset reports a kernel panic whose record carries `want`, and the +/// kernel reached `Boot: complete` before it. +fn panicked_boot(loader: &str, log: &str, want: &str) -> Result { + let refuse = |why: &str| Refusal::Panic { want: want.to_string(), why: why.to_string() }; + if bootlog::handed_back(loader).is_ok() { + return Err(refuse("it reached the shutdown's own last word, so it never panicked")); + } + let record = bootlog::panic_record(loader) + .ok_or_else(|| refuse("the pass after the reset reports no kernel panic"))?; + let Some(said) = record.iter().find(|line| line.contains(want)) else { + return Err(refuse(&format!( + "the kernel panicked with a record that does not carry it:\n {}", + record.join("\n ") + ))); + }; + let ms = bootlog::boot_millis(log).ok_or_else(|| { + refuse("the kernel wrote no `Boot: complete`, so it panicked before the jobs ran") + })?; + println!("{}", said.trim()); + Ok(ms) +} + /// What an image armed to stop itself owes instead of `Rebooting.`, and the /// `Boot: complete` it still owes as well. /// @@ -3380,4 +3452,96 @@ mod tests { let early = locked.replace("for 60004 ms", "for 59000 ms"); assert_eq!(lockup_lateness_ms(&early), None); } + + /// The pass after the reset of the T14's `perf-request-diverges` boot, cut + /// to the lines the verdict reads: the panic record, and the log ring the + /// loader files after it. + fn diverged_pass(named: &str) -> String { + format!( + "Loader log: the kernel handoff begins, so this file ends here\n\ + {}\n\ + Black box: the record below is from the boot armed at 2026-09-29-072056\n\ + {} 15052 bytes off 0x8000000\n\ + | {}2): panicked at src/arch/x86_64/control_regs.rs:440:9: \n\ + | older records dropped to fit this page: 177\n\ + | {named}\n\ + | usb-quiesce: xHCI 00:14.0 bus mastering off\n\ + {} 191 record(s)\n\ + | [1.519 cpu1] control_regs: cpu1 pm_enable=1 hwp_request=0x80002a05\n\ + {}\n", + bootlog::SEPARATOR, + bootlog::PREVIOUS_PANIC, + bootlog::PANIC_RECORD, + bootlog::TAIL_IN_THE_FILE, + bootlog::CHAIN_ENDS_LINE, + ) + } + + /// **A boot told to end in a panic passes on that panic and on nothing + /// else**: the line has to be in the kernel's own panic record, not in the + /// ring filed after it, not in a wedge's record, and not in a boot that + /// handed the machine back. + #[test] + fn a_boot_expected_to_panic_is_judged_by_its_panic_record() { + const WANT: &str = + "control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04"; + let booted = "[2026-09-29 07:20:58 1.156 cpu0] Boot: complete (1156ms)\n"; + let armed = vec!["perf-request-diverges".to_string()]; + let ending = Ending::of(&armed, Some(WANT)).expect("a panic to expect"); + assert_eq!(ending.judge(&diverged_pass(WANT), booted), Ok(1156)); + + let other = diverged_pass("control_regs: cpu1 holds nothing it was not declared"); + let why = ending.judge(&other, booted).unwrap_err().to_string(); + assert!(why.contains("does not carry it"), "{why}"); + + // Only in the ring filed after the record, which is not the panic's. + let ringed = diverged_pass("usb-recovery: the log ring holds no transport break") + .replace("pm_enable=1 hwp_request=0x80002a05", WANT); + let why = ending.judge(&ringed, booted).unwrap_err().to_string(); + assert!(why.contains("does not carry it"), "{why}"); + + let wedged = format!( + "{}\n{} the last boot read WEDGED\n| {}: a bound of 120000 ms\n| {WANT}\n", + bootlog::SEPARATOR, + bootlog::PREVIOUS_PANIC, + bootlog::DEADLINE_EXPIRED + ); + let why = ending.judge(&wedged, booted).unwrap_err().to_string(); + assert!(why.contains("reports no kernel panic"), "{why}"); + + let done = format!( + "{}\nBlack box: {}\n| {}[kernel 1.3 cpu0] {}\n", + bootlog::SEPARATOR, + bootlog::HANDED_BACK, + bootlog::LOG_TAIL, + bootlog::REBOOTING + ); + let why = ending.judge(&done, booted).unwrap_err().to_string(); + assert!(why.contains("never panicked"), "{why}"); + + let why = ending.judge(&diverged_pass(WANT), "nothing at all\n").unwrap_err(); + assert!(why.to_string().contains("no `Boot: complete`"), "{why}"); + assert!(why.about_the_boot()); + } + + /// The expectation is the command line's, and a wedge arm's image is one + /// it cannot describe. + #[test] + fn a_boot_ends_one_way() { + let armed = |names: &[&str]| -> Vec { names.iter().map(|n| n.to_string()).collect() }; + let plain = armed(&["perf-request-diverges"]); + assert_eq!(Ending::of(&plain, None), Ok(Ending::HandedBack)); + assert_eq!(Ending::of(&plain, Some("x")), Ok(Ending::Panicked("x"))); + assert_eq!(Ending::of(&armed(&[WEDGE_ARM]), None), Ok(Ending::Wedged)); + let refusal = Ending::of(&armed(&[WEDGE_ARM]), Some("x")).unwrap_err(); + assert!(refusal.to_string().contains("a boot ends one way"), "{refusal}"); + + let words = |w: &[&str]| -> Vec { w.iter().map(|s| s.to_string()).collect() }; + let args = Args::parse(&words(&["--image", "i.img", "--expect-panic", "a line"])) + .expect("an image and the panic it ends in"); + assert_eq!(args.expect_panic.as_deref(), Some("a line")); + let refusal = + Args::parse(&words(&["--install-sudoers", "p", "--expect-panic", "a line"])).unwrap_err(); + assert!(refusal.to_string().contains("--expect-panic"), "{refusal}"); + } } diff --git a/tests/common/metal.rs b/tests/common/metal.rs index 346cb502d5c..f46c4a92476 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -24,11 +24,18 @@ use toyos_build::metalprofile::{job_ms_row, Profile, AROUND_THE_LIST_MS}; use super::serial::Serial; /// The fields only the boots that took that path produce: the two bounds' -/// lateness, and the stop's own count. Absent **and** unpriced is a boot that -/// did not take the path and owes nothing; absent and priced is a boot armed -/// for one path that ended on another, which is a red the pricing loop names. -const PATH_TAKEN: &[&str] = - &["deadline_lateness_ms", "lockup_lateness_ms", "park_open_operations"]; +/// lateness, the stop's own count, and the panel's census, which a stop or a +/// bound seals and a kernel panic does not. Absent **and** unpriced is a boot +/// that did not take the path and owes nothing; absent and priced is a boot +/// armed for one path that ended on another, which is a red the pricing loop +/// names. +const PATH_TAKEN: &[&str] = &[ + "deadline_lateness_ms", + "lockup_lateness_ms", + "park_open_operations", + "panel_max_us", + "panel_us", +]; /// One boot a metal test needs. pub struct Arm { @@ -82,6 +89,11 @@ pub struct Arm { /// service and writes [`toyos_build::metal::READBACK_SWAP`] beside the /// stick's files. pub swap: Option<&'static str>, + /// **The boot ends in a kernel panic whose record carries this line**, and + /// the loop judges it by that record rather than by the shutdown's last + /// word. `None` on every boot that hands the machine back or is ended by a + /// bound its image is armed with. + pub panics: Option<&'static str>, } /// The ordinary arm: one boot, and the fields a caller must still say. @@ -95,7 +107,17 @@ pub const fn once( params: &'static [&'static str], jobs: &'static [&'static str], ) -> Arm { - Arm { boot, config, params, jobs, features: &[], nic: None, talk: false, swap: None } + Arm { + boot, + config, + params, + jobs, + features: &[], + nic: None, + talk: false, + swap: None, + panics: None, + } } /// One boot carrying members that are **discovered rather than registered**. @@ -555,6 +577,8 @@ struct Batch { talk: bool, /// [`Arm::swap`], carried to the image, the invocation and the second one. swap: Option<&'static str>, + /// [`Arm::panics`], carried to the invocation. + panics: Option<&'static str>, } impl Batch { @@ -602,6 +626,7 @@ fn batches( nic: None, talk: false, swap: None, + panics: None, }, ); if was.is_some() { @@ -621,6 +646,7 @@ fn batches( nic: arm.nic, talk: arm.talk, swap: arm.swap, + panics: arm.panics, }); if batch.config != arm.config || batch.params != arm.params @@ -628,21 +654,25 @@ fn batches( || batch.nic != arm.nic || batch.talk != arm.talk || batch.swap != arm.swap + || batch.panics != arm.panics { return Err(format!( - "{name} rides the boot {:?} as ({}, {:?}, {:?}, {:?}, talk={}) and another row \ - rides it as ({}, {:?}, {:?}, {:?}, talk={}); one boot is one image", + "{name} rides the boot {:?} as ({}, {:?}, {:?}, {:?}, talk={}, \ + panics={:?}) and another row rides it as ({}, {:?}, {:?}, {:?}, talk={}, \ + panics={:?}); one boot is one image", arm.boot, arm.config, arm.params, arm.features, arm.nic, arm.talk, + arm.panics, batch.config, batch.params, batch.features, batch.nic, - batch.talk + batch.talk, + batch.panics )); } batch.add(arm.jobs.iter().map(|j| (*j).to_string())); @@ -823,7 +853,20 @@ fn talk_home(home: &Path) -> PathBuf { home.join("ssh") } -fn invocation(image: &Path, home: &Path, nic: Option<&str>, talk: bool) -> Vec { +/// `words` as a shell reads them back, for the request a hand runs: a word with +/// a space in it, which an expected panic line has, is quoted. +fn command_line(words: &[String]) -> String { + let word = |w: &String| { + if w.contains(char::is_whitespace) { + format!("'{}'", w.replace('\'', "'\\''")) + } else { + w.clone() + } + }; + words.iter().map(word).collect::>().join(" ") +} + +fn invocation(image: &Path, home: &Path, batch: &Batch) -> Vec { let mut words = vec![ "run".to_string(), "--bin".to_string(), @@ -839,11 +882,15 @@ fn invocation(image: &Path, home: &Path, nic: Option<&str>, talk: bool) -> Vec Vec { + let mut found = Vec::new(); + let panel = back.panel(); + // A boot the file prices a path-taken field for and that produced none is + // a boot some *other* bound ended. + for (field, value) in [ + ("complete_ms", back.boot_ms), + ("back_secs", Some(back.back_secs)), + ("stick_secs", Some(back.stick_secs)), + ("deadline_lateness_ms", back.deadline_lateness_ms()), + ("lockup_lateness_ms", back.lockup_lateness_ms()), + ("panel_max_us", panel.map(|panel| panel.max_micros)), + ("panel_us", panel.map(|panel| panel.micros)), + ("park_open_operations", back.park_open_operations()), + ] { + let name = format!("boot.{label}.{field}"); + let priced = profile.row(&name).is_some(); + if value.is_none() && !priced && PATH_TAKEN.contains(&field) { + continue; + } + let Some(value) = value else { + found.push(format!( + "{name}: this boot recorded none, and the profile prices it — so the bound this \ + boot was armed for is not the one that ended it" + )); + continue; + }; + if let Err(why) = profile.judge(&name, value) { + found.push(why.to_string()); + } + } + // **Every boot, and before any verdict is read out of its log.** A test's + // judge reads the file the stick came back with, so a file that stops + // before the boot does turns a machine fact into a missing line — and the + // missing line is what a reader would have to guess about. + found.extend(back.log_reached_the_stick().err()); + found.extend(back.stop_completed().err()); + found +} + /// The whole metal profile: batch, build, drive, judge, report. // Each argument is one of the suite's own flags or tables, passed through // once; a struct holding them would be a second name for the command line. @@ -1024,7 +1113,7 @@ pub fn run( request.push_str(&format!( "\n{label}\n image: {}\n cargo {}\n", image.display(), - invocation(image, &at(dir, label), batches[*label].nic, batches[*label].talk).join(" ") + command_line(&invocation(image, &at(dir, label), &batches[*label])) )); if let Some(service) = batches[*label].swap { request.push_str(&format!( @@ -1057,7 +1146,7 @@ pub fn run( let mut refused: BTreeMap<&str, String> = BTreeMap::new(); if mode == Mode::Drive { for (label, image) in &images { - let words = invocation(image, &at(dir, label), batches[*label].nic, batches[*label].talk); + let words = invocation(image, &at(dir, label), &batches[*label]); // A swapping boot's second invocation is started first: it dials // the machine under its own name for as long as it takes, and // waits for the boot. @@ -1066,7 +1155,7 @@ pub fn run( eprintln!("[metal] {label}, beside it: cargo {}", words.join(" ")); Command::new("cargo").args(&words).current_dir(&root).spawn() }); - eprintln!("[metal] {label}: cargo {}", words.join(" ")); + eprintln!("[metal] {label}: cargo {}", command_line(&words)); let booted = Command::new("cargo").args(&words).current_dir(&root).status(); if let Some(swap) = beside { match swap.and_then(|mut child| child.wait()) { @@ -1125,47 +1214,7 @@ pub fn run( panel.paints, panel.pixels ); } - // A boot the file prices a path-taken field for and that - // produced none is a boot some *other* bound ended. - for (field, value) in [ - ("complete_ms", back.boot_ms), - ("back_secs", Some(back.back_secs)), - ("stick_secs", Some(back.stick_secs)), - ("deadline_lateness_ms", back.deadline_lateness_ms()), - ("lockup_lateness_ms", back.lockup_lateness_ms()), - ("panel_max_us", panel.map(|panel| panel.max_micros)), - ("panel_us", panel.map(|panel| panel.micros)), - ("park_open_operations", back.park_open_operations()), - ] { - let name = format!("boot.{label}.{field}"); - let priced = profile.row(&name).is_some(); - if value.is_none() && !priced && PATH_TAKEN.contains(&field) { - continue; - } - let Some(value) = value else { - eprintln!( - " FAIL {name}: this boot recorded none, and the profile prices \ - it — so the bound this boot was armed for is not the one that \ - ended it" - ); - red = true; - continue; - }; - if let Err(why) = profile.judge(&name, value) { - eprintln!(" FAIL {why}"); - red = true; - } - } - // **Every boot, and before any verdict is read out of its - // log.** A test's judge reads the file the stick came back - // with, so a file that stops before the boot does turns a - // machine fact into a missing line — and the missing line is - // what a reader would have to guess about. - if let Err(why) = back.log_reached_the_stick() { - eprintln!(" FAIL {why}"); - red = true; - } - if let Err(why) = back.stop_completed() { + for why in boot_findings(label, back, &profile) { eprintln!(" FAIL {why}"); red = true; } @@ -1254,3 +1303,94 @@ pub fn run( } } +#[cfg(test)] +mod tests { + use super::*; + + /// The pass after the reset of the T14's `perfdiverge` boot, less the log + /// ring the loader files after the record. + const DIVERGED: &str = r#" +Loader log: the kernel handoff begins, so this file ends here +--- the pass after the reset, reading what the boot above left +ToyOS Bootloader 1.0 +Boot attempts: this image has had the machine 1 time(s) without reporting; now 0 +Slot A: its image 4cc5b5b9b215561977a75452953bd9752a5c373447d48c3830762c3eb5e68624 died on its last boot, so no pass boots it again until an update replaces it +Anti-rollback floor: ToyOSImageFloor-Icc9ccdd1ac0cc468 (image scope) holds 0 +Black box: the record below is from the boot armed at 2026-09-29-072056 +Previous boot's panic: 15052 bytes off 0x8000000 +| PANIC (apic 2): panicked at src/arch/x86_64/control_regs.rs:440:9: +| usb-recovery: the log ring holds no transport break +| older records dropped to fit this page: 177 +| control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04 +| usb-quiesce: no barrier was taken, so this reset is not the shutdown's +| usb-quiesce: no Bulk-Only command was open, so this reset cuts none +| usb-quiesce: xHCI 00:0d.0 0/0 connected port(s) reset +| usb-quiesce: xHCI 00:0d.0 halted=true USBSTS=0x00000001 +| usb-quiesce: xHCI 00:0d.0 reset=true ready=true +| usb-quiesce: xHCI 00:0d.0 5/5 port(s) unpowered +| usb-quiesce: xHCI 00:0d.0 bus mastering off +| usb-quiesce: xHCI 00:14.0 5/5 connected port(s) reset +| usb-quiesce: xHCI 00:14.0 halted=true USBSTS=0x00000019 +| usb-quiesce: xHCI 00:14.0 reset=true ready=true +| usb-quiesce: xHCI 00:14.0 16/16 port(s) unpowered +| usb-quiesce: xHCI 00:14.0 bus mastering off +| usb-quiesce: 0/0 disk cache(s) flushed, 0 with no cache to flush, 5/5 connected port(s) reset, 2/2 controller(s) halted, 2 reset, 21/21 port(s) unpowered +Black box: that record's log ring is in loader.log, not on a console the firmware scrolls: 191 record(s) +Loader log: the last boot is accounted for, so this pass resets the machine +"#; + + fn perf_request_alone() -> Result<(BTreeMap, Profile), String> { + let profile = Profile::load(&super::super::compile::repo_root()).map_err(|e| e.to_string())?; + let selected: Vec<(&str, &'static Metal)> = + crate::METAL.iter().filter(|(name, _)| *name == "perf_request").map(|(n, d)| (*n, d)).collect(); + assert_eq!(selected.len(), 1, "one perf_request row"); + Ok((batches(&selected, &[], &profile)?, profile)) + } + + /// **A run that selects `perf_request` and no other rider of `testcases` + /// still carries the job the row reads there.** + #[test] + fn a_filtered_run_carries_the_job_its_row_reads() -> Result<(), String> { + let (boots, _) = perf_request_alone()?; + let testcases = boots.get("testcases").ok_or("no testcases boot")?; + if !testcases.jobs.iter().any(|job| job == "test_rs_perf_state") { + return Err(format!("testcases carries {:?} and not test_rs_perf_state", testcases.jobs)); + } + Ok(()) + } + + /// **The boot that is to panic is driven as one and judged as one**: its + /// invocation tells the loop which line the panic record carries, and the + /// boot's own facts, read off the record the T14 sealed, owe nothing a + /// panic does not seal. + #[test] + fn a_boot_that_is_to_panic_is_told_so_and_owes_no_stop() -> Result<(), String> { + let (boots, profile) = perf_request_alone()?; + let diverge = boots.get("perfdiverge").ok_or("no perfdiverge boot")?; + let want = diverge.panics.ok_or("perfdiverge expects no panic")?; + let words = invocation(Path::new("image.img"), Path::new("perfdiverge"), diverge); + if !words.windows(2).any(|pair| pair[0] == "--expect-panic" && pair[1] == want) { + return Err(format!("the invocation does not expect {want:?}: {words:?}")); + } + if !DIVERGED.contains(want) { + return Err(format!("the record the T14 sealed does not carry {want:?}")); + } + let kernel = "[2026-09-29 07:20:58 1.156 cpu0] Boot: complete (1156ms)\n".to_string(); + let back = Readback { + label: "perfdiverge".to_string(), + home: PathBuf::new(), + loader: DIVERGED.to_string(), + boot_ms: bootlog::boot_millis(&kernel), + log: kernel.clone(), + kernel, + back_secs: 101, + stick_secs: 0, + cable: None, + }; + let found = boot_findings("perfdiverge", &back, &profile); + if !found.is_empty() { + return Err(found.join("\n")); + } + Ok(()) + } +} diff --git a/tests/metal-profile.toml b/tests/metal-profile.toml index 1fd4e206933..3c5a1efc538 100644 --- a/tests/metal-profile.toml +++ b/tests/metal-profile.toml @@ -1025,12 +1025,6 @@ ceiling = 16667 ceiling_from = "as boot.deadlinewedge.panel_max_us" measured = 3890 -[[number]] -name = "boot.perfdiverge.panel_max_us" -unit = "us" -ceiling = 16667 -ceiling_from = "as boot.testcases.panel_max_us" - [[number]] name = "boot.usbbreak.panel_max_us" unit = "us" @@ -1178,12 +1172,6 @@ ceiling = 100000 ceiling_from = "as boot.testcases.panel_us" measured = 19488 -[[number]] -name = "boot.perfdiverge.panel_us" -unit = "us" -ceiling = 100000 -ceiling_from = "as boot.testcases.panel_us" - # --- what the block layer had open where the stop ended # How long the stop took is priced nowhere: a stop that gave up spends its # budget and no more, so the record's own shortfall clause is the verdict, and diff --git a/tests/toyos.rs b/tests/toyos.rs index f8e7a3ea441..60f09293c0b 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2112,14 +2112,22 @@ const TESTCASES: &[metal::Arm] = &[metal::once( ], )]; -/// `perf_request`'s two boots. The first is [`TESTCASES`], whose list carries -/// `test_rs_perf_state` because a job this row added would land after -/// `log-close`; the second is its own, since it ends in a panic. +/// `perf_request`'s two boots. The first is [`TESTCASES`]'s, and names the one +/// job this row reads so a run that selects no other rider still carries it; +/// the second is its own, since it ends in a panic. const PERF_REQUEST: &[metal::Arm] = &[ - metal::once("testcases", "tests/testcases", &[], &[]), - metal::once("perfdiverge", "tests/testcases", &["perf-request-diverges"], &["test_rs_perf_state"]), + metal::once("testcases", "tests/testcases", &[], &["test_rs_perf_state"]), + metal::Arm { + panics: Some(PERF_REQUEST_DIVERGED), + ..metal::once("perfdiverge", "tests/testcases", &["perf-request-diverges"], &["test_rs_perf_state"]) + }, ]; +/// What `perfdiverge`'s panic says, in `kernel/src/arch/x86_64/control_regs.rs`'s +/// `hwp_check`: cpu1 holds the request one ratio off the declaration. +const PERF_REQUEST_DIVERGED: &str = + "control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04"; + /// **Two boots of one config, because these two cannot share one.** Each fills /// a machine-wide cap and leaves it filled: `mkdir_cap` fills the directory cap, /// and `readdir_bound`'s own `create_dir("/tmp/empty")` is then refused with @@ -17770,9 +17778,8 @@ fn perf_request_on_metal(boot: &metal::Readback) -> Result<(), String> { /// was declared — the page after the reset carries it — and a boot whose check /// did not assert reaches no such panic. fn perf_request_diverged(boot: &metal::Readback) -> Result<(), String> { - const NAMED: &str = "control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04"; let after = boot.after_the_reset()?; - let said = after.must_say_after(bootlog::PREVIOUS_PANIC, NAMED)?.to_string(); + let said = after.must_say_after(bootlog::PREVIOUS_PANIC, PERF_REQUEST_DIVERGED)?.to_string(); eprintln!(" [perf_request] a request moved off the declaration panicked: {}", said.trim()); Ok(()) } From 185f3ea00bb0f52ad619dc0009923e92dc026ddb Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 09:52:40 +0200 Subject: [PATCH 08/12] metal: the harness's host tests hold no item a harness-less build leaves unused `cargo clippy --all-targets` builds tests/toyos.rs, which has no libtest harness, with cfg(test) set and every #[test] dropped, so the test module's shared fixture, selection helper and glob import were dead there and `-D warnings` refused them. The two tests now sit at module level and carry their own fixture and selection. Also files usbload's unpriced panel census, found beside this: a deadline seals the census, and no boot.usbload.panel_* row prices it. Co-Authored-By: Claude Opus 5.5 --- ...panel-census-its-profile-does-not-price.md | 25 +++++ tests/common/metal.rs | 106 ++++++++---------- 2 files changed, 74 insertions(+), 57 deletions(-) create mode 100644 issues/build/usbload-seals-a-panel-census-its-profile-does-not-price.md diff --git a/issues/build/usbload-seals-a-panel-census-its-profile-does-not-price.md b/issues/build/usbload-seals-a-panel-census-its-profile-does-not-price.md new file mode 100644 index 00000000000..d8199a8fb06 --- /dev/null +++ b/issues/build/usbload-seals-a-panel-census-its-profile-does-not-price.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-09-29 +--- + +# `usbload` seals a panel census that `tests/metal-profile.toml` does not price, so its next metal run reds on it + +Evidence, read from the tree at `00d6966a`: +- `tests/metal-profile.toml` prices `boot.usbload.complete_ms`, `back_secs`, + `stick_secs` and `deadline_lateness_ms`, and no `boot.usbload.panel_max_us` + or `boot.usbload.panel_us`. Every other boot but `perfdiverge`, which ends in + a panic and seals none, is priced for both. +- `usbload` is ended by the boot deadline, and `seal_wedge` + (`kernel/src/drivers/panic_console/mod.rs`) seals `{said}{Census}` on that + path, so the page after the reset carries `panel: paints=`. +- `tests/common/metal.rs`'s `boot_findings` judges every census a boot carries, + and `Profile::judge` answers `Unfit::Unpriced` for a name with no row. + +Not measured on the T14. No `usbload` run has been read since the census +existed. + +**Exit**: `boot.usbload.panel_max_us` and `boot.usbload.panel_us` priced as +`boot.deadlinewedge.*`'s are, since both are ended by the same bound, and a +`usbload` metal run that reads green on both. diff --git a/tests/common/metal.rs b/tests/common/metal.rs index f46c4a92476..ee3883ad743 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -1303,12 +1303,29 @@ pub fn run( } } -#[cfg(test)] -mod tests { - use super::*; +/// **A run that selects `perf_request` and no other rider of `testcases` +/// still carries the job the row reads there.** +#[test] +fn a_filtered_run_carries_the_job_its_row_reads() -> Result<(), String> { + let profile = Profile::load(&super::compile::repo_root()).map_err(|e| e.to_string())?; + let alone: Vec<(&str, &'static Metal)> = + crate::METAL.iter().filter(|(name, _)| *name == "perf_request").map(|(n, d)| (*n, d)).collect(); + let boots = batches(&alone, &[], &profile)?; + let testcases = boots.get("testcases").ok_or("no testcases boot")?; + if !testcases.jobs.iter().any(|job| job == "test_rs_perf_state") { + return Err(format!("testcases carries {:?} and not test_rs_perf_state", testcases.jobs)); + } + Ok(()) +} - /// The pass after the reset of the T14's `perfdiverge` boot, less the log - /// ring the loader files after the record. +/// **The boot that is to panic is driven as one and judged as one**: its +/// invocation tells the loop which line the panic record carries, and the +/// boot's own facts, read off the record the T14 sealed, owe nothing a +/// panic does not seal. +#[test] +fn a_boot_that_is_to_panic_is_told_so_and_owes_no_stop() -> Result<(), String> { + // The pass after the reset of the T14's `perfdiverge` boot, less the log + // ring the loader files after the record. const DIVERGED: &str = r#" Loader log: the kernel handoff begins, so this file ends here --- the pass after the reset, reading what the boot above left @@ -1338,59 +1355,34 @@ Previous boot's panic: 15052 bytes off 0x8000000 Black box: that record's log ring is in loader.log, not on a console the firmware scrolls: 191 record(s) Loader log: the last boot is accounted for, so this pass resets the machine "#; - - fn perf_request_alone() -> Result<(BTreeMap, Profile), String> { - let profile = Profile::load(&super::super::compile::repo_root()).map_err(|e| e.to_string())?; - let selected: Vec<(&str, &'static Metal)> = - crate::METAL.iter().filter(|(name, _)| *name == "perf_request").map(|(n, d)| (*n, d)).collect(); - assert_eq!(selected.len(), 1, "one perf_request row"); - Ok((batches(&selected, &[], &profile)?, profile)) + let profile = Profile::load(&super::compile::repo_root()).map_err(|e| e.to_string())?; + let alone: Vec<(&str, &'static Metal)> = + crate::METAL.iter().filter(|(name, _)| *name == "perf_request").map(|(n, d)| (*n, d)).collect(); + let boots = batches(&alone, &[], &profile)?; + let diverge = boots.get("perfdiverge").ok_or("no perfdiverge boot")?; + let want = diverge.panics.ok_or("perfdiverge expects no panic")?; + let words = invocation(Path::new("image.img"), Path::new("perfdiverge"), diverge); + if !words.windows(2).any(|pair| pair[0] == "--expect-panic" && pair[1] == want) { + return Err(format!("the invocation does not expect {want:?}: {words:?}")); } - - /// **A run that selects `perf_request` and no other rider of `testcases` - /// still carries the job the row reads there.** - #[test] - fn a_filtered_run_carries_the_job_its_row_reads() -> Result<(), String> { - let (boots, _) = perf_request_alone()?; - let testcases = boots.get("testcases").ok_or("no testcases boot")?; - if !testcases.jobs.iter().any(|job| job == "test_rs_perf_state") { - return Err(format!("testcases carries {:?} and not test_rs_perf_state", testcases.jobs)); - } - Ok(()) + if !DIVERGED.contains(want) { + return Err(format!("the record the T14 sealed does not carry {want:?}")); } - - /// **The boot that is to panic is driven as one and judged as one**: its - /// invocation tells the loop which line the panic record carries, and the - /// boot's own facts, read off the record the T14 sealed, owe nothing a - /// panic does not seal. - #[test] - fn a_boot_that_is_to_panic_is_told_so_and_owes_no_stop() -> Result<(), String> { - let (boots, profile) = perf_request_alone()?; - let diverge = boots.get("perfdiverge").ok_or("no perfdiverge boot")?; - let want = diverge.panics.ok_or("perfdiverge expects no panic")?; - let words = invocation(Path::new("image.img"), Path::new("perfdiverge"), diverge); - if !words.windows(2).any(|pair| pair[0] == "--expect-panic" && pair[1] == want) { - return Err(format!("the invocation does not expect {want:?}: {words:?}")); - } - if !DIVERGED.contains(want) { - return Err(format!("the record the T14 sealed does not carry {want:?}")); - } - let kernel = "[2026-09-29 07:20:58 1.156 cpu0] Boot: complete (1156ms)\n".to_string(); - let back = Readback { - label: "perfdiverge".to_string(), - home: PathBuf::new(), - loader: DIVERGED.to_string(), - boot_ms: bootlog::boot_millis(&kernel), - log: kernel.clone(), - kernel, - back_secs: 101, - stick_secs: 0, - cable: None, - }; - let found = boot_findings("perfdiverge", &back, &profile); - if !found.is_empty() { - return Err(found.join("\n")); - } - Ok(()) + let kernel = "[2026-09-29 07:20:58 1.156 cpu0] Boot: complete (1156ms)\n".to_string(); + let back = Readback { + label: "perfdiverge".to_string(), + home: PathBuf::new(), + loader: DIVERGED.to_string(), + boot_ms: bootlog::boot_millis(&kernel), + log: kernel.clone(), + kernel, + back_secs: 101, + stick_secs: 0, + cable: None, + }; + let found = boot_findings("perfdiverge", &back, &profile); + if !found.is_empty() { + return Err(found.join("\n")); } + Ok(()) } From b2b0b4cbc74d82ecb00d83e20a61242d13b99725 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 29 Sep 2026 11:34:19 +0200 Subject: [PATCH 09/12] perf-state: each record names the CPU it was read on, and the metal row derives from the machine's own lines Review round 5 found that nothing tested the ABI's claim that each CPU's record is read on that CPU: an asker that samples every slot while the kicked CPUs answer nothing stayed green on every arm, and every T14 read-back showed all eight CPUs at cpu1's hwp_capabilities=0x0110182a. The sampling was right. In the one T14 read taken during a divergence (590r2-metal-noasserts.log:474-475, the reader on cpu1, which had just moved its own request to 0x80002a05) cpu0's record holds 0x80002a04. A record sampled on cpu1 would hold 0x80002a05. Both records hold 0x0110182a because IA32_HWP_CAPABILITIES[23:16], Most_Efficient_Performance, is dynamic. The boot lines were taken 11 ms apart over 233 ms, and cpu0's own value moved from 0x04 at boot to 0x10 by the read. So the flaw was the missing test, not the sampling. - CpuRegisters gains hardware_id, the CPU's x2APIC ID from CPUID (arch::cpu::hardware_id, the ID panic records already carry). Only the CPU itself can read it. The guest read-back prints it on every record. The host holds each record to the kernel's roster, the BSP's `percpu: BSP ... lapic_id=` and each `SMP: AP cpuN lapic=`. On the T14 that roster does not number CPUs in ID order (cpu1 is lapic 2). - perf-state-deaf-cpu now leaves only the boot's first three asks to their asker. perf_state_silent_cpu's fourth read is therefore answered by every CPU, and its records are held to the roster under QEMU. perf_request's QEMU boot has no claim to read (NotFound is what it asserts), so the QEMU identity check lives here. - perf_request's metal judges hold the machine to its own lines. Each CPU's boot line must hold what toyos_perfstate declares from that line's hwp_capabilities and platform_info. The perfdiverge panic must name the declaration cpu1's own boot line makes, and a request other than it. --expect-panic carries only the machine-independent head. No T14 literal is left in the judge. - The panel census is exempt only on a boot told to panic (NOT_SEALED_BY_A_PANIC, keyed on Arm::panics). PATH_TAKEN is main's again. - command_line quotes every word that holds a character a shell reads specially, not only words with whitespace. - Filed issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md, and the track's "Not covered" points at it. The track's owed-row clause and its Linux provenance are deleted. Not done: a check that hwp_capabilities still differs across CPUs in the read-back wherever the boot lines differ. The noasserts read above shows the correct kernel reading equal values there, so that check would fail on a correct kernel. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + Cargo.toml | 3 + .../the-kernel-owns-cpu-performance-state.md | 18 +- ...ation-refuses-hybrid-intel-and-amd-cppc.md | 32 +++ kernel/src/actuator.rs | 6 +- kernel/src/arch/x86_64/perf_state.rs | 3 +- kernel/src/perf_state.rs | 52 +++- tests/common/metal.rs | 69 +++-- .../src/bin/perf_state_silent.rs | 20 +- tests/toyos.rs | 240 ++++++++++++++++-- toyos-abi/src/perf.rs | 15 +- userland/perfstate/src/lib.rs | 3 +- 12 files changed, 370 insertions(+), 92 deletions(-) create mode 100644 issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md diff --git a/Cargo.lock b/Cargo.lock index d10c19d40c6..ac16a6f900b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1006,6 +1006,7 @@ dependencies = [ "toyos-keymap", "toyos-logstream", "toyos-manifest", + "toyos-perfstate", "toyos-quiesce", "toyos-sched", "toyos-swap", diff --git a/Cargo.toml b/Cargo.toml index fba382f55d7..ec0f54b9a49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -185,6 +185,9 @@ toyos-sched = { path = "toyos-sched", features = ["check"] } # The Bulk-Only phases, so the harness judging a wedge reads the word # `toyos_xhci::bot::Phase` declares instead of spelling it a second time. toyos-xhci = { path = "toyos-xhci" } +# The performance request, so `perf_request`'s metal row holds each CPU's boot +# line to the declaration the kernel makes from that line's own inputs. +toyos-perfstate = { path = "toyos-perfstate" } # The second reader `pkg_install_gbae` is judged against: what is read back off # the guest's volume is compared with a third party's decoding of the committed # archive, never with `userland/pkg`'s own. The archive itself is committed diff --git a/issues/kernel/the-kernel-owns-cpu-performance-state.md b/issues/kernel/the-kernel-owns-cpu-performance-state.md index 033c33889ff..3538c865262 100644 --- a/issues/kernel/the-kernel-owns-cpu-performance-state.md +++ b/issues/kernel/the-kernel-owns-cpu-performance-state.md @@ -27,11 +27,7 @@ enumerated by CPUID or read at boot under the declaration's proof `perf-state` claim, together with the turbo bit and package thermal status (`/system/bin/perfstate`). *Exit*: in QEMU, `perf_request` (the refusal) and `perf_state_silent_cpu` (a CPU that never answers is refused `Io` by name); - on the T14, **owed**, `perf_request`'s metal row: on `testcases` every CPU - logs `control_regs: cpuN pm_enable=1 hwp_request=0x80002a04 - hwp_request_pkg=0x8000ff01 epb=6` and `test_rs_perf_state` exits 0; on - `perfdiverge` the page after the reset carries the panic `control_regs: - cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04`. + on the T14, `perf_request`'s metal row. No test launches `/system/bin/perfstate`, so its row's `devices` is unmeasured. - **2 — RAPL, declared.** PL1, PL2 and their windows through @@ -57,14 +53,10 @@ enumerated by CPUID or read at boot under the declaration's proof the owner rules on, never the `perfstate` row any session can launch. *Exit*: one valid span on the T14. -**Not covered.** A hybrid CPU is refused: its HWP scale is not its ratio scale -and the declared minimum is a ratio. AMD's CPPC and AArch64 declare nothing; -the AArch64 kernel refuses the claim by name. +**Not covered.** Hybrid Intel and AMD CPPC are refused: +`issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md`. +AArch64 declares nothing, and the AArch64 kernel refuses the claim by name. **What only the T14 proves.** No QEMU CPU enumerates HWP (TCG's `qemu64`, and KVM, which reduces leaf 6 to `ARAT`), so every write and every read of these -registers runs only there. Under Linux on the T14 every CPU held -`IA32_HWP_REQUEST` `0x80002a04` with `IA32_HWP_CAPABILITIES` `0x010d182a` or -`0x010e182a`, and the package `IA32_HWP_REQUEST_PKG` `0x8000ff01`. -`MSR_PLATFORM_INFO` was not read there; its ratio 4 is inferred from Linux's -`cpuinfo_min_freq` of 400000 kHz, and ToyOS's boot line prints the register. +registers runs only there. diff --git a/issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md b/issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md new file mode 100644 index 00000000000..36b706667fc --- /dev/null +++ b/issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md @@ -0,0 +1,32 @@ +--- +status: open +kind: defect +opened: 2026-09-29 +--- + +# The perf-state declaration refuses hybrid Intel and AMD CPPC + +Every Intel client CPU since Alder Lake is hybrid, and AMD CPUs name their +performance controls through CPPC, not HWP. On both, the kernel declares no +performance request, firmware's values stand, and a `perf-state` claim is +refused `NotFound`. + +The refusal sites, all in `toyos-perfstate/src/lib.rs`'s `refusal`: +- `Refusal::Hybrid`, for CPUID.07H:EDX[15]. The reason it gives is that the + declared minimum is a ratio (`MSR_PLATFORM_INFO[47:40]`), and a hybrid + CPU's HWP scale is not its ratio scale. +- `Refusal::NoHwp`, for CPUID.06H:EAX[7] clear. AMD does not set that bit, so + this is where an AMD CPU with CPPC is refused. The CPPC controls are not + read at all. +- `Refusal::NotIntel`, for HWP on any vendor but `GenuineIntel`, because the + minimum comes from `MSR_PLATFORM_INFO`, which is Intel's. + +`kernel/src/arch/x86_64/control_regs.rs`'s `hwp_declared` logs the refusal once +and declares nothing on any CPU. + +**Exit**: on a hybrid Intel CPU, each core type's request is declared on that +core's own HWP scale, and the minimum is not read as a ratio. On an AMD CPU +with CPPC, `MSR_AMD_CPPC_ENABLE` and `MSR_AMD_CPPC_REQ` are declared from its +`MSR_AMD_CPPC_CAP1` and asserted on every CPU. On each machine the claim reads +the declaration back, and `perf_request`'s metal row passes on one machine of +each kind. diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 740280503be..2b15877b6cf 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -291,9 +291,9 @@ actuators! { dump_deaf_cpu = "dump-deaf-cpu"; /// Grant a `perf-state` claim where no performance request was declared, - /// answering zeros, and have no CPU answer a kick, so every CPU but a - /// read's asker is silent: what the read's bound refuses, on a machine - /// QEMU can stage. + /// each record its CPU's identity and zeros, and have no CPU but its asker + /// answer the boot's first three asks: what the read's bound refuses, and + /// then a read every CPU answers, on a machine QEMU can stage. perf_state_deaf_cpu = "perf-state-deaf-cpu"; /// Have cpu1 move its HWP request off the declaration when it answers a diff --git a/kernel/src/arch/x86_64/perf_state.rs b/kernel/src/arch/x86_64/perf_state.rs index 7da5cb70747..49821d12138 100644 --- a/kernel/src/arch/x86_64/perf_state.rs +++ b/kernel/src/arch/x86_64/perf_state.rs @@ -14,9 +14,10 @@ pub fn declared() -> Result { Declared::ask().map_err(toyos_perfstate::Refusal::reason) } -/// This CPU's own registers. +/// This CPU's own identity and registers. pub fn read_cpu(_: &Declared) -> CpuRegisters { CpuRegisters { + hardware_id: u64::from(cpu::hardware_id()), pm_enable: cpu::rdmsr(msr::PM_ENABLE), hwp_capabilities: cpu::rdmsr(msr::HWP_CAPABILITIES), hwp_request: cpu::rdmsr(msr::HWP_REQUEST), diff --git a/kernel/src/perf_state.rs b/kernel/src/perf_state.rs index 07ace0573b7..a0f8da9243b 100644 --- a/kernel/src/perf_state.rs +++ b/kernel/src/perf_state.rs @@ -15,7 +15,8 @@ //! stack, so a read that ends — answered, refused, cancelled or not waiting — //! leaves nothing a later read could be answered from or refused by. -use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use core::sync::atomic::AtomicU64; +use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; use toyos_abi::perf::{answer_len, CpuRegisters}; use toyos_abi::syscall::SyscallError; @@ -37,31 +38,45 @@ const ANSWER: Budget = Budget::of( "the read is refused `Io`, and the CPUs that did not answer are named", ); -struct Slot([AtomicU64; 5]); +struct Slot([AtomicU64; 6]); impl Slot { const fn new() -> Self { - Self([const { AtomicU64::new(0) }; 5]) + Self([const { AtomicU64::new(0) }; 6]) } fn store(&self, r: CpuRegisters) { - let words = [r.pm_enable, r.hwp_capabilities, r.hwp_request, r.energy_perf_bias, r.misc_enable]; + let words = [ + r.hardware_id, + r.pm_enable, + r.hwp_capabilities, + r.hwp_request, + r.energy_perf_bias, + r.misc_enable, + ]; for (slot, word) in self.0.iter().zip(words) { slot.store(word, Relaxed); } } fn load(&self) -> CpuRegisters { - let [pm_enable, hwp_capabilities, hwp_request, energy_perf_bias, misc_enable] = + let [hardware_id, pm_enable, hwp_capabilities, hwp_request, energy_perf_bias, misc_enable] = self.0.each_ref().map(|word| word.load(Relaxed)); - CpuRegisters { pm_enable, hwp_capabilities, hwp_request, energy_perf_bias, misc_enable } + CpuRegisters { + hardware_id, + pm_enable, + hwp_capabilities, + hwp_request, + energy_perf_bias, + misc_enable, + } } } /// One claim's side of the protocol: the proof its reads need. pub struct Reader { /// `None` only under `perf-state-deaf-cpu`, whose claim has no - /// declaration behind it and answers zeros. + /// declaration behind it and answers each CPU's identity and zeros. declared: Option, } @@ -77,6 +92,9 @@ impl Ask { /// order, so no kicked CPU can look before the generation it owes exists. fn issue(cpus: usize, now: Instant) -> Self { let generation = ASKS.issue(); + if crate::actuator::perf_state_deaf_cpu() { + DEAF_ASKED.fetch_add(1, Release); + } let me = crate::arch::percpu::cpu_id() as usize; ASKS.serve(me, || SLOTS[me].store(sample(me))); for cpu in (0..cpus).filter(|&cpu| cpu != me) { @@ -136,19 +154,28 @@ impl Reader { } } +/// Under `perf-state-deaf-cpu`, how many of the boot's asks only their asker +/// answers. +const DEAF_ASKS: u64 = 3; + +/// The asks issued under `perf-state-deaf-cpu`. +static DEAF_ASKED: AtomicU64 = AtomicU64::new(0); + /// This CPU's answer, if one is owed. Called from `drain_irqs` every pass. -/// Under `perf-state-deaf-cpu` no CPU answers here, so a read is answered by -/// its asker alone. +/// Under `perf-state-deaf-cpu` no CPU answers here until the boot's first +/// [`DEAF_ASKS`] asks are made, so each of those is answered by its asker alone. pub fn serve_if_owed() { let me = crate::arch::percpu::cpu_id() as usize; - if !ASKS.owes(me) || crate::actuator::perf_state_deaf_cpu() { + let deaf = crate::actuator::perf_state_deaf_cpu() && DEAF_ASKED.load(Acquire) <= DEAF_ASKS; + if !ASKS.owes(me) || deaf { return; } ASKS.serve(me, || SLOTS[me].store(sample(me))); WATCH.post(); } -/// This CPU's registers, or zeros where the claim has no declaration behind it. +/// This CPU's identity and registers, the registers zeros where the claim has +/// no declaration behind it. fn sample(me: usize) -> CpuRegisters { // The proof is used inside the closure only: on an architecture where it // is uninhabited, binding one here would make the rest unreachable. @@ -168,7 +195,8 @@ fn sample(me: usize) -> CpuRegisters { "an ask is made only through a claim, and a claim only where the request is \ declared: {why}", ); - CpuRegisters::default() + let hardware_id = u64::from(crate::arch::cpu::hardware_id()); + CpuRegisters { hardware_id, ..CpuRegisters::default() } } } } diff --git a/tests/common/metal.rs b/tests/common/metal.rs index ee3883ad743..e8019a392ef 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -24,18 +24,16 @@ use toyos_build::metalprofile::{job_ms_row, Profile, AROUND_THE_LIST_MS}; use super::serial::Serial; /// The fields only the boots that took that path produce: the two bounds' -/// lateness, the stop's own count, and the panel's census, which a stop or a -/// bound seals and a kernel panic does not. Absent **and** unpriced is a boot -/// that did not take the path and owes nothing; absent and priced is a boot -/// armed for one path that ended on another, which is a red the pricing loop -/// names. -const PATH_TAKEN: &[&str] = &[ - "deadline_lateness_ms", - "lockup_lateness_ms", - "park_open_operations", - "panel_max_us", - "panel_us", -]; +/// lateness, and the stop's own count. Absent **and** unpriced is a boot that +/// did not take the path and owes nothing; absent and priced is a boot armed +/// for one path that ended on another, which is a red the pricing loop names. +const PATH_TAKEN: &[&str] = + &["deadline_lateness_ms", "lockup_lateness_ms", "park_open_operations"]; + +/// The panel's census, which a stop or a bound seals and a kernel panic does +/// not: path-taken fields on a boot told to end in a panic, and owed by every +/// other boot. +const NOT_SEALED_BY_A_PANIC: &[&str] = &["panel_max_us", "panel_us"]; /// One boot a metal test needs. pub struct Arm { @@ -89,7 +87,7 @@ pub struct Arm { /// service and writes [`toyos_build::metal::READBACK_SWAP`] beside the /// stick's files. pub swap: Option<&'static str>, - /// **The boot ends in a kernel panic whose record carries this line**, and + /// **The boot ends in a kernel panic whose record carries this text**, and /// the loop judges it by that record rather than by the shutdown's last /// word. `None` on every boot that hands the machine back or is ended by a /// bound its image is armed with. @@ -853,14 +851,15 @@ fn talk_home(home: &Path) -> PathBuf { home.join("ssh") } -/// `words` as a shell reads them back, for the request a hand runs: a word with -/// a space in it, which an expected panic line has, is quoted. +/// `words` as a shell reads them back, for the request a hand runs: a word +/// holding anything but the characters no shell treats specially is quoted. fn command_line(words: &[String]) -> String { + let plain = |c: char| c.is_ascii_alphanumeric() || "@%+:,./_-".contains(c); let word = |w: &String| { - if w.contains(char::is_whitespace) { - format!("'{}'", w.replace('\'', "'\\''")) - } else { + if !w.is_empty() && w.chars().all(plain) { w.clone() + } else { + format!("'{}'", w.replace('\'', "'\\''")) } }; words.iter().map(word).collect::>().join(" ") @@ -964,8 +963,9 @@ pub enum Verdict { } /// What one boot's own facts say against the profile and against what every -/// boot owes, one finding a line; none is a boot whose facts pass. -fn boot_findings(label: &str, back: &Readback, profile: &Profile) -> Vec { +/// boot owes, one finding a line; none is a boot whose facts pass. `panicked` +/// is a boot told to end in a kernel panic ([`Arm::panics`]). +fn boot_findings(label: &str, back: &Readback, profile: &Profile, panicked: bool) -> Vec { let mut found = Vec::new(); let panel = back.panel(); // A boot the file prices a path-taken field for and that produced none is @@ -982,7 +982,8 @@ fn boot_findings(label: &str, back: &Readback, profile: &Profile) -> Vec ] { let name = format!("boot.{label}.{field}"); let priced = profile.row(&name).is_some(); - if value.is_none() && !priced && PATH_TAKEN.contains(&field) { + let taken = PATH_TAKEN.contains(&field) || (panicked && NOT_SEALED_BY_A_PANIC.contains(&field)); + if value.is_none() && !priced && taken { continue; } let Some(value) = value else { @@ -1214,7 +1215,7 @@ pub fn run( panel.paints, panel.pixels ); } - for why in boot_findings(label, back, &profile) { + for why in boot_findings(label, back, &profile, batches[label].panics.is_some()) { eprintln!(" FAIL {why}"); red = true; } @@ -1319,9 +1320,10 @@ fn a_filtered_run_carries_the_job_its_row_reads() -> Result<(), String> { } /// **The boot that is to panic is driven as one and judged as one**: its -/// invocation tells the loop which line the panic record carries, and the +/// invocation tells the loop what the panic record carries, and the /// boot's own facts, read off the record the T14 sealed, owe nothing a -/// panic does not seal. +/// panic does not seal — and the same facts from a boot not told to panic +/// still owe the panel's census. #[test] fn a_boot_that_is_to_panic_is_told_so_and_owes_no_stop() -> Result<(), String> { // The pass after the reset of the T14's `perfdiverge` boot, less the log @@ -1380,9 +1382,26 @@ Loader log: the last boot is accounted for, so this pass resets the machine stick_secs: 0, cable: None, }; - let found = boot_findings("perfdiverge", &back, &profile); + let found = boot_findings("perfdiverge", &back, &profile, true); if !found.is_empty() { return Err(found.join("\n")); } + let owed = boot_findings("perfdiverge", &back, &profile, false); + for field in NOT_SEALED_BY_A_PANIC { + if !owed.iter().any(|why| why.contains(&format!("boot.perfdiverge.{field}:"))) { + return Err(format!("a boot not told to panic owes no {field}: {owed:?}")); + } + } Ok(()) } + +/// **The request a hand runs is the invocation, word for word**: a word a +/// shell would read specially is quoted, and a plain one is not. +#[test] +fn a_command_line_quotes_every_word_a_shell_would_read() { + let words = ["--image", "a/b-1.img", "it's", "$HOME", "a;b", "a b", "", "x=y"].map(str::to_string); + assert_eq!( + command_line(&words), + r#"--image a/b-1.img 'it'\''s' '$HOME' 'a;b' 'a b' '' 'x=y'"#, + ); +} diff --git a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs index 0114be445aa..e820fd6d417 100644 --- a/tests/toyos-rust-tests/src/bin/perf_state_silent.rs +++ b/tests/toyos-rust-tests/src/bin/perf_state_silent.rs @@ -2,13 +2,15 @@ //! which also grants the claim where no request was declared): refused `Io` //! once the kernel's bound has passed, never answered and never left waiting. //! A read that does not wait goes first, and each read after it must make an -//! ask of its own. `perf_state_silent_cpu` drives it and reads which ask and -//! which CPU the kernel named. +//! ask of its own. The boot's fourth ask is every CPU's to answer, and each +//! record names the CPU that read it. `perf_state_silent_cpu` drives it, reads +//! which ask and which CPU the kernel named, and holds each record's identity +//! to the kernel's roster. use toyos::endow::Endowments; use toyos::syscap::SysCap; use toyos::{AsHandle, Device}; -use toyos_abi::perf::answer_len; +use toyos_abi::perf::{answer_len, CpuRegisters}; use toyos_abi::syscall::{self, DeviceType, SyscallError, SYSCAP_LABEL}; fn main() { @@ -18,11 +20,19 @@ fn main() { let claim = cap .claim::(DeviceType::PerfState) .expect("perf-state-deaf-cpu grants the claim on any machine"); - let mut buf = vec![0u8; answer_len(syscall::cpu_count() as usize)]; + let cpus = syscall::cpu_count() as usize; + let mut buf = vec![0u8; answer_len(cpus)]; assert_eq!(syscall::read_nonblock(claim.as_handle(), &mut buf), Err(SyscallError::WouldBlock)); - // Twice: the claim still answers after a refusal. for _ in 0..2 { assert_eq!(claim.read(&mut buf), Err(SyscallError::Io)); } + // The claim still answers after a refusal. + let whole = buf.len(); + assert_eq!(claim.read(&mut buf), Ok(whole)); + for cpu in 0..cpus { + let regs = CpuRegisters::read_from(&buf[answer_len(cpu)..]) + .expect("the buffer holds every CPU's record"); + println!("cpu{cpu} hardware_id={}", regs.hardware_id); + } println!("===PERF_STATE_SILENT_OK==="); } diff --git a/tests/toyos.rs b/tests/toyos.rs index 60f09293c0b..8948d0398c9 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2123,10 +2123,11 @@ const PERF_REQUEST: &[metal::Arm] = &[ }, ]; -/// What `perfdiverge`'s panic says, in `kernel/src/arch/x86_64/control_regs.rs`'s -/// `hwp_check`: cpu1 holds the request one ratio off the declaration. -const PERF_REQUEST_DIVERGED: &str = - "control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04"; +/// How `perfdiverge`'s panic opens, in `kernel/src/arch/x86_64/control_regs.rs`'s +/// `hwp_check`: cpu1 holds a request other than its declaration. The values +/// are the machine's, so [`perf_request_diverged`] reads the rest against +/// cpu1's own boot line. +const PERF_REQUEST_DIVERGED: &str = "control_regs: cpu1 holds hwp_request="; /// **Two boots of one config, because these two cannot share one.** Each fills /// a machine-wide cap and leaves it filled: `mkdir_cap` fills the directory cap, @@ -17750,48 +17751,223 @@ fn perf_request( Ok(()) } -/// The same on the T14, whose CPUs have every register the request names: -/// each CPU holds the bar's power envelope — the `IA32_HWP_REQUEST`, -/// `IA32_HWP_REQUEST_PKG` and EPB the Linux run it is held against held — and -/// the guest binary read every CPU back holding its declaration. +/// The same on a machine whose CPUs have every register the request names: +/// each CPU's boot line holds the declaration `toyos_perfstate` makes from the +/// two inputs the line carries, and the guest binary read every CPU back +/// holding it, each record read on the CPU the kernel's roster names. fn perf_request_on_metal(boot: &metal::Readback) -> Result<(), String> { - const BAR: &str = "pm_enable=1 hwp_request=0x80002a04 hwp_request_pkg=0x8000ff01 epb=6 "; let cpus = boot.cpus()?; let log = boot.kernel(); for cpu in 0..cpus { - let head = format!("control_regs: cpu{cpu} pm_enable="); - let Some(line) = log.text().lines().find(|l| l.contains(&head)) else { - return Err(format!("cpu{cpu} logged no performance request:\n{}", log.text())); - }; - if !line.contains(&format!("control_regs: cpu{cpu} {BAR}")) { - return Err(format!("cpu{cpu} does not hold the bar's envelope {BAR:?}: {line}")); - } + hwp_boot_line(log.text(), cpu)?; } boot.job_passed("test_rs_perf_state")?; - eprintln!(" [perf_request] {cpus} CPUs hold the bar's request and read it back"); + let roster = roster(log.text())?; + if roster.len() != cpus as usize { + return Err(format!("the roster names {} CPUs and {cpus} came up", roster.len())); + } + let records = records_name_their_cpus(&bootlog::lines_of(boot.log().text(), "test-runner"), &roster)?; + eprintln!( + " [perf_request] {cpus} CPUs hold their declared request, and {records} records read \ + back each named its own CPU" + ); Ok(()) } +/// One CPU's `control_regs:` boot line, as `hwp_check` writes it, held to the +/// declaration `toyos_perfstate` makes from the two inputs the line carries. +/// Answers that declared request. +fn hwp_boot_line(log: &str, cpu: u32) -> Result { + let head = format!("control_regs: cpu{cpu} pm_enable="); + let Some(line) = log.lines().find(|l| l.contains(&head)) else { + return Err(format!("cpu{cpu} logged no performance request:\n{log}")); + }; + let field = |name: &str| -> Result { + let word = line + .split_once(&format!(" {name}=")) + .and_then(|(_, rest)| rest.split_whitespace().next()) + .ok_or_else(|| format!("cpu{cpu}'s line carries no {name}: {line}"))?; + match word.strip_prefix("0x") { + Some(hex) => u64::from_str_radix(hex, 16), + None => word.parse(), + } + .map_err(|e| format!("cpu{cpu}'s {name}={word}: {e}")) + }; + let declared = toyos_perfstate::hwp_request(field("hwp_capabilities")?, field("platform_info")?); + for (name, want) in [ + ("pm_enable", toyos_perfstate::PM_ENABLE), + ("hwp_request", declared), + ("hwp_request_pkg", toyos_perfstate::HWP_REQUEST_PKG), + ("epb", toyos_perfstate::ENERGY_PERF_BIAS), + ] { + let holds = field(name)?; + if holds != want { + return Err(format!( + "cpu{cpu} holds {name}={holds:#x}, and its line's own inputs declare {want:#x}: {line}" + )); + } + } + Ok(declared) +} + +/// The kernel's roster in CPU order: each CPU's hardware ID as its bring-up +/// record names it. +fn roster(log: &str) -> Result, String> { + let number = |line: &str, head: &str| -> Option { + line.split_once(head)?.1.split_whitespace().next()?.parse().ok() + }; + let Some(bsp) = log.lines().find_map(|l| number(l, "percpu: BSP cpu_id=0 lapic_id=")) else { + return Err(format!("no `percpu: BSP` record names the BSP's hardware ID:\n{log}")); + }; + let mut ids = vec![bsp]; + loop { + let head = format!("{}{} lapic=", bootlog::AP_BRINGUP, ids.len()); + let online = |l: &&str| l.trim_end().ends_with(" online"); + match log.lines().filter(online).find_map(|l| number(l, &head)) { + Some(id) => ids.push(id), + None => return Ok(ids), + } + } +} + +/// Every `cpuN hardware_id=K` record line in `text`, held to `roster`: each +/// record was read on the CPU it is filed under, and every CPU's record was +/// read as often as every other's. Answers how many records there were. +fn records_name_their_cpus(text: &str, roster: &[u64]) -> Result { + let mut seen = vec![0usize; roster.len()]; + for line in text.lines() { + let Some((head, rest)) = line.split_once(" hardware_id=") else { continue }; + let cpu: usize = head + .rsplit_once("cpu") + .and_then(|(_, n)| n.parse().ok()) + .ok_or_else(|| format!("a record names no CPU: {line}"))?; + let id: u64 = rest + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .ok_or_else(|| format!("a record carries no hardware ID: {line}"))?; + let Some(&want) = roster.get(cpu) else { + return Err(format!("a record for cpu{cpu}, and the roster has {} CPUs: {line}", roster.len())); + }; + if id != want { + return Err(format!( + "cpu{cpu}'s record was read on the CPU whose hardware ID is {id}, and the roster \ + names cpu{cpu} {want}: {line}" + )); + } + seen[cpu] += 1; + } + if seen[0] == 0 || seen.iter().any(|&n| n != seen[0]) { + return Err(format!("want every CPU's record read alike, got {seen:?} per CPU:\n{text}")); + } + Ok(seen.iter().sum()) +} + /// The second boot: `perf-request-diverges` moves cpu1's request one ratio off /// the declaration when it answers `test_rs_perf_state`'s read, and runs the /// check boot runs. That check panics naming what cpu1 holds against what it /// was declared — the page after the reset carries it — and a boot whose check /// did not assert reaches no such panic. fn perf_request_diverged(boot: &metal::Readback) -> Result<(), String> { - let after = boot.after_the_reset()?; - let said = after.must_say_after(bootlog::PREVIOUS_PANIC, PERF_REQUEST_DIVERGED)?.to_string(); - eprintln!(" [perf_request] a request moved off the declaration panicked: {}", said.trim()); + let said = diverged_panic(boot.kernel().text(), boot.loader().text())?; + eprintln!(" [perf_request] a request moved off the declaration panicked: {said}"); + Ok(()) +} + +/// The kernel panic's line in `loader`, held to the declaration cpu1's own boot +/// line in `log` makes: it names that declaration, and a request other than it. +fn diverged_panic(log: &str, loader: &str) -> Result { + let declared = hwp_boot_line(log, 1)?; + let Some(record) = bootlog::panic_record(loader) else { + return Err(format!("the pass after the reset reports no kernel panic:\n{loader}")); + }; + let Some(said) = record.iter().map(|l| l.trim_end()).find(|l| l.starts_with(PERF_REQUEST_DIVERGED)) else { + return Err(format!("the panic record carries no {PERF_REQUEST_DIVERGED:?}:\n{}", record.join("\n"))); + }; + let tail = format!(", the declaration is {declared:#x}"); + let held = said + .strip_prefix(PERF_REQUEST_DIVERGED) + .and_then(|rest| rest.strip_suffix(&tail)) + .and_then(|held| u64::from_str_radix(held.strip_prefix("0x")?, 16).ok()); + match held { + Some(held) if held != declared => Ok(said.to_string()), + _ => Err(format!("cpu1's boot line declares {declared:#x}, and the panic says {said:?}")), + } +} + +/// **`perf_request`'s metal judges read the machine's own lines**: the T14's +/// roster and boot lines pass, a record read on its asker's CPU is refused, a +/// line holding anything but what its own inputs declare is refused, and the +/// panic must name the declaration cpu1's own line makes. +#[test] +fn perf_request_judges_hold_the_machine_to_its_own_lines() -> Result<(), String> { + // Two of the T14's CPUs as its `testcases` boot logged them: the roster + // does not number the CPUs in hardware-ID order, and the capabilities + // differ in the most efficient level. + const LOG: &str = "\ +[2026-09-29 08:10:06 0.000 cpu0 boot] control_regs: cpu0 pm_enable=1 hwp_request=0x80002a04 hwp_request_pkg=0x8000ff01 epb=6 hwp_interrupt=Some(0) hwp_capabilities=0x0104182a platform_info=0x0004043df0811800 +[2026-09-29 08:10:06 0.000 cpu0] percpu: BSP cpu_id=0 lapic_id=0 +[2026-09-29 08:10:06 0.166 cpu1] control_regs: cpu1 pm_enable=1 hwp_request=0x80002a04 hwp_request_pkg=0x8000ff01 epb=6 hwp_interrupt=Some(0) hwp_capabilities=0x0110182a platform_info=0x0004043df0811800 +[2026-09-29 08:10:06 0.166 cpu0] SMP: AP cpu1 lapic=2 online +"; + const PANIC: &str = "\ +--- the pass after the reset, reading what the boot above left +Previous boot's panic: 15052 bytes off 0x8000000 +| PANIC (apic 2): panicked at src/arch/x86_64/control_regs.rs:440:9: +| control_regs: cpu1 holds hwp_request=0x80002a05, the declaration is 0x80002a04 +"; + let roster = roster(LOG)?; + if roster != [0, 2] { + return Err(format!("the roster read {roster:?}")); + } + let records = + |ids: [u64; 2]| format!("cpu0 hardware_id={} epb=6\ncpu1 hardware_id={} epb=6\n", ids[0], ids[1]); + if records_name_their_cpus(&records([0, 2]), &roster)? != 2 { + return Err("two records were not counted as two".to_string()); + } + // Every record read on cpu1, the asker. + records_name_their_cpus(&records([2, 2]), &roster) + .err() + .ok_or("a record read on its asker's CPU passed")?; + records_name_their_cpus("cpu0 hardware_id=0\n", &roster).err().ok_or("cpu1's missing record passed")?; + + if hwp_boot_line(LOG, 1)? != 0x8000_2a04 { + return Err("cpu1's line declares 0x80002a04".to_string()); + } + let moved = + LOG.replace("cpu1 pm_enable=1 hwp_request=0x80002a04", "cpu1 pm_enable=1 hwp_request=0x80002a05"); + hwp_boot_line(&moved, 1).err().ok_or("a request its own inputs do not declare passed")?; + let pkg = LOG.replace( + "hwp_request_pkg=0x8000ff01 epb=6 hwp_interrupt=Some(0) hwp_capabilities=0x0110182a", + "hwp_request_pkg=0x8000ff02 epb=6 hwp_interrupt=Some(0) hwp_capabilities=0x0110182a", + ); + hwp_boot_line(&pkg, 1).err().ok_or("a package request other than the declaration passed")?; + + diverged_panic(LOG, PANIC)?; + // cpu1's line declaring a minimum of 3, which it holds. + let other = LOG.replace( + "cpu1 pm_enable=1 hwp_request=0x80002a04 hwp_request_pkg=0x8000ff01 epb=6 hwp_interrupt=Some(0) hwp_capabilities=0x0110182a platform_info=0x0004043df0811800", + "cpu1 pm_enable=1 hwp_request=0x80002a03 hwp_request_pkg=0x8000ff01 epb=6 hwp_interrupt=Some(0) hwp_capabilities=0x0110182a platform_info=0x0004033df0811800", + ); + if hwp_boot_line(&other, 1)? != 0x8000_2a03 { + return Err("cpu1's altered line declares 0x80002a03".to_string()); + } + diverged_panic(&other, PANIC).err().ok_or("a panic naming another declaration passed")?; + let held = PANIC.replace("hwp_request=0x80002a05", "hwp_request=0x80002a04"); + diverged_panic(LOG, &held).err().ok_or("a panic holding the declaration passed")?; Ok(()) } /// A read only its asker answers is refused `Io` once the kernel's bound has -/// passed, naming the one other CPU — twice, so the claim still answers after -/// a refusal. `perf-state-deaf-cpu` grants the claim on QEMU's CPUs, which have -/// no HWP, and has no CPU answer a kick, so which CPU the test runs on decides -/// nothing. A read that did not wait goes first and makes the boot's first ask; -/// the two refusals must name the second and the third, so a read answered or -/// refused from an ask that is not its own reds. A read with no bound waits for -/// ever, and this reds at its ceiling. +/// passed, naming the one other CPU — twice. `perf-state-deaf-cpu` grants the +/// claim on QEMU's CPUs, which have no HWP, and has no CPU but the asker answer +/// the boot's first three asks, so which CPU the test runs on decides nothing. +/// A read that did not wait goes first and makes the boot's first ask; the two +/// refusals must name the second and the third, so a read answered or refused +/// from an ask that is not its own reds. A read with no bound waits for ever, +/// and this reds at its ceiling. The fourth ask every CPU answers, and each +/// record must carry the hardware ID the roster gives its CPU, so a record +/// read anywhere but on its own CPU reds. fn perf_state_silent_cpu( test_config: &Path, c_bins: &[(String, Vec)], @@ -17838,6 +18014,14 @@ fn perf_state_silent_cpu( result.serial, )); } - eprintln!(" [perf_state_silent_cpu] the second and third asks refused Io, each naming one CPU"); + let roster = roster(qemu.boot_log())?; + if roster.len() != CPUS as usize { + return Err(format!("the roster names {} CPUs and {CPUS} were launched", roster.len())); + } + records_name_their_cpus(&result.stdout, &roster)?; + eprintln!( + " [perf_state_silent_cpu] the second and third asks refused Io, each naming one CPU; \ + the fourth's records each named its own CPU" + ); Ok(()) } diff --git a/toyos-abi/src/perf.rs b/toyos-abi/src/perf.rs index 9cbd0b4be6a..ed278b8f729 100644 --- a/toyos-abi/src/perf.rs +++ b/toyos-abi/src/perf.rs @@ -1,8 +1,9 @@ //! What a read of a `perf-state` claim answers: one [`PackageRegisters`], then //! one [`CpuRegisters`] per CPU in CPU order — the machine's whole CPU count, //! or the read is refused whole with `ResourceExhausted`, and with `Io` when a -//! CPU did not answer within the kernel's bound. Every field is the -//! register's raw value, named by its x86-64 MSR; decoding is the reader's. +//! CPU did not answer within the kernel's bound. Every field but +//! [`CpuRegisters::hardware_id`] is the register's raw value, named by its +//! x86-64 MSR; decoding is the reader's. /// The package-wide registers, read on whichever CPU answered the read. #[repr(C)] @@ -20,6 +21,11 @@ pub struct PackageRegisters { #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct CpuRegisters { + /// The CPU's own hardware identity, on x86-64 its x2APIC ID from CPUID + /// (leaf 1FH or 0BH `EDX`, else 01H `EBX[31:24]`): what the kernel's roster + /// names it by. Only that CPU can read it, so a record taken anywhere else + /// carries another CPU's. + pub hardware_id: u64, /// `IA32_PM_ENABLE`, 0x770. pub pm_enable: u64, /// `IA32_HWP_CAPABILITIES`, 0x771. @@ -35,7 +41,7 @@ pub struct CpuRegisters { // Every byte belongs to a field: both cross the boundary as bytes, so a gap // would publish whatever the kernel stack held. const _: () = assert!(core::mem::size_of::() == 3 * 8); -const _: () = assert!(core::mem::size_of::() == 5 * 8); +const _: () = assert!(core::mem::size_of::() == 6 * 8); /// The bytes a read answers on a machine of `cpus` CPUs. pub const fn answer_len(cpus: usize) -> usize { @@ -86,6 +92,7 @@ mod tests { #[test] fn a_record_round_trips_through_its_bytes() { let cpu = CpuRegisters { + hardware_id: 2, pm_enable: 1, hwp_capabilities: 0x010d_182a, hwp_request: 0x8000_2a04, @@ -96,6 +103,6 @@ mod tests { assert_eq!(CpuRegisters::read_from(&cpu.as_bytes()[1..]), None); let pkg = PackageRegisters { package_therm_status: 0x8830_0000, ..Default::default() }; assert_eq!(PackageRegisters::read_from(pkg.as_bytes()), Some(pkg)); - assert_eq!(answer_len(8), 24 + 8 * 40); + assert_eq!(answer_len(8), 24 + 8 * 48); } } diff --git a/userland/perfstate/src/lib.rs b/userland/perfstate/src/lib.rs index 3bddc6af84b..641cf48db6b 100644 --- a/userland/perfstate/src/lib.rs +++ b/userland/perfstate/src/lib.rs @@ -27,8 +27,9 @@ pub fn read_back(claim: &Device) -> Result<(), String> { let regs = CpuRegisters::read_from(&buf[answer_len(cpu)..]) .expect("the buffer holds every CPU's record"); println!( - "cpu{cpu} pm_enable={} hwp_request={:#010x} {:?} epb={} turbo={} \ + "cpu{cpu} hardware_id={} pm_enable={} hwp_request={:#010x} {:?} epb={} turbo={} \ hwp_capabilities={:#010x}", + regs.hardware_id, regs.pm_enable, regs.hwp_request, HwpRequest::of(regs.hwp_request), From 8c4d1f57fb851c2e17a33d3aea888c2e81c9e051 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 4 Oct 2026 08:21:23 +0200 Subject: [PATCH 10/12] The kernel declares the CPU's HWP request, and the counters round reads the power envelope back under TRACE The CPU-state declaration in control_regs gains the performance request: on a CPU toyos_cpuvuln::hwp admits (HWP, EPP, the package request and EPB enumerated, Intel family 6, not hybrid), every CPU clears IA32_HWP_INTERRUPT where it exists, sets IA32_PM_ENABLE, and only then reads IA32_HWP_CAPABILITIES and MSR_PLATFORM_INFO and writes IA32_HWP_REQUEST (intel_pstate's: min the package's maximum-efficiency ratio, max the highest level, EPP 0x80), IA32_HWP_REQUEST_PKG and EPB 6 whole, then logs and asserts each. A refused machine logs the reason once and writes none of them. The read-back rides #705's one round instead of a second: three counters, hwp_request, hwp_request_pkg and energy_perf_bias, each under Rights::TRACE (the owner put power under the strict right), present only where the declaration declared them. CPU identity is toyos-cpuvuln's: its third verdict, `hwp`, beside the vulnerabilities and the counters, and cpu::vendor now decodes CPUID.0 once for both kernel callers. The counters metal row holds every CPU's declaration (pm_enable=1) and its envelope in every read to its boot line, and every CPU's busy clock across the spin to the lowest Linux reached loaded on the same machine, which closes the defect that the T14 ran below it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcU2Dsw6mDYtwYfzVHPzM8 --- .../the-kernel-owns-cpu-performance-state.md | 43 ++++ ...ation-refuses-hybrid-intel-and-amd-cppc.md | 31 +++ ...the-scheduler-and-the-clocks-never-talk.md | 4 +- ...nder-load-below-the-clock-linux-reaches.md | 28 --- kernel/src/arch/aarch64/counters.rs | 5 +- kernel/src/arch/x86_64/control_regs.rs | 174 +++++++++++++++- kernel/src/arch/x86_64/counters.rs | 11 +- kernel/src/arch/x86_64/cpu.rs | 6 + kernel/src/counters.rs | 30 ++- kernel/src/syscall/machine.rs | 3 +- tests/toyos.rs | 33 ++- toyos-abi/src/counters.rs | 51 +++-- toyos-abi/src/handle.rs | 4 +- toyos-abi/src/syscall.rs | 4 +- toyos-cpuvuln/Cargo.toml | 2 +- toyos-cpuvuln/src/hwp.rs | 196 ++++++++++++++++++ toyos-cpuvuln/src/lib.rs | 5 +- toyos-inspect/src/kernel.rs | 6 +- toyos/src/syscap.rs | 5 +- 19 files changed, 560 insertions(+), 81 deletions(-) create mode 100644 issues/kernel/the-kernel-owns-cpu-performance-state.md create mode 100644 issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md delete mode 100644 issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md create mode 100644 toyos-cpuvuln/src/hwp.rs diff --git a/issues/kernel/the-kernel-owns-cpu-performance-state.md b/issues/kernel/the-kernel-owns-cpu-performance-state.md new file mode 100644 index 00000000000..81d3c82caa7 --- /dev/null +++ b/issues/kernel/the-kernel-owns-cpu-performance-state.md @@ -0,0 +1,43 @@ +--- +status: open +kind: track +opened: 2026-09-28 +--- + +# The kernel owns CPU performance state + +The self-hosting bar is measured under a fixed power envelope that must be +read back for a whole build span. The kernel declares the envelope from the +one CPU-state declaration (`kernel/src/arch/x86_64/control_regs.rs`) and +`SYS_COUNTERS` reads it back per CPU, under `Rights::TRACE`. + +**A register reaches a read only once boot has proven it**: the kernel has no +`rdmsr` fault fixup, so a register a read is the first to touch is a kernel +`#GP` a holder of the right can cause. Each register is enumerated by CPUID +or read at boot by the declaration's own check before a read can reach it. + +Declared and read back: the HWP request, its package's and the energy/ +performance bias, on every CPU that has all of them +(`toyos_cpuvuln::hwp`). Still to build: + +- **RAPL**: PL1, PL2 and their windows through `MSR_PKG_POWER_LIMIT` and the + MMIO mirror in the host bridge's MCHBAR, at the bar's values; a limit + firmware locked (bit 63) is refused by name. Neither register is + enumerated by CPUID. *Exit*: the T14 reads both back at the bar's values. +- **Turbo**: `IA32_MISC_ENABLE` bit 38 is firmware's. Its other bits are + model-specific, so declaring the one bit needs the owner's ruling on + writing that register whole. *Exit*: the ruling, and the bit declared or + recorded as firmware's. +- **The sampler**: a program that reads the envelope every 60 s and at a + span's start and end, with package power from `MSR_PKG_ENERGY_STATUS` and + temperature from `IA32_PACKAGE_THERM_STATUS` and `MSR_TEMPERATURE_TARGET`, + the bar's validity conditions. The energy counter is a side channel + (CVE-2020-8694), under `Rights::TRACE` like the rest. *Exit*: one valid + span on the T14. + +Not covered: hybrid Intel and AMD CPPC, +`issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md`; +AArch64 declares nothing. No QEMU CPU enumerates HWP, so every write and read +of these registers runs only on the T14. + +Owner: the orchestrator, which holds the T14. diff --git a/issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md b/issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md new file mode 100644 index 00000000000..92b339856ab --- /dev/null +++ b/issues/kernel/the-perf-state-declaration-refuses-hybrid-intel-and-amd-cppc.md @@ -0,0 +1,31 @@ +--- +status: open +kind: defect +opened: 2026-09-29 +--- + +# The perf-state declaration refuses hybrid Intel and AMD CPPC + +Every Intel client CPU since Alder Lake is hybrid, and AMD CPUs name their +performance controls through CPPC, not HWP. On both the kernel declares no +performance request, firmware's values stand, and `SYS_COUNTERS` carries no +power envelope. + +The refusals are `toyos_cpuvuln::hwp`'s (`toyos-cpuvuln/src/hwp.rs`): +- `HwpRefusal::Hybrid`, for CPUID.(7,0):EDX[15]: the declared minimum is a + ratio (`MSR_PLATFORM_INFO[47:40]`), and a hybrid CPU's HWP scale is not its + ratio scale. +- `HwpRefusal::NoHwp`, for CPUID.6:EAX[7] clear, which is where an AMD CPU + with CPPC is refused; its controls are not read. +- `HwpRefusal::NotIntel`, for HWP on any vendor but Intel, because the + minimum comes from `MSR_PLATFORM_INFO`, which is Intel's. + +`control_regs::hwp_declared` logs the refusal once and declares nothing on +any CPU. + +Owner: the orchestrator. **Exit**: on a hybrid Intel CPU each core type's +request is declared on that core's own HWP scale; on an AMD CPU with CPPC, +`MSR_AMD_CPPC_ENABLE` and `MSR_AMD_CPPC_REQ` are declared from its +`MSR_AMD_CPPC_CAP1` and asserted on every CPU; and on one machine of each +kind the `counters` metal row holds the envelope it reads back to the +declaration. diff --git a/issues/kernel/the-scheduler-and-the-clocks-never-talk.md b/issues/kernel/the-scheduler-and-the-clocks-never-talk.md index c9a9165e011..752d501f70f 100644 --- a/issues/kernel/the-scheduler-and-the-clocks-never-talk.md +++ b/issues/kernel/the-scheduler-and-the-clocks-never-talk.md @@ -21,6 +21,8 @@ Two couplings every production scheduler has and ours lacks: Both wait for an AML interpreter (`issues/kernel/toyos-runs-the-machine-in-acpi-mode-and-interprets-its-aml.md`), -because today the kernel neither reads nor sets P-states or C-states on metal. +because the kernel reads no C-state on metal and chooses no P-state: it +declares HWP's bounds (`kernel/src/arch/x86_64/control_regs.rs`), and the CPU +chooses within them. Filed so the placement track (see `placement-is-blind-to-caches-and-topology.md`) leaves room for an idle-state input when it designs wake placement. diff --git a/issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md b/issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md deleted file mode 100644 index 8a74e88ce38..00000000000 --- a/issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-04 ---- - -# ToyOS runs the T14 under load below the clock Linux reaches - -The `counters` metal row, at `ce1786ff0` (pull request #705), spun one thread -per CPU for its `spin` phase and read every CPU's busy clock as 2318 MHz -(TSC 2419 MHz times APERF over MPERF, MPERF at 0.986 to 0.990 of the stamp, -so every CPU was in C0 the whole span). Linux's turbostat on the same machine -under one `yes` per CPU (`tests/t14-linux/turbostat-loaded.txt`) reads -3800 MHz for its first 20 s and 3075 to 3094 MHz once the package settles near -20 W. ToyOS runs a loaded T14 at 61 to 75% of Linux's busy clock, below the -TSC's own rate. - -The CPU enumerates HWP (`hwp`, `hwp_epp` in -`toyos-cpuvuln/fixtures/t14/cpuinfo.txt`), and the kernel writes neither -`IA32_PM_ENABLE` nor `IA32_PERF_CTL`: the clock is whatever the firmware left. -`issues/kernel/the-scheduler-and-the-clocks-never-talk.md` is the track that -would couple the scheduler to frequency; this is the floor beneath it, a -machine that never leaves the firmware's operating point. - -Owner: the orchestrator, which holds the T14. **Exit**: the `counters` row -holds every CPU's busy clock across its spin to -Linux's loaded range in `tests/t14-linux/`, and the reading that held it is in -the closing pull request. diff --git a/kernel/src/arch/aarch64/counters.rs b/kernel/src/arch/aarch64/counters.rs index b808d6c166b..38e9eeb8333 100644 --- a/kernel/src/arch/aarch64/counters.rs +++ b/kernel/src/arch/aarch64/counters.rs @@ -1,10 +1,11 @@ //! No model-specific counter is read on this architecture: the activity -//! monitors' cycle counters (`FEAT_AMUv1`) are unread. +//! monitors' cycle counters (`FEAT_AMUv1`) are unread, and no performance +//! request is declared. use crate::counters::Hardware; pub fn bring_up() {} pub fn read() -> Hardware { - Hardware { smi: None, aperf: None, mperf: None } + Hardware { smi: None, aperf: None, mperf: None, envelope: None } } diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index d1c42c00d0d..0c110b18343 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -1,11 +1,22 @@ -//! What `CR0`, `CR4` and `IA32_EFER` hold on every CPU in this machine. One -//! declaration, applied by the BSP and every AP and checked on each; nothing -//! else may write any of the three. Each register is written whole: `CR0` -//! and `EFER` are constants, `CR4` is required bits plus whatever optional -//! bits this CPU offers. `EFER.NXE` lets bit 63 of a paging entry mean *not -//! executable* ([`Prot`](crate::mm::policy::Prot)). - -use core::sync::atomic::{AtomicU64, Ordering}; +//! What `CR0`, `CR4`, `IA32_EFER` and the performance request hold on every +//! CPU in this machine. One declaration, applied by the BSP and every AP and +//! checked on each; nothing else may write any of them. Each register is +//! written whole: `CR0` and `EFER` are constants, `CR4` is required bits plus +//! whatever optional bits this CPU offers. `EFER.NXE` lets bit 63 of a paging +//! entry mean *not executable* ([`Prot`](crate::mm::policy::Prot)). +//! +//! The performance request is HWP's: `IA32_HWP_INTERRUPT` where it exists, +//! `IA32_PM_ENABLE`, `IA32_HWP_REQUEST` as `toyos_cpuvuln::hwp_request` derives +//! it from this CPU's own registers, `IA32_HWP_REQUEST_PKG` and +//! `IA32_ENERGY_PERF_BIAS`, on a machine whose CPUs have every one of them, +//! and none on one that does not: refused by name once, firmware's values +//! standing. `IA32_MISC_ENABLE`'s turbo bit is not declared: that register's +//! other bits are model-specific and firmware's, and writing it whole would +//! decide them. + +use core::sync::atomic::{AtomicU64, AtomicU8, Ordering}; + +use toyos_cpuvuln::HwpFacts; use super::cpu; use crate::log; @@ -47,6 +58,33 @@ mod cr4 { pub const SMAP: u64 = 1 << 21; } +/// The performance request's registers, SDM Vol. 4. +mod hwp { + pub const PLATFORM_INFO: u32 = 0xCE; + pub const ENERGY_PERF_BIAS: u32 = 0x1B0; + pub const PM_ENABLE: u32 = 0x770; + pub const HWP_CAPABILITIES: u32 = 0x771; + pub const HWP_REQUEST_PKG: u32 = 0x772; + pub const HWP_INTERRUPT: u32 = 0x773; + pub const HWP_REQUEST: u32 = 0x774; +} + +/// `IA32_PM_ENABLE` on every CPU: HWP on, which only a reset turns off. +const PM_ENABLE: u64 = 1; + +/// `IA32_HWP_INTERRUPT` on every CPU that has it: nothing in this kernel takes +/// an HWP notification. +const HWP_INTERRUPT: u64 = 0; + +/// `IA32_HWP_REQUEST_PKG` on every CPU: the widest range at the request's own +/// preference. Inert, since no CPU's request sets package control, and +/// declared so that firmware does not decide it either. +const HWP_REQUEST_PKG: u64 = 0x01 | 0xff << 8 | toyos_cpuvuln::HWP_EPP << 24; + +/// `IA32_ENERGY_PERF_BIAS` on every CPU: Linux's `ENERGY_PERF_BIAS_NORMAL` +/// (`arch/x86/include/asm/msr-index.h:858`). +const ENERGY_PERF_BIAS: u64 = 6; + /// `CR0` on every CPU: `TS` stays clear because lazy FP switching would leak /// a register file across `#NM`; `AM` stays clear because Ring 3 has no `#AC` path. pub const CR0: u64 = cr0::PE | cr0::MP | cr0::ET | cr0::NE | cr0::WP | cr0::PG; @@ -106,10 +144,50 @@ pub fn init_cr0(cpu_id: u32) { bench::report(cpu_id, before); } -/// Puts this CPU's `CR4` and `EFER` into the declaration and checks all -/// three against it. Must run after [`init_cr0`] and before `arch::syscall::init`, which needs `SCE` set. +/// Whether the machine's declaration carries the performance request: the +/// BSP's verdict, which every AP must reach too. +static HWP: AtomicU8 = AtomicU8::new(HWP_UNDECIDED); +const HWP_UNDECIDED: u8 = 0; +const HWP_DECLARED: u8 = 1; +const HWP_REFUSED: u8 = 2; + +/// Whether this CPU gets a performance request, from CPUID alone, and whether +/// it has `IA32_HWP_INTERRUPT`: the machine's answer, a CPU that disagrees +/// named. +fn hwp_declared(cpu_id: u32) -> Option { + let (max_leaf, _, _, _) = cpu::cpuid(0, 0); + // A leaf above the maximum answers with the highest basic leaf's data. + let (cpuid_6_eax, _, cpuid_6_ecx, _) = if max_leaf >= 6 { cpu::cpuid(6, 0) } else { (0, 0, 0, 0) }; + let verdict = toyos_cpuvuln::hwp(&HwpFacts { + vendor: cpu::vendor(), + signature: cpu::cpuid(1, 0).0, + cpuid_6_eax, + cpuid_6_ecx, + cpuid_7_0_edx: if max_leaf >= 7 { cpu::cpuid(7, 0).3 } else { 0 }, + }); + let mine = if verdict.is_ok() { HWP_DECLARED } else { HWP_REFUSED }; + match HWP.compare_exchange(HWP_UNDECIDED, mine, Ordering::Release, Ordering::Acquire) { + Ok(_) => { + if let Err(refusal) = verdict { + log!("control_regs: no performance request is declared: {}", refusal.reason()); + } + } + Err(machine) => assert!( + machine == mine, + "control_regs: cpu{cpu_id} {} a performance request and the BSP {} one", + if mine == HWP_DECLARED { "can hold" } else { "cannot hold" }, + if machine == HWP_DECLARED { "declared" } else { "refused" }, + ), + } + verdict.ok() +} + +/// Puts this CPU's `CR4`, `EFER` and performance request into the declaration +/// and checks every register against it. Must run after [`init_cr0`] and +/// before `arch::syscall::init`, which needs `SCE` set. pub fn init(cpu_id: u32) { let declared = declaration(cpu_id); + let hwp = hwp_declared(cpu_id); if !skipped(cpu_id) { // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked @@ -126,6 +204,46 @@ pub fn init(cpu_id: u32) { } } self_check(cpu_id, declared); + if let Some(hwp) = hwp { + hwp_init(cpu_id, hwp); + } +} + +/// Puts this CPU's performance request into the declaration and checks it. +/// The interrupt is cleared and HWP enabled before any other HWP register is +/// touched, the request's input included: SDM Vol. 3B §17.4.2 (253669-093US), +/// "Additional MSRs associated with HWP may only be accessed after HWP is +/// enabled, with the exception of IA32_HWP_INTERRUPT and MSR_PPERF", and +/// `intel_pstate_hwp_enable`'s order. +fn hwp_init(cpu_id: u32, hwp: toyos_cpuvuln::Hwp) { + // SAFETY: `hwp_declared` admitted this CPU only where CPUID enumerates HWP, + // and `IA32_HWP_INTERRUPT` where it enumerates that too; both values hold + // only defined bits. + unsafe { + if hwp.notify { + cpu::wrmsr(hwp::HWP_INTERRUPT, HWP_INTERRUPT); + } + cpu::wrmsr(hwp::PM_ENABLE, PM_ENABLE); + } + let request = toyos_cpuvuln::hwp_request(cpu::rdmsr(hwp::HWP_CAPABILITIES), cpu::rdmsr(hwp::PLATFORM_INFO)); + // SAFETY: HWP is enabled, so its registers are live; CPUID enumerated EPP, + // the package request and EPB, and every value holds only defined bits. + unsafe { + cpu::wrmsr(hwp::HWP_REQUEST, request); + cpu::wrmsr(hwp::HWP_REQUEST_PKG, HWP_REQUEST_PKG); + cpu::wrmsr(hwp::ENERGY_PERF_BIAS, ENERGY_PERF_BIAS); + } + hwp_check(cpu_id, request, hwp.notify); +} + +/// The power envelope this CPU runs under, where the performance request is +/// declared: read only after this CPU's [`init`], which enabled HWP. +pub fn envelope() -> Option { + (HWP.load(Ordering::Acquire) == HWP_DECLARED).then(|| crate::counters::Envelope { + hwp_request: cpu::rdmsr(hwp::HWP_REQUEST), + hwp_request_pkg: cpu::rdmsr(hwp::HWP_REQUEST_PKG), + energy_perf_bias: cpu::rdmsr(hwp::ENERGY_PERF_BIAS), + }) } /// Whether the declaration carries `PCIDE`, and therefore whether `INVPCID` is this machine's flush. @@ -272,6 +390,42 @@ fn self_check(cpu_id: u32, declared_cr4: u64) { CHECKED.fetch_add(1, Ordering::Relaxed); } +/// [`self_check`] for the performance request, logged first for the same +/// reason; the line carries the request's two inputs, so a reader can +/// recompute it. +fn hwp_check(cpu_id: u32, request: u64, notify: bool) { + let pm_enable = cpu::rdmsr(hwp::PM_ENABLE); + let live = cpu::rdmsr(hwp::HWP_REQUEST); + let pkg = cpu::rdmsr(hwp::HWP_REQUEST_PKG); + let epb = cpu::rdmsr(hwp::ENERGY_PERF_BIAS); + let interrupt = notify.then(|| cpu::rdmsr(hwp::HWP_INTERRUPT)); + log!( + "control_regs: cpu{} pm_enable={} hwp_request={:#010x} hwp_request_pkg={:#010x} epb={} \ + hwp_interrupt={:x?} hwp_capabilities={:#010x} platform_info={:#018x}", + cpu_id, + pm_enable, + live, + pkg, + epb, + interrupt, + cpu::rdmsr(hwp::HWP_CAPABILITIES), + cpu::rdmsr(hwp::PLATFORM_INFO), + ); + for (name, holds, declared) in [ + ("pm_enable", Some(pm_enable), PM_ENABLE), + ("hwp_request", Some(live), request), + ("hwp_request_pkg", Some(pkg), HWP_REQUEST_PKG), + ("epb", Some(epb), ENERGY_PERF_BIAS), + ("hwp_interrupt", interrupt, HWP_INTERRUPT), + ] { + let Some(holds) = holds else { continue }; + assert!( + holds == declared, + "control_regs: cpu{cpu_id} holds {name}={holds:#x}, the declaration is {declared:#x}", + ); + } +} + /// How many CPUs hold the declaration, said once after the last of them has /// been checked. A divergent CPU panics inside [`self_check`], so what this /// line adds is the *count*: a CPU that never reached [`init`] at all is diff --git a/kernel/src/arch/x86_64/counters.rs b/kernel/src/arch/x86_64/counters.rs index efbc1eb891e..c4d423d7464 100644 --- a/kernel/src/arch/x86_64/counters.rs +++ b/kernel/src/arch/x86_64/counters.rs @@ -1,9 +1,10 @@ //! Which of this CPU's model-specific counters it reads: decided on the CPU -//! itself at bring-up, by `toyos_cpuvuln::counters` over its own CPUID. +//! itself at bring-up, by `toyos_cpuvuln::counters` over its own CPUID, and +//! the power envelope where `control_regs` declared it. use core::sync::atomic::{AtomicU8, Ordering::Relaxed}; -use toyos_cpuvuln::{CounterFacts, Vendor}; +use toyos_cpuvuln::CounterFacts; use super::{cpu, percpu}; use crate::counters::Hardware; @@ -20,13 +21,12 @@ const APERF_MPERF: u8 = 1 << 1; static ADMITTED: [AtomicU8; MAX_CPUS] = [const { AtomicU8::new(0) }; MAX_CPUS]; pub fn bring_up() { - let (max_leaf, ebx, ecx, edx) = cpu::cpuid(0, 0); - let id: [u8; 12] = core::array::from_fn(|i| [ebx, edx, ecx][i / 4].to_le_bytes()[i % 4]); + let max_leaf = cpu::cpuid(0, 0).0; let (signature, _, cpuid_1_ecx, _) = cpu::cpuid(1, 0); // As `init_scattered_cpuid_features` reads it: a level past the range's own is no level. let cpuid_6_ecx = if (6..=0xFFFF).contains(&max_leaf) { cpu::cpuid(6, 0).2 } else { 0 }; let verdict = toyos_cpuvuln::counters(&CounterFacts { - vendor: Vendor::from_id(&id), + vendor: cpu::vendor(), signature, cpuid_1_ecx, cpuid_6_ecx, @@ -44,5 +44,6 @@ pub fn read() -> Hardware { smi: msr(SMI, MSR_SMI_COUNT).map(|count| count & 0xFFFF_FFFF), aperf: msr(APERF_MPERF, IA32_APERF), mperf: msr(APERF_MPERF, IA32_MPERF), + envelope: super::control_regs::envelope(), } } diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 6e67f58bbf2..e3f40f26ee7 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -127,6 +127,12 @@ pub fn df_witness(site: &str) { ); } +/// CPUID.0's vendor string, as `toyos_cpuvuln` names it. +pub fn vendor() -> toyos_cpuvuln::Vendor { + let (_, ebx, ecx, edx) = cpuid(0, 0); + toyos_cpuvuln::Vendor::from_id(&core::array::from_fn(|i| [ebx, edx, ecx][i / 4].to_le_bytes()[i % 4])) +} + /// CPUID with both index registers; `rbx` is saved by hand since Rust reserves it as an operand. pub fn cpuid(leaf: u32, subleaf: u32) -> (u32, u32, u32, u32) { let eax: u32; diff --git a/kernel/src/counters.rs b/kernel/src/counters.rs index b0a43211113..7d7cd8c953b 100644 --- a/kernel/src/counters.rs +++ b/kernel/src/counters.rs @@ -12,7 +12,8 @@ //! [`JOIN`]. //! //! A model-specific counter is read only where the CPU's own verdict at -//! [`bring_up`] admitted it, and that bring-up's sample reads it once: a wrong +//! [`bring_up`] admitted it, and the power envelope only where the CPU-state +//! declaration declared it; that bring-up's sample reads each once, so a wrong //! verdict is `#GP` at boot, never inside a read. use kernel::sched::task::WaitClass; @@ -48,6 +49,15 @@ pub struct Hardware { pub smi: Option, pub aperf: Option, pub mperf: Option, + pub envelope: Option, +} + +/// The power envelope's registers, where the CPU's performance request is +/// declared. +pub struct Envelope { + pub hwp_request: u64, + pub hwp_request_pkg: u64, + pub energy_perf_bias: u64, } /// A block's words: the hardware id, a bit per counter it holds, and a word @@ -68,8 +78,19 @@ pub fn bring_up() { let me = percpu::cpu_id() as usize; let words = sample(me); BLOCKS[me].publish(words); - let [smi, aperf, mperf] = [Counter::Smi, Counter::Aperf, Counter::Mperf].map(|c| held(&words, c)); - log!("counters: cpu{me} reads smi={smi} aperf={aperf} mperf={mperf}"); + let [smi, aperf, mperf, request, pkg, epb] = [ + Counter::Smi, + Counter::Aperf, + Counter::Mperf, + Counter::HwpRequest, + Counter::HwpRequestPkg, + Counter::EnergyPerfBias, + ] + .map(|c| held(&words, c)); + log!( + "counters: cpu{me} reads smi={smi} aperf={aperf} mperf={mperf} hwp_request={request} \ + hwp_request_pkg={pkg} energy_perf_bias={epb}" + ); } fn held(words: &[u64; WORDS], counter: Counter) -> bool { @@ -89,6 +110,9 @@ fn sample(me: usize) -> [u64; WORDS] { Counter::Aperf => hardware.aperf, Counter::Mperf => hardware.mperf, Counter::Kicks => crate::irq_census::deliveries(me as u32, Source::Kick), + Counter::HwpRequest => hardware.envelope.as_ref().map(|e| e.hwp_request), + Counter::HwpRequestPkg => hardware.envelope.as_ref().map(|e| e.hwp_request_pkg), + Counter::EnergyPerfBias => hardware.envelope.as_ref().map(|e| e.energy_perf_bias), }; if let Some(value) = value { words[1] |= 1 << counter as usize; diff --git a/kernel/src/syscall/machine.rs b/kernel/src/syscall/machine.rs index df469aa4150..da3a809c58d 100644 --- a/kernel/src/syscall/machine.rs +++ b/kernel/src/syscall/machine.rs @@ -277,7 +277,8 @@ pub(super) fn sys_sched_info() -> toyos_abi::syscall::SchedInfo { /// Every online CPU's counters into `out`, or the CPU count when `out` is /// empty: requires a `SysCap` carrying `Rights::COUNTERS`, and answers the -/// counters that time programs only where it carries `Rights::TRACE` too. +/// counters that time programs and the power envelope only where it carries +/// `Rights::TRACE` too. pub(super) fn sys_counters(syscap: RawHandle, out: &mut UserBytesMut) -> u64 { let rights = match demand_syscap(syscap, Rights::COUNTERS) { Ok(rights) => rights, diff --git a/tests/toyos.rs b/tests/toyos.rs index 7fac3b67117..26c252dae98 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -3264,17 +3264,20 @@ const SMI_SPAN_NS: u64 = 4_444_000_000; /// /// Held: a record per CPU the bring-up started, naming the local APIC id the /// bring-up gave that CPU and carrying the counters its `counters: cpuN -/// reads` line names, and none stale. From `idle0` to `spin`, at least +/// reads` line names, and none stale; every CPU's performance request +/// declared at boot, `pm_enable=1`, and its power envelope in every read the +/// one its `control_regs:` line holds. From `idle0` to `spin`, at least /// [`SMI_SPAN_NS`] apart, every CPU's SMI count rose alike and by two or more: /// the firmware's legacy mode, the positive control ACPI stage 1's flatness /// is read against, and the row that stage changes. Across the spin every /// CPU's MPERF ran nine tenths of its stamp or more [e], a CPU in C0 the whole /// span: MPERF counts at the TSC's rate there (SDM Vol. 3B, "Hardware -/// Coordination Feedback"). +/// Coordination Feedback"); and every CPU's busy frequency reached the lowest +/// Linux's turbostat read under one `yes` per CPU on the same machine +/// (`tests/t14-linux/turbostat-loaded.txt`). /// -/// Read and not held, beside Linux's turbostat on the same machine -/// (`tests/t14-linux/`): each CPU's idle busy fraction, its busy -/// frequency under the spin, and what one round cost its reader. +/// Read and not held, beside Linux's turbostat: each CPU's idle busy +/// fraction, and what one round cost its reader. fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { type Read<'a> = BTreeMap>; back.job_passed("test_rs_counters_metal")?; @@ -3320,7 +3323,7 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { } for (cpu, counters) in read { let reads = kernel.must_say(&format!("counters: cpu{cpu} reads "))?; - for counter in ["smi", "aperf", "mperf"] { + for counter in ["smi", "aperf", "mperf", "hwp_request", "hwp_request_pkg", "energy_perf_bias"] { if !reads.contains(&format!("{counter}={}", counters.contains_key(counter))) { return Err(format!("{name}: cpu{cpu} carries {counters:?} and its bring-up said {reads:?}")); } @@ -3328,6 +3331,18 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { if counters.get("stale") != Some(&0) || counters.get("hardware_id").map(u64::to_string) != Some(roster[*cpu].clone()) { return Err(format!("{name}: cpu{cpu} is stale or not lapic {}: {counters:?}", roster[*cpu])); } + let declared = kernel.must_say(&format!("control_regs: cpu{cpu} pm_enable=1 "))?; + for (counter, field, radix) in + [("hwp_request", "hwp_request=0x", 16), ("hwp_request_pkg", "hwp_request_pkg=0x", 16), ("energy_perf_bias", "epb=", 10)] + { + let held = declared + .split(' ') + .find_map(|w| u64::from_str_radix(w.strip_prefix(field)?, radix).ok()) + .ok_or_else(|| format!("cpu{cpu}'s declaration carries no {field}: {declared:?}"))?; + if counters.get(counter) != Some(&held) { + return Err(format!("{name}: cpu{cpu} reads {counter} {:?} and boot declared {held}: {declared:?}", counters.get(counter))); + } + } } } for cpu in 0..cpus { @@ -3362,6 +3377,7 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { }; let idle = linux(include_str!("t14-linux/turbostat-idle.txt"), "Busy%")?; let loaded = linux(include_str!("t14-linux/turbostat-loaded.txt"), "Bzy_MHz")?; + let spinning: Vec = (0..cpus).map(|cpu| tsc_mhz * ratio(idle1, spin, cpu, "aperf", "mperf")).collect(); eprintln!( " [counters] {cpus} cpus, SMI +{} each over {} ms; TSC {tsc_mhz:.0} MHz", smis[0], @@ -3374,12 +3390,15 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { ratio(idle0, idle1, cpu, "mperf", "stamp") * 100.0, idle.0, idle.1, - tsc_mhz * ratio(idle1, spin, cpu, "aperf", "mperf"), + spinning[cpu], loaded.0, loaded.1, ); } eprintln!(" [counters] the spin's read, a whole round, took its reader {} ns", clock["spin"].1); + if spinning.iter().any(|&mhz| mhz < loaded.0) { + return Err(format!("spinning at {spinning:.0?} MHz, some cpu below the {:.0} Linux reached loaded", loaded.0)); + } Ok(()) } diff --git a/toyos-abi/src/counters.rs b/toyos-abi/src/counters.rs index fd7109da628..69643bd7ff6 100644 --- a/toyos-abi/src/counters.rs +++ b/toyos-abi/src/counters.rs @@ -9,7 +9,9 @@ //! absent and never zero; a record with no stamp is a CPU whose answer could //! not be read whole, and says nothing about it but its index. //! -//! Every value counts up from boot, so the difference of two reads is exact. +//! Every counter counts up from boot, so the difference of two reads is exact; +//! the power envelope's registers ([`Counter::HwpRequest`] and those after it) +//! count nothing and are what the CPU held when it read them. //! //! [`SYS_COUNTERS`]: crate::syscall::SYS_COUNTERS @@ -31,12 +33,27 @@ pub enum Counter { Mperf, /// Kicks the CPU took: the interrupt another CPU sends it to wake it. Kicks, + /// The CPU's performance request (`IA32_HWP_REQUEST`), where the kernel + /// declares one. + HwpRequest, + /// Its package's request (`IA32_HWP_REQUEST_PKG`), beside it. + HwpRequestPkg, + /// Its energy/performance bias (`IA32_ENERGY_PERF_BIAS`), beside it. + EnergyPerfBias, } impl Counter { - pub const COUNT: usize = 5; - pub const ALL: [Counter; Self::COUNT] = - [Self::Stamp, Self::Smi, Self::Aperf, Self::Mperf, Self::Kicks]; + pub const COUNT: usize = 8; + pub const ALL: [Counter; Self::COUNT] = [ + Self::Stamp, + Self::Smi, + Self::Aperf, + Self::Mperf, + Self::Kicks, + Self::HwpRequest, + Self::HwpRequestPkg, + Self::EnergyPerfBias, + ]; pub const fn name(self) -> &'static str { match self { @@ -45,6 +62,9 @@ impl Counter { Self::Aperf => "aperf", Self::Mperf => "mperf", Self::Kicks => "kicks", + Self::HwpRequest => "hwp_request", + Self::HwpRequestPkg => "hwp_request_pkg", + Self::EnergyPerfBias => "energy_perf_bias", } } @@ -52,7 +72,12 @@ impl Counter { pub const fn needs(self) -> Rights { match self { Self::Stamp | Self::Smi => Rights::COUNTERS, - Self::Aperf | Self::Mperf | Self::Kicks => Rights::COUNTERS.union(Rights::TRACE), + Self::Aperf + | Self::Mperf + | Self::Kicks + | Self::HwpRequest + | Self::HwpRequestPkg + | Self::EnergyPerfBias => Rights::COUNTERS.union(Rights::TRACE), } } } @@ -154,8 +179,8 @@ mod tests { #[test] fn every_shape_of_record_reads_back_as_written() { for r in [ - record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0)]), - record(true, [Some(9), None, None, None, Some(4)]), + record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0), Some(0x8000_2a04), Some(6), Some(0)]), + record(true, [Some(9), None, None, None, Some(4), None, Some(0x8000_ff01), None]), record(true, [None; Counter::COUNT]), ] { assert_eq!(Record::decode(&r.encode()), Ok(r)); @@ -165,7 +190,7 @@ mod tests { /// Absent is not zero: a zero that is there reads back there. #[test] fn a_present_zero_is_not_an_absent_counter() { - let r = record(false, [Some(5), Some(0), None, None, None]); + let r = record(false, [Some(5), Some(0), None, None, None, None, None, None]); assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Smi), Some(0)); assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Aperf), None); } @@ -182,20 +207,20 @@ mod tests { #[test] fn a_word_under_an_absent_counter_is_refused() { - let mut raw = record(false, [Some(1), None, None, None, None]).encode(); + let mut raw = record(false, [Some(1), None, None, None, None, None, None, None]).encode(); raw.0[16 + 8 * Counter::Mperf as usize] = 1; assert_eq!(Record::decode(&raw), Err(Undecodable::Absent(Counter::Mperf))); } - /// The counters that time programs are the trace right's, and the two that - /// do not are readable on `COUNTERS` alone. + /// The counters that time programs and the power envelope are the trace + /// right's, and the two that are neither are readable on `COUNTERS` alone. #[test] - fn what_times_a_program_needs_the_trace_right() { + fn what_times_a_program_or_is_power_needs_the_trace_right() { for counter in Counter::ALL { assert!(counter.needs().contains(Rights::COUNTERS), "{counter:?}"); assert_eq!( counter.needs().contains(Rights::TRACE), - matches!(counter, Counter::Aperf | Counter::Mperf | Counter::Kicks), + !matches!(counter, Counter::Stamp | Counter::Smi), "{counter:?}" ); } diff --git a/toyos-abi/src/handle.rs b/toyos-abi/src/handle.rs index 5c503c112a7..220b683b98b 100644 --- a/toyos-abi/src/handle.rs +++ b/toyos-abi/src/handle.rs @@ -151,8 +151,8 @@ impl Rights { /// /// Beside [`COUNTERS`](Self::COUNTERS), [`SYS_COUNTERS`] answers each /// CPU's `APERF` and `MPERF` — its frequency and busy fraction, the - /// Hertzbleed power channel — and the kicks it took, the wake-ups that time - /// keystrokes. Admin tools hold it, `inspect` and `test-runner`; no + /// Hertzbleed power channel — the kicks it took, the wake-ups that time + /// keystrokes, and the power envelope it runs under. Admin tools hold it, `inspect` and `test-runner`; no /// ordinary program and no toybox applet does while a manifest row grants /// a whole binary. /// diff --git a/toyos-abi/src/syscall.rs b/toyos-abi/src/syscall.rs index d63e214f3c3..162b831292f 100644 --- a/toyos-abi/src/syscall.rs +++ b/toyos-abi/src/syscall.rs @@ -260,8 +260,8 @@ pub const SYS_DEVICE_DMA_MAP: u64 = 122; pub const SYS_DEVICE_DMA_UNMAP: u64 = 123; /// Every online CPU's counters, as [`crate::counters`] records. Gated by -/// [`Rights::COUNTERS`] on a `SysCap`, and the counters that time programs by -/// [`Rights::TRACE`] beside it. See [`counters`]. +/// [`Rights::COUNTERS`] on a `SysCap`, and the counters that time programs and +/// the power envelope by [`Rights::TRACE`] beside it. See [`counters`]. /// /// [`Rights::COUNTERS`]: crate::handle::Rights::COUNTERS /// [`Rights::TRACE`]: crate::handle::Rights::TRACE diff --git a/toyos-cpuvuln/Cargo.toml b/toyos-cpuvuln/Cargo.toml index 2bee37a101e..6ce1bb2b9f3 100644 --- a/toyos-cpuvuln/Cargo.toml +++ b/toyos-cpuvuln/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "toyos-cpuvuln" -description = "Linux's verdicts from a CPU's CPUID and MSR facts, pure: each speculative-execution vulnerability and its mitigation choice, and which model-specific counters the CPU has." +description = "Linux's verdicts from a CPU's CPUID and MSR facts, pure: each speculative-execution vulnerability and its mitigation choice, which model-specific counters the CPU has, and the HWP request intel_pstate asks of it." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/toyos-cpuvuln/src/hwp.rs b/toyos-cpuvuln/src/hwp.rs new file mode 100644 index 00000000000..636f0763913 --- /dev/null +++ b/toyos-cpuvuln/src/hwp.rs @@ -0,0 +1,196 @@ +//! Whether a CPU's performance request is declared through HWP, and the +//! request: the one `intel_pstate` asks of it under the `powersave` governor +//! at its defaults, read at upstream Linux `v6.8`'s +//! `drivers/cpufreq/intel_pstate.c` and `arch/x86/include/asm/msr-index.h`, +//! not at this crate's pinned tag. The minimum is `MSR_PLATFORM_INFO[47:40]` +//! (`core_get_min_pstate`, `:1859-1864`), the maximum the CPU's highest level, +//! turbo included (`__intel_pstate_get_hwp_cap`, `:940-948`), and the +//! preference `HWP_EPP_BALANCE_PERFORMANCE` (`msr-index.h:500`), the +//! `balance_performance` it leaves on every CPU `intel_epp_balance_perf` +//! (`:3410-3419`) does not list. +//! +//! **The refusals are ToyOS's, not Linux's**: `intel_pstate` drives hybrid +//! CPUs on a scale of their own and leaves other vendors to other drivers, +//! and neither is modelled here. + +use crate::{Ident, Vendor}; + +/// What one CPU says about itself, read on that CPU. +#[derive(Clone, Copy, Debug)] +pub struct HwpFacts { + pub vendor: Vendor, + /// CPUID.1:EAX. + pub signature: u32, + /// 0 unless CPUID.0:EAX reaches 6. + pub cpuid_6_eax: u32, + /// 0 unless CPUID.0:EAX reaches 6. + pub cpuid_6_ecx: u32, + /// 0 unless CPUID.0:EAX reaches 7. + pub cpuid_7_0_edx: u32, +} + +/// A CPU whose request is declared. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Hwp { + /// `IA32_HWP_INTERRUPT` exists (CPUID.6:EAX[8]); it is `#GP` where not. + pub notify: bool, +} + +/// Why a CPU's request is not declared, and its performance state stays what +/// firmware left. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum HwpRefusal { + NoHwp, + NotIntel, + NotFamily6, + NoEpp, + NoPackageRequest, + NoEnergyPerfBias, + Hybrid, +} + +impl HwpRefusal { + pub const fn reason(self) -> &'static str { + match self { + Self::NoHwp => "no HWP (CPUID.6:EAX[7])", + Self::NotIntel => "HWP on a CPU that is not Intel's, and the minimum is MSR_PLATFORM_INFO's, which is Intel's", + Self::NotFamily6 => "an Intel CPU outside family 6, which no CPUID bit enumerates MSR_PLATFORM_INFO on", + Self::NoEpp => "HWP without an energy/performance preference (CPUID.6:EAX[10])", + Self::NoPackageRequest => "HWP without a package request (CPUID.6:EAX[11])", + Self::NoEnergyPerfBias => "no energy/performance bias (CPUID.6:ECX[3])", + Self::Hybrid => "a hybrid CPU (CPUID.(7,0):EDX[15]), whose HWP scale is not its ratio scale", + } + } +} + +// `arch/x86/include/asm/cpufeatures.h`'s bits at their CPUID positions. +const CPUID_6_EAX_HWP: u32 = 1 << 7; +const CPUID_6_EAX_HWP_NOTIFY: u32 = 1 << 8; +const CPUID_6_EAX_HWP_EPP: u32 = 1 << 10; +const CPUID_6_EAX_HWP_PKG_REQ: u32 = 1 << 11; +const CPUID_6_ECX_EPB: u32 = 1 << 3; +const CPUID_7_0_EDX_HYBRID_CPU: u32 = 1 << 15; + +/// **All or nothing**: a CPU lacking one register the declaration writes gets +/// none of them, so no reader takes firmware's value of one for the +/// declaration's. +pub fn hwp(facts: &HwpFacts) -> Result { + let eax = facts.cpuid_6_eax; + let refusal = if eax & CPUID_6_EAX_HWP == 0 { + Some(HwpRefusal::NoHwp) + } else if facts.vendor != Vendor::Intel { + Some(HwpRefusal::NotIntel) + } else if Ident::new(facts.vendor, facts.signature).family != 6 { + Some(HwpRefusal::NotFamily6) + } else if eax & CPUID_6_EAX_HWP_EPP == 0 { + Some(HwpRefusal::NoEpp) + } else if eax & CPUID_6_EAX_HWP_PKG_REQ == 0 { + Some(HwpRefusal::NoPackageRequest) + } else if facts.cpuid_6_ecx & CPUID_6_ECX_EPB == 0 { + Some(HwpRefusal::NoEnergyPerfBias) + } else if facts.cpuid_7_0_edx & CPUID_7_0_EDX_HYBRID_CPU != 0 { + Some(HwpRefusal::Hybrid) + } else { + None + }; + match refusal { + Some(refusal) => Err(refusal), + None => Ok(Hwp { notify: eax & CPUID_6_EAX_HWP_NOTIFY != 0 }), + } +} + +/// `HWP_EPP_BALANCE_PERFORMANCE`. +pub const HWP_EPP: u64 = 0x80; + +/// `IA32_HWP_REQUEST` for a CPU whose `IA32_HWP_CAPABILITIES` reads +/// `capabilities`, in a package whose `MSR_PLATFORM_INFO` reads +/// `platform_info`: no desired level, no activity window, no package control, +/// so the CPU chooses between the two levels itself (SDM Vol. 3B, "Managing +/// HWP"; `msr-index.h:495-504`). +pub const fn hwp_request(capabilities: u64, platform_info: u64) -> u64 { + let min = (platform_info >> 40) & 0xff; + let max = capabilities & 0xff; + min | max << 8 | HWP_EPP << 24 +} + +#[cfg(test)] +mod tests { + extern crate std; + + use std::vec::Vec; + + use super::*; + + /// The T14's CPU 0, off `fixtures/t14/cpuid.txt`. + fn t14() -> HwpFacts { + let leaves: Vec> = include_str!("../fixtures/t14/cpuid.txt") + .lines() + .map(|l| l.split(' ').map(|w| u32::from_str_radix(w, 16).expect(w)).collect()) + .collect(); + let leaf = |n: u32| leaves.iter().find(|l| l[..2] == [n, 0]).expect("a leaf the capture read"); + let id: Vec = [leaf(0)[3], leaf(0)[5], leaf(0)[4]].iter().flat_map(|r| r.to_le_bytes()).collect(); + assert!(leaf(0)[2] >= 7, "the T14 reaches leaf 7"); + HwpFacts { + vendor: Vendor::from_id(id[..].try_into().unwrap()), + signature: leaf(1)[2], + cpuid_6_eax: leaf(6)[2], + cpuid_6_ecx: leaf(6)[4], + cpuid_7_0_edx: leaf(7)[5], + } + } + + /// `cpuinfo.txt` lists `hwp hwp_notify hwp_act_window hwp_epp + /// hwp_pkg_req` and `epb`, and no `hybrid_cpu`. + #[test] + fn the_t14_is_declared_with_its_notification() { + let flags = include_str!("../fixtures/t14/cpuinfo.txt") + .lines() + .find_map(|l| l.strip_prefix("flags\t\t: ")) + .expect("a flags line"); + for flag in ["hwp", "hwp_notify", "hwp_epp", "hwp_pkg_req", "epb"] { + assert!(flags.split(' ').any(|f| f == flag), "{flag}"); + } + assert!(!flags.split(' ').any(|f| f == "hybrid_cpu")); + assert_eq!(hwp(&t14()), Ok(Hwp { notify: true })); + assert_eq!(hwp(&HwpFacts { cpuid_6_eax: t14().cpuid_6_eax & !CPUID_6_EAX_HWP_NOTIFY, ..t14() }), Ok(Hwp { notify: false })); + } + + /// QEMU's `qemu64`, the TCG model: `AuthenticAMD`, and no HWP in leaf 6. + #[test] + fn the_tcg_model_is_refused() { + let tcg = HwpFacts { vendor: Vendor::Amd, signature: 0x0006_0fb1, cpuid_6_eax: 1 << 2, cpuid_6_ecx: 0, cpuid_7_0_edx: 0 }; + assert_eq!(hwp(&tcg), Err(HwpRefusal::NoHwp)); + } + + #[test] + fn a_missing_register_refuses_the_whole_request_by_name() { + let without = |eax: u32, ecx: u32| hwp(&HwpFacts { cpuid_6_eax: t14().cpuid_6_eax & !eax, cpuid_6_ecx: t14().cpuid_6_ecx & !ecx, ..t14() }); + assert_eq!(without(CPUID_6_EAX_HWP, 0), Err(HwpRefusal::NoHwp)); + assert_eq!(without(CPUID_6_EAX_HWP_EPP, 0), Err(HwpRefusal::NoEpp)); + assert_eq!(without(CPUID_6_EAX_HWP_PKG_REQ, 0), Err(HwpRefusal::NoPackageRequest)); + assert_eq!(without(0, CPUID_6_ECX_EPB), Err(HwpRefusal::NoEnergyPerfBias)); + assert_eq!(hwp(&HwpFacts { cpuid_7_0_edx: CPUID_7_0_EDX_HYBRID_CPU, ..t14() }), Err(HwpRefusal::Hybrid)); + assert_eq!(hwp(&HwpFacts { vendor: Vendor::Amd, ..t14() }), Err(HwpRefusal::NotIntel)); + } + + /// The family is `x86_family`'s: 0xF plus the extended field, never the + /// extended field beside family 6. + #[test] + fn a_cpu_outside_family_6_is_refused_by_name() { + let of = |signature: u32| hwp(&HwpFacts { signature, ..t14() }); + assert_eq!(of(0x0000_0f41), Err(HwpRefusal::NotFamily6)); + assert_eq!(of(0x0030_0f00), Err(HwpRefusal::NotFamily6)); + assert_eq!(of(0x0ff0_0600), Ok(Hwp { notify: true })); + } + + /// Each field where `msr-index.h` puts it, from the bits its source names + /// and no other. + #[test] + fn the_request_is_the_minimum_ratio_the_highest_level_and_the_preference() { + assert_eq!(hwp_request(0, 0), 0x8000_0000); + assert_eq!(hwp_request(0xffff_ff2a, 0), 0x8000_2a00); + assert_eq!(hwp_request(0, 0x04 << 40), 0x8000_0004); + assert_eq!(hwp_request(0, !(0xff << 40)), 0x8000_0000); + assert_eq!(hwp_request(u64::MAX, u64::MAX), 0x8000_ffff); + } +} diff --git a/toyos-cpuvuln/src/lib.rs b/toyos-cpuvuln/src/lib.rs index 181e3bd1125..20c9e7270d8 100644 --- a/toyos-cpuvuln/src/lib.rs +++ b/toyos-cpuvuln/src/lib.rs @@ -16,7 +16,8 @@ //! [`Unmodelled`], never approximated. //! //! [`counters`] is the same tag's other verdict on a CPU: which of its -//! model-specific counters exist. +//! model-specific counters exist. [`hwp`] is the third: whether its +//! performance request is declared, and [`hwp_request`] the request. //! //! Pure: no I/O, no allocation, no `unsafe`. The caller reads the facts. @@ -26,9 +27,11 @@ use core::fmt; mod counters; +mod hwp; mod table; pub use counters::{counters, CounterFacts, Counters}; +pub use hwp::{hwp, hwp_request, Hwp, HwpFacts, HwpRefusal, HWP_EPP}; #[cfg(test)] mod tests; diff --git a/toyos-inspect/src/kernel.rs b/toyos-inspect/src/kernel.rs index ca5ff1513ff..47b8de70d8b 100644 --- a/toyos-inspect/src/kernel.rs +++ b/toyos-inspect/src/kernel.rs @@ -2,7 +2,7 @@ //! //! A root no port answers for, as `dev` is: the reader asks the kernel on a //! `SysCap` carrying `Rights::COUNTERS`, and `Rights::TRACE` for the counters -//! that time programs, and renders its records here. +//! that time programs and the power envelope, and renders its records here. //! //! ```text //! kernel.cpu..stale whether the CPU missed this read's bound @@ -65,7 +65,7 @@ mod tests { /// The T14's shape: every counter on every CPU. fn whole(cpu: u32) -> Record { - record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3)]) + record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3), Some(0x8000_2a04), Some(0x8000_ff01), Some(6)]) } #[test] @@ -84,7 +84,7 @@ mod tests { /// `TRACE`: only what the record carries, and no zero in place of the rest. #[test] fn an_absent_counter_has_no_path() { - let got = render(&[record(0, false, [Some(10), None, None, None, None])]).unwrap(); + let got = render(&[record(0, false, [Some(10), None, None, None, None, None, None, None])]).unwrap(); let paths: Vec<&str> = got.keys().map(String::as_str).collect(); assert_eq!(paths, vec!["kernel.cpu.0.hardware_id", "kernel.cpu.0.stale", "kernel.cpu.0.stamp"]); } diff --git a/toyos/src/syscap.rs b/toyos/src/syscap.rs index 9e20f70f690..50e2824dc15 100644 --- a/toyos/src/syscap.rs +++ b/toyos/src/syscap.rs @@ -134,8 +134,9 @@ impl SysCap { /// how many. An empty `buf` asks how many CPUs there are — see /// [`syscall::counters`]. /// - /// Needs [`Rights::COUNTERS`]; the counters that time programs come back - /// only where the capability carries [`Rights::TRACE`] too. + /// Needs [`Rights::COUNTERS`]; the counters that time programs and the + /// power envelope come back only where the capability carries + /// [`Rights::TRACE`] too. pub fn counters( &self, buf: &mut [toyos_abi::counters::RawRecord], From 14fa36de14aa87d94872b018e64082f7832baff7 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 4 Oct 2026 10:26:28 +0200 Subject: [PATCH 11/12] Answer the rebuild's review: HWP after the IDT, one leaf-6 reader, the row held to Linux's request and to Linux's clock over the spin's span - The performance request moves out of control_regs::init into control_regs::init_performance, called on the BSP after idt::init and on every AP after control_regs::init (an AP's IDT is loaded by its trampoline). A #GP in one of its wrmsr or rdmsr now panics by name instead of stopping the BSP under firmware's table. CR4.OSFXSR still precedes the IDT. - CPUID leaf 6 is read once, by cpu::leaf_6, under the rule the counters' bring-up already used (a max leaf outside 6..=0xFFFF reaches no leaf 6); control_regs::hwp_declared and counters::bring_up both call it. - tests/t14-linux/hwp-request.txt commits Linux's IA32_HWP_REQUEST on the T14, 0x80002a04 on all eight CPUs, from commit 868378b87's message (#568); counters_on_metal holds every CPU's boot hwp_request to it. - The clock floor is Linux's busy clock over the intervals that open within the spin's own span (turbostat's first opens 2 s into the load, each 10 s), not the lowest after the package settles: 3800 MHz for the 11.2 s spin, where it was 3075. Judged against the T14 readbacks of 8c4d1f57f: the head's EXIT=0, the control's (d47b383cf) EXIT=1; the head's with every spin APERF scaled to 3100 MHz EXIT=1 under this judge and EXIT=0 under 8c4d1f57f's; the oracle file set to 0x80002a05 EXIT=1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcU2Dsw6mDYtwYfzVHPzM8 --- kernel/src/arch/x86_64/control_regs.rs | 25 +++++++------- kernel/src/arch/x86_64/counters.rs | 4 +-- kernel/src/arch/x86_64/cpu.rs | 12 +++++++ kernel/src/arch/x86_64/percpu.rs | 2 ++ tests/t14-linux/SOURCE | 10 ++++++ tests/t14-linux/hwp-request.txt | 1 + tests/toyos.rs | 45 ++++++++++++++++++-------- 7 files changed, 69 insertions(+), 30 deletions(-) create mode 100644 tests/t14-linux/hwp-request.txt diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 0c110b18343..d5e31beea78 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -155,15 +155,14 @@ const HWP_REFUSED: u8 = 2; /// it has `IA32_HWP_INTERRUPT`: the machine's answer, a CPU that disagrees /// named. fn hwp_declared(cpu_id: u32) -> Option { - let (max_leaf, _, _, _) = cpu::cpuid(0, 0); - // A leaf above the maximum answers with the highest basic leaf's data. - let (cpuid_6_eax, _, cpuid_6_ecx, _) = if max_leaf >= 6 { cpu::cpuid(6, 0) } else { (0, 0, 0, 0) }; + let (cpuid_6_eax, cpuid_6_ecx) = cpu::leaf_6(); let verdict = toyos_cpuvuln::hwp(&HwpFacts { vendor: cpu::vendor(), signature: cpu::cpuid(1, 0).0, cpuid_6_eax, cpuid_6_ecx, - cpuid_7_0_edx: if max_leaf >= 7 { cpu::cpuid(7, 0).3 } else { 0 }, + // A leaf above the maximum answers with the highest basic leaf's data. + cpuid_7_0_edx: if cpu::cpuid(0, 0).0 >= 7 { cpu::cpuid(7, 0).3 } else { 0 }, }); let mine = if verdict.is_ok() { HWP_DECLARED } else { HWP_REFUSED }; match HWP.compare_exchange(HWP_UNDECIDED, mine, Ordering::Release, Ordering::Acquire) { @@ -182,12 +181,11 @@ fn hwp_declared(cpu_id: u32) -> Option { verdict.ok() } -/// Puts this CPU's `CR4`, `EFER` and performance request into the declaration -/// and checks every register against it. Must run after [`init_cr0`] and -/// before `arch::syscall::init`, which needs `SCE` set. +/// Puts this CPU's `CR4` and `EFER` into the declaration and checks both +/// against it. Must run after [`init_cr0`] and before `arch::syscall::init`, +/// which needs `SCE` set. pub fn init(cpu_id: u32) { let declared = declaration(cpu_id); - let hwp = hwp_declared(cpu_id); if !skipped(cpu_id) { // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked @@ -204,18 +202,19 @@ pub fn init(cpu_id: u32) { } } self_check(cpu_id, declared); - if let Some(hwp) = hwp { - hwp_init(cpu_id, hwp); - } } /// Puts this CPU's performance request into the declaration and checks it. +/// Must run after [`init`] and after this CPU's IDT is loaded, so a fault in +/// its `wrmsr` or `rdmsr` panics by name. +/// /// The interrupt is cleared and HWP enabled before any other HWP register is /// touched, the request's input included: SDM Vol. 3B §17.4.2 (253669-093US), /// "Additional MSRs associated with HWP may only be accessed after HWP is /// enabled, with the exception of IA32_HWP_INTERRUPT and MSR_PPERF", and /// `intel_pstate_hwp_enable`'s order. -fn hwp_init(cpu_id: u32, hwp: toyos_cpuvuln::Hwp) { +pub fn init_performance(cpu_id: u32) { + let Some(hwp) = hwp_declared(cpu_id) else { return }; // SAFETY: `hwp_declared` admitted this CPU only where CPUID enumerates HWP, // and `IA32_HWP_INTERRUPT` where it enumerates that too; both values hold // only defined bits. @@ -237,7 +236,7 @@ fn hwp_init(cpu_id: u32, hwp: toyos_cpuvuln::Hwp) { } /// The power envelope this CPU runs under, where the performance request is -/// declared: read only after this CPU's [`init`], which enabled HWP. +/// declared: read only after this CPU's [`init_performance`], which enabled HWP. pub fn envelope() -> Option { (HWP.load(Ordering::Acquire) == HWP_DECLARED).then(|| crate::counters::Envelope { hwp_request: cpu::rdmsr(hwp::HWP_REQUEST), diff --git a/kernel/src/arch/x86_64/counters.rs b/kernel/src/arch/x86_64/counters.rs index c4d423d7464..4c4b25dae3b 100644 --- a/kernel/src/arch/x86_64/counters.rs +++ b/kernel/src/arch/x86_64/counters.rs @@ -21,10 +21,8 @@ const APERF_MPERF: u8 = 1 << 1; static ADMITTED: [AtomicU8; MAX_CPUS] = [const { AtomicU8::new(0) }; MAX_CPUS]; pub fn bring_up() { - let max_leaf = cpu::cpuid(0, 0).0; let (signature, _, cpuid_1_ecx, _) = cpu::cpuid(1, 0); - // As `init_scattered_cpuid_features` reads it: a level past the range's own is no level. - let cpuid_6_ecx = if (6..=0xFFFF).contains(&max_leaf) { cpu::cpuid(6, 0).2 } else { 0 }; + let (_, cpuid_6_ecx) = cpu::leaf_6(); let verdict = toyos_cpuvuln::counters(&CounterFacts { vendor: cpu::vendor(), signature, diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index e3f40f26ee7..83872bffe5f 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -133,6 +133,18 @@ pub fn vendor() -> toyos_cpuvuln::Vendor { toyos_cpuvuln::Vendor::from_id(&core::array::from_fn(|i| [ebx, edx, ecx][i / 4].to_le_bytes()[i % 4])) } +/// CPUID.6's EAX and ECX, the thermal and power leaf; zero where CPUID.0 +/// does not reach it, as `init_scattered_cpuid_features` reads it: a level +/// past the range's own is no level. +pub fn leaf_6() -> (u32, u32) { + if (6..=0xFFFF).contains(&cpuid(0, 0).0) { + let (eax, _, ecx, _) = cpuid(6, 0); + (eax, ecx) + } else { + (0, 0) + } +} + /// CPUID with both index registers; `rbx` is saved by hand since Rust reserves it as an operand. pub fn cpuid(leaf: u32, subleaf: u32) -> (u32, u32, u32, u32) { let eax: u32; diff --git a/kernel/src/arch/x86_64/percpu.rs b/kernel/src/arch/x86_64/percpu.rs index 07cc73c15a7..5bdfdf2bea3 100644 --- a/kernel/src/arch/x86_64/percpu.rs +++ b/kernel/src/arch/x86_64/percpu.rs @@ -518,6 +518,7 @@ pub fn init_bsp(lapic_id: u32) { // handlers report on no channel of this kernel's — so a fault in `fpu` // below would stop the machine with the panel holding the record before it. super::idt::init(); + super::control_regs::init_performance(0); super::fpu::init(0); // Between `fpu::init` and this function's own line: the facts `fpu::init` @@ -550,6 +551,7 @@ pub fn init_ap(percpu_ptr: *mut PerCpu) { // SAFETY: `load_gdt`'s once-per-CPU contract; this is this AP's call. unsafe { percpu.load_gdt(); } super::control_regs::init(percpu.cpu_id); + super::control_regs::init_performance(percpu.cpu_id); super::fpu::init(percpu.cpu_id); super::fpu::log_state(percpu.cpu_id); } diff --git a/tests/t14-linux/SOURCE b/tests/t14-linux/SOURCE index d59763d1e66..20e69eb8c43 100644 --- a/tests/t14-linux/SOURCE +++ b/tests/t14-linux/SOURCE @@ -12,3 +12,13 @@ beside them. sleep 2; turbostat --quiet --interval 10 --num_iterations 6 --show Core,CPU,Avg_MHz,Busy%,Bzy_MHz,TSC_MHz,IRQ,SMI,CPU%c1,CPU%c6,CPU%c7,CoreTmp,PkgTmp,Pkg%pc2,Pkg%pc6,Pk%pc10,PkgWatt,CorWatt; wait` + + hwp-request.txt IA32_HWP_REQUEST (0x774), the same on all eight CPUs, + read as root through /dev/cpu/N/msr under the same + Ubuntu with intel_pstate `active`, the `powersave` + governor and EPP `balance_performance`; recorded in + the message of commit 868378b87 (#568, branch + `wt/toyos-llvmbar`): "IA32_HWP_REQUEST 0x80002a04 on + all eight CPUs (min 4, max 42, desired 0, EPP 128)". + `counters_on_metal` holds every CPU's boot + `hwp_request` to it. diff --git a/tests/t14-linux/hwp-request.txt b/tests/t14-linux/hwp-request.txt new file mode 100644 index 00000000000..3e9991a79bb --- /dev/null +++ b/tests/t14-linux/hwp-request.txt @@ -0,0 +1 @@ +0x80002a04 diff --git a/tests/toyos.rs b/tests/toyos.rs index 26c252dae98..32269893a20 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -3265,16 +3265,17 @@ const SMI_SPAN_NS: u64 = 4_444_000_000; /// Held: a record per CPU the bring-up started, naming the local APIC id the /// bring-up gave that CPU and carrying the counters its `counters: cpuN /// reads` line names, and none stale; every CPU's performance request -/// declared at boot, `pm_enable=1`, and its power envelope in every read the -/// one its `control_regs:` line holds. From `idle0` to `spin`, at least +/// declared at boot, `pm_enable=1`, the request Linux makes on this machine +/// (`tests/t14-linux/hwp-request.txt`), and its power envelope in every read +/// the one its `control_regs:` line holds. From `idle0` to `spin`, at least /// [`SMI_SPAN_NS`] apart, every CPU's SMI count rose alike and by two or more: /// the firmware's legacy mode, the positive control ACPI stage 1's flatness /// is read against, and the row that stage changes. Across the spin every /// CPU's MPERF ran nine tenths of its stamp or more [e], a CPU in C0 the whole /// span: MPERF counts at the TSC's rate there (SDM Vol. 3B, "Hardware /// Coordination Feedback"); and every CPU's busy frequency reached the lowest -/// Linux's turbostat read under one `yes` per CPU on the same machine -/// (`tests/t14-linux/turbostat-loaded.txt`). +/// Linux's turbostat read under one `yes` per CPU on the same machine over the +/// same span of load (`tests/t14-linux/turbostat-loaded.txt`). /// /// Read and not held, beside Linux's turbostat: each CPU's idle busy /// fraction, and what one round cost its reader. @@ -3310,8 +3311,10 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { Ok((read, clock.get(name).ok_or_else(|| format!("counters_metal printed no {name} clock"))?.0)) }; let (idle0, at0) = phase("idle0")?; - let (idle1, _) = phase("idle1")?; + let (idle1, at1) = phase("idle1")?; let (spin, at2) = phase("spin")?; + let linux_request = u64::from_str_radix(include_str!("t14-linux/hwp-request.txt").trim().trim_start_matches("0x"), 16) + .map_err(|e| format!("t14-linux/hwp-request.txt: {e}"))?; let bsp = kernel.must_say("percpu: BSP cpu_id=0 lapic_id=")?; let mut roster = vec![bsp.rsplit("lapic_id=").next().unwrap_or_default().trim().to_string()]; for cpu in 1..cpus { @@ -3332,6 +3335,9 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { return Err(format!("{name}: cpu{cpu} is stale or not lapic {}: {counters:?}", roster[*cpu])); } let declared = kernel.must_say(&format!("control_regs: cpu{cpu} pm_enable=1 "))?; + if !declared.contains(&format!(" hwp_request={linux_request:#010x} ")) { + return Err(format!("cpu{cpu} declared {declared:?}, and Linux requests {linux_request:#010x} on this machine")); + } for (counter, field, radix) in [("hwp_request", "hwp_request=0x", 16), ("hwp_request_pkg", "hwp_request_pkg=0x", 16), ("energy_perf_bias", "epb=", 10)] { @@ -3368,15 +3374,26 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { if busy.iter().any(|&b| b < 0.9) { return Err(format!("MPERF ran {busy:?} of each stamp across the spin, some cpu under 0.9")); } - let linux = |file: &str, column: &str| -> Result<(f64, f64), String> { + // Each interval's machine-wide row, in order. + let linux = |file: &str, column: &str| -> Result, String> { let mut rows = file.lines().map(|l| l.split('\t').collect::>()); let header = rows.next().ok_or("an empty turbostat reading")?; let at = header.iter().position(|c| *c == column).ok_or_else(|| format!("turbostat read no {column}"))?; - let values: Vec = rows.filter(|r| r[0] == "-").filter_map(|r| r.get(at)?.parse().ok()).collect(); - Ok(values.iter().fold((f64::MAX, f64::MIN), |(lo, hi), &v| (lo.min(v), hi.max(v)))) + Ok(rows.filter(|r| r[0] == "-").filter_map(|r| r.get(at)?.parse().ok()).collect()) }; - let idle = linux(include_str!("t14-linux/turbostat-idle.txt"), "Busy%")?; - let loaded = linux(include_str!("t14-linux/turbostat-loaded.txt"), "Bzy_MHz")?; + let range = |values: &[f64]| values.iter().fold((f64::MAX, f64::MIN), |(lo, hi), &v| (lo.min(v), hi.max(v))); + let idle = range(&linux(include_str!("t14-linux/turbostat-idle.txt"), "Busy%")?); + let loaded_rows = linux(include_str!("t14-linux/turbostat-loaded.txt"), "Bzy_MHz")?; + let loaded = range(&loaded_rows); + // The floor is Linux's clock over the spin's own span of load, not after + // the package settles: the intervals that open within it, the first 2 s + // into Linux's load and each 10 s long (`tests/t14-linux/SOURCE`). + let spin_ns = at2 - at1; + let opened = loaded_rows.iter().enumerate().take_while(|&(k, _)| 2_000_000_000 + k as u64 * 10_000_000_000 < spin_ns); + let floor = opened.map(|(_, &mhz)| mhz).fold(f64::MAX, f64::min); + if floor == f64::MAX { + return Err(format!("the spin lasted {spin_ns} ns, and no Linux interval opens within it")); + } let spinning: Vec = (0..cpus).map(|cpu| tsc_mhz * ratio(idle1, spin, cpu, "aperf", "mperf")).collect(); eprintln!( " [counters] {cpus} cpus, SMI +{} each over {} ms; TSC {tsc_mhz:.0} MHz", @@ -3385,8 +3402,8 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { ); for (cpu, busy) in busy.iter().enumerate() { eprintln!( - " [counters] cpu{cpu}: idle busy {:.2}% (Linux {:.2}-{:.2}%), spinning {:.0} MHz (Linux {:.0}-{:.0}), \ - busy {busy:.3}", + " [counters] cpu{cpu}: idle busy {:.2}% (Linux {:.2}-{:.2}%), spinning {:.0} MHz (Linux {floor:.0} over \ + this span, {:.0}-{:.0} loaded), busy {busy:.3}", ratio(idle0, idle1, cpu, "mperf", "stamp") * 100.0, idle.0, idle.1, @@ -3396,8 +3413,8 @@ fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { ); } eprintln!(" [counters] the spin's read, a whole round, took its reader {} ns", clock["spin"].1); - if spinning.iter().any(|&mhz| mhz < loaded.0) { - return Err(format!("spinning at {spinning:.0?} MHz, some cpu below the {:.0} Linux reached loaded", loaded.0)); + if spinning.iter().any(|&mhz| mhz < floor) { + return Err(format!("spinning {spin_ns} ns at {spinning:.0?} MHz, some cpu below the {floor:.0} Linux held over that span")); } Ok(()) } From b69e713c29df396b4bbc66ab150f0d18c11beb71 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 4 Oct 2026 12:00:33 +0200 Subject: [PATCH 12/12] Answer the rebuild's review round 2: CPUID leaf 7 is read by one function cpu::leaf_7 reads CPUID.(7,0) under the rule both control_regs sites and boot::report_counter_origin each wrote out (a maximum leaf below 7 reaches no leaf 7, and the registers read zero), as cpu::leaf_6 does for leaf 6. control_regs::hwp_declared, control_regs::supported and boot::report_counter_origin call it. On any CPU whose maximum leaf reaches 7 every caller reads the same registers as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcU2Dsw6mDYtwYfzVHPzM8 --- kernel/src/arch/x86_64/boot.rs | 3 +-- kernel/src/arch/x86_64/control_regs.rs | 8 ++------ kernel/src/arch/x86_64/cpu.rs | 7 +++++++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index a606ea3f475..b58d961a7f5 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -119,8 +119,7 @@ pub fn clock(args: &KernelArgs) { /// and the TSC is time since power-on. pub fn report_counter_origin() { const IA32_TSC_ADJUST: u32 = 0x3b; - // Leaf 7 exists when the maximum leaf reaches it. - if super::cpu::cpuid(0, 0).0 < 7 || super::cpu::cpuid(7, 0).1 & (1 << 1) == 0 { + if super::cpu::leaf_7().1 & (1 << 1) == 0 { log!("boot: IA32_TSC_ADJUST not on this CPU"); return; } diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index d5e31beea78..81267b30008 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -161,8 +161,7 @@ fn hwp_declared(cpu_id: u32) -> Option { signature: cpu::cpuid(1, 0).0, cpuid_6_eax, cpuid_6_ecx, - // A leaf above the maximum answers with the highest basic leaf's data. - cpuid_7_0_edx: if cpu::cpuid(0, 0).0 >= 7 { cpu::cpuid(7, 0).3 } else { 0 }, + cpuid_7_0_edx: cpu::leaf_7().3, }); let mine = if verdict.is_ok() { HWP_DECLARED } else { HWP_REFUSED }; match HWP.compare_exchange(HWP_UNDECIDED, mine, Ordering::Release, Ordering::Acquire) { @@ -322,11 +321,8 @@ fn supported() -> u64 { [(0, cr4::FSGSBASE), (7, cr4::SMEP), (20, cr4::SMAP)]; const CPUID_7_ECX: [(u32, u64); 1] = [(2, cr4::UMIP)]; - let (max_leaf, _, _, _) = cpu::cpuid(0, 0); let (_, _, ecx1, edx1) = cpu::cpuid(1, 0); - // A leaf above the maximum answers with the highest basic leaf's data - // instead of faulting, so an unguarded read here would misreport bits. - let (_, ebx7, ecx7, _) = if max_leaf >= 7 { cpu::cpuid(7, 0) } else { (0, 0, 0, 0) }; + let (_, ebx7, ecx7, _) = cpu::leaf_7(); let mut have = 0; for (bit, flag) in CPUID_1_EDX { diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 83872bffe5f..0f514f69b12 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -145,6 +145,13 @@ pub fn leaf_6() -> (u32, u32) { } } +/// CPUID.(7,0)'s four registers, the structured extended features; zero +/// where CPUID.0 does not reach leaf 7, since a leaf above the maximum answers +/// with the highest basic leaf's data instead of faulting. +pub fn leaf_7() -> (u32, u32, u32, u32) { + if cpuid(0, 0).0 >= 7 { cpuid(7, 0) } else { (0, 0, 0, 0) } +} + /// CPUID with both index registers; `rbx` is saved by hand since Rust reserves it as an operand. pub fn cpuid(leaf: u32, subleaf: u32) -> (u32, u32, u32, u32) { let eax: u32;