From c41340acaff745edf2f395bafa216948c651e308 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 06:57:02 +0200 Subject: [PATCH 1/2] Every entry from Ring 3 clears branch history where the CPU's decision names it The decision named the BHB cleared on entry for a CPU with enhanced IBRS and no BHI_DIS_S, or a guest whose IBRS or returns could be undercut, and no entry cleared it: user code's branch history reached the kernel's first indirect branch. - arch::mitigation::clear_branch_history is Intel's short sequence ("Branch History Injection and Intra-mode Branch Target Injection", Listing 1), between a save and a restore of the two registers it writes. - SYSCALL has two entries, one calling it as soon as the kernel's stack is under it. Each CPU's LSTAR names the one its decision selects, so a CPU that does not clear runs no added instruction, not even a branch. - Every gate the IDT names opens with a fixed 12-byte prefix: a test of the frame's CS, a short jz over the call, the call. idt::init builds the table that enters each gate there and a second that enters past it, refusing a gate that does not open with the prefix; each CPU loads the one its decision selects (the boot CPU in init, an AP after its own decision, over the boot CPU's table the trampoline loaded). A jz mispredicted past the call meets no load through GS and no indirect branch before the gate's own lfence or serializing wrmsr. - The #DB that a MOV SS holds over an entry's first instruction is resumed at either entry of a gate or of SYSCALL. - The line says the clear in force where the CPU enumerates it, as each CPU's entries are the ones its decision selects. On the test kernel the routine counts its runs, and SYS_DEBUG's BRANCH_HISTORY_CLEARS reads them where this CPU clears; branch_history_cleared_per_syscall holds at least one per syscall. Co-Authored-By: Claude Opus 5.5 --- ...the-bhb-fills-the-rsb-or-issues-an-ibpb.md | 73 ++++++---- kernel/pure/mitigation.rs | 14 +- kernel/src/arch/aarch64/mitigation.rs | 6 + kernel/src/arch/x86_64/boot.rs | 2 +- kernel/src/arch/x86_64/entry.rs | 40 +++++- kernel/src/arch/x86_64/idt/debug.rs | 9 +- kernel/src/arch/x86_64/idt/device_irq.rs | 9 +- kernel/src/arch/x86_64/idt/mod.rs | 79 ++++++++--- kernel/src/arch/x86_64/idt/nmi.rs | 5 +- kernel/src/arch/x86_64/idt/spurious.rs | 2 +- kernel/src/arch/x86_64/idt/timer.rs | 2 +- kernel/src/arch/x86_64/idt/tlb.rs | 2 +- kernel/src/arch/x86_64/idt/unclaimed.rs | 2 +- kernel/src/arch/x86_64/idt/user_dev.rs | 14 +- kernel/src/arch/x86_64/mitigation.rs | 80 ++++++++++- kernel/src/arch/x86_64/percpu.rs | 2 +- kernel/src/arch/x86_64/smp.rs | 4 +- kernel/src/arch/x86_64/syscall.rs | 129 ++++++++++-------- kernel/src/syscall/dispatch.rs | 3 + sdk/abi/src/syscall.rs | 3 + tests/checks.rs | 2 +- .../bin/branch_history_cleared_per_syscall.rs | 27 ++++ tests/toyos.rs | 3 + 23 files changed, 387 insertions(+), 125 deletions(-) create mode 100644 tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs diff --git a/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md b/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md index d3b8e5148b8..7bb0064641c 100644 --- a/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md +++ b/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md @@ -7,35 +7,56 @@ opened: 2026-09-29 # No entry or switch clears the BHB, fills the RSB or issues an IBPB -**Built**: a switch into a program's address space issues an IBPB where the -space this CPU last ran a program in is another and either asked for -isolated speculation (the ask of -`issues/no-program-runs-with-speculative-store-bypass-disabled.md`) and the -CPU's decision names IBPB (`kernel::mitigation::LastAsked`, -`arch::x86_64::mitigation::switch_to`). On an AMD CPU whose IBPB leaves the -return address predictor's entries, which `IBPB_RET` (APM Vol. 3 -Fn8000_0008_EBX[30]) does not enumerate, AMD-SB-1040 asks 70586's V2-3 -after it: the decision names the predictor filled after IBPB there, and the -switch follows each IBPB with 32 calls. Intel's IBPB covers the RSB -("Speculative Execution Side Channel Mitigations", the IBPB note on near -returns), and no other fill on a switch is a vendor's for a CPU the decision -covers. `an_ibpb_goes_between_two_spaces_one_of_which_asked` holds two IBPBs -over B, C, idle, B, A, idle, A, B, where A asked, and one under the -mutation that decides by the previous space alone. `speculation_on_request` -counts IBPBs on a CPU where a program that asked takes turns with this one, -and none where neither asked. Linux at `Ubuntu-6.8.0-142.142` issues an IBPB -on a switch between address spaces one of which asked for it -(`arch/x86/mm/tlb.c:384`). +**Built**: +- Where a CPU's decision names the BHB cleared on entry, every entry from + Ring 3 runs Intel's short branch history clearing sequence ("Branch + History Injection and Intra-mode Branch Target Injection", Listing 1) + before any indirect branch (`arch::x86_64::mitigation::clear_branch_history`). + `LSTAR` names the `SYSCALL` entry that calls it, and every gate the IDT + names opens with a 12-byte prefix that calls it where the frame's `CS` is + Ring 3's; such a CPU loads the table that enters each gate there, and + every other CPU the one that enters past it, so it runs none of it + (`arch::x86_64::entry::gate_prefix`, `arch::x86_64::idt`). + `branch_history_cleared_per_syscall` counts at least one clear per + syscall where the CPU's decision names it. +- A switch into a program's address space issues an IBPB where the space + this CPU last ran a program in is another and either asked for isolated + speculation (the ask of + `issues/no-program-runs-with-speculative-store-bypass-disabled.md`) and the + CPU's decision names IBPB (`kernel::mitigation::LastAsked`, + `arch::x86_64::mitigation::switch_to`). On an AMD CPU whose IBPB leaves the + return address predictor's entries, which `IBPB_RET` (APM Vol. 3 + Fn8000_0008_EBX[30]) does not enumerate, AMD-SB-1040 asks 70586's V2-3 + after it: the decision names the predictor filled after IBPB there, and + the switch follows each IBPB with 32 calls. Intel's IBPB covers the RSB + ("Speculative Execution Side Channel Mitigations", the IBPB note on near + returns), and no other fill on a switch is a vendor's for a CPU the + decision covers. `an_ibpb_goes_between_two_spaces_one_of_which_asked` + holds two IBPBs over B, C, idle, B, A, idle, A, B, where A asked, and one + under the mutation that decides by the previous space alone. + `speculation_on_request` counts IBPBs on a CPU where a program that asked + takes turns with this one, and none where neither asked. +- Linux at `Ubuntu-6.8.0-142.142` runs `clear_bhb_loop` on `SYSCALL` entry + where its decision selects the BHI loop (`arch/x86/entry/entry_64.S:1534`), + and issues an IBPB on a switch between address spaces one of which asked + for it (`arch/x86/mm/tlb.c:384`). **Left**: -- Nothing clears branch history on entry. Linux runs `clear_bhb_loop` on - entry where its decision selects the BHI loop - (`arch/x86/entry/entry_64.S:1534`). -- **Owed on metal:** `speculation_on_request`, a member of main's - `shared-debug` boot. +- A guest that clears with the short sequence does not tell its VMM so: + Intel's guidance asks it to set `BHB_CLEAR_SEQ_S_USED` in + `MSR_VIRTUAL_MITIGATION_CTRL` where `MSR_VIRTUAL_MITIGATION_ENUM` + enumerates `BHB_CLEAR_SEQ_S_SUPPORT`, so that a VMM migrating it to a part + the short sequence does not cover sets `BHI_DIS_S` under it. A guest on + the guest runner is one; no proving machine is. +- **Owed on metal:** `branch_history_cleared_per_syscall` and + `speculation_on_request`, members of main's `shared-debug` boot. - Benches: the T14 and the AMD bench. - - Main must show the member exit 0, its line naming at least 2 IBPBs - beside the probe and none beside the child that did not ask. + - Main must show both members exit 0: on the T14 the first naming at + least 1000 clears over 1000 syscalls, on the AMD bench naming no clear; + the second naming at least 2 IBPBs beside the probe and none beside the + child that did not ask. + - The T14's `mitigations` row, the `metalcase` boot of main, with + `branch_history_injection: mitigated: BHB cleared on entry` on every CPU. **Exit**: a gate over `kernel.elf`, on the decoder of `issues/no-gate-decodes-kernel-elfs-instructions.md`, matches it against diff --git a/kernel/pure/mitigation.rs b/kernel/pure/mitigation.rs index 0a6eb146bdc..69d430e2454 100644 --- a/kernel/pure/mitigation.rs +++ b/kernel/pure/mitigation.rs @@ -497,6 +497,10 @@ impl Decision { for one in Mitigation::ALL.into_iter().filter(|&one| ON_REQUEST.has(one) && self.enumerates(one)) { on = on.with(one); } + // Each CPU's entries are those its decision selects. + if self.enumerates(Mitigation::BhbClear) { + on = on.with(Mitigation::BhbClear); + } if let Some(spec_ctrl) = registers.spec_ctrl { for (one, bit) in [ (Mitigation::EnhancedIbrs, msr::IBRS), @@ -1759,14 +1763,14 @@ mod tests { ); } - /// What every CPU has, what it applies on request where it enumerates - /// it, and each control its register shows on: a register the CPU was - /// not read for shows nothing. + /// What every CPU has, what it applies on request or on entry where it + /// enumerates it, and each control its register shows on: a register the + /// CPU was not read for shows nothing. #[test] fn what_is_in_force_is_what_the_registers_show() { let unread = Registers { spec_ctrl: None, mcu_opt_ctrl: None, de_cfg2: None, c001_1029: None }; let intel = decide(&INTEL_806C1); - let always = ALWAYS.with(Ibpb).with(Ssbd); + let always = ALWAYS.with(Ibpb).with(Ssbd).with(BhbClear); assert_eq!(intel.in_force(&unread), always); assert_eq!(intel.in_force(&Registers { spec_ctrl: Some(msr::IBRS), ..unread }), always.with(EnhancedIbrs)); // A bit the decision does not name shows nothing. @@ -1778,7 +1782,7 @@ mod tests { assert_eq!(intel.in_force(&opted_out), always); // What the CPU does not enumerate is not applied for anyone. let bare = decide(&Facts { cpuid_7_0_edx: 0, ..INTEL_806C1 }); - assert_eq!(bare.in_force(&unread), ALWAYS); + assert_eq!(bare.in_force(&unread), ALWAYS.with(BhbClear)); let amd = decide(&AMD_17H_60H); let always = ALWAYS.with(Ibpb).with(ReturnPredictorFilled).with(Stibp).with(Ssbd); let both = Registers { de_cfg2: Some(msr::SUPPRESS_BP_ON_NON_BR), c001_1029: Some(C001_1029_LFENCE_SERIALIZES_DISPATCH), ..unread }; diff --git a/kernel/src/arch/aarch64/mitigation.rs b/kernel/src/arch/aarch64/mitigation.rs index d4cf26343de..0e1475b24b3 100644 --- a/kernel/src/arch/aarch64/mitigation.rs +++ b/kernel/src/arch/aarch64/mitigation.rs @@ -26,3 +26,9 @@ pub fn debug_spec_ctrl_on_request() -> u64 { pub fn debug_switch_ibpbs() -> Option { None } + +/// x86-64's counts its entries' branch history clears, and none runs here. +#[cfg(feature = "test-actuators")] +pub fn debug_branch_history_clears() -> Option { + None +} diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index c87984f50a2..08049e7304d 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -116,7 +116,7 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { super::power::init_control(rsdp_addr); ioapic::init(&madt); idt::enable_interrupts(); - super::syscall::init(); + super::syscall::init(super::mitigation::boot_cpu_clears_branch_history()); Platform { madt, facs } } diff --git a/kernel/src/arch/x86_64/entry.rs b/kernel/src/arch/x86_64/entry.rs index e225af1a3dc..902c5a6d0ce 100644 --- a/kernel/src/arch/x86_64/entry.rs +++ b/kernel/src/arch/x86_64/entry.rs @@ -39,12 +39,31 @@ macro_rules! cld { }; } +/// The [`GATE_PREFIX`] bytes every gate the IDT names opens with: where the +/// frame's `CS`, `{gate_cs}` bytes above `rsp`, is Ring 3's, a call of +/// `{clear_branch_history}` (`arch::mitigation::clear_branch_history`). A CPU +/// whose decision names the clear has its IDT enter each gate here; every +/// other CPU's enters past it and runs none of it (`arch::idt`). A `jz` +/// mispredicted past the clear is followed by no load through GS and no +/// indirect branch before the gate's own `lfence` or serializing `wrmsr`. +macro_rules! gate_prefix { + () => { + "test byte ptr [rsp + {gate_cs}], 3\njz 78f\ncall {clear_branch_history}\n78:" + }; +} + +/// [`gate_prefix`]'s length: `test` with an 8-bit displacement and +/// immediate, a short `jz` and a `call` with a 32-bit displacement. `arch::idt` +/// reads it back off each gate before it points a table past it. +pub(crate) const GATE_PREFIX: usize = 12; + /// `naked_asm!` for an interrupt or trap gate a Ring 3 thread can arrive /// through, `cs = N;` bytes below its frame's `CS` on entry: prepends `cld`, /// `clac`, and a `swapgs` when that `CS` is Ring 3's. The body must end with a /// trailing comma, and swap back before every `iretq` that can return to /// Ring 3: [`gate_iretq`] where the frame decides, a bare `swapgs` on a path -/// only a Ring 3 frame takes. +/// only a Ring 3 frame takes. `gate cs = N;` makes it a gate the IDT names, +/// opening with [`gate_prefix`]; `cs = N;` one a stub jumps to. /// /// Such a gate clears neither `DF` nor `AC` (Intel SDM Vol. 3A §6.12.1.3 names /// `TF`, `VM`, `RF`, `NT`, and `IF` for an interrupt gate), and Ring 3 sets @@ -57,6 +76,21 @@ macro_rules! cld { /// of SWAPGS and Segment Registers", ID 758371). On AMD it holds where /// `LFENCE` is dispatch-serializing (AMD 70586 G-2). macro_rules! ring3_naked_asm { + (gate cs = $cs:expr; $($body:tt)*) => { + core::arch::naked_asm!( + $crate::arch::entry::gate_prefix!(), + $crate::arch::entry::cld!(), + "clac", + "test byte ptr [rsp + {gate_cs}], 3", + "jz 70f", + "swapgs", + "70:", + "lfence", + $($body)* + gate_cs = const $cs, + clear_branch_history = sym $crate::arch::mitigation::clear_branch_history, + ) + }; (cs = $cs:expr; $($body:tt)*) => { core::arch::naked_asm!( $crate::arch::entry::cld!(), @@ -82,7 +116,7 @@ macro_rules! gate_iretq { }; } -/// `naked_asm!` for `SYSCALL`'s entry: prepends `swapgs`, since only Ring 3 +/// `naked_asm!` for `SYSCALL`'s entries: prepends `swapgs`, since only Ring 3 /// executes `SYSCALL`, then `cld`. `IA32_FMASK` clears `AC` /// (`arch::syscall::init`), so the entry's next instructions are the switch /// off the user's stack. @@ -116,7 +150,7 @@ macro_rules! install_kernel_gs { }; } -pub(crate) use {cld, gate_iretq, install_kernel_gs, ring3_naked_asm, syscall_naked_asm}; +pub(crate) use {cld, gate_iretq, gate_prefix, install_kernel_gs, ring3_naked_asm, syscall_naked_asm}; /// A thread's first entry to Ring 3, at `r12` on the stack `r13`, `$align` /// applied to it, with `rdi` = `r14`: a process's argument is zero, a thread's diff --git a/kernel/src/arch/x86_64/idt/debug.rs b/kernel/src/arch/x86_64/idt/debug.rs index 434b46e59d9..3b950c52916 100644 --- a/kernel/src/arch/x86_64/idt/debug.rs +++ b/kernel/src/arch/x86_64/idt/debug.rs @@ -18,7 +18,7 @@ use core::arch::naked_asm; -use crate::arch::entry::install_kernel_gs; +use crate::arch::entry::{gate_prefix, install_kernel_gs}; use crate::arch::percpu::{MSR_GS_BASE, OFF_GDT, OFF_KERNEL_RSP, USER_CS, USER_DS}; /// Eleven pushes place the CPU frame's `rip` at 88. @@ -27,6 +27,7 @@ const RIP_OFFSET: usize = 88; #[unsafe(naked)] pub(super) extern "sysv64" fn debug_entry() { naked_asm!( + gate_prefix!(), "cld", "clac", // Neither path addresses GS before it has made it the kernel's, so a @@ -105,6 +106,8 @@ pub(super) extern "sysv64" fn debug_entry() { vector = const super::Vector::Debug as usize, deferred = sym deferred, common = sym super::common_entry, + gate_cs = const 8, + clear_branch_history = sym crate::arch::mitigation::clear_branch_history, ); } @@ -122,11 +125,11 @@ struct Frame { /// reaches, whose trap it is. On `debug_entry` itself the frame becomes a Ring /// 3 one, its lost `rip` and `rsp` zero, so its thread ends as a trap. extern "sysv64" fn deferred(frame: &mut Frame) { - if frame.rip == debug_entry as *const () as u64 { + if super::is_gate_of(frame.rip, debug_entry as *const () as u64) { *frame = Frame { rip: 0, cs: u64::from(USER_CS), rflags: 0x202, rsp: 0, ss: u64::from(USER_DS) }; return; } - if frame.rip != crate::arch::syscall::entry_address() && !super::is_gate_target(frame.rip) { + if !crate::arch::syscall::is_entry(frame.rip) && !super::is_gate_target(frame.rip) { panic!("#DB at {:#x} in Ring 0, which is no entry: nothing in this kernel arms a debug trap", frame.rip); } } diff --git a/kernel/src/arch/x86_64/idt/device_irq.rs b/kernel/src/arch/x86_64/idt/device_irq.rs index 9cf46756f2e..47a70d9227d 100644 --- a/kernel/src/arch/x86_64/idt/device_irq.rs +++ b/kernel/src/arch/x86_64/idt/device_irq.rs @@ -11,18 +11,19 @@ /// CPU's frame, which `$handler` takes as its argument and the exit drops. macro_rules! device_irq_entry { ($(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path) => { - device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; 88; []; []); + device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; [gate]; 88; []; []); }; ($(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path, numbered) => { - device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; 96; ["mov rdi, [rbp + 80]"]; ["add rsp, 8"]); + device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; []; 96; ["mov rdi, [rbp + 80]"]; ["add rsp, 8"]); }; // `$cs` is the CS word's offset from `rbp`: 10 GPRs and RIP, and the stub's word if there is one. - (@entry $(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path; $cs:literal; [$($argument:literal)?]; [$($drop:literal)?]) => { + // A numbered entry is a stub's jump target, and the stub is the gate. + (@entry $(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path; [$($gate:ident)?]; $cs:literal; [$($argument:literal)?]; [$($drop:literal)?]) => { $(#[$meta])* #[unsafe(naked)] $vis extern "sysv64" fn $name() { // On entry the 10 GPRs are not yet pushed. - $crate::arch::entry::ring3_naked_asm!(cs = $cs - 80; + $crate::arch::entry::ring3_naked_asm!($($gate)? cs = $cs - 80; // Handler may clobber any scratch GPR; one left unsaved leaks kernel state into a user register on iretq. "push rax", "push rcx", diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index 5bc05eff25d..4a3f4ec59d0 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -18,7 +18,7 @@ pub(crate) use user_dev::ClaimVector; use core::arch::naked_asm; use super::cpu; -use super::entry::{gate_iretq, install_kernel_gs, ring3_naked_asm}; +use super::entry::{gate_iretq, gate_prefix, install_kernel_gs, ring3_naked_asm, GATE_PREFIX}; use super::cpu::{outb, io_wait}; use super::pio::{Port, PIC_PRIMARY, PIC_SECONDARY}; use super::percpu; @@ -107,6 +107,21 @@ impl IdtEntry { self.ist = ist_index; self } + + /// The same gate entered past its prefix (`entry::gate_prefix`), which + /// it is refused unless it opens with: the bytes of the `test` of the + /// frame's `CS`, the short `jz` over the call and the call. + fn past_prefix(self) -> Self { + let gate = self.target(); + // SAFETY: a gate's first bytes are kernel text, mapped readable. + let opens = unsafe { core::ptr::read(gate as *const [u8; GATE_PREFIX]) }; + assert!( + matches!(opens, [0xF6, 0x44, 0x24, _, 0x03, 0x74, 0x05, 0xE8, ..]), + "idt: the gate at {gate:#x} opens with {opens:02x?}, not the branch history prefix" + ); + let past = gate + GATE_PREFIX as u64; + Self { offset_low: past as u16, offset_mid: (past >> 16) as u16, offset_high: (past >> 32) as u32, ..self } + } } #[repr(C, align(16))] @@ -114,10 +129,17 @@ struct Idt { entries: [IdtEntry; 256], } +/// Every gate entered at its prefix: the table a CPU whose decision names the +/// branch history clear loads. static IDT: Lock = Lock::new(Idt { entries: [IdtEntry::EMPTY; 256], }); +/// Every gate entered past its prefix: every other CPU's table. +static PAST_PREFIX: Lock = Lock::new(Idt { + entries: [IdtEntry::EMPTY; 256], +}); + #[repr(C, packed)] pub(crate) struct IdtPointer { pub(crate) limit: u16, @@ -165,11 +187,14 @@ macro_rules! exception_stub { #[unsafe(naked)] extern "sysv64" fn $stub() { naked_asm!( + gate_prefix!(), "push 0", "push {vector}", "jmp {common}", vector = const Vector::$variant as usize, common = sym $common, + gate_cs = const 8, + clear_branch_history = sym super::mitigation::clear_branch_history, ); } }; @@ -177,10 +202,13 @@ macro_rules! exception_stub { #[unsafe(naked)] extern "sysv64" fn $stub() { naked_asm!( + gate_prefix!(), "push {vector}", "jmp {common}", vector = const Vector::$variant as usize, common = sym $common, + gate_cs = const 16, + clear_branch_history = sym super::mitigation::clear_branch_history, ); } }; @@ -301,17 +329,29 @@ const _: () = { } }; -/// Whether `rip` is where some gate of this CPU's table enters. +/// Whether `rip` is where some gate of either table enters. fn is_gate_target(rip: u64) -> bool { - // SAFETY: the table is written only by `init`, before any gate is taken. - let idt = unsafe { &*IDT.data_ptr() }; - idt.entries.iter().any(|gate| gate.type_attr != 0 && gate.target() == rip) + // SAFETY: the tables are written only by `init`, before any gate is taken. + let tables = unsafe { [&*IDT.data_ptr(), &*PAST_PREFIX.data_ptr()] }; + tables.iter().flat_map(|idt| &idt.entries).any(|gate| gate.type_attr != 0 && gate.target() == rip) +} + +/// Whether `rip` is where either table enters the gate at `gate`. +fn is_gate_of(rip: u64, gate: u64) -> bool { + rip == gate || rip == gate + GATE_PREFIX as u64 } /// Halt IPI from `halt_all_cpus()`; never returns. #[unsafe(naked)] extern "sysv64" fn stub_halt_all() { - naked_asm!("cli", "2: hlt", "jmp 2b"); + naked_asm!( + gate_prefix!(), + "cli", + "2: hlt", + "jmp 2b", + gate_cs = const 8, + clear_branch_history = sym super::mitigation::clear_branch_history, + ); } /// Every exception vector's second half: the [`TrapFrame`], 22 words below a @@ -449,7 +489,9 @@ fn disable_pic() { } } -pub fn init() { +/// Builds both tables and loads the one `clears_branch_history` names: the +/// boot CPU's. +pub fn init(clears_branch_history: bool) { disable_pic(); install_gates(&mut IDT.lock()); @@ -470,15 +512,9 @@ pub fn init() { unclaimed += 1; } } - let ptr = IdtPointer { - limit: (core::mem::size_of::() - 1) as u16, - base: IDT.data_ptr() as u64, - }; - - // SAFETY: `ptr` is a valid IDT descriptor built from `size_of::()` and the just-filled static IDT. - unsafe { - cpu::lidt(&ptr as *const IdtPointer as *const u8); - } + let gates = IDT.lock().entries; + PAST_PREFIX.lock().entries = gates.map(IdtEntry::past_prefix); + let ptr = load(clears_branch_history); // The table this CPU is now loaded with: how many vectors carry a handler // this kernel named, and how many carry only the unclaimed stub, which is @@ -497,6 +533,17 @@ pub fn init() { ); } +/// Loads on this CPU the table whose gates open with the branch history +/// clear where `clears_branch_history`, and the one past it elsewhere: what +/// this CPU's decision names. After [`init`]. +pub fn load(clears_branch_history: bool) -> IdtPointer { + let table = if clears_branch_history { &IDT } else { &PAST_PREFIX }; + let ptr = IdtPointer { limit: (core::mem::size_of::() - 1) as u16, base: table.data_ptr() as u64 }; + // SAFETY: `ptr` names one of the two static tables `init` filled whole. + unsafe { cpu::lidt(&ptr as *const IdtPointer as *const u8) }; + ptr +} + /// Take IF=1 on this CPU; split from `init` so `ioapic::init` can mask firmware-left entries before interrupts are live. pub fn enable_interrupts() { cpu::enable_interrupts(); diff --git a/kernel/src/arch/x86_64/idt/nmi.rs b/kernel/src/arch/x86_64/idt/nmi.rs index 9fb53054b78..bcb8cdac3ab 100644 --- a/kernel/src/arch/x86_64/idt/nmi.rs +++ b/kernel/src/arch/x86_64/idt/nmi.rs @@ -12,7 +12,7 @@ use core::arch::naked_asm; -use crate::arch::entry::install_kernel_gs; +use crate::arch::entry::{gate_prefix, install_kernel_gs}; use crate::arch::percpu::{MSR_GS_BASE, OFF_GDT, OFF_NMI_ACTIVE}; /// The words the entry pushes before it can tell a nested NMI: three @@ -31,6 +31,7 @@ const RFLAGS_OFFSET: usize = 8 * (FIRST_PUSHES + 7) + 16; #[unsafe(naked)] pub(super) extern "sysv64" fn nmi_entry() { naked_asm!( + gate_prefix!(), // Not routed through `arch::entry::ring3_naked_asm`: cld must run here, since `note`'s sysv64 call requires DF clear. "cld", // And from Ring 3 it arrives with that thread's `AC`, as every gate does. @@ -95,6 +96,8 @@ pub(super) extern "sysv64" fn nmi_entry() { nested_rsp = const NESTED_RSP_OFFSET, note = sym note, nested = sym nested_nmi, + gate_cs = const 8, + clear_branch_history = sym crate::arch::mitigation::clear_branch_history, ); } diff --git a/kernel/src/arch/x86_64/idt/spurious.rs b/kernel/src/arch/x86_64/idt/spurious.rs index 65adafb4d80..c70144b3474 100644 --- a/kernel/src/arch/x86_64/idt/spurious.rs +++ b/kernel/src/arch/x86_64/idt/spurious.rs @@ -15,7 +15,7 @@ pub const SPURIOUS_VECTOR: u8 = 0xFF; #[unsafe(naked)] pub(super) extern "sysv64" fn spurious_entry() { - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; "push rax", "push rcx", "push rdx", diff --git a/kernel/src/arch/x86_64/idt/timer.rs b/kernel/src/arch/x86_64/idt/timer.rs index dffecac0938..5b082bc3254 100644 --- a/kernel/src/arch/x86_64/idt/timer.rs +++ b/kernel/src/arch/x86_64/idt/timer.rs @@ -8,7 +8,7 @@ use crate::hw::HW; #[unsafe(naked)] pub(super) extern "sysv64" fn timer_entry() { // No error code for interrupts. CS is at [rsp + 8]. - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; "test dword ptr [rsp + 8], 3", "jz 2f", diff --git a/kernel/src/arch/x86_64/idt/tlb.rs b/kernel/src/arch/x86_64/idt/tlb.rs index ace06db7ad6..54804f7ea04 100644 --- a/kernel/src/arch/x86_64/idt/tlb.rs +++ b/kernel/src/arch/x86_64/idt/tlb.rs @@ -3,7 +3,7 @@ use crate::arch::entry::{gate_iretq, ring3_naked_asm}; /// IDT entry point for the TLB-shootdown IPI vector. #[unsafe(naked)] pub(super) extern "sysv64" fn tlb_flush_entry() { - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; // Every pushed register must be popped before `iretq`: `flush` may clobber any System V scratch register. "push rax", "push rcx", diff --git a/kernel/src/arch/x86_64/idt/unclaimed.rs b/kernel/src/arch/x86_64/idt/unclaimed.rs index 13e7e1d4038..30bf21ab075 100644 --- a/kernel/src/arch/x86_64/idt/unclaimed.rs +++ b/kernel/src/arch/x86_64/idt/unclaimed.rs @@ -21,7 +21,7 @@ static NO_ISR: AtomicU64 = AtomicU64::new(0); #[unsafe(naked)] pub(super) extern "sysv64" fn unclaimed_entry() { - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; "push rax", "push rcx", "push rdx", diff --git a/kernel/src/arch/x86_64/idt/user_dev.rs b/kernel/src/arch/x86_64/idt/user_dev.rs index ceea39e8e66..1201931676b 100644 --- a/kernel/src/arch/x86_64/idt/user_dev.rs +++ b/kernel/src/arch/x86_64/idt/user_dev.rs @@ -22,8 +22,8 @@ use super::{Idt, IdtEntry}; const FIRST: u8 = 0x40; const COUNT: usize = 0xFC - FIRST as usize; -/// Each stub's bytes: a push and a jump fit in 10. -const STRIDE: usize = 16; +/// Each stub's bytes: the gate prefix, a push and a jump fit in 22. +const STRIDE: usize = 32; /// The `pcidev` function each vector is held for. static HELD: Owners = Owners::new(); @@ -65,14 +65,19 @@ pub(super) fn install(idt: &mut Idt) { } } -/// `COUNT` stubs, `STRIDE` apart from the first `STRIDE` boundary: each pushes -/// its number and jumps to [`claim_entry`]. Never called. +/// `COUNT` stubs, `STRIDE` apart from the first `STRIDE` boundary: each opens +/// with the gate prefix (`arch::entry::gate_prefix`, here in AT&T syntax), +/// pushes its number and jumps to [`claim_entry`]. Never called. #[unsafe(naked)] extern "sysv64" fn stubs() { core::arch::naked_asm!( ".set toyos_claim_vector, 0", ".rept {count}", ".balign {stride}", + "testb $3, 8(%rsp)", + "jz 78f", + "call {clear_branch_history}", + "78:", "pushq $toyos_claim_vector", "jmp {entry}", ".set toyos_claim_vector, toyos_claim_vector + 1", @@ -80,6 +85,7 @@ extern "sysv64" fn stubs() { count = const COUNT, stride = const STRIDE, entry = sym claim_entry, + clear_branch_history = sym crate::arch::mitigation::clear_branch_history, options(att_syntax), ); } diff --git a/kernel/src/arch/x86_64/mitigation.rs b/kernel/src/arch/x86_64/mitigation.rs index a9fd47ab4d8..bb36bb5b53a 100644 --- a/kernel/src/arch/x86_64/mitigation.rs +++ b/kernel/src/arch/x86_64/mitigation.rs @@ -9,7 +9,9 @@ //! opt-outs and AMD's LFENCE bit. The boot CPU runs it before the allocator //! is up, so what it returns and where the boot CPU keeps it hold no heap. //! [`switch_to`] applies at each switch what the decision names for a program -//! that asked for isolated speculation. +//! that asked for isolated speculation. Where the decision names the branch +//! history clear, this CPU's `LSTAR` and IDT enter through +//! [`clear_branch_history`] (`arch::syscall`, `arch::idt`). use core::fmt; use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::Relaxed}; @@ -155,6 +157,20 @@ pub fn record_boot_cpu(read: Read) { *BOOT_CPU.lock() = Some(read); } +impl Read { + /// Whether this CPU's entries from Ring 3 clear branch history: what its + /// decision names, which its core's other threads do not change. + pub fn clears_branch_history(&self) -> bool { + decide(&self.facts).enumerates(Mitigation::BhbClear) + } +} + +/// [`Read::clears_branch_history`] for the boot CPU, once [`record_boot_cpu`] +/// has kept its words. +pub fn boot_cpu_clears_branch_history() -> bool { + BOOT_CPU.lock().expect("mitigation: the boot CPU's words are read before its entries are installed").clears_branch_history() +} + /// Keeps a later CPU's [`apply`]. pub fn record(cpu_id: u32, read: Read) { *LATER[cpu_id as usize].lock() = Some(read); @@ -183,6 +199,8 @@ pub fn install() { for (cpu_id, read) in online() { let local = &crate::cpu_local::of(cpu_id).arch.mitigation; local.install(read.registers.spec_ctrl, decide(&read.facts).on_request()); + #[cfg(feature = "test-actuators")] + local.clears_branch_history.store(read.clears_branch_history(), Relaxed); } } @@ -280,6 +298,9 @@ pub struct Local { /// Whether this CPU has an `IA32_SPEC_CTRL` to read. #[cfg(feature = "test-actuators")] has_spec_ctrl: AtomicBool, + /// Whether this CPU's entries clear branch history. + #[cfg(feature = "test-actuators")] + clears_branch_history: AtomicBool, } impl Local { @@ -297,6 +318,8 @@ impl Local { ibpbs: AtomicU64::new(0), #[cfg(feature = "test-actuators")] has_spec_ctrl: AtomicBool::new(false), + #[cfg(feature = "test-actuators")] + clears_branch_history: AtomicBool::new(false), } } @@ -400,3 +423,58 @@ pub fn debug_switch_ibpbs() -> Option { let local = &crate::cpu_local::here().arch.mitigation; local.ibpb.load(Relaxed).then(|| local.ibpbs.load(Relaxed)) } + +/// Intel's short branch history clearing sequence ("Branch History +/// Injection and Intra-mode Branch Target Injection", Listing 1, "sufficient +/// to clear the BHB on all processors prior to Alder Lake"), between a save +/// and a restore of the two registers it writes: callable from the first +/// instruction of a gate, before anything of the interrupted context is +/// saved. Its own `call`s use the stack it is called on; it reads no memory +/// through GS and leaves the flags as it found them not. +#[unsafe(naked)] +pub(crate) extern "sysv64" fn clear_branch_history() { + core::arch::naked_asm!( + "push rax", + "push rcx", + #[cfg(feature = "test-actuators")] + "lock inc qword ptr [rip + {clears}]", + "mov ecx, 5", + "call 2f", + "jmp 6f", + ".balign 64", + "2:", + "call 3f", + "ret", + ".balign 64", + "3:", + "mov eax, 5", + "4:", + "jmp 5f", + "nop", + "5:", + "sub eax, 1", + "jnz 4b", + "sub ecx, 1", + "jnz 2b", + "ret", + "6:", + "lfence", + "pop rcx", + "pop rax", + "ret", + #[cfg(feature = "test-actuators")] + clears = sym CLEARS, + ); +} + +/// Clears [`clear_branch_history`] has run, on every CPU. +#[cfg(feature = "test-actuators")] +static CLEARS: AtomicU64 = AtomicU64::new(0); + +/// `SYS_DEBUG`'s `BRANCH_HISTORY_CLEARS`: the clears every CPU's entries have +/// run, where this CPU's decision names the clear. +#[cfg(feature = "test-actuators")] +pub fn debug_branch_history_clears() -> Option { + let local = &crate::cpu_local::here().arch.mitigation; + local.clears_branch_history.load(Relaxed).then(|| CLEARS.load(Relaxed)) +} diff --git a/kernel/src/arch/x86_64/percpu.rs b/kernel/src/arch/x86_64/percpu.rs index b9d8423144c..261f38ad027 100644 --- a/kernel/src/arch/x86_64/percpu.rs +++ b/kernel/src/arch/x86_64/percpu.rs @@ -595,7 +595,7 @@ pub fn init_bsp(lapic_id: u32) { // loaded table is the one firmware left across `ExitBootServices`, whose // handlers report on no channel of this kernel's — so a fault in `fpu` // below would stop the machine with the panel holding the record before it. - super::idt::init(); + super::idt::init(super::mitigation::boot_cpu_clears_branch_history()); super::control_regs::init_performance(0); super::fpu::init(0); diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index d72c3d856b4..9a84b1a1fb9 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -269,7 +269,9 @@ extern "C" fn ap_entry() -> ! { // GS base was set by the trampoline; finish percpu init (GDT, CR4). percpu::init_ap(percpu::percpu_ptr()); - syscall::init(); + // The trampoline loaded the boot CPU's table; this CPU's decision names its own. + crate::arch::idt::load(mitigation.clears_branch_history()); + syscall::init(mitigation.clears_branch_history()); // Calibration is a one-time BSP measurement; nothing left for an AP to do here. apic::init_ap(); super::clocksource::start_ap(percpu::cpu_id()); diff --git a/kernel/src/arch/x86_64/syscall.rs b/kernel/src/arch/x86_64/syscall.rs index 78e33757514..7b8786ad292 100644 --- a/kernel/src/arch/x86_64/syscall.rs +++ b/kernel/src/arch/x86_64/syscall.rs @@ -14,13 +14,18 @@ const MSR_LSTAR: u32 = 0xC000_0082; const MSR_FMASK: u32 = 0xC000_0084; /// `EFER.SCE` is applied and asserted on this CPU before `init` is called, on both the BSP's path and an AP's. -pub fn init() { +/// `LSTAR` names the entry that clears branch history where +/// `clears_branch_history`, what this CPU's decision names, and the other +/// elsewhere, which runs none of it. +pub fn init(clears_branch_history: bool) { let star = ((percpu::STAR_SYSRET_BASE as u64) << 48) | ((percpu::KERNEL_CS as u64) << 32); // SAFETY: this function owns all three `SYSCALL` MSRs. // All three `wrmsr` writes stay in one `unsafe` block because they are one declaration: a CPU holding only some of them has its `SYSCALL` gate aimed by something this file did not decide, and no point between the writes is a state the machine may be left in. unsafe { cpu::wrmsr(MSR_STAR, star); - cpu::wrmsr(MSR_LSTAR, syscall_entry as *const () as u64); + let entry: extern "sysv64" fn() = + if clears_branch_history { syscall_entry_clearing_branch_history } else { syscall_entry }; + cpu::wrmsr(MSR_LSTAR, entry as *const () as u64); // `SYSCALL` clears exactly the bits named here; every bit left out carries a Ring 3 thread's flag into Ring 0. // The pre-mask `RFLAGS` survives in `r11` and `sysretq` restores it to the thread regardless — `FMASK` only decides what the kernel itself runs with. const TF: u64 = 1 << 8; @@ -40,64 +45,80 @@ pub fn init() { // `SYSCALL` switches no stack: before the `rsp` switch below runs, the CPU is at CPL 0 on the user's stack, so nothing that can fault may execute there. // The same holds from the exit's `swapgs` on, with the user's GS base live (`arch::entry`): `SYSRET` cannot fault, since no instruction ends at the user half's top for `rcx` to be non-canonical (`toyos_userbound`'s placement). -#[unsafe(naked)] -extern "sysv64" fn syscall_entry() { - syscall_naked_asm!( - "mov gs:[{user_rsp}], rsp", - "mov rsp, gs:[{kernel_rsp}]", - "mov gs:[{syscall_rip}], rcx", - "mov gs:[{syscall_num}], rdi", - "mov gs:[{syscall_rbp}], rbp", - "push gs:[{user_rsp}]", // user RSP on kernel stack - "push rcx", // return RIP - "push r11", // return RFLAGS - "push rdi", - "push rsi", - "push rdx", - "push r8", - "push r9", - "push r10", - // Nine words below a 16-byte-aligned top: the pad aligns `rsp` for - // both calls, and parks the return value across `exit_to_user`. - "sub rsp, 8", +/// One of `SYSCALL`'s two entries, which `LSTAR` names per CPU (`init`): +/// with a template line and its operand, the branch history clear, which runs +/// as soon as the kernel's stack is under it and before any branch. +macro_rules! syscall_entry { + ($name:ident $(, $clear:literal, $operand:ident = $path:path)?) => { + #[unsafe(naked)] + extern "sysv64" fn $name() { + syscall_naked_asm!( + "mov gs:[{user_rsp}], rsp", + "mov rsp, gs:[{kernel_rsp}]", + $($clear,)? + "mov gs:[{syscall_rip}], rcx", + "mov gs:[{syscall_num}], rdi", + "mov gs:[{syscall_rbp}], rbp", + "push gs:[{user_rsp}]", // user RSP on kernel stack + "push rcx", // return RIP + "push r11", // return RFLAGS + "push rdi", + "push rsi", + "push rdx", + "push r8", + "push r9", + "push r10", + // Nine words below a 16-byte-aligned top: the pad aligns `rsp` for + // both calls, and parks the return value across `exit_to_user`. + "sub rsp, 8", - "add dword ptr gs:[{preempt_count}], 1", + "add dword ptr gs:[{preempt_count}], 1", - "call {handler}", + "call {handler}", - "sub dword ptr gs:[{preempt_count}], 1", - // The helper called before `pop rsp`/`sysretq` (`exit_to_user`) preserves `IF=0` across its return. - // Runs before GPR restore: the sysv64 call would otherwise clobber rcx/r11 (sysretq's RIP/RFLAGS) and the restored args. - "mov [rsp], rax", - "call {exit_to_user}", - "mov rax, [rsp]", - "add rsp, 8", + "sub dword ptr gs:[{preempt_count}], 1", + // The helper called before `pop rsp`/`sysretq` (`exit_to_user`) preserves `IF=0` across its return. + // Runs before GPR restore: the sysv64 call would otherwise clobber rcx/r11 (sysretq's RIP/RFLAGS) and the restored args. + "mov [rsp], rax", + "call {exit_to_user}", + "mov rax, [rsp]", + "add rsp, 8", - "pop r10", - "pop r9", - "pop r8", - "pop rdx", - "pop rsi", - "pop rdi", - "pop r11", - "pop rcx", - "pop rsp", // restore user RSP from kernel stack - "swapgs", - "sysretq", - handler = sym syscall_handler, - exit_to_user = sym crate::arch::idt::kernel_exit_to_user_check, - kernel_rsp = const percpu::OFF_KERNEL_RSP, - user_rsp = const percpu::OFF_USER_RSP, - syscall_rip = const percpu::OFF_SYSCALL_RIP, - syscall_num = const percpu::OFF_SYSCALL_NUM, - syscall_rbp = const percpu::OFF_SYSCALL_RBP, - preempt_count = const percpu::OFF_PREEMPT_COUNT, - ); + "pop r10", + "pop r9", + "pop r8", + "pop rdx", + "pop rsi", + "pop rdi", + "pop r11", + "pop rcx", + "pop rsp", // restore user RSP from kernel stack + "swapgs", + "sysretq", + handler = sym syscall_handler, + exit_to_user = sym crate::arch::idt::kernel_exit_to_user_check, + kernel_rsp = const percpu::OFF_KERNEL_RSP, + user_rsp = const percpu::OFF_USER_RSP, + syscall_rip = const percpu::OFF_SYSCALL_RIP, + syscall_num = const percpu::OFF_SYSCALL_NUM, + syscall_rbp = const percpu::OFF_SYSCALL_RBP, + preempt_count = const percpu::OFF_PREEMPT_COUNT, + $($operand = sym $path,)? + ); + } + }; } -/// Where `SYSCALL` enters, as `LSTAR` holds it. -pub(crate) fn entry_address() -> u64 { - syscall_entry as *const () as u64 +syscall_entry!(syscall_entry); +syscall_entry!( + syscall_entry_clearing_branch_history, + "call {clear_branch_history}", + clear_branch_history = crate::arch::mitigation::clear_branch_history +); + +/// Whether `rip` is where either entry begins. +pub(crate) fn is_entry(rip: u64) -> bool { + [syscall_entry as *const () as u64, syscall_entry_clearing_branch_history as *const () as u64].contains(&rip) } /// The syscall bracket: the entry's diagnostic stores stay readable only while [`percpu::in_syscall`] is true. diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 2f86bb32bd0..dcfd3eb73f4 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -458,6 +458,9 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> DA::SWITCH_IBPBS => { crate::arch::mitigation::debug_switch_ibpbs().unwrap_or(SyscallError::NotSupported.to_u64()) } + DA::BRANCH_HISTORY_CLEARS => { + crate::arch::mitigation::debug_branch_history_clears().unwrap_or(SyscallError::NotSupported.to_u64()) + } _ => SyscallError::InvalidArgument.to_u64(), }, SYS_JOB_FLOOR => sys_job_floor(RawHandle(a1 as u32), RawHandle(a2 as u32), a3), diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 07be5d2084b..bd91696d295 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -1038,6 +1038,9 @@ pub mod debug_action { /// spaces one of which asked; `NotSupported` where its decision names /// no IBPB. pub const SWITCH_IBPBS: u64 = 32; + /// The branch history clears every CPU's entries from Ring 3 have run; + /// `NotSupported` where this CPU's decision names none. + pub const BRANCH_HISTORY_CLEARS: u64 = 33; } /// Create a pipe. Returns the read and write ends. diff --git a/tests/checks.rs b/tests/checks.rs index a90f2809538..bc2fd2f6dc5 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -562,7 +562,7 @@ mod checks { const T14_BENCH_LINES: [&str; 17] = [ "spectre_v1: mitigated: user pointers clamped without a branch", "spectre_v2: mitigated: SMEP, enhanced IBRS, IBPB between programs that ask", - "branch_history_injection: vulnerable: needs BHB cleared on entry", + "branch_history_injection: mitigated: BHB cleared on entry", "retbleed: not affected", "spec_rstack_overflow: not affected", "indirect_target_selection: vulnerable: needs indirect branches in a line's upper half", diff --git a/tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs b/tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs new file mode 100644 index 00000000000..4ee8af2a85d --- /dev/null +++ b/tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs @@ -0,0 +1,27 @@ +//! Each syscall this job makes runs its CPU's branch history clear where the +//! CPU's decision names one, read off the kernel's count of clears through +//! `SYS_DEBUG`: at least one per syscall, since every other CPU's entries +//! only add to it. On a CPU whose decision names none the count is refused, +//! and there is nothing to hold. + +use toyos_abi::syscall::{self, debug_action, SyscallError}; + +const SYSCALLS: u64 = 1000; + +fn main() { + let cpu = syscall::current_cpu(); + let mut mask = [0u64; syscall::CPU_MASK_WORDS]; + mask[cpu as usize / 64] = 1 << (cpu % 64); + syscall::thread_set_affinity(0, &mask).expect("pinned to one CPU"); + let before = syscall::debug(debug_action::BRANCH_HISTORY_CLEARS); + if SyscallError::from_u64(before) == Some(SyscallError::NotSupported) { + println!("branch_history_cleared_per_syscall: CPU {cpu}'s decision names no branch history clear"); + return; + } + for _ in 0..SYSCALLS { + syscall::current_cpu(); + } + let cleared = syscall::debug(debug_action::BRANCH_HISTORY_CLEARS) - before; + assert!(cleared >= SYSCALLS, "CPU {cpu} cleared branch history {cleared} times over {SYSCALLS} syscalls"); + println!("branch_history_cleared_per_syscall: {cleared} clears over {SYSCALLS} syscalls on CPU {cpu}"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 75c793e103f..03bbafb305f 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -71,6 +71,9 @@ const ACTUATOR_TESTS: &[&str] = &[ // SSBD: both benches and the KVM guest runner; under TCG's `qemu64`, which // enumerates neither, it says so and holds nothing. "speculation_on_request", + // Action 33: the branch history clears the entries ran, which leave + // nothing a guest can read. + "branch_history_cleared_per_syscall", ]; /// What [`ACTUATOR_TESTS`] boots: the one kernel that carries `SYS_DEBUG`. From a9892c2a86eb8adbebbd5c2c12577bf769a8a5d7 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 07:54:18 +0200 Subject: [PATCH 2/2] Review r1 of #1279: the short sequence only where Intel scopes it, and the record renamed to what is left - The decision names the short BHB-clearing sequence, Listing 1, only where Intel scopes it: hosts its table gives BHI's mitigation as "Software" alone, the processors before Alder Lake, and guests, whose VMM sets BHI_DIS_S under it. A later or unlisted host without BHI_CTRL, whose microcode predates BHI_DIS_S, says vulnerable and lacking BHI_DIS_S rather than mitigated; the record keeps the gap (B5). The T14 and the guest runner's guest decide as before. - in_force names one set, BY_CONSTRUCTION, for what the kernel applies wherever the decision names it and no register shows: the on-request controls and the entries' clear (B2). - clear_branch_history's returns each go back to a call it made, at most seven deep, so the RSB predicts them and Indirect Target Selection, which reaches a return only through an empty RSB's alternate predictors, does not; the site and the thunks record say so (B3). - The BHB/RSB/IBPB record is renamed to what remains, the gate, the metal runs and the VMM notice, with its three citations moved, and records the NMI/#MC window before a clear finishes (B1, S1). - The build's entry-window gate reads both SYSCALL entries (S2). - The member, now branch_history_cleared_on_each_entry, also holds a clear per page fault on a fresh page, through the IDT's gates (S3). - The table that enters each gate at its prefix is AT_PREFIX (S5). Co-Authored-By: Claude Opus 5.5 --- build/build.rs | 98 ++++++++++------ ...branches-and-returns-run-without-thunks.md | 6 +- ...o-gate-decodes-kernel-elfs-instructions.md | 2 +- ...return-fill-to-their-vendors-sequences.md} | 29 +++-- ...s-linux-on-every-machine-toyos-supports.md | 2 +- ...st-linux-on-both-benches-the-scoreboard.md | 2 +- kernel/pure/mitigation.rs | 111 ++++++++++++------ kernel/src/arch/x86_64/idt/mod.rs | 18 +-- kernel/src/arch/x86_64/mitigation.rs | 8 +- kernel/src/arch/x86_64/syscall.rs | 6 +- .../branch_history_cleared_on_each_entry.rs | 49 ++++++++ .../bin/branch_history_cleared_per_syscall.rs | 27 ----- tests/toyos.rs | 2 +- 13 files changed, 232 insertions(+), 128 deletions(-) rename issues/{no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md => no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md} (72%) create mode 100644 tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs delete mode 100644 tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs diff --git a/build/build.rs b/build/build.rs index 0fc6becb549..72c5c9e2ebb 100644 --- a/build/build.rs +++ b/build/build.rs @@ -1783,12 +1783,19 @@ fn assert_actuators_match_features(root: &Path, features: &str, kernel: &[u8]) { ); } -/// `arch::syscall::syscall_entry`'s v0-mangled path, less the crate -/// disambiguator that stands in front of it. -const SYSCALL_ENTRY_SYMBOL: &str = "6kernel4arch6x86_647syscall13syscall_entry"; +/// `SYSCALL`'s two entries, which `LSTAR` names per CPU: their names, and +/// their v0-mangled paths less the crate disambiguator that stands in front +/// of each. +const SYSCALL_ENTRIES: [(&str, &str); 2] = [ + ("syscall_entry", "6kernel4arch6x86_647syscall13syscall_entry"), + ( + "syscall_entry_clearing_branch_history", + "6kernel4arch6x86_647syscall37syscall_entry_clearing_branch_history", + ), +]; /// `swapgs` then `cld`, which `arch::entry::syscall_naked_asm` puts first in -/// `syscall_entry`: the kernel's GS base before anything addresses GS. +/// each entry: the kernel's GS base before anything addresses GS. const OPENS: [u8; 4] = [0x0f, 0x01, 0xf8, 0xfc]; /// `mov gs:[disp32], rsp` less its displacement: the `gs` override, `REX.W`, /// the opcode, and the ModRM/SIB pair that says `rsp` against an absolute @@ -1799,28 +1806,26 @@ const LOAD_RSP_FROM_GS: [u8; 5] = [0x65, 0x48, 0x8b, 0x24, 0x25]; /// Bytes either `mov` occupies, displacement included. const MOV_RSP_GS_LEN: usize = 9; -/// A kernel's `syscall_entry` from its first instruction to the end of the -/// segment it is in, found at its symbol. -fn syscall_entry_bytes(kernel: &[u8]) -> Result<&[u8], String> { +/// The entry at `symbol` from its first instruction to the end of the +/// segment it is in. +fn syscall_entry_bytes<'a>(kernel: &'a [u8], (name, symbol): (&str, &str)) -> Result<&'a [u8], String> { use toyos::elf::header::{ProgramHeader, PT_LOAD}; let (syms, strs) = toyos::elf::section::symbol_tables(kernel).ok_or("the kernel has no readable `.symtab`")?; let table = toyos::elf::SymTab::new(syms, strs); let mut entries = table.defined().filter(|&(i, sym)| { - sym.kind() == toyos::elf::sym::STT_FUNC && table.name(i).ends_with(SYSCALL_ENTRY_SYMBOL) + sym.kind() == toyos::elf::sym::STT_FUNC && table.name(i).ends_with(symbol) }); let (Some((_, entry)), None) = (entries.next(), entries.next()) else { - return Err(format!( - "the kernel's `.symtab` does not name exactly one function `…{SYSCALL_ENTRY_SYMBOL}`" - )); + return Err(format!("the kernel's `.symtab` does not name exactly one function `…{symbol}`")); }; let header = toyos::elf::FileHeader::parse(kernel).map_err(|e| format!("{e:?}"))?; let extent = toyos::elf::Layout::parse(kernel, header.machine).map_err(|e| format!("{e}"))?.extent(); let value = entry .address(extent) .map(|at| extent.min() + at.get()) - .ok_or("the kernel's `syscall_entry` lies outside its own image")?; + .ok_or_else(|| format!("the kernel's `{name}` lies outside its own image"))?; let segments = header.program_headers(kernel).map_err(|e| format!("{e:?}"))?; (0..header.phnum as usize) .filter_map(|i| ProgramHeader::parse(segments, i)) @@ -1830,7 +1835,7 @@ fn syscall_entry_bytes(kernel: &[u8]) -> Result<&[u8], String> { let left = segment.filesz.checked_sub(within).filter(|&left| left != 0)?; toyos::elf::file_range(kernel, segment.offset.checked_add(within)?, left) }) - .ok_or_else(|| format!("no `PT_LOAD` holds `syscall_entry` at {value:#x} in the file")) + .ok_or_else(|| format!("no `PT_LOAD` holds `{name}` at {value:#x} in the file")) } /// Whether an entry switches to the kernel's `rsp` in the instruction after it @@ -1843,7 +1848,7 @@ fn entry_window(entry: &[u8]) -> Result, String> { let after_save = OPENS.len() + MOV_RSP_GS_LEN; if !entry.starts_with(&OPENS) || !entry[OPENS.len()..].starts_with(&SAVE_RSP_TO_GS) { return Err(format!( - "`syscall_entry` does not open `swapgs`, `cld`, `mov gs:[…], rsp`: it opens {:02x?}", + "a `SYSCALL` entry does not open `swapgs`, `cld`, `mov gs:[…], rsp`: it opens {:02x?}", &entry[..entry.len().min(after_save)], )); } @@ -1860,25 +1865,28 @@ fn entry_window(entry: &[u8]) -> Result, String> { Ok(Err(&rest[..between])) } -/// Refuse to write a shipping image whose `syscall_entry` does anything between -/// saving the user's `rsp` and switching to the kernel's. +/// Refuse to write a shipping image either of whose `SYSCALL` entries does +/// anything between saving the user's `rsp` and switching to the kernel's. /// -/// Read at the entry's own symbol, the shipping kernel's first four +/// Read at each entry's own symbol, the shipping kernel's first four /// instructions are `swapgs`, `cld`, the save and the switch, with nothing /// between. fn judge_entry_window(features: &str, kernel: &[u8]) -> Result<(), String> { if !features.is_empty() { return Ok(()); } - match entry_window(syscall_entry_bytes(kernel)?)? { - Ok(()) => Ok(()), - Err(between) => Err(format!( - "the shipping kernel's `syscall_entry` does not switch to the kernel's `rsp` in the \ - instruction after it saves the user's; between them stand {between:02x?}.\nEvery \ - instruction there runs at CPL 0 on a user's stack, and an image that ships must \ - not be able to be asked to stop there." - )), + for entry in SYSCALL_ENTRIES { + if let Err(between) = entry_window(syscall_entry_bytes(kernel, entry)?)? { + return Err(format!( + "the shipping kernel's `{}` does not switch to the kernel's `rsp` in the \ + instruction after it saves the user's; between them stand {between:02x?}.\nEvery \ + instruction there runs at CPL 0 on a user's stack, and an image that ships must \ + not be able to be asked to stop there.", + entry.0, + )); + } } + Ok(()) } fn assert_entry_window_matches_features(features: &str, kernel: &[u8]) { @@ -3874,22 +3882,27 @@ mod tests { ]; /// A PIE of one `PT_LOAD` over the whole file, whose `.symtab` names one - /// function per `names` entry, each at `entry`'s address. + /// function per `names` entry, each at a copy of `entry`. fn kernel_with_entry(names: &[&str], entry: &[u8]) -> Vec { + kernel_with_entries(&names.iter().map(|name| (*name, entry)).collect::>()) + } + + /// The same with each function at its own code, one after another. + fn kernel_with_entries(entries: &[(&str, &[u8])]) -> Vec { const TEXT: u64 = 0x200; let mut strtab = vec![0u8]; let mut symtab = vec![0u8; 24]; - for name in names { + let mut file = vec![0u8; TEXT as usize]; + for (name, code) in entries { symtab.extend((strtab.len() as u32).to_le_bytes()); // `STB_GLOBAL`, `STT_FUNC`; `st_other`; defined in section 1. symtab.extend([0x12, 0, 1, 0]); - symtab.extend(TEXT.to_le_bytes()); - symtab.extend((entry.len() as u64).to_le_bytes()); + symtab.extend((file.len() as u64).to_le_bytes()); + symtab.extend((code.len() as u64).to_le_bytes()); strtab.extend(name.as_bytes()); strtab.push(0); + file.extend(*code); } - let mut file = vec![0u8; TEXT as usize]; - file.extend(entry); let symtab_at = file.len() as u64; file.extend(&symtab); let strtab_at = file.len() as u64; @@ -3932,6 +3945,9 @@ mod tests { } const ENTRY_NAME: &str = "_RNvNtNtNtCs2TF9wDo3GXK_6kernel4arch6x86_647syscall13syscall_entry"; + const CLEARING_NAME: &str = + "_RNvNtNtNtCs2TF9wDo3GXK_6kernel4arch6x86_647syscall37syscall_entry_clearing_branch_history"; + const BOTH: [&str; 2] = [ENTRY_NAME, CLEARING_NAME]; fn entry_with(between: &[u8]) -> Vec { [&ENTRY_OPENS[..], between, &SWITCH[..], &[0x90; 32][..]].concat() @@ -3939,30 +3955,40 @@ mod tests { #[test] fn a_clean_entry_is_the_shipping_kernels() { - let kernel = kernel_with_entry(&[ENTRY_NAME], &entry_with(&[])); + let kernel = kernel_with_entry(&BOTH, &entry_with(&[])); assert_eq!(judge_entry_window("", &kernel), Ok(())); } #[test] fn a_hold_is_refused_in_a_shipping_entry_however_its_labels_are_spelled() { for hold in [&LABELLED_HOLD[..], &LOCAL_LABEL_HOLD[..]] { - let kernel = kernel_with_entry(&[ENTRY_NAME], &entry_with(hold)); + let kernel = kernel_with_entry(&BOTH, &entry_with(hold)); let refusal = judge_entry_window("", &kernel).unwrap_err(); assert!(refusal.contains(&format!("between them stand {hold:02x?}")), "{refusal}"); } } + /// The entry that clears branch history is judged as the other is: a + /// clear moved in front of the switch would run on the user's stack. + #[test] + fn a_hold_in_the_clearing_entry_alone_is_refused() { + let hold = entry_with(&LABELLED_HOLD); + let kernel = kernel_with_entries(&[(ENTRY_NAME, &entry_with(&[])[..]), (CLEARING_NAME, &hold[..])]); + let refusal = judge_entry_window("", &kernel).unwrap_err(); + assert!(refusal.contains("`syscall_entry_clearing_branch_history` does not switch"), "{refusal}"); + } + #[test] fn a_hold_in_front_of_the_save_is_an_entry_the_judge_refuses_to_read() { let entry = [&ENTRY_OPENS[..4], &[0xf3, 0x90][..], &ENTRY_OPENS[4..], &SWITCH[..]].concat(); - let kernel = kernel_with_entry(&[ENTRY_NAME], &entry); + let kernel = kernel_with_entry(&BOTH, &entry); let refusal = judge_entry_window("", &kernel).unwrap_err(); assert!(refusal.contains("does not open `swapgs`, `cld`"), "{refusal}"); } #[test] fn an_entry_without_its_swapgs_is_refused() { - let kernel = kernel_with_entry(&[ENTRY_NAME], &[&ENTRY_OPENS[3..], &SWITCH[..]].concat()); + let kernel = kernel_with_entry(&BOTH, &[&ENTRY_OPENS[3..], &SWITCH[..]].concat()); let refusal = judge_entry_window("", &kernel).unwrap_err(); assert!(refusal.contains("does not open `swapgs`, `cld`"), "{refusal}"); } @@ -3970,7 +3996,7 @@ mod tests { #[test] fn a_kernel_that_does_not_name_exactly_one_entry_is_refused() { let clean = entry_with(&[]); - for names in [&["_RNvCs1_6kernel4main"][..], &[ENTRY_NAME, ENTRY_NAME][..]] { + for names in [&["_RNvCs1_6kernel4main"][..], &[ENTRY_NAME, ENTRY_NAME][..], &[ENTRY_NAME][..]] { let refusal = judge_entry_window("", &kernel_with_entry(names, &clean)).unwrap_err(); assert!(refusal.contains("does not name exactly one function"), "{refusal}"); } diff --git a/issues/indirect-branches-and-returns-run-without-thunks.md b/issues/indirect-branches-and-returns-run-without-thunks.md index bcee2082272..e759efac3d1 100644 --- a/issues/indirect-branches-and-returns-run-without-thunks.md +++ b/issues/indirect-branches-and-returns-run-without-thunks.md @@ -23,7 +23,11 @@ script asserts (`arch/x86/kernel/vmlinux.lds.S:510-539`): pair's addresses differing in exactly bits 2, 8, 14 and 20, and each ITS thunk's branch in its line's upper half. Each proving machine's boot installs the thunks the decision selects for it: ITS's on the T14, SRSO's alias on a -family-0x19 EPYC guest. No proving machine is family 0x17, so the untrained +family-0x19 EPYC guest. The branch history clear's returns +(`arch::x86_64::mitigation::clear_branch_history`) are Intel's sequence and +take no thunk; three lie in the lower halves of their lines, and each returns +to a call the sequence made, so the RSB predicts it and ITS, which reaches a +return only through an empty RSB's alternate predictors, does not. No proving machine is family 0x17, so the untrained return is proven by the gate and the decision's host test alone. **Mutation**, each red: an ITS thunk at a line start; `retbleed_return_thunk` one byte past its line start; the SRSO alias one cache line off its pair; an diff --git a/issues/no-gate-decodes-kernel-elfs-instructions.md b/issues/no-gate-decodes-kernel-elfs-instructions.md index 2bf4971685b..8d59f06a9a5 100644 --- a/issues/no-gate-decodes-kernel-elfs-instructions.md +++ b/issues/no-gate-decodes-kernel-elfs-instructions.md @@ -8,7 +8,7 @@ opened: 2026-09-29 Five stages each owe a gate over `kernel.elf`'s instructions: -- `issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md` +- `issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md` - `issues/indirect-branches-and-returns-run-without-thunks.md` - `issues/kernel-functions-return-with-their-used-registers-intact.md` - `issues/the-kernel-runs-without-indirect-branch-tracking.md` diff --git a/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md b/issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md similarity index 72% rename from issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md rename to issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md index 7bb0064641c..c4668fb1acc 100644 --- a/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md +++ b/issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md @@ -5,7 +5,7 @@ severity: [security] opened: 2026-09-29 --- -# No entry or switch clears the BHB, fills the RSB or issues an IBPB +# No gate holds the BHB clear and the return fill to their vendors' sequences **Built**: - Where a CPU's decision names the BHB cleared on entry, every entry from @@ -16,9 +16,12 @@ opened: 2026-09-29 names opens with a 12-byte prefix that calls it where the frame's `CS` is Ring 3's; such a CPU loads the table that enters each gate there, and every other CPU the one that enters past it, so it runs none of it - (`arch::x86_64::entry::gate_prefix`, `arch::x86_64::idt`). - `branch_history_cleared_per_syscall` counts at least one clear per - syscall where the CPU's decision names it. + (`arch::x86_64::entry::gate_prefix`, `arch::x86_64::idt`). The decision + names the short sequence only where Intel scopes it: its table's + processors whose BHI mitigation is software alone, those before Alder + Lake, and a guest, whose VMM sets `BHI_DIS_S` under it. + `branch_history_cleared_on_each_entry` counts at least one clear per + syscall and one per page fault where the CPU's decision names it. - A switch into a program's address space issues an IBPB where the space this CPU last ran a program in is another and either asked for isolated speculation (the ask of @@ -42,19 +45,29 @@ opened: 2026-09-29 for it (`arch/x86/mm/tlb.c:384`). **Left**: +- An NMI or `#MC` that lands on an entry from Ring 3 before its clear + finishes, in `SYSCALL`'s first four instructions or while a gate's clear + runs, arrives as a Ring 0 frame, skips the clear and runs its handler on + the user's branch history. User code cannot time either. +- An Intel host that the table gives `BHI_DIS_S` (its "MCU+Software" and + "Hardware+Software" rows, Alder Lake on), or one the table omits, whose + microcode predates `BHI_CTRL` says branch history injection vulnerable, + lacking `BHI_DIS_S`: the short sequence is not Intel's for it, and the + long sequence, Listing 2, which covers every part without `BHI_NO`, is not + built. ToyOS ships microcode for no such part. - A guest that clears with the short sequence does not tell its VMM so: Intel's guidance asks it to set `BHB_CLEAR_SEQ_S_USED` in `MSR_VIRTUAL_MITIGATION_CTRL` where `MSR_VIRTUAL_MITIGATION_ENUM` enumerates `BHB_CLEAR_SEQ_S_SUPPORT`, so that a VMM migrating it to a part the short sequence does not cover sets `BHI_DIS_S` under it. A guest on the guest runner is one; no proving machine is. -- **Owed on metal:** `branch_history_cleared_per_syscall` and +- **Owed on metal:** `branch_history_cleared_on_each_entry` and `speculation_on_request`, members of main's `shared-debug` boot. - Benches: the T14 and the AMD bench. - Main must show both members exit 0: on the T14 the first naming at - least 1000 clears over 1000 syscalls, on the AMD bench naming no clear; - the second naming at least 2 IBPBs beside the probe and none beside the - child that did not ask. + least 1000 clears over 1000 syscalls and 256 over 256 page faults, on + the AMD bench naming no clear; the second naming at least 2 IBPBs beside + the probe and none beside the child that did not ask. - The T14's `mitigations` row, the `metalcase` boot of main, with `branch_history_injection: mitigated: BHB cleared on entry` on every CPU. diff --git a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md index ef25c57fb51..f61b4391d96 100644 --- a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md +++ b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md @@ -26,7 +26,7 @@ work needs. - `issues/no-program-runs-with-speculative-store-bypass-disabled.md` - `issues/no-aarch64-program-runs-with-speculative-store-bypass-disabled.md` - `issues/no-gate-decodes-kernel-elfs-instructions.md` -- `issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md` +- `issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md` - `issues/user-pointer-checks-have-no-spectre-v1-fence.md` - `issues/indirect-branches-and-returns-run-without-thunks.md` - `issues/tsx-stays-as-firmware-left-it.md` diff --git a/issues/toyos-against-linux-on-both-benches-the-scoreboard.md b/issues/toyos-against-linux-on-both-benches-the-scoreboard.md index 5008bba48e3..f939135c5cf 100644 --- a/issues/toyos-against-linux-on-both-benches-the-scoreboard.md +++ b/issues/toyos-against-linux-on-both-benches-the-scoreboard.md @@ -306,7 +306,7 @@ there: reads 159 against 49 ns, and its run lines state retpolines, conditional IBPB, STIBP and Safe RET (`issues/indirect-branches-and-returns-run-without-thunks.md`, - `issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md`). + `issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md`). - After 23 of the 76, all shapes that block and wake, the clock Linux's reporting thread read was 1.37 to 1.9 GHz in the median run, where ToyOS's read 4.2 to 4.3 GHz. The probe reads the reporting thread's CPU, so that diff --git a/kernel/pure/mitigation.rs b/kernel/pure/mitigation.rs index 69d430e2454..c449a1bd2b3 100644 --- a/kernel/pure/mitigation.rs +++ b/kernel/pure/mitigation.rs @@ -488,19 +488,15 @@ impl Decision { /// What is in force on a CPU whose registers read `registers`: [`ALWAYS`]; /// what the kernel applies by construction wherever this decision names - /// it and the CPU enumerates it, which no register shows at bring-up (the - /// controls a program that asks runs with, [`ON_REQUEST`]); and each - /// control a register shows on. Intel's two `IA32_MCU_OPT_CTRL` bits opt - /// out of a mitigation its microcode turns on; the others turn one on. + /// it and the CPU enumerates it, which no register shows at bring-up + /// ([`BY_CONSTRUCTION`]); and each control a register shows on. Intel's + /// two `IA32_MCU_OPT_CTRL` bits opt out of a mitigation its microcode + /// turns on; the others turn one on. pub fn in_force(&self, registers: &Registers) -> Mitigations { let mut on = ALWAYS; - for one in Mitigation::ALL.into_iter().filter(|&one| ON_REQUEST.has(one) && self.enumerates(one)) { + for one in Mitigation::ALL.into_iter().filter(|&one| BY_CONSTRUCTION.has(one) && self.enumerates(one)) { on = on.with(one); } - // Each CPU's entries are those its decision selects. - if self.enumerates(Mitigation::BhbClear) { - on = on.with(Mitigation::BhbClear); - } if let Some(spec_ctrl) = registers.spec_ctrl { for (one, bit) in [ (Mitigation::EnhancedIbrs, msr::IBRS), @@ -534,15 +530,17 @@ impl Decision { /// CPU without `CR4.SMEP`, and enters no VMX or SVM operation. pub const ALWAYS: Mitigations = Mitigations::of(&[Mitigation::UserPointerClamp, Mitigation::Smep, Mitigation::NoGuest]); -/// What the kernel applies for a program that asked, on every CPU that -/// enumerates it ([`Decision::on_request`]). No register shows it at -/// bring-up, before any program has. -const ON_REQUEST: Mitigations = Mitigations::of(&[ +/// What the kernel applies on every CPU whose decision names it and that +/// enumerates it, and no register shows at bring-up: what a program that +/// asks runs with ([`Decision::on_request`]), and the branch history clear of +/// the entries each CPU loads for its decision. +const BY_CONSTRUCTION: Mitigations = Mitigations::of(&[ Mitigation::Ibpb, Mitigation::ReturnPredictorFilled, Mitigation::Stibp, Mitigation::Ssbd, Mitigation::VirtSsbd, + Mitigation::BhbClear, ]); /// What a CPU applies for a program that asked, and for no other. @@ -887,7 +885,17 @@ fn intel(f: &Facts, of: Vulnerability) -> Verdict { } else if f.cpuid_7_2_edx & BHI_CTRL != 0 { Asks::of(Mitigation::BhiDisS, true).affected() } else if f.cap(cap::IBRS_ALL) { - Asks::of(Mitigation::BhbClear, true).affected() + // Its short sequence is "sufficient to clear the BHB on all + // processors prior to Alder Lake", the table's software rows; a + // guest's VMM sets `BHI_DIS_S` under it on any other ("Software + // Mitigations in Migration Pools"). A later part names + // `BHI_DIS_S`, which microcode older than its BHI update lacks. + match f.intel_lists(BHI_SOFTWARE) { + _ if f.hypervisor => Asks::of(Mitigation::BhbClear, true).affected(), + Some(true) => Asks::of(Mitigation::BhbClear, true).affected(), + Some(false) => Asks::of(Mitigation::BhiDisS, false).affected(), + None => Asks::of(Mitigation::BhiDisS, false).assumed(true), + } } else if f.hypervisor && ibrs && (f.intel_rsba() || f.cap(cap::RRSBA)) { // A guest that enumerates IBRS and not IBRS_ALL may run on // hardware that has it: the sequence, where the guest relies @@ -1211,7 +1219,10 @@ const fn any_stepping(signature: u32, affected: u16) -> Row { // Check Error Avoidance (CVE-2018-12207); Special Register Buffer Data // Sampling; the two MMIO columns; Gather Data Sampling; Register File Data // Sampling; the three Indirect Target Selection columns; RSBU's RRSBA -// column; Branch History Injection. +// column; Branch History Injection, and where its every row gives BHI's +// mitigation as "Software" alone, which the table gives every processor +// before Alder Lake, against "MCU+Software" or "Hardware+Software" for one +// with `BHI_DIS_S`. const MELTDOWN: u16 = 1 << 0; const L1TF: u16 = 1 << 1; const MDS: u16 = 1 << 2; @@ -1223,6 +1234,7 @@ const RFDS: u16 = 1 << 7; const ITS: u16 = 1 << 8; const RRSBA: u16 = 1 << 9; const BHI: u16 = 1 << 10; +const BHI_SOFTWARE: u16 = 1 << 11; /// Intel's "Affected Processors: Guidance for Security Issues on Intel /// Processors" as its machine-readable list gave it on 2026-09-04: each @@ -1236,46 +1248,46 @@ const INTEL_ROWS: [Row; 67] = [ row(0x406f1, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50653, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50654, MELTDOWN | L1TF | MDS | MCEPSC | MMIO | GDS), - row(0x5065b, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), + row(0x5065b, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), row(0x50663, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50664, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50665, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x506ca, RFDS), row(0x506e3, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), any_stepping(0x506f0, RFDS), - row(0x606a6, MMIO | GDS | ITS | BHI), - any_stepping(0x606c0, MMIO | GDS | ITS | BHI), - row(0x706a8, RFDS | BHI), - row(0x706e5, MDS | MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0x80665, MMIO | RFDS | BHI), - row(0x80667, MMIO | RFDS | BHI), - row(0x806c1, MMIO | GDS | ITS | BHI), - row(0x806c2, GDS | ITS | BHI), - row(0x806d1, MMIO | GDS | ITS | BHI), + row(0x606a6, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), + any_stepping(0x606c0, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), + row(0x706a8, RFDS | BHI | BHI_SOFTWARE), + row(0x706e5, MDS | MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0x80665, MMIO | RFDS | BHI | BHI_SOFTWARE), + row(0x80667, MMIO | RFDS | BHI | BHI_SOFTWARE), + row(0x806c1, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), + row(0x806c2, GDS | ITS | BHI | BHI_SOFTWARE), + row(0x806d1, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), row(0x806e9, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x806ea, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x806eb, MDS | MCEPSC | SRBDS | MMIO | GDS), - row(0x806ec, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI), + row(0x806ec, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), row(0x806f5, RRSBA | BHI), row(0x806f6, RRSBA | BHI), row(0x806f7, RRSBA | BHI), row(0x806f8, RRSBA | BHI), - any_stepping(0x90660, MMIO | RFDS | BHI), + any_stepping(0x90660, MMIO | RFDS | BHI | BHI_SOFTWARE), row(0x90672, RFDS | BHI), row(0x90675, RFDS | BHI), row(0x906a3, RFDS | BHI), row(0x906a4, RFDS | BHI), - any_stepping(0x906c0, MMIO | RFDS | BHI), + any_stepping(0x906c0, MMIO | RFDS | BHI | BHI_SOFTWARE), row(0x906e9, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x906ea, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x906ec, MDS | MCEPSC | SRBDS | MMIO | GDS), - row(0x906ed, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0652, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0653, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0655, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0660, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0661, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0671, MMIO | GDS | ITS | BHI), + row(0x906ed, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0652, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0653, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0655, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0660, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0661, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0671, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), row(0xa06a4, RRSBA | BHI), row(0xa06d1, RRSBA | BHI), row(0xa06e1, RRSBA | BHI), @@ -1527,14 +1539,37 @@ mod tests { (SpecStoreBypass, needs(&[Ssbd])), ], ); - // Without the controls enhanced IBRS holds its RRSBA, and the - // sequence its branch history. + // Without the controls enhanced IBRS holds its RRSBA, and its branch + // history needs the `BHI_DIS_S` it lacks: the short sequence is not + // Intel's for a part this late. let without = decide(&Facts { cpuid_7_2_edx: 0, ..facts }); assert_eq!(without.verdict(Retbleed), needs(&[EnhancedIbrs])); - assert_eq!(without.verdict(BranchHistoryInjection), needs(&[BhbClear])); + assert_eq!(without.verdict(BranchHistoryInjection), lacking(&[BhiDisS], &[BhiDisS])); assert_eq!(decide(&Facts { arch_capabilities: facts.arch_capabilities | cap::BHI_NO, ..facts }).verdict(BranchHistoryInjection), Verdict::NotAffected); } + /// Intel's short sequence is the BHB clear only where its table gives + /// BHI's mitigation as software alone, the processors before Alder Lake, + /// and in a guest, whose VMM sets `BHI_DIS_S` under it. A later part, or + /// one the table omits, whose microcode predates `BHI_CTRL` is said to + /// lack `BHI_DIS_S` rather than to be mitigated. + #[test] + fn the_short_sequence_is_named_only_where_intel_scopes_it() { + let bhi = |facts: &Facts| decide(facts).verdict(BranchHistoryInjection); + assert_eq!(bhi(&INTEL_806C1), needs(&[BhbClear])); + let alder_lake = Facts { signature: 0x906a3, ..INTEL_806C1 }; + assert_eq!(bhi(&alder_lake), lacking(&[BhiDisS], &[BhiDisS])); + assert_eq!(bhi(&Facts { cpuid_7_2_edx: BHI_CTRL, ..alder_lake }), needs(&[BhiDisS])); + assert_eq!(bhi(&Facts { signature: 0x50657, ..INTEL_806C1 }), assumed(lacking(&[BhiDisS], &[BhiDisS]))); + assert_eq!(bhi(&Facts { hypervisor: true, ..alder_lake }), needs(&[BhbClear])); + let decision = decide(&alder_lake); + let unread = Registers { spec_ctrl: None, mcu_opt_ctrl: None, de_cfg2: None, c001_1029: None }; + assert_eq!( + std::format!("{}", decision.line(BranchHistoryInjection, decision.in_force(&unread))), + "vulnerable: needs BHI_DIS_S; this CPU enumerates no BHI_DIS_S" + ); + } + #[test] fn rfds_is_enumerated_by_rfds_clear_and_listed_without_it() { let listed = Facts { signature: 0xb06e0, ..NOTHING }; diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index 4a3f4ec59d0..545d87ec66b 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -131,7 +131,7 @@ struct Idt { /// Every gate entered at its prefix: the table a CPU whose decision names the /// branch history clear loads. -static IDT: Lock = Lock::new(Idt { +static AT_PREFIX: Lock = Lock::new(Idt { entries: [IdtEntry::EMPTY; 256], }); @@ -332,7 +332,7 @@ const _: () = { /// Whether `rip` is where some gate of either table enters. fn is_gate_target(rip: u64) -> bool { // SAFETY: the tables are written only by `init`, before any gate is taken. - let tables = unsafe { [&*IDT.data_ptr(), &*PAST_PREFIX.data_ptr()] }; + let tables = unsafe { [&*AT_PREFIX.data_ptr(), &*PAST_PREFIX.data_ptr()] }; tables.iter().flat_map(|idt| &idt.entries).any(|gate| gate.type_attr != 0 && gate.target() == rip) } @@ -494,32 +494,32 @@ fn disable_pic() { pub fn init(clears_branch_history: bool) { disable_pic(); - install_gates(&mut IDT.lock()); + install_gates(&mut AT_PREFIX.lock()); // `INT3` from Ring 3 arrives as the `#BP` it is, and ends its process as a // trap, as `BRK` does on AArch64. { - let mut idt = IDT.lock(); + let mut idt = AT_PREFIX.lock(); let breakpoint = &mut idt.entries[Vector::Breakpoint as usize]; *breakpoint = breakpoint.reachable_from_ring3(); } - user_dev::install(&mut IDT.lock()); + user_dev::install(&mut AT_PREFIX.lock()); // Every slot no row filled: delivery through a P = 0 gate is a // contributory fault, and the machine would halt as #DF with no name. let mut unclaimed = 0u32; - for entry in IDT.lock().entries.iter_mut() { + for entry in AT_PREFIX.lock().entries.iter_mut() { if entry.type_attr == 0 { *entry = IdtEntry::at(unclaimed::unclaimed_entry); unclaimed += 1; } } - let gates = IDT.lock().entries; + let gates = AT_PREFIX.lock().entries; PAST_PREFIX.lock().entries = gates.map(IdtEntry::past_prefix); let ptr = load(clears_branch_history); // The table this CPU is now loaded with: how many vectors carry a handler // this kernel named, and how many carry only the unclaimed stub, which is // what a machine delivering an interrupt nobody declared would land on. - let entries = IDT.lock().entries.len() as u32; + let entries = AT_PREFIX.lock().entries.len() as u32; // Copied out first: `IdtPointer` is packed, so a formatting argument would // be a reference to an unaligned field. let (base, limit) = (ptr.base, ptr.limit); @@ -537,7 +537,7 @@ pub fn init(clears_branch_history: bool) { /// clear where `clears_branch_history`, and the one past it elsewhere: what /// this CPU's decision names. After [`init`]. pub fn load(clears_branch_history: bool) -> IdtPointer { - let table = if clears_branch_history { &IDT } else { &PAST_PREFIX }; + let table = if clears_branch_history { &AT_PREFIX } else { &PAST_PREFIX }; let ptr = IdtPointer { limit: (core::mem::size_of::() - 1) as u16, base: table.data_ptr() as u64 }; // SAFETY: `ptr` names one of the two static tables `init` filled whole. unsafe { cpu::lidt(&ptr as *const IdtPointer as *const u8) }; diff --git a/kernel/src/arch/x86_64/mitigation.rs b/kernel/src/arch/x86_64/mitigation.rs index bb36bb5b53a..70ed957621c 100644 --- a/kernel/src/arch/x86_64/mitigation.rs +++ b/kernel/src/arch/x86_64/mitigation.rs @@ -429,8 +429,12 @@ pub fn debug_switch_ibpbs() -> Option { /// to clear the BHB on all processors prior to Alder Lake"), between a save /// and a restore of the two registers it writes: callable from the first /// instruction of a gate, before anything of the interrupted context is -/// saved. Its own `call`s use the stack it is called on; it reads no memory -/// through GS and leaves the flags as it found them not. +/// saved. Its own `call`s use the stack it is called on; it reads nothing +/// through GS and clobbers the flags, which no caller holds across it. Each +/// of its returns goes back to a call it made, at most seven deep, so the RSB +/// holds its target: Intel's "Indirect Target Selection" reaches a return, +/// wherever it lies in its line, only through the alternate predictors an +/// empty RSB falls back on. #[unsafe(naked)] pub(crate) extern "sysv64" fn clear_branch_history() { core::arch::naked_asm!( diff --git a/kernel/src/arch/x86_64/syscall.rs b/kernel/src/arch/x86_64/syscall.rs index 7b8786ad292..caea47cd0fa 100644 --- a/kernel/src/arch/x86_64/syscall.rs +++ b/kernel/src/arch/x86_64/syscall.rs @@ -45,9 +45,9 @@ pub fn init(clears_branch_history: bool) { // `SYSCALL` switches no stack: before the `rsp` switch below runs, the CPU is at CPL 0 on the user's stack, so nothing that can fault may execute there. // The same holds from the exit's `swapgs` on, with the user's GS base live (`arch::entry`): `SYSRET` cannot fault, since no instruction ends at the user half's top for `rcx` to be non-canonical (`toyos_userbound`'s placement). -/// One of `SYSCALL`'s two entries, which `LSTAR` names per CPU (`init`): -/// with a template line and its operand, the branch history clear, which runs -/// as soon as the kernel's stack is under it and before any branch. +/// One of `SYSCALL`'s two entries, which `LSTAR` names per CPU (`init`): the +/// second calls the branch history clear as soon as the kernel's stack is +/// under it and before any branch. macro_rules! syscall_entry { ($name:ident $(, $clear:literal, $operand:ident = $path:path)?) => { #[unsafe(naked)] diff --git a/tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs b/tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs new file mode 100644 index 00000000000..b2876a7d2a0 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs @@ -0,0 +1,49 @@ +//! Each entry from Ring 3 runs its CPU's branch history clear where the CPU's +//! decision names one, read off the kernel's count of clears through +//! `SYS_DEBUG`: at least one per syscall, through `LSTAR`'s entry, and one per +//! page fault on a fresh page, through the IDT's, since every other CPU's +//! entries only add to the count. On a CPU whose decision names none the +//! count is refused, and there is nothing to hold. + +use toyos_abi::syscall::{self, debug_action, SyscallError, VmAt, VmProt}; + +const SYSCALLS: u64 = 1000; +const PAGES: usize = 256; +const PAGE: usize = 4096; + +fn clears() -> u64 { + syscall::debug(debug_action::BRANCH_HISTORY_CLEARS) +} + +fn main() { + let cpu = syscall::current_cpu(); + let mut mask = [0u64; syscall::CPU_MASK_WORDS]; + mask[cpu as usize / 64] = 1 << (cpu % 64); + syscall::thread_set_affinity(0, &mask).expect("pinned to one CPU"); + let before = clears(); + if SyscallError::from_u64(before) == Some(SyscallError::NotSupported) { + println!("branch_history_cleared_on_each_entry: CPU {cpu}'s decision names no branch history clear"); + return; + } + for _ in 0..SYSCALLS { + syscall::current_cpu(); + } + let by_syscalls = clears() - before; + assert!(by_syscalls >= SYSCALLS, "CPU {cpu} cleared branch history {by_syscalls} times over {SYSCALLS} syscalls"); + + // SAFETY: a fresh range nothing else names; committed, so each page is + // populated by its first touch below, a Ring 3 page fault. + let region = unsafe { syscall::vm_reserve(PAGES * PAGE, VmProt::ReadWrite, VmAt::ANYWHERE) }.expect("reserve"); + let before = clears(); + for page in 0..PAGES { + // SAFETY: inside the range just reserved, which nothing else names. + unsafe { region.add(page * PAGE).write_volatile(1) }; + } + let by_faults = clears() - before; + // SAFETY: the range reserved above, which nothing names any more. + unsafe { syscall::vm_release(region, PAGES * PAGE) }.expect("release"); + assert!(by_faults >= PAGES as u64, "CPU {cpu} cleared branch history {by_faults} times over {PAGES} page faults"); + println!( + "branch_history_cleared_on_each_entry: {by_syscalls} clears over {SYSCALLS} syscalls and {by_faults} over {PAGES} page faults on CPU {cpu}" + ); +} diff --git a/tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs b/tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs deleted file mode 100644 index 4ee8af2a85d..00000000000 --- a/tests/guest/rust-tests/src/bin/branch_history_cleared_per_syscall.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Each syscall this job makes runs its CPU's branch history clear where the -//! CPU's decision names one, read off the kernel's count of clears through -//! `SYS_DEBUG`: at least one per syscall, since every other CPU's entries -//! only add to it. On a CPU whose decision names none the count is refused, -//! and there is nothing to hold. - -use toyos_abi::syscall::{self, debug_action, SyscallError}; - -const SYSCALLS: u64 = 1000; - -fn main() { - let cpu = syscall::current_cpu(); - let mut mask = [0u64; syscall::CPU_MASK_WORDS]; - mask[cpu as usize / 64] = 1 << (cpu % 64); - syscall::thread_set_affinity(0, &mask).expect("pinned to one CPU"); - let before = syscall::debug(debug_action::BRANCH_HISTORY_CLEARS); - if SyscallError::from_u64(before) == Some(SyscallError::NotSupported) { - println!("branch_history_cleared_per_syscall: CPU {cpu}'s decision names no branch history clear"); - return; - } - for _ in 0..SYSCALLS { - syscall::current_cpu(); - } - let cleared = syscall::debug(debug_action::BRANCH_HISTORY_CLEARS) - before; - assert!(cleared >= SYSCALLS, "CPU {cpu} cleared branch history {cleared} times over {SYSCALLS} syscalls"); - println!("branch_history_cleared_per_syscall: {cleared} clears over {SYSCALLS} syscalls on CPU {cpu}"); -} diff --git a/tests/toyos.rs b/tests/toyos.rs index 03bbafb305f..98e51966b53 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -73,7 +73,7 @@ const ACTUATOR_TESTS: &[&str] = &[ "speculation_on_request", // Action 33: the branch history clears the entries ran, which leave // nothing a guest can read. - "branch_history_cleared_per_syscall", + "branch_history_cleared_on_each_entry", ]; /// What [`ACTUATOR_TESTS`] boots: the one kernel that carries `SYS_DEBUG`.