diff --git a/build/build.rs b/build/build.rs index 0fc6becb54..72c5c9e2eb 100644 --- a/build/build.rs +++ b/build/build.rs @@ -1783,12 +1783,19 @@ fn assert_actuators_match_features(root: &Path, features: &str, kernel: &[u8]) { ); } -/// `arch::syscall::syscall_entry`'s v0-mangled path, less the crate -/// disambiguator that stands in front of it. -const SYSCALL_ENTRY_SYMBOL: &str = "6kernel4arch6x86_647syscall13syscall_entry"; +/// `SYSCALL`'s two entries, which `LSTAR` names per CPU: their names, and +/// their v0-mangled paths less the crate disambiguator that stands in front +/// of each. +const SYSCALL_ENTRIES: [(&str, &str); 2] = [ + ("syscall_entry", "6kernel4arch6x86_647syscall13syscall_entry"), + ( + "syscall_entry_clearing_branch_history", + "6kernel4arch6x86_647syscall37syscall_entry_clearing_branch_history", + ), +]; /// `swapgs` then `cld`, which `arch::entry::syscall_naked_asm` puts first in -/// `syscall_entry`: the kernel's GS base before anything addresses GS. +/// each entry: the kernel's GS base before anything addresses GS. const OPENS: [u8; 4] = [0x0f, 0x01, 0xf8, 0xfc]; /// `mov gs:[disp32], rsp` less its displacement: the `gs` override, `REX.W`, /// the opcode, and the ModRM/SIB pair that says `rsp` against an absolute @@ -1799,28 +1806,26 @@ const LOAD_RSP_FROM_GS: [u8; 5] = [0x65, 0x48, 0x8b, 0x24, 0x25]; /// Bytes either `mov` occupies, displacement included. const MOV_RSP_GS_LEN: usize = 9; -/// A kernel's `syscall_entry` from its first instruction to the end of the -/// segment it is in, found at its symbol. -fn syscall_entry_bytes(kernel: &[u8]) -> Result<&[u8], String> { +/// The entry at `symbol` from its first instruction to the end of the +/// segment it is in. +fn syscall_entry_bytes<'a>(kernel: &'a [u8], (name, symbol): (&str, &str)) -> Result<&'a [u8], String> { use toyos::elf::header::{ProgramHeader, PT_LOAD}; let (syms, strs) = toyos::elf::section::symbol_tables(kernel).ok_or("the kernel has no readable `.symtab`")?; let table = toyos::elf::SymTab::new(syms, strs); let mut entries = table.defined().filter(|&(i, sym)| { - sym.kind() == toyos::elf::sym::STT_FUNC && table.name(i).ends_with(SYSCALL_ENTRY_SYMBOL) + sym.kind() == toyos::elf::sym::STT_FUNC && table.name(i).ends_with(symbol) }); let (Some((_, entry)), None) = (entries.next(), entries.next()) else { - return Err(format!( - "the kernel's `.symtab` does not name exactly one function `…{SYSCALL_ENTRY_SYMBOL}`" - )); + return Err(format!("the kernel's `.symtab` does not name exactly one function `…{symbol}`")); }; let header = toyos::elf::FileHeader::parse(kernel).map_err(|e| format!("{e:?}"))?; let extent = toyos::elf::Layout::parse(kernel, header.machine).map_err(|e| format!("{e}"))?.extent(); let value = entry .address(extent) .map(|at| extent.min() + at.get()) - .ok_or("the kernel's `syscall_entry` lies outside its own image")?; + .ok_or_else(|| format!("the kernel's `{name}` lies outside its own image"))?; let segments = header.program_headers(kernel).map_err(|e| format!("{e:?}"))?; (0..header.phnum as usize) .filter_map(|i| ProgramHeader::parse(segments, i)) @@ -1830,7 +1835,7 @@ fn syscall_entry_bytes(kernel: &[u8]) -> Result<&[u8], String> { let left = segment.filesz.checked_sub(within).filter(|&left| left != 0)?; toyos::elf::file_range(kernel, segment.offset.checked_add(within)?, left) }) - .ok_or_else(|| format!("no `PT_LOAD` holds `syscall_entry` at {value:#x} in the file")) + .ok_or_else(|| format!("no `PT_LOAD` holds `{name}` at {value:#x} in the file")) } /// Whether an entry switches to the kernel's `rsp` in the instruction after it @@ -1843,7 +1848,7 @@ fn entry_window(entry: &[u8]) -> Result, String> { let after_save = OPENS.len() + MOV_RSP_GS_LEN; if !entry.starts_with(&OPENS) || !entry[OPENS.len()..].starts_with(&SAVE_RSP_TO_GS) { return Err(format!( - "`syscall_entry` does not open `swapgs`, `cld`, `mov gs:[…], rsp`: it opens {:02x?}", + "a `SYSCALL` entry does not open `swapgs`, `cld`, `mov gs:[…], rsp`: it opens {:02x?}", &entry[..entry.len().min(after_save)], )); } @@ -1860,25 +1865,28 @@ fn entry_window(entry: &[u8]) -> Result, String> { Ok(Err(&rest[..between])) } -/// Refuse to write a shipping image whose `syscall_entry` does anything between -/// saving the user's `rsp` and switching to the kernel's. +/// Refuse to write a shipping image either of whose `SYSCALL` entries does +/// anything between saving the user's `rsp` and switching to the kernel's. /// -/// Read at the entry's own symbol, the shipping kernel's first four +/// Read at each entry's own symbol, the shipping kernel's first four /// instructions are `swapgs`, `cld`, the save and the switch, with nothing /// between. fn judge_entry_window(features: &str, kernel: &[u8]) -> Result<(), String> { if !features.is_empty() { return Ok(()); } - match entry_window(syscall_entry_bytes(kernel)?)? { - Ok(()) => Ok(()), - Err(between) => Err(format!( - "the shipping kernel's `syscall_entry` does not switch to the kernel's `rsp` in the \ - instruction after it saves the user's; between them stand {between:02x?}.\nEvery \ - instruction there runs at CPL 0 on a user's stack, and an image that ships must \ - not be able to be asked to stop there." - )), + for entry in SYSCALL_ENTRIES { + if let Err(between) = entry_window(syscall_entry_bytes(kernel, entry)?)? { + return Err(format!( + "the shipping kernel's `{}` does not switch to the kernel's `rsp` in the \ + instruction after it saves the user's; between them stand {between:02x?}.\nEvery \ + instruction there runs at CPL 0 on a user's stack, and an image that ships must \ + not be able to be asked to stop there.", + entry.0, + )); + } } + Ok(()) } fn assert_entry_window_matches_features(features: &str, kernel: &[u8]) { @@ -3874,22 +3882,27 @@ mod tests { ]; /// A PIE of one `PT_LOAD` over the whole file, whose `.symtab` names one - /// function per `names` entry, each at `entry`'s address. + /// function per `names` entry, each at a copy of `entry`. fn kernel_with_entry(names: &[&str], entry: &[u8]) -> Vec { + kernel_with_entries(&names.iter().map(|name| (*name, entry)).collect::>()) + } + + /// The same with each function at its own code, one after another. + fn kernel_with_entries(entries: &[(&str, &[u8])]) -> Vec { const TEXT: u64 = 0x200; let mut strtab = vec![0u8]; let mut symtab = vec![0u8; 24]; - for name in names { + let mut file = vec![0u8; TEXT as usize]; + for (name, code) in entries { symtab.extend((strtab.len() as u32).to_le_bytes()); // `STB_GLOBAL`, `STT_FUNC`; `st_other`; defined in section 1. symtab.extend([0x12, 0, 1, 0]); - symtab.extend(TEXT.to_le_bytes()); - symtab.extend((entry.len() as u64).to_le_bytes()); + symtab.extend((file.len() as u64).to_le_bytes()); + symtab.extend((code.len() as u64).to_le_bytes()); strtab.extend(name.as_bytes()); strtab.push(0); + file.extend(*code); } - let mut file = vec![0u8; TEXT as usize]; - file.extend(entry); let symtab_at = file.len() as u64; file.extend(&symtab); let strtab_at = file.len() as u64; @@ -3932,6 +3945,9 @@ mod tests { } const ENTRY_NAME: &str = "_RNvNtNtNtCs2TF9wDo3GXK_6kernel4arch6x86_647syscall13syscall_entry"; + const CLEARING_NAME: &str = + "_RNvNtNtNtCs2TF9wDo3GXK_6kernel4arch6x86_647syscall37syscall_entry_clearing_branch_history"; + const BOTH: [&str; 2] = [ENTRY_NAME, CLEARING_NAME]; fn entry_with(between: &[u8]) -> Vec { [&ENTRY_OPENS[..], between, &SWITCH[..], &[0x90; 32][..]].concat() @@ -3939,30 +3955,40 @@ mod tests { #[test] fn a_clean_entry_is_the_shipping_kernels() { - let kernel = kernel_with_entry(&[ENTRY_NAME], &entry_with(&[])); + let kernel = kernel_with_entry(&BOTH, &entry_with(&[])); assert_eq!(judge_entry_window("", &kernel), Ok(())); } #[test] fn a_hold_is_refused_in_a_shipping_entry_however_its_labels_are_spelled() { for hold in [&LABELLED_HOLD[..], &LOCAL_LABEL_HOLD[..]] { - let kernel = kernel_with_entry(&[ENTRY_NAME], &entry_with(hold)); + let kernel = kernel_with_entry(&BOTH, &entry_with(hold)); let refusal = judge_entry_window("", &kernel).unwrap_err(); assert!(refusal.contains(&format!("between them stand {hold:02x?}")), "{refusal}"); } } + /// The entry that clears branch history is judged as the other is: a + /// clear moved in front of the switch would run on the user's stack. + #[test] + fn a_hold_in_the_clearing_entry_alone_is_refused() { + let hold = entry_with(&LABELLED_HOLD); + let kernel = kernel_with_entries(&[(ENTRY_NAME, &entry_with(&[])[..]), (CLEARING_NAME, &hold[..])]); + let refusal = judge_entry_window("", &kernel).unwrap_err(); + assert!(refusal.contains("`syscall_entry_clearing_branch_history` does not switch"), "{refusal}"); + } + #[test] fn a_hold_in_front_of_the_save_is_an_entry_the_judge_refuses_to_read() { let entry = [&ENTRY_OPENS[..4], &[0xf3, 0x90][..], &ENTRY_OPENS[4..], &SWITCH[..]].concat(); - let kernel = kernel_with_entry(&[ENTRY_NAME], &entry); + let kernel = kernel_with_entry(&BOTH, &entry); let refusal = judge_entry_window("", &kernel).unwrap_err(); assert!(refusal.contains("does not open `swapgs`, `cld`"), "{refusal}"); } #[test] fn an_entry_without_its_swapgs_is_refused() { - let kernel = kernel_with_entry(&[ENTRY_NAME], &[&ENTRY_OPENS[3..], &SWITCH[..]].concat()); + let kernel = kernel_with_entry(&BOTH, &[&ENTRY_OPENS[3..], &SWITCH[..]].concat()); let refusal = judge_entry_window("", &kernel).unwrap_err(); assert!(refusal.contains("does not open `swapgs`, `cld`"), "{refusal}"); } @@ -3970,7 +3996,7 @@ mod tests { #[test] fn a_kernel_that_does_not_name_exactly_one_entry_is_refused() { let clean = entry_with(&[]); - for names in [&["_RNvCs1_6kernel4main"][..], &[ENTRY_NAME, ENTRY_NAME][..]] { + for names in [&["_RNvCs1_6kernel4main"][..], &[ENTRY_NAME, ENTRY_NAME][..], &[ENTRY_NAME][..]] { let refusal = judge_entry_window("", &kernel_with_entry(names, &clean)).unwrap_err(); assert!(refusal.contains("does not name exactly one function"), "{refusal}"); } diff --git a/issues/indirect-branches-and-returns-run-without-thunks.md b/issues/indirect-branches-and-returns-run-without-thunks.md index bcee208227..e759efac3d 100644 --- a/issues/indirect-branches-and-returns-run-without-thunks.md +++ b/issues/indirect-branches-and-returns-run-without-thunks.md @@ -23,7 +23,11 @@ script asserts (`arch/x86/kernel/vmlinux.lds.S:510-539`): pair's addresses differing in exactly bits 2, 8, 14 and 20, and each ITS thunk's branch in its line's upper half. Each proving machine's boot installs the thunks the decision selects for it: ITS's on the T14, SRSO's alias on a -family-0x19 EPYC guest. No proving machine is family 0x17, so the untrained +family-0x19 EPYC guest. The branch history clear's returns +(`arch::x86_64::mitigation::clear_branch_history`) are Intel's sequence and +take no thunk; three lie in the lower halves of their lines, and each returns +to a call the sequence made, so the RSB predicts it and ITS, which reaches a +return only through an empty RSB's alternate predictors, does not. No proving machine is family 0x17, so the untrained return is proven by the gate and the decision's host test alone. **Mutation**, each red: an ITS thunk at a line start; `retbleed_return_thunk` one byte past its line start; the SRSO alias one cache line off its pair; an diff --git a/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md b/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md deleted file mode 100644 index d3b8e5148b..0000000000 --- a/issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -status: open -kind: defect -severity: [security] -opened: 2026-09-29 ---- - -# No entry or switch clears the BHB, fills the RSB or issues an IBPB - -**Built**: a switch into a program's address space issues an IBPB where the -space this CPU last ran a program in is another and either asked for -isolated speculation (the ask of -`issues/no-program-runs-with-speculative-store-bypass-disabled.md`) and the -CPU's decision names IBPB (`kernel::mitigation::LastAsked`, -`arch::x86_64::mitigation::switch_to`). On an AMD CPU whose IBPB leaves the -return address predictor's entries, which `IBPB_RET` (APM Vol. 3 -Fn8000_0008_EBX[30]) does not enumerate, AMD-SB-1040 asks 70586's V2-3 -after it: the decision names the predictor filled after IBPB there, and the -switch follows each IBPB with 32 calls. Intel's IBPB covers the RSB -("Speculative Execution Side Channel Mitigations", the IBPB note on near -returns), and no other fill on a switch is a vendor's for a CPU the decision -covers. `an_ibpb_goes_between_two_spaces_one_of_which_asked` holds two IBPBs -over B, C, idle, B, A, idle, A, B, where A asked, and one under the -mutation that decides by the previous space alone. `speculation_on_request` -counts IBPBs on a CPU where a program that asked takes turns with this one, -and none where neither asked. Linux at `Ubuntu-6.8.0-142.142` issues an IBPB -on a switch between address spaces one of which asked for it -(`arch/x86/mm/tlb.c:384`). - -**Left**: -- Nothing clears branch history on entry. Linux runs `clear_bhb_loop` on - entry where its decision selects the BHI loop - (`arch/x86/entry/entry_64.S:1534`). -- **Owed on metal:** `speculation_on_request`, a member of main's - `shared-debug` boot. - - Benches: the T14 and the AMD bench. - - Main must show the member exit 0, its line naming at least 2 IBPBs - beside the probe and none beside the child that did not ask. - -**Exit**: a gate over `kernel.elf`, on the decoder of -`issues/no-gate-decodes-kernel-elfs-instructions.md`, matches it against -the branch history clearing and return predictor filling sequences Intel's -and AMD's published guidance prescribes; on every proving machine probes -count what -`issues/a-pure-function-decides-a-cpus-speculation-mitigations-from-its-vendors-guidance.md` -selects for it: a clear per syscall where it selects the loop, and IBPBs -where a program that asked takes turns with another on one CPU and none -where neither asked, each followed by the fill where it selects one. A host -test holds the IBPB's decision over B, C, idle, B, A, idle, A, B to two. -**Mutation**: either sequence edited, the clear's condition inverted, no -fill, the IBPB decided by the previous space alone. **Oracle**: the vendors' -published sequences. Linux's sequences are a goal compared in evidence, -never the gate's expected text. diff --git a/issues/no-gate-decodes-kernel-elfs-instructions.md b/issues/no-gate-decodes-kernel-elfs-instructions.md index 2bf4971685..8d59f06a9a 100644 --- a/issues/no-gate-decodes-kernel-elfs-instructions.md +++ b/issues/no-gate-decodes-kernel-elfs-instructions.md @@ -8,7 +8,7 @@ opened: 2026-09-29 Five stages each owe a gate over `kernel.elf`'s instructions: -- `issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md` +- `issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md` - `issues/indirect-branches-and-returns-run-without-thunks.md` - `issues/kernel-functions-return-with-their-used-registers-intact.md` - `issues/the-kernel-runs-without-indirect-branch-tracking.md` diff --git a/issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md b/issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md new file mode 100644 index 0000000000..c4668fb1ac --- /dev/null +++ b/issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md @@ -0,0 +1,87 @@ +--- +status: open +kind: defect +severity: [security] +opened: 2026-09-29 +--- + +# No gate holds the BHB clear and the return fill to their vendors' sequences + +**Built**: +- Where a CPU's decision names the BHB cleared on entry, every entry from + Ring 3 runs Intel's short branch history clearing sequence ("Branch + History Injection and Intra-mode Branch Target Injection", Listing 1) + before any indirect branch (`arch::x86_64::mitigation::clear_branch_history`). + `LSTAR` names the `SYSCALL` entry that calls it, and every gate the IDT + names opens with a 12-byte prefix that calls it where the frame's `CS` is + Ring 3's; such a CPU loads the table that enters each gate there, and + every other CPU the one that enters past it, so it runs none of it + (`arch::x86_64::entry::gate_prefix`, `arch::x86_64::idt`). The decision + names the short sequence only where Intel scopes it: its table's + processors whose BHI mitigation is software alone, those before Alder + Lake, and a guest, whose VMM sets `BHI_DIS_S` under it. + `branch_history_cleared_on_each_entry` counts at least one clear per + syscall and one per page fault where the CPU's decision names it. +- A switch into a program's address space issues an IBPB where the space + this CPU last ran a program in is another and either asked for isolated + speculation (the ask of + `issues/no-program-runs-with-speculative-store-bypass-disabled.md`) and the + CPU's decision names IBPB (`kernel::mitigation::LastAsked`, + `arch::x86_64::mitigation::switch_to`). On an AMD CPU whose IBPB leaves the + return address predictor's entries, which `IBPB_RET` (APM Vol. 3 + Fn8000_0008_EBX[30]) does not enumerate, AMD-SB-1040 asks 70586's V2-3 + after it: the decision names the predictor filled after IBPB there, and + the switch follows each IBPB with 32 calls. Intel's IBPB covers the RSB + ("Speculative Execution Side Channel Mitigations", the IBPB note on near + returns), and no other fill on a switch is a vendor's for a CPU the + decision covers. `an_ibpb_goes_between_two_spaces_one_of_which_asked` + holds two IBPBs over B, C, idle, B, A, idle, A, B, where A asked, and one + under the mutation that decides by the previous space alone. + `speculation_on_request` counts IBPBs on a CPU where a program that asked + takes turns with this one, and none where neither asked. +- Linux at `Ubuntu-6.8.0-142.142` runs `clear_bhb_loop` on `SYSCALL` entry + where its decision selects the BHI loop (`arch/x86/entry/entry_64.S:1534`), + and issues an IBPB on a switch between address spaces one of which asked + for it (`arch/x86/mm/tlb.c:384`). + +**Left**: +- An NMI or `#MC` that lands on an entry from Ring 3 before its clear + finishes, in `SYSCALL`'s first four instructions or while a gate's clear + runs, arrives as a Ring 0 frame, skips the clear and runs its handler on + the user's branch history. User code cannot time either. +- An Intel host that the table gives `BHI_DIS_S` (its "MCU+Software" and + "Hardware+Software" rows, Alder Lake on), or one the table omits, whose + microcode predates `BHI_CTRL` says branch history injection vulnerable, + lacking `BHI_DIS_S`: the short sequence is not Intel's for it, and the + long sequence, Listing 2, which covers every part without `BHI_NO`, is not + built. ToyOS ships microcode for no such part. +- A guest that clears with the short sequence does not tell its VMM so: + Intel's guidance asks it to set `BHB_CLEAR_SEQ_S_USED` in + `MSR_VIRTUAL_MITIGATION_CTRL` where `MSR_VIRTUAL_MITIGATION_ENUM` + enumerates `BHB_CLEAR_SEQ_S_SUPPORT`, so that a VMM migrating it to a part + the short sequence does not cover sets `BHI_DIS_S` under it. A guest on + the guest runner is one; no proving machine is. +- **Owed on metal:** `branch_history_cleared_on_each_entry` and + `speculation_on_request`, members of main's `shared-debug` boot. + - Benches: the T14 and the AMD bench. + - Main must show both members exit 0: on the T14 the first naming at + least 1000 clears over 1000 syscalls and 256 over 256 page faults, on + the AMD bench naming no clear; the second naming at least 2 IBPBs beside + the probe and none beside the child that did not ask. + - The T14's `mitigations` row, the `metalcase` boot of main, with + `branch_history_injection: mitigated: BHB cleared on entry` on every CPU. + +**Exit**: a gate over `kernel.elf`, on the decoder of +`issues/no-gate-decodes-kernel-elfs-instructions.md`, matches it against +the branch history clearing and return predictor filling sequences Intel's +and AMD's published guidance prescribes; on every proving machine probes +count what +`issues/a-pure-function-decides-a-cpus-speculation-mitigations-from-its-vendors-guidance.md` +selects for it: a clear per syscall where it selects the loop, and IBPBs +where a program that asked takes turns with another on one CPU and none +where neither asked, each followed by the fill where it selects one. A host +test holds the IBPB's decision over B, C, idle, B, A, idle, A, B to two. +**Mutation**: either sequence edited, the clear's condition inverted, no +fill, the IBPB decided by the previous space alone. **Oracle**: the vendors' +published sequences. Linux's sequences are a goal compared in evidence, +never the gate's expected text. diff --git a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md index ef25c57fb5..f61b4391d9 100644 --- a/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md +++ b/issues/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md @@ -26,7 +26,7 @@ work needs. - `issues/no-program-runs-with-speculative-store-bypass-disabled.md` - `issues/no-aarch64-program-runs-with-speculative-store-bypass-disabled.md` - `issues/no-gate-decodes-kernel-elfs-instructions.md` -- `issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md` +- `issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md` - `issues/user-pointer-checks-have-no-spectre-v1-fence.md` - `issues/indirect-branches-and-returns-run-without-thunks.md` - `issues/tsx-stays-as-firmware-left-it.md` diff --git a/issues/toyos-against-linux-on-both-benches-the-scoreboard.md b/issues/toyos-against-linux-on-both-benches-the-scoreboard.md index 5008bba48e..f939135c5c 100644 --- a/issues/toyos-against-linux-on-both-benches-the-scoreboard.md +++ b/issues/toyos-against-linux-on-both-benches-the-scoreboard.md @@ -306,7 +306,7 @@ there: reads 159 against 49 ns, and its run lines state retpolines, conditional IBPB, STIBP and Safe RET (`issues/indirect-branches-and-returns-run-without-thunks.md`, - `issues/no-entry-or-switch-clears-the-bhb-fills-the-rsb-or-issues-an-ibpb.md`). + `issues/no-gate-holds-the-bhb-clear-and-the-return-fill-to-their-vendors-sequences.md`). - After 23 of the 76, all shapes that block and wake, the clock Linux's reporting thread read was 1.37 to 1.9 GHz in the median run, where ToyOS's read 4.2 to 4.3 GHz. The probe reads the reporting thread's CPU, so that diff --git a/kernel/pure/mitigation.rs b/kernel/pure/mitigation.rs index 0a6eb146bd..c449a1bd2b 100644 --- a/kernel/pure/mitigation.rs +++ b/kernel/pure/mitigation.rs @@ -488,13 +488,13 @@ impl Decision { /// What is in force on a CPU whose registers read `registers`: [`ALWAYS`]; /// what the kernel applies by construction wherever this decision names - /// it and the CPU enumerates it, which no register shows at bring-up (the - /// controls a program that asks runs with, [`ON_REQUEST`]); and each - /// control a register shows on. Intel's two `IA32_MCU_OPT_CTRL` bits opt - /// out of a mitigation its microcode turns on; the others turn one on. + /// it and the CPU enumerates it, which no register shows at bring-up + /// ([`BY_CONSTRUCTION`]); and each control a register shows on. Intel's + /// two `IA32_MCU_OPT_CTRL` bits opt out of a mitigation its microcode + /// turns on; the others turn one on. pub fn in_force(&self, registers: &Registers) -> Mitigations { let mut on = ALWAYS; - for one in Mitigation::ALL.into_iter().filter(|&one| ON_REQUEST.has(one) && self.enumerates(one)) { + for one in Mitigation::ALL.into_iter().filter(|&one| BY_CONSTRUCTION.has(one) && self.enumerates(one)) { on = on.with(one); } if let Some(spec_ctrl) = registers.spec_ctrl { @@ -530,15 +530,17 @@ impl Decision { /// CPU without `CR4.SMEP`, and enters no VMX or SVM operation. pub const ALWAYS: Mitigations = Mitigations::of(&[Mitigation::UserPointerClamp, Mitigation::Smep, Mitigation::NoGuest]); -/// What the kernel applies for a program that asked, on every CPU that -/// enumerates it ([`Decision::on_request`]). No register shows it at -/// bring-up, before any program has. -const ON_REQUEST: Mitigations = Mitigations::of(&[ +/// What the kernel applies on every CPU whose decision names it and that +/// enumerates it, and no register shows at bring-up: what a program that +/// asks runs with ([`Decision::on_request`]), and the branch history clear of +/// the entries each CPU loads for its decision. +const BY_CONSTRUCTION: Mitigations = Mitigations::of(&[ Mitigation::Ibpb, Mitigation::ReturnPredictorFilled, Mitigation::Stibp, Mitigation::Ssbd, Mitigation::VirtSsbd, + Mitigation::BhbClear, ]); /// What a CPU applies for a program that asked, and for no other. @@ -883,7 +885,17 @@ fn intel(f: &Facts, of: Vulnerability) -> Verdict { } else if f.cpuid_7_2_edx & BHI_CTRL != 0 { Asks::of(Mitigation::BhiDisS, true).affected() } else if f.cap(cap::IBRS_ALL) { - Asks::of(Mitigation::BhbClear, true).affected() + // Its short sequence is "sufficient to clear the BHB on all + // processors prior to Alder Lake", the table's software rows; a + // guest's VMM sets `BHI_DIS_S` under it on any other ("Software + // Mitigations in Migration Pools"). A later part names + // `BHI_DIS_S`, which microcode older than its BHI update lacks. + match f.intel_lists(BHI_SOFTWARE) { + _ if f.hypervisor => Asks::of(Mitigation::BhbClear, true).affected(), + Some(true) => Asks::of(Mitigation::BhbClear, true).affected(), + Some(false) => Asks::of(Mitigation::BhiDisS, false).affected(), + None => Asks::of(Mitigation::BhiDisS, false).assumed(true), + } } else if f.hypervisor && ibrs && (f.intel_rsba() || f.cap(cap::RRSBA)) { // A guest that enumerates IBRS and not IBRS_ALL may run on // hardware that has it: the sequence, where the guest relies @@ -1207,7 +1219,10 @@ const fn any_stepping(signature: u32, affected: u16) -> Row { // Check Error Avoidance (CVE-2018-12207); Special Register Buffer Data // Sampling; the two MMIO columns; Gather Data Sampling; Register File Data // Sampling; the three Indirect Target Selection columns; RSBU's RRSBA -// column; Branch History Injection. +// column; Branch History Injection, and where its every row gives BHI's +// mitigation as "Software" alone, which the table gives every processor +// before Alder Lake, against "MCU+Software" or "Hardware+Software" for one +// with `BHI_DIS_S`. const MELTDOWN: u16 = 1 << 0; const L1TF: u16 = 1 << 1; const MDS: u16 = 1 << 2; @@ -1219,6 +1234,7 @@ const RFDS: u16 = 1 << 7; const ITS: u16 = 1 << 8; const RRSBA: u16 = 1 << 9; const BHI: u16 = 1 << 10; +const BHI_SOFTWARE: u16 = 1 << 11; /// Intel's "Affected Processors: Guidance for Security Issues on Intel /// Processors" as its machine-readable list gave it on 2026-09-04: each @@ -1232,46 +1248,46 @@ const INTEL_ROWS: [Row; 67] = [ row(0x406f1, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50653, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50654, MELTDOWN | L1TF | MDS | MCEPSC | MMIO | GDS), - row(0x5065b, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), + row(0x5065b, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), row(0x50663, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50664, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x50665, MELTDOWN | L1TF | MDS | MCEPSC | MMIO), row(0x506ca, RFDS), row(0x506e3, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), any_stepping(0x506f0, RFDS), - row(0x606a6, MMIO | GDS | ITS | BHI), - any_stepping(0x606c0, MMIO | GDS | ITS | BHI), - row(0x706a8, RFDS | BHI), - row(0x706e5, MDS | MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0x80665, MMIO | RFDS | BHI), - row(0x80667, MMIO | RFDS | BHI), - row(0x806c1, MMIO | GDS | ITS | BHI), - row(0x806c2, GDS | ITS | BHI), - row(0x806d1, MMIO | GDS | ITS | BHI), + row(0x606a6, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), + any_stepping(0x606c0, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), + row(0x706a8, RFDS | BHI | BHI_SOFTWARE), + row(0x706e5, MDS | MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0x80665, MMIO | RFDS | BHI | BHI_SOFTWARE), + row(0x80667, MMIO | RFDS | BHI | BHI_SOFTWARE), + row(0x806c1, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), + row(0x806c2, GDS | ITS | BHI | BHI_SOFTWARE), + row(0x806d1, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), row(0x806e9, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x806ea, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x806eb, MDS | MCEPSC | SRBDS | MMIO | GDS), - row(0x806ec, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI), + row(0x806ec, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), row(0x806f5, RRSBA | BHI), row(0x806f6, RRSBA | BHI), row(0x806f7, RRSBA | BHI), row(0x806f8, RRSBA | BHI), - any_stepping(0x90660, MMIO | RFDS | BHI), + any_stepping(0x90660, MMIO | RFDS | BHI | BHI_SOFTWARE), row(0x90672, RFDS | BHI), row(0x90675, RFDS | BHI), row(0x906a3, RFDS | BHI), row(0x906a4, RFDS | BHI), - any_stepping(0x906c0, MMIO | RFDS | BHI), + any_stepping(0x906c0, MMIO | RFDS | BHI | BHI_SOFTWARE), row(0x906e9, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x906ea, MELTDOWN | L1TF | MDS | MCEPSC | SRBDS | MMIO | GDS), row(0x906ec, MDS | MCEPSC | SRBDS | MMIO | GDS), - row(0x906ed, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0652, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0653, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0655, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0660, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0661, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI), - row(0xa0671, MMIO | GDS | ITS | BHI), + row(0x906ed, MCEPSC | SRBDS | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0652, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0653, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0655, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0660, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0661, MCEPSC | MMIO | GDS | ITS | RRSBA | BHI | BHI_SOFTWARE), + row(0xa0671, MMIO | GDS | ITS | BHI | BHI_SOFTWARE), row(0xa06a4, RRSBA | BHI), row(0xa06d1, RRSBA | BHI), row(0xa06e1, RRSBA | BHI), @@ -1523,14 +1539,37 @@ mod tests { (SpecStoreBypass, needs(&[Ssbd])), ], ); - // Without the controls enhanced IBRS holds its RRSBA, and the - // sequence its branch history. + // Without the controls enhanced IBRS holds its RRSBA, and its branch + // history needs the `BHI_DIS_S` it lacks: the short sequence is not + // Intel's for a part this late. let without = decide(&Facts { cpuid_7_2_edx: 0, ..facts }); assert_eq!(without.verdict(Retbleed), needs(&[EnhancedIbrs])); - assert_eq!(without.verdict(BranchHistoryInjection), needs(&[BhbClear])); + assert_eq!(without.verdict(BranchHistoryInjection), lacking(&[BhiDisS], &[BhiDisS])); assert_eq!(decide(&Facts { arch_capabilities: facts.arch_capabilities | cap::BHI_NO, ..facts }).verdict(BranchHistoryInjection), Verdict::NotAffected); } + /// Intel's short sequence is the BHB clear only where its table gives + /// BHI's mitigation as software alone, the processors before Alder Lake, + /// and in a guest, whose VMM sets `BHI_DIS_S` under it. A later part, or + /// one the table omits, whose microcode predates `BHI_CTRL` is said to + /// lack `BHI_DIS_S` rather than to be mitigated. + #[test] + fn the_short_sequence_is_named_only_where_intel_scopes_it() { + let bhi = |facts: &Facts| decide(facts).verdict(BranchHistoryInjection); + assert_eq!(bhi(&INTEL_806C1), needs(&[BhbClear])); + let alder_lake = Facts { signature: 0x906a3, ..INTEL_806C1 }; + assert_eq!(bhi(&alder_lake), lacking(&[BhiDisS], &[BhiDisS])); + assert_eq!(bhi(&Facts { cpuid_7_2_edx: BHI_CTRL, ..alder_lake }), needs(&[BhiDisS])); + assert_eq!(bhi(&Facts { signature: 0x50657, ..INTEL_806C1 }), assumed(lacking(&[BhiDisS], &[BhiDisS]))); + assert_eq!(bhi(&Facts { hypervisor: true, ..alder_lake }), needs(&[BhbClear])); + let decision = decide(&alder_lake); + let unread = Registers { spec_ctrl: None, mcu_opt_ctrl: None, de_cfg2: None, c001_1029: None }; + assert_eq!( + std::format!("{}", decision.line(BranchHistoryInjection, decision.in_force(&unread))), + "vulnerable: needs BHI_DIS_S; this CPU enumerates no BHI_DIS_S" + ); + } + #[test] fn rfds_is_enumerated_by_rfds_clear_and_listed_without_it() { let listed = Facts { signature: 0xb06e0, ..NOTHING }; @@ -1759,14 +1798,14 @@ mod tests { ); } - /// What every CPU has, what it applies on request where it enumerates - /// it, and each control its register shows on: a register the CPU was - /// not read for shows nothing. + /// What every CPU has, what it applies on request or on entry where it + /// enumerates it, and each control its register shows on: a register the + /// CPU was not read for shows nothing. #[test] fn what_is_in_force_is_what_the_registers_show() { let unread = Registers { spec_ctrl: None, mcu_opt_ctrl: None, de_cfg2: None, c001_1029: None }; let intel = decide(&INTEL_806C1); - let always = ALWAYS.with(Ibpb).with(Ssbd); + let always = ALWAYS.with(Ibpb).with(Ssbd).with(BhbClear); assert_eq!(intel.in_force(&unread), always); assert_eq!(intel.in_force(&Registers { spec_ctrl: Some(msr::IBRS), ..unread }), always.with(EnhancedIbrs)); // A bit the decision does not name shows nothing. @@ -1778,7 +1817,7 @@ mod tests { assert_eq!(intel.in_force(&opted_out), always); // What the CPU does not enumerate is not applied for anyone. let bare = decide(&Facts { cpuid_7_0_edx: 0, ..INTEL_806C1 }); - assert_eq!(bare.in_force(&unread), ALWAYS); + assert_eq!(bare.in_force(&unread), ALWAYS.with(BhbClear)); let amd = decide(&AMD_17H_60H); let always = ALWAYS.with(Ibpb).with(ReturnPredictorFilled).with(Stibp).with(Ssbd); let both = Registers { de_cfg2: Some(msr::SUPPRESS_BP_ON_NON_BR), c001_1029: Some(C001_1029_LFENCE_SERIALIZES_DISPATCH), ..unread }; diff --git a/kernel/src/arch/aarch64/mitigation.rs b/kernel/src/arch/aarch64/mitigation.rs index d4cf26343d..0e1475b24b 100644 --- a/kernel/src/arch/aarch64/mitigation.rs +++ b/kernel/src/arch/aarch64/mitigation.rs @@ -26,3 +26,9 @@ pub fn debug_spec_ctrl_on_request() -> u64 { pub fn debug_switch_ibpbs() -> Option { None } + +/// x86-64's counts its entries' branch history clears, and none runs here. +#[cfg(feature = "test-actuators")] +pub fn debug_branch_history_clears() -> Option { + None +} diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index c87984f50a..08049e7304 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -116,7 +116,7 @@ pub fn interrupts(rsdp_addr: u64) -> Platform { super::power::init_control(rsdp_addr); ioapic::init(&madt); idt::enable_interrupts(); - super::syscall::init(); + super::syscall::init(super::mitigation::boot_cpu_clears_branch_history()); Platform { madt, facs } } diff --git a/kernel/src/arch/x86_64/entry.rs b/kernel/src/arch/x86_64/entry.rs index e225af1a3d..902c5a6d0c 100644 --- a/kernel/src/arch/x86_64/entry.rs +++ b/kernel/src/arch/x86_64/entry.rs @@ -39,12 +39,31 @@ macro_rules! cld { }; } +/// The [`GATE_PREFIX`] bytes every gate the IDT names opens with: where the +/// frame's `CS`, `{gate_cs}` bytes above `rsp`, is Ring 3's, a call of +/// `{clear_branch_history}` (`arch::mitigation::clear_branch_history`). A CPU +/// whose decision names the clear has its IDT enter each gate here; every +/// other CPU's enters past it and runs none of it (`arch::idt`). A `jz` +/// mispredicted past the clear is followed by no load through GS and no +/// indirect branch before the gate's own `lfence` or serializing `wrmsr`. +macro_rules! gate_prefix { + () => { + "test byte ptr [rsp + {gate_cs}], 3\njz 78f\ncall {clear_branch_history}\n78:" + }; +} + +/// [`gate_prefix`]'s length: `test` with an 8-bit displacement and +/// immediate, a short `jz` and a `call` with a 32-bit displacement. `arch::idt` +/// reads it back off each gate before it points a table past it. +pub(crate) const GATE_PREFIX: usize = 12; + /// `naked_asm!` for an interrupt or trap gate a Ring 3 thread can arrive /// through, `cs = N;` bytes below its frame's `CS` on entry: prepends `cld`, /// `clac`, and a `swapgs` when that `CS` is Ring 3's. The body must end with a /// trailing comma, and swap back before every `iretq` that can return to /// Ring 3: [`gate_iretq`] where the frame decides, a bare `swapgs` on a path -/// only a Ring 3 frame takes. +/// only a Ring 3 frame takes. `gate cs = N;` makes it a gate the IDT names, +/// opening with [`gate_prefix`]; `cs = N;` one a stub jumps to. /// /// Such a gate clears neither `DF` nor `AC` (Intel SDM Vol. 3A §6.12.1.3 names /// `TF`, `VM`, `RF`, `NT`, and `IF` for an interrupt gate), and Ring 3 sets @@ -57,6 +76,21 @@ macro_rules! cld { /// of SWAPGS and Segment Registers", ID 758371). On AMD it holds where /// `LFENCE` is dispatch-serializing (AMD 70586 G-2). macro_rules! ring3_naked_asm { + (gate cs = $cs:expr; $($body:tt)*) => { + core::arch::naked_asm!( + $crate::arch::entry::gate_prefix!(), + $crate::arch::entry::cld!(), + "clac", + "test byte ptr [rsp + {gate_cs}], 3", + "jz 70f", + "swapgs", + "70:", + "lfence", + $($body)* + gate_cs = const $cs, + clear_branch_history = sym $crate::arch::mitigation::clear_branch_history, + ) + }; (cs = $cs:expr; $($body:tt)*) => { core::arch::naked_asm!( $crate::arch::entry::cld!(), @@ -82,7 +116,7 @@ macro_rules! gate_iretq { }; } -/// `naked_asm!` for `SYSCALL`'s entry: prepends `swapgs`, since only Ring 3 +/// `naked_asm!` for `SYSCALL`'s entries: prepends `swapgs`, since only Ring 3 /// executes `SYSCALL`, then `cld`. `IA32_FMASK` clears `AC` /// (`arch::syscall::init`), so the entry's next instructions are the switch /// off the user's stack. @@ -116,7 +150,7 @@ macro_rules! install_kernel_gs { }; } -pub(crate) use {cld, gate_iretq, install_kernel_gs, ring3_naked_asm, syscall_naked_asm}; +pub(crate) use {cld, gate_iretq, gate_prefix, install_kernel_gs, ring3_naked_asm, syscall_naked_asm}; /// A thread's first entry to Ring 3, at `r12` on the stack `r13`, `$align` /// applied to it, with `rdi` = `r14`: a process's argument is zero, a thread's diff --git a/kernel/src/arch/x86_64/idt/debug.rs b/kernel/src/arch/x86_64/idt/debug.rs index 434b46e59d..3b950c5291 100644 --- a/kernel/src/arch/x86_64/idt/debug.rs +++ b/kernel/src/arch/x86_64/idt/debug.rs @@ -18,7 +18,7 @@ use core::arch::naked_asm; -use crate::arch::entry::install_kernel_gs; +use crate::arch::entry::{gate_prefix, install_kernel_gs}; use crate::arch::percpu::{MSR_GS_BASE, OFF_GDT, OFF_KERNEL_RSP, USER_CS, USER_DS}; /// Eleven pushes place the CPU frame's `rip` at 88. @@ -27,6 +27,7 @@ const RIP_OFFSET: usize = 88; #[unsafe(naked)] pub(super) extern "sysv64" fn debug_entry() { naked_asm!( + gate_prefix!(), "cld", "clac", // Neither path addresses GS before it has made it the kernel's, so a @@ -105,6 +106,8 @@ pub(super) extern "sysv64" fn debug_entry() { vector = const super::Vector::Debug as usize, deferred = sym deferred, common = sym super::common_entry, + gate_cs = const 8, + clear_branch_history = sym crate::arch::mitigation::clear_branch_history, ); } @@ -122,11 +125,11 @@ struct Frame { /// reaches, whose trap it is. On `debug_entry` itself the frame becomes a Ring /// 3 one, its lost `rip` and `rsp` zero, so its thread ends as a trap. extern "sysv64" fn deferred(frame: &mut Frame) { - if frame.rip == debug_entry as *const () as u64 { + if super::is_gate_of(frame.rip, debug_entry as *const () as u64) { *frame = Frame { rip: 0, cs: u64::from(USER_CS), rflags: 0x202, rsp: 0, ss: u64::from(USER_DS) }; return; } - if frame.rip != crate::arch::syscall::entry_address() && !super::is_gate_target(frame.rip) { + if !crate::arch::syscall::is_entry(frame.rip) && !super::is_gate_target(frame.rip) { panic!("#DB at {:#x} in Ring 0, which is no entry: nothing in this kernel arms a debug trap", frame.rip); } } diff --git a/kernel/src/arch/x86_64/idt/device_irq.rs b/kernel/src/arch/x86_64/idt/device_irq.rs index 9cf46756f2..47a70d9227 100644 --- a/kernel/src/arch/x86_64/idt/device_irq.rs +++ b/kernel/src/arch/x86_64/idt/device_irq.rs @@ -11,18 +11,19 @@ /// CPU's frame, which `$handler` takes as its argument and the exit drops. macro_rules! device_irq_entry { ($(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path) => { - device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; 88; []; []); + device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; [gate]; 88; []; []); }; ($(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path, numbered) => { - device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; 96; ["mov rdi, [rbp + 80]"]; ["add rsp, 8"]); + device_irq_entry!(@entry $(#[$meta])* $vis fn $name => $handler; []; 96; ["mov rdi, [rbp + 80]"]; ["add rsp, 8"]); }; // `$cs` is the CS word's offset from `rbp`: 10 GPRs and RIP, and the stub's word if there is one. - (@entry $(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path; $cs:literal; [$($argument:literal)?]; [$($drop:literal)?]) => { + // A numbered entry is a stub's jump target, and the stub is the gate. + (@entry $(#[$meta:meta])* $vis:vis fn $name:ident => $handler:path; [$($gate:ident)?]; $cs:literal; [$($argument:literal)?]; [$($drop:literal)?]) => { $(#[$meta])* #[unsafe(naked)] $vis extern "sysv64" fn $name() { // On entry the 10 GPRs are not yet pushed. - $crate::arch::entry::ring3_naked_asm!(cs = $cs - 80; + $crate::arch::entry::ring3_naked_asm!($($gate)? cs = $cs - 80; // Handler may clobber any scratch GPR; one left unsaved leaks kernel state into a user register on iretq. "push rax", "push rcx", diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index 5bc05eff25..545d87ec66 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -18,7 +18,7 @@ pub(crate) use user_dev::ClaimVector; use core::arch::naked_asm; use super::cpu; -use super::entry::{gate_iretq, install_kernel_gs, ring3_naked_asm}; +use super::entry::{gate_iretq, gate_prefix, install_kernel_gs, ring3_naked_asm, GATE_PREFIX}; use super::cpu::{outb, io_wait}; use super::pio::{Port, PIC_PRIMARY, PIC_SECONDARY}; use super::percpu; @@ -107,6 +107,21 @@ impl IdtEntry { self.ist = ist_index; self } + + /// The same gate entered past its prefix (`entry::gate_prefix`), which + /// it is refused unless it opens with: the bytes of the `test` of the + /// frame's `CS`, the short `jz` over the call and the call. + fn past_prefix(self) -> Self { + let gate = self.target(); + // SAFETY: a gate's first bytes are kernel text, mapped readable. + let opens = unsafe { core::ptr::read(gate as *const [u8; GATE_PREFIX]) }; + assert!( + matches!(opens, [0xF6, 0x44, 0x24, _, 0x03, 0x74, 0x05, 0xE8, ..]), + "idt: the gate at {gate:#x} opens with {opens:02x?}, not the branch history prefix" + ); + let past = gate + GATE_PREFIX as u64; + Self { offset_low: past as u16, offset_mid: (past >> 16) as u16, offset_high: (past >> 32) as u32, ..self } + } } #[repr(C, align(16))] @@ -114,7 +129,14 @@ struct Idt { entries: [IdtEntry; 256], } -static IDT: Lock = Lock::new(Idt { +/// Every gate entered at its prefix: the table a CPU whose decision names the +/// branch history clear loads. +static AT_PREFIX: Lock = Lock::new(Idt { + entries: [IdtEntry::EMPTY; 256], +}); + +/// Every gate entered past its prefix: every other CPU's table. +static PAST_PREFIX: Lock = Lock::new(Idt { entries: [IdtEntry::EMPTY; 256], }); @@ -165,11 +187,14 @@ macro_rules! exception_stub { #[unsafe(naked)] extern "sysv64" fn $stub() { naked_asm!( + gate_prefix!(), "push 0", "push {vector}", "jmp {common}", vector = const Vector::$variant as usize, common = sym $common, + gate_cs = const 8, + clear_branch_history = sym super::mitigation::clear_branch_history, ); } }; @@ -177,10 +202,13 @@ macro_rules! exception_stub { #[unsafe(naked)] extern "sysv64" fn $stub() { naked_asm!( + gate_prefix!(), "push {vector}", "jmp {common}", vector = const Vector::$variant as usize, common = sym $common, + gate_cs = const 16, + clear_branch_history = sym super::mitigation::clear_branch_history, ); } }; @@ -301,17 +329,29 @@ const _: () = { } }; -/// Whether `rip` is where some gate of this CPU's table enters. +/// Whether `rip` is where some gate of either table enters. fn is_gate_target(rip: u64) -> bool { - // SAFETY: the table is written only by `init`, before any gate is taken. - let idt = unsafe { &*IDT.data_ptr() }; - idt.entries.iter().any(|gate| gate.type_attr != 0 && gate.target() == rip) + // SAFETY: the tables are written only by `init`, before any gate is taken. + let tables = unsafe { [&*AT_PREFIX.data_ptr(), &*PAST_PREFIX.data_ptr()] }; + tables.iter().flat_map(|idt| &idt.entries).any(|gate| gate.type_attr != 0 && gate.target() == rip) +} + +/// Whether `rip` is where either table enters the gate at `gate`. +fn is_gate_of(rip: u64, gate: u64) -> bool { + rip == gate || rip == gate + GATE_PREFIX as u64 } /// Halt IPI from `halt_all_cpus()`; never returns. #[unsafe(naked)] extern "sysv64" fn stub_halt_all() { - naked_asm!("cli", "2: hlt", "jmp 2b"); + naked_asm!( + gate_prefix!(), + "cli", + "2: hlt", + "jmp 2b", + gate_cs = const 8, + clear_branch_history = sym super::mitigation::clear_branch_history, + ); } /// Every exception vector's second half: the [`TrapFrame`], 22 words below a @@ -449,41 +489,37 @@ fn disable_pic() { } } -pub fn init() { +/// Builds both tables and loads the one `clears_branch_history` names: the +/// boot CPU's. +pub fn init(clears_branch_history: bool) { disable_pic(); - install_gates(&mut IDT.lock()); + install_gates(&mut AT_PREFIX.lock()); // `INT3` from Ring 3 arrives as the `#BP` it is, and ends its process as a // trap, as `BRK` does on AArch64. { - let mut idt = IDT.lock(); + let mut idt = AT_PREFIX.lock(); let breakpoint = &mut idt.entries[Vector::Breakpoint as usize]; *breakpoint = breakpoint.reachable_from_ring3(); } - user_dev::install(&mut IDT.lock()); + user_dev::install(&mut AT_PREFIX.lock()); // Every slot no row filled: delivery through a P = 0 gate is a // contributory fault, and the machine would halt as #DF with no name. let mut unclaimed = 0u32; - for entry in IDT.lock().entries.iter_mut() { + for entry in AT_PREFIX.lock().entries.iter_mut() { if entry.type_attr == 0 { *entry = IdtEntry::at(unclaimed::unclaimed_entry); unclaimed += 1; } } - let ptr = IdtPointer { - limit: (core::mem::size_of::() - 1) as u16, - base: IDT.data_ptr() as u64, - }; - - // SAFETY: `ptr` is a valid IDT descriptor built from `size_of::()` and the just-filled static IDT. - unsafe { - cpu::lidt(&ptr as *const IdtPointer as *const u8); - } + let gates = AT_PREFIX.lock().entries; + PAST_PREFIX.lock().entries = gates.map(IdtEntry::past_prefix); + let ptr = load(clears_branch_history); // The table this CPU is now loaded with: how many vectors carry a handler // this kernel named, and how many carry only the unclaimed stub, which is // what a machine delivering an interrupt nobody declared would land on. - let entries = IDT.lock().entries.len() as u32; + let entries = AT_PREFIX.lock().entries.len() as u32; // Copied out first: `IdtPointer` is packed, so a formatting argument would // be a reference to an unaligned field. let (base, limit) = (ptr.base, ptr.limit); @@ -497,6 +533,17 @@ pub fn init() { ); } +/// Loads on this CPU the table whose gates open with the branch history +/// clear where `clears_branch_history`, and the one past it elsewhere: what +/// this CPU's decision names. After [`init`]. +pub fn load(clears_branch_history: bool) -> IdtPointer { + let table = if clears_branch_history { &AT_PREFIX } else { &PAST_PREFIX }; + let ptr = IdtPointer { limit: (core::mem::size_of::() - 1) as u16, base: table.data_ptr() as u64 }; + // SAFETY: `ptr` names one of the two static tables `init` filled whole. + unsafe { cpu::lidt(&ptr as *const IdtPointer as *const u8) }; + ptr +} + /// Take IF=1 on this CPU; split from `init` so `ioapic::init` can mask firmware-left entries before interrupts are live. pub fn enable_interrupts() { cpu::enable_interrupts(); diff --git a/kernel/src/arch/x86_64/idt/nmi.rs b/kernel/src/arch/x86_64/idt/nmi.rs index 9fb53054b7..bcb8cdac3a 100644 --- a/kernel/src/arch/x86_64/idt/nmi.rs +++ b/kernel/src/arch/x86_64/idt/nmi.rs @@ -12,7 +12,7 @@ use core::arch::naked_asm; -use crate::arch::entry::install_kernel_gs; +use crate::arch::entry::{gate_prefix, install_kernel_gs}; use crate::arch::percpu::{MSR_GS_BASE, OFF_GDT, OFF_NMI_ACTIVE}; /// The words the entry pushes before it can tell a nested NMI: three @@ -31,6 +31,7 @@ const RFLAGS_OFFSET: usize = 8 * (FIRST_PUSHES + 7) + 16; #[unsafe(naked)] pub(super) extern "sysv64" fn nmi_entry() { naked_asm!( + gate_prefix!(), // Not routed through `arch::entry::ring3_naked_asm`: cld must run here, since `note`'s sysv64 call requires DF clear. "cld", // And from Ring 3 it arrives with that thread's `AC`, as every gate does. @@ -95,6 +96,8 @@ pub(super) extern "sysv64" fn nmi_entry() { nested_rsp = const NESTED_RSP_OFFSET, note = sym note, nested = sym nested_nmi, + gate_cs = const 8, + clear_branch_history = sym crate::arch::mitigation::clear_branch_history, ); } diff --git a/kernel/src/arch/x86_64/idt/spurious.rs b/kernel/src/arch/x86_64/idt/spurious.rs index 65adafb4d8..c70144b347 100644 --- a/kernel/src/arch/x86_64/idt/spurious.rs +++ b/kernel/src/arch/x86_64/idt/spurious.rs @@ -15,7 +15,7 @@ pub const SPURIOUS_VECTOR: u8 = 0xFF; #[unsafe(naked)] pub(super) extern "sysv64" fn spurious_entry() { - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; "push rax", "push rcx", "push rdx", diff --git a/kernel/src/arch/x86_64/idt/timer.rs b/kernel/src/arch/x86_64/idt/timer.rs index dffecac093..5b082bc325 100644 --- a/kernel/src/arch/x86_64/idt/timer.rs +++ b/kernel/src/arch/x86_64/idt/timer.rs @@ -8,7 +8,7 @@ use crate::hw::HW; #[unsafe(naked)] pub(super) extern "sysv64" fn timer_entry() { // No error code for interrupts. CS is at [rsp + 8]. - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; "test dword ptr [rsp + 8], 3", "jz 2f", diff --git a/kernel/src/arch/x86_64/idt/tlb.rs b/kernel/src/arch/x86_64/idt/tlb.rs index ace06db7ad..54804f7ea0 100644 --- a/kernel/src/arch/x86_64/idt/tlb.rs +++ b/kernel/src/arch/x86_64/idt/tlb.rs @@ -3,7 +3,7 @@ use crate::arch::entry::{gate_iretq, ring3_naked_asm}; /// IDT entry point for the TLB-shootdown IPI vector. #[unsafe(naked)] pub(super) extern "sysv64" fn tlb_flush_entry() { - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; // Every pushed register must be popped before `iretq`: `flush` may clobber any System V scratch register. "push rax", "push rcx", diff --git a/kernel/src/arch/x86_64/idt/unclaimed.rs b/kernel/src/arch/x86_64/idt/unclaimed.rs index 13e7e1d403..30bf21ab07 100644 --- a/kernel/src/arch/x86_64/idt/unclaimed.rs +++ b/kernel/src/arch/x86_64/idt/unclaimed.rs @@ -21,7 +21,7 @@ static NO_ISR: AtomicU64 = AtomicU64::new(0); #[unsafe(naked)] pub(super) extern "sysv64" fn unclaimed_entry() { - ring3_naked_asm!(cs = 8; + ring3_naked_asm!(gate cs = 8; "push rax", "push rcx", "push rdx", diff --git a/kernel/src/arch/x86_64/idt/user_dev.rs b/kernel/src/arch/x86_64/idt/user_dev.rs index ceea39e8e6..1201931676 100644 --- a/kernel/src/arch/x86_64/idt/user_dev.rs +++ b/kernel/src/arch/x86_64/idt/user_dev.rs @@ -22,8 +22,8 @@ use super::{Idt, IdtEntry}; const FIRST: u8 = 0x40; const COUNT: usize = 0xFC - FIRST as usize; -/// Each stub's bytes: a push and a jump fit in 10. -const STRIDE: usize = 16; +/// Each stub's bytes: the gate prefix, a push and a jump fit in 22. +const STRIDE: usize = 32; /// The `pcidev` function each vector is held for. static HELD: Owners = Owners::new(); @@ -65,14 +65,19 @@ pub(super) fn install(idt: &mut Idt) { } } -/// `COUNT` stubs, `STRIDE` apart from the first `STRIDE` boundary: each pushes -/// its number and jumps to [`claim_entry`]. Never called. +/// `COUNT` stubs, `STRIDE` apart from the first `STRIDE` boundary: each opens +/// with the gate prefix (`arch::entry::gate_prefix`, here in AT&T syntax), +/// pushes its number and jumps to [`claim_entry`]. Never called. #[unsafe(naked)] extern "sysv64" fn stubs() { core::arch::naked_asm!( ".set toyos_claim_vector, 0", ".rept {count}", ".balign {stride}", + "testb $3, 8(%rsp)", + "jz 78f", + "call {clear_branch_history}", + "78:", "pushq $toyos_claim_vector", "jmp {entry}", ".set toyos_claim_vector, toyos_claim_vector + 1", @@ -80,6 +85,7 @@ extern "sysv64" fn stubs() { count = const COUNT, stride = const STRIDE, entry = sym claim_entry, + clear_branch_history = sym crate::arch::mitigation::clear_branch_history, options(att_syntax), ); } diff --git a/kernel/src/arch/x86_64/mitigation.rs b/kernel/src/arch/x86_64/mitigation.rs index a9fd47ab4d..70ed957621 100644 --- a/kernel/src/arch/x86_64/mitigation.rs +++ b/kernel/src/arch/x86_64/mitigation.rs @@ -9,7 +9,9 @@ //! opt-outs and AMD's LFENCE bit. The boot CPU runs it before the allocator //! is up, so what it returns and where the boot CPU keeps it hold no heap. //! [`switch_to`] applies at each switch what the decision names for a program -//! that asked for isolated speculation. +//! that asked for isolated speculation. Where the decision names the branch +//! history clear, this CPU's `LSTAR` and IDT enter through +//! [`clear_branch_history`] (`arch::syscall`, `arch::idt`). use core::fmt; use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::Relaxed}; @@ -155,6 +157,20 @@ pub fn record_boot_cpu(read: Read) { *BOOT_CPU.lock() = Some(read); } +impl Read { + /// Whether this CPU's entries from Ring 3 clear branch history: what its + /// decision names, which its core's other threads do not change. + pub fn clears_branch_history(&self) -> bool { + decide(&self.facts).enumerates(Mitigation::BhbClear) + } +} + +/// [`Read::clears_branch_history`] for the boot CPU, once [`record_boot_cpu`] +/// has kept its words. +pub fn boot_cpu_clears_branch_history() -> bool { + BOOT_CPU.lock().expect("mitigation: the boot CPU's words are read before its entries are installed").clears_branch_history() +} + /// Keeps a later CPU's [`apply`]. pub fn record(cpu_id: u32, read: Read) { *LATER[cpu_id as usize].lock() = Some(read); @@ -183,6 +199,8 @@ pub fn install() { for (cpu_id, read) in online() { let local = &crate::cpu_local::of(cpu_id).arch.mitigation; local.install(read.registers.spec_ctrl, decide(&read.facts).on_request()); + #[cfg(feature = "test-actuators")] + local.clears_branch_history.store(read.clears_branch_history(), Relaxed); } } @@ -280,6 +298,9 @@ pub struct Local { /// Whether this CPU has an `IA32_SPEC_CTRL` to read. #[cfg(feature = "test-actuators")] has_spec_ctrl: AtomicBool, + /// Whether this CPU's entries clear branch history. + #[cfg(feature = "test-actuators")] + clears_branch_history: AtomicBool, } impl Local { @@ -297,6 +318,8 @@ impl Local { ibpbs: AtomicU64::new(0), #[cfg(feature = "test-actuators")] has_spec_ctrl: AtomicBool::new(false), + #[cfg(feature = "test-actuators")] + clears_branch_history: AtomicBool::new(false), } } @@ -400,3 +423,62 @@ pub fn debug_switch_ibpbs() -> Option { let local = &crate::cpu_local::here().arch.mitigation; local.ibpb.load(Relaxed).then(|| local.ibpbs.load(Relaxed)) } + +/// Intel's short branch history clearing sequence ("Branch History +/// Injection and Intra-mode Branch Target Injection", Listing 1, "sufficient +/// to clear the BHB on all processors prior to Alder Lake"), between a save +/// and a restore of the two registers it writes: callable from the first +/// instruction of a gate, before anything of the interrupted context is +/// saved. Its own `call`s use the stack it is called on; it reads nothing +/// through GS and clobbers the flags, which no caller holds across it. Each +/// of its returns goes back to a call it made, at most seven deep, so the RSB +/// holds its target: Intel's "Indirect Target Selection" reaches a return, +/// wherever it lies in its line, only through the alternate predictors an +/// empty RSB falls back on. +#[unsafe(naked)] +pub(crate) extern "sysv64" fn clear_branch_history() { + core::arch::naked_asm!( + "push rax", + "push rcx", + #[cfg(feature = "test-actuators")] + "lock inc qword ptr [rip + {clears}]", + "mov ecx, 5", + "call 2f", + "jmp 6f", + ".balign 64", + "2:", + "call 3f", + "ret", + ".balign 64", + "3:", + "mov eax, 5", + "4:", + "jmp 5f", + "nop", + "5:", + "sub eax, 1", + "jnz 4b", + "sub ecx, 1", + "jnz 2b", + "ret", + "6:", + "lfence", + "pop rcx", + "pop rax", + "ret", + #[cfg(feature = "test-actuators")] + clears = sym CLEARS, + ); +} + +/// Clears [`clear_branch_history`] has run, on every CPU. +#[cfg(feature = "test-actuators")] +static CLEARS: AtomicU64 = AtomicU64::new(0); + +/// `SYS_DEBUG`'s `BRANCH_HISTORY_CLEARS`: the clears every CPU's entries have +/// run, where this CPU's decision names the clear. +#[cfg(feature = "test-actuators")] +pub fn debug_branch_history_clears() -> Option { + let local = &crate::cpu_local::here().arch.mitigation; + local.clears_branch_history.load(Relaxed).then(|| CLEARS.load(Relaxed)) +} diff --git a/kernel/src/arch/x86_64/percpu.rs b/kernel/src/arch/x86_64/percpu.rs index b9d8423144..261f38ad02 100644 --- a/kernel/src/arch/x86_64/percpu.rs +++ b/kernel/src/arch/x86_64/percpu.rs @@ -595,7 +595,7 @@ pub fn init_bsp(lapic_id: u32) { // loaded table is the one firmware left across `ExitBootServices`, whose // handlers report on no channel of this kernel's — so a fault in `fpu` // below would stop the machine with the panel holding the record before it. - super::idt::init(); + super::idt::init(super::mitigation::boot_cpu_clears_branch_history()); super::control_regs::init_performance(0); super::fpu::init(0); diff --git a/kernel/src/arch/x86_64/smp.rs b/kernel/src/arch/x86_64/smp.rs index d72c3d856b..9a84b1a1fb 100644 --- a/kernel/src/arch/x86_64/smp.rs +++ b/kernel/src/arch/x86_64/smp.rs @@ -269,7 +269,9 @@ extern "C" fn ap_entry() -> ! { // GS base was set by the trampoline; finish percpu init (GDT, CR4). percpu::init_ap(percpu::percpu_ptr()); - syscall::init(); + // The trampoline loaded the boot CPU's table; this CPU's decision names its own. + crate::arch::idt::load(mitigation.clears_branch_history()); + syscall::init(mitigation.clears_branch_history()); // Calibration is a one-time BSP measurement; nothing left for an AP to do here. apic::init_ap(); super::clocksource::start_ap(percpu::cpu_id()); diff --git a/kernel/src/arch/x86_64/syscall.rs b/kernel/src/arch/x86_64/syscall.rs index 78e3375751..caea47cd0f 100644 --- a/kernel/src/arch/x86_64/syscall.rs +++ b/kernel/src/arch/x86_64/syscall.rs @@ -14,13 +14,18 @@ const MSR_LSTAR: u32 = 0xC000_0082; const MSR_FMASK: u32 = 0xC000_0084; /// `EFER.SCE` is applied and asserted on this CPU before `init` is called, on both the BSP's path and an AP's. -pub fn init() { +/// `LSTAR` names the entry that clears branch history where +/// `clears_branch_history`, what this CPU's decision names, and the other +/// elsewhere, which runs none of it. +pub fn init(clears_branch_history: bool) { let star = ((percpu::STAR_SYSRET_BASE as u64) << 48) | ((percpu::KERNEL_CS as u64) << 32); // SAFETY: this function owns all three `SYSCALL` MSRs. // All three `wrmsr` writes stay in one `unsafe` block because they are one declaration: a CPU holding only some of them has its `SYSCALL` gate aimed by something this file did not decide, and no point between the writes is a state the machine may be left in. unsafe { cpu::wrmsr(MSR_STAR, star); - cpu::wrmsr(MSR_LSTAR, syscall_entry as *const () as u64); + let entry: extern "sysv64" fn() = + if clears_branch_history { syscall_entry_clearing_branch_history } else { syscall_entry }; + cpu::wrmsr(MSR_LSTAR, entry as *const () as u64); // `SYSCALL` clears exactly the bits named here; every bit left out carries a Ring 3 thread's flag into Ring 0. // The pre-mask `RFLAGS` survives in `r11` and `sysretq` restores it to the thread regardless — `FMASK` only decides what the kernel itself runs with. const TF: u64 = 1 << 8; @@ -40,64 +45,80 @@ pub fn init() { // `SYSCALL` switches no stack: before the `rsp` switch below runs, the CPU is at CPL 0 on the user's stack, so nothing that can fault may execute there. // The same holds from the exit's `swapgs` on, with the user's GS base live (`arch::entry`): `SYSRET` cannot fault, since no instruction ends at the user half's top for `rcx` to be non-canonical (`toyos_userbound`'s placement). -#[unsafe(naked)] -extern "sysv64" fn syscall_entry() { - syscall_naked_asm!( - "mov gs:[{user_rsp}], rsp", - "mov rsp, gs:[{kernel_rsp}]", - "mov gs:[{syscall_rip}], rcx", - "mov gs:[{syscall_num}], rdi", - "mov gs:[{syscall_rbp}], rbp", - "push gs:[{user_rsp}]", // user RSP on kernel stack - "push rcx", // return RIP - "push r11", // return RFLAGS - "push rdi", - "push rsi", - "push rdx", - "push r8", - "push r9", - "push r10", - // Nine words below a 16-byte-aligned top: the pad aligns `rsp` for - // both calls, and parks the return value across `exit_to_user`. - "sub rsp, 8", +/// One of `SYSCALL`'s two entries, which `LSTAR` names per CPU (`init`): the +/// second calls the branch history clear as soon as the kernel's stack is +/// under it and before any branch. +macro_rules! syscall_entry { + ($name:ident $(, $clear:literal, $operand:ident = $path:path)?) => { + #[unsafe(naked)] + extern "sysv64" fn $name() { + syscall_naked_asm!( + "mov gs:[{user_rsp}], rsp", + "mov rsp, gs:[{kernel_rsp}]", + $($clear,)? + "mov gs:[{syscall_rip}], rcx", + "mov gs:[{syscall_num}], rdi", + "mov gs:[{syscall_rbp}], rbp", + "push gs:[{user_rsp}]", // user RSP on kernel stack + "push rcx", // return RIP + "push r11", // return RFLAGS + "push rdi", + "push rsi", + "push rdx", + "push r8", + "push r9", + "push r10", + // Nine words below a 16-byte-aligned top: the pad aligns `rsp` for + // both calls, and parks the return value across `exit_to_user`. + "sub rsp, 8", - "add dword ptr gs:[{preempt_count}], 1", + "add dword ptr gs:[{preempt_count}], 1", - "call {handler}", + "call {handler}", - "sub dword ptr gs:[{preempt_count}], 1", - // The helper called before `pop rsp`/`sysretq` (`exit_to_user`) preserves `IF=0` across its return. - // Runs before GPR restore: the sysv64 call would otherwise clobber rcx/r11 (sysretq's RIP/RFLAGS) and the restored args. - "mov [rsp], rax", - "call {exit_to_user}", - "mov rax, [rsp]", - "add rsp, 8", + "sub dword ptr gs:[{preempt_count}], 1", + // The helper called before `pop rsp`/`sysretq` (`exit_to_user`) preserves `IF=0` across its return. + // Runs before GPR restore: the sysv64 call would otherwise clobber rcx/r11 (sysretq's RIP/RFLAGS) and the restored args. + "mov [rsp], rax", + "call {exit_to_user}", + "mov rax, [rsp]", + "add rsp, 8", - "pop r10", - "pop r9", - "pop r8", - "pop rdx", - "pop rsi", - "pop rdi", - "pop r11", - "pop rcx", - "pop rsp", // restore user RSP from kernel stack - "swapgs", - "sysretq", - handler = sym syscall_handler, - exit_to_user = sym crate::arch::idt::kernel_exit_to_user_check, - kernel_rsp = const percpu::OFF_KERNEL_RSP, - user_rsp = const percpu::OFF_USER_RSP, - syscall_rip = const percpu::OFF_SYSCALL_RIP, - syscall_num = const percpu::OFF_SYSCALL_NUM, - syscall_rbp = const percpu::OFF_SYSCALL_RBP, - preempt_count = const percpu::OFF_PREEMPT_COUNT, - ); + "pop r10", + "pop r9", + "pop r8", + "pop rdx", + "pop rsi", + "pop rdi", + "pop r11", + "pop rcx", + "pop rsp", // restore user RSP from kernel stack + "swapgs", + "sysretq", + handler = sym syscall_handler, + exit_to_user = sym crate::arch::idt::kernel_exit_to_user_check, + kernel_rsp = const percpu::OFF_KERNEL_RSP, + user_rsp = const percpu::OFF_USER_RSP, + syscall_rip = const percpu::OFF_SYSCALL_RIP, + syscall_num = const percpu::OFF_SYSCALL_NUM, + syscall_rbp = const percpu::OFF_SYSCALL_RBP, + preempt_count = const percpu::OFF_PREEMPT_COUNT, + $($operand = sym $path,)? + ); + } + }; } -/// Where `SYSCALL` enters, as `LSTAR` holds it. -pub(crate) fn entry_address() -> u64 { - syscall_entry as *const () as u64 +syscall_entry!(syscall_entry); +syscall_entry!( + syscall_entry_clearing_branch_history, + "call {clear_branch_history}", + clear_branch_history = crate::arch::mitigation::clear_branch_history +); + +/// Whether `rip` is where either entry begins. +pub(crate) fn is_entry(rip: u64) -> bool { + [syscall_entry as *const () as u64, syscall_entry_clearing_branch_history as *const () as u64].contains(&rip) } /// The syscall bracket: the entry's diagnostic stores stay readable only while [`percpu::in_syscall`] is true. diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 2f86bb32bd..dcfd3eb73f 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -458,6 +458,9 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> DA::SWITCH_IBPBS => { crate::arch::mitigation::debug_switch_ibpbs().unwrap_or(SyscallError::NotSupported.to_u64()) } + DA::BRANCH_HISTORY_CLEARS => { + crate::arch::mitigation::debug_branch_history_clears().unwrap_or(SyscallError::NotSupported.to_u64()) + } _ => SyscallError::InvalidArgument.to_u64(), }, SYS_JOB_FLOOR => sys_job_floor(RawHandle(a1 as u32), RawHandle(a2 as u32), a3), diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 07be5d2084..bd91696d29 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -1038,6 +1038,9 @@ pub mod debug_action { /// spaces one of which asked; `NotSupported` where its decision names /// no IBPB. pub const SWITCH_IBPBS: u64 = 32; + /// The branch history clears every CPU's entries from Ring 3 have run; + /// `NotSupported` where this CPU's decision names none. + pub const BRANCH_HISTORY_CLEARS: u64 = 33; } /// Create a pipe. Returns the read and write ends. diff --git a/tests/checks.rs b/tests/checks.rs index a90f280953..bc2fd2f6dc 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -562,7 +562,7 @@ mod checks { const T14_BENCH_LINES: [&str; 17] = [ "spectre_v1: mitigated: user pointers clamped without a branch", "spectre_v2: mitigated: SMEP, enhanced IBRS, IBPB between programs that ask", - "branch_history_injection: vulnerable: needs BHB cleared on entry", + "branch_history_injection: mitigated: BHB cleared on entry", "retbleed: not affected", "spec_rstack_overflow: not affected", "indirect_target_selection: vulnerable: needs indirect branches in a line's upper half", diff --git a/tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs b/tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs new file mode 100644 index 0000000000..b2876a7d2a --- /dev/null +++ b/tests/guest/rust-tests/src/bin/branch_history_cleared_on_each_entry.rs @@ -0,0 +1,49 @@ +//! Each entry from Ring 3 runs its CPU's branch history clear where the CPU's +//! decision names one, read off the kernel's count of clears through +//! `SYS_DEBUG`: at least one per syscall, through `LSTAR`'s entry, and one per +//! page fault on a fresh page, through the IDT's, since every other CPU's +//! entries only add to the count. On a CPU whose decision names none the +//! count is refused, and there is nothing to hold. + +use toyos_abi::syscall::{self, debug_action, SyscallError, VmAt, VmProt}; + +const SYSCALLS: u64 = 1000; +const PAGES: usize = 256; +const PAGE: usize = 4096; + +fn clears() -> u64 { + syscall::debug(debug_action::BRANCH_HISTORY_CLEARS) +} + +fn main() { + let cpu = syscall::current_cpu(); + let mut mask = [0u64; syscall::CPU_MASK_WORDS]; + mask[cpu as usize / 64] = 1 << (cpu % 64); + syscall::thread_set_affinity(0, &mask).expect("pinned to one CPU"); + let before = clears(); + if SyscallError::from_u64(before) == Some(SyscallError::NotSupported) { + println!("branch_history_cleared_on_each_entry: CPU {cpu}'s decision names no branch history clear"); + return; + } + for _ in 0..SYSCALLS { + syscall::current_cpu(); + } + let by_syscalls = clears() - before; + assert!(by_syscalls >= SYSCALLS, "CPU {cpu} cleared branch history {by_syscalls} times over {SYSCALLS} syscalls"); + + // SAFETY: a fresh range nothing else names; committed, so each page is + // populated by its first touch below, a Ring 3 page fault. + let region = unsafe { syscall::vm_reserve(PAGES * PAGE, VmProt::ReadWrite, VmAt::ANYWHERE) }.expect("reserve"); + let before = clears(); + for page in 0..PAGES { + // SAFETY: inside the range just reserved, which nothing else names. + unsafe { region.add(page * PAGE).write_volatile(1) }; + } + let by_faults = clears() - before; + // SAFETY: the range reserved above, which nothing names any more. + unsafe { syscall::vm_release(region, PAGES * PAGE) }.expect("release"); + assert!(by_faults >= PAGES as u64, "CPU {cpu} cleared branch history {by_faults} times over {PAGES} page faults"); + println!( + "branch_history_cleared_on_each_entry: {by_syscalls} clears over {SYSCALLS} syscalls and {by_faults} over {PAGES} page faults on CPU {cpu}" + ); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 75c793e103..98e51966b5 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -71,6 +71,9 @@ const ACTUATOR_TESTS: &[&str] = &[ // SSBD: both benches and the KVM guest runner; under TCG's `qemu64`, which // enumerates neither, it says so and holds nothing. "speculation_on_request", + // Action 33: the branch history clears the entries ran, which leave + // nothing a guest can read. + "branch_history_cleared_on_each_entry", ]; /// What [`ACTUATOR_TESTS`] boots: the one kernel that carries `SYS_DEBUG`.